chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,2 @@
|
||||
index.html -linguist-detectable
|
||||
assets/** linguist-vendored
|
||||
@@ -0,0 +1,49 @@
|
||||
name: "🐛 Bug Report"
|
||||
description: Submit a bug report to help us improve transformers.js
|
||||
labels: ["bug"]
|
||||
body:
|
||||
- type: textarea
|
||||
id: system-info
|
||||
attributes:
|
||||
label: System Info
|
||||
description: Please share your system info with us. If you are using other JS libraries/frameworks (e.g., React or Next.js), please include their versions too.
|
||||
placeholder: transformers.js version, browser (if applicable), operating system, Node.js version, bundlers, ...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: checkboxes
|
||||
id: environment-or-platform
|
||||
attributes:
|
||||
label: Environment/Platform
|
||||
description: "The environment I am running in, or platform I am developing for:"
|
||||
options:
|
||||
- label: "Website/web-app"
|
||||
- label: "Browser extension"
|
||||
- label: "Server-side (e.g., Node.js, Deno, Bun)"
|
||||
- label: "Desktop app (e.g., Electron)"
|
||||
- label: "Other (e.g., VSCode extension)"
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Description
|
||||
description: A clear and concise description of the bug, as well as what you expected to happen.
|
||||
|
||||
- type: textarea
|
||||
id: reproduction
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Reproduction
|
||||
description: |
|
||||
Please provide a code sample that reproduces the problem you ran into. If you have code snippets, error messages, or stack traces, please provide them here as well.
|
||||
Important! Use [code tags](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#syntax-highlighting) to correctly format your code.
|
||||
|
||||
placeholder: |
|
||||
Steps to reproduce the behavior:
|
||||
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
@@ -0,0 +1,40 @@
|
||||
name: "🌟 New model addition"
|
||||
description: Submit a proposal/request to implement a new model
|
||||
labels: ["new model"]
|
||||
|
||||
body:
|
||||
- type: textarea
|
||||
id: description-request
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Model description
|
||||
description: |
|
||||
Include important information about the model.
|
||||
|
||||
- type: checkboxes
|
||||
id: information-tasks
|
||||
attributes:
|
||||
label: Prerequisites
|
||||
description: |
|
||||
Please note that Transformers.js relies on the model first being supported in [🤗 Transformers](https://github.com/huggingface/transformers) and [🤗 Optimum](https://github.com/huggingface/optimum). If the model you are requesting is not yet supported by either of them, feel free to open up a model request there too.
|
||||
options:
|
||||
- label: "The model is supported in Transformers (i.e., listed [here](https://huggingface.co/docs/transformers/index#supported-models-and-frameworks))"
|
||||
- label: "The model can be exported to ONNX with Optimum (i.e., listed [here](https://huggingface.co/docs/optimum/main/en/exporters/onnx/overview))"
|
||||
|
||||
- type: textarea
|
||||
id: additional-info
|
||||
attributes:
|
||||
label: Additional information
|
||||
description: |
|
||||
Please provide additional information about the model here.
|
||||
If the model is already supported in Transformers, you can provide example Python code to help ensure the JavaScript implementation (and output) matches the original version.
|
||||
|
||||
- type: textarea
|
||||
id: contribution
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Your contribution
|
||||
description: |
|
||||
Is there any way that you could help, e.g. by submitting a PR?
|
||||
@@ -0,0 +1,39 @@
|
||||
name: "🔧 New pipeline addition"
|
||||
description: Submit a proposal/request to implement a new pipeline
|
||||
labels: ["new pipeline"]
|
||||
|
||||
body:
|
||||
- type: textarea
|
||||
id: description-request
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Pipeline description
|
||||
description: |
|
||||
Include important information about the pipeline.
|
||||
|
||||
- type: checkboxes
|
||||
id: information-tasks
|
||||
attributes:
|
||||
label: Prerequisites
|
||||
description: |
|
||||
Please note that Transformers.js relies on the pipeline first being supported in [🤗 Transformers](https://github.com/huggingface/transformers). If the pipeline you are requesting is not yet supported by Transformers, feel free to open up a feature request for it there too.
|
||||
options:
|
||||
- label: "The pipeline is supported in Transformers (i.e., listed [here](https://huggingface.co/docs/transformers/main_classes/pipelines))"
|
||||
- label: "The task is listed [here](https://huggingface.co/tasks)"
|
||||
|
||||
- type: textarea
|
||||
id: additional-info
|
||||
attributes:
|
||||
label: Additional information
|
||||
description: |
|
||||
Please provide additional information about the pipeline here. If the pipeline is already supported in Transformers, you can provide example Python code to help ensure the JavaScript implementation (and output) matches the original version.
|
||||
|
||||
- type: textarea
|
||||
id: contribution
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Your contribution
|
||||
description: |
|
||||
Is there any way that you could help, e.g. by submitting a PR?
|
||||
@@ -0,0 +1,30 @@
|
||||
name: "🚀 Feature request"
|
||||
description: Submit a proposal/request for a new transformers.js feature
|
||||
labels: ["enhancement"]
|
||||
body:
|
||||
- type: textarea
|
||||
id: feature-request
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Feature request
|
||||
description: |
|
||||
A clear and concise description of the feature proposal. If the feature is already part of the [Transformers](https://github.com/huggingface/transformers) library, please provide relevant links or example usage.
|
||||
|
||||
- type: textarea
|
||||
id: motivation
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Motivation
|
||||
description: |
|
||||
Please outline the motivation for the proposal. Why is it important that we add this feature? What is your intended use case?
|
||||
|
||||
- type: textarea
|
||||
id: contribution
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Your contribution
|
||||
description: |
|
||||
Is there any way that you could help, e.g. by submitting a PR?
|
||||
@@ -0,0 +1,13 @@
|
||||
name: "🙋 Question"
|
||||
description: Ask a question about the library
|
||||
labels: ["question"]
|
||||
|
||||
body:
|
||||
- type: textarea
|
||||
id: question
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Question
|
||||
description: |
|
||||
Please enter your question here...
|
||||
@@ -0,0 +1,9 @@
|
||||
blank_issues_enabled: true
|
||||
version: 2.1
|
||||
contact_links:
|
||||
- name: Models on the Hugging Face Hub
|
||||
url: https://huggingface.co/models?library=transformers.js
|
||||
about: Open a Pull request / Discussion related to a specific model checkpoint directly on the Hugging Face Hub
|
||||
- name: Documentation
|
||||
url: https://huggingface.co/docs/transformers.js
|
||||
about: View the transformers.js documentation
|
||||
@@ -0,0 +1,11 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
groups:
|
||||
actions:
|
||||
patterns: ["*"]
|
||||
@@ -0,0 +1,19 @@
|
||||
name: Build documentation
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
build:
|
||||
uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@90b4ee2c10b81b5c1a6367c4e6fc9e2fb510a7e3 # main
|
||||
with:
|
||||
commit_sha: ${{ github.sha }}
|
||||
package: transformers.js
|
||||
path_to_docs: transformers.js/packages/transformers/docs/source
|
||||
pre_command: cd transformers.js && corepack enable && ONNXRUNTIME_NODE_INSTALL=skip pnpm install --frozen-lockfile && pnpm --filter @huggingface/transformers docs-api
|
||||
additional_args: --not_python_module
|
||||
secrets:
|
||||
hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }}
|
||||
@@ -0,0 +1,19 @@
|
||||
name: Build PR Documentation
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@90b4ee2c10b81b5c1a6367c4e6fc9e2fb510a7e3 # main
|
||||
with:
|
||||
commit_sha: ${{ github.sha }}
|
||||
pr_number: ${{ github.event.number }}
|
||||
package: transformers.js
|
||||
path_to_docs: transformers.js/packages/transformers/docs/source
|
||||
pre_command: cd transformers.js && corepack enable && ONNXRUNTIME_NODE_INSTALL=skip pnpm install --frozen-lockfile && pnpm --filter @huggingface/transformers docs-api
|
||||
additional_args: --not_python_module
|
||||
@@ -0,0 +1,28 @@
|
||||
name: Publish Package to npmjs
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
env:
|
||||
ONNXRUNTIME_NODE_INSTALL: skip
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
|
||||
# Setup .npmrc file to publish to npm
|
||||
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
|
||||
with:
|
||||
node-version: "24.10.0"
|
||||
registry-url: "https://registry.npmjs.org"
|
||||
cache: "pnpm"
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- run: pnpm build
|
||||
- name: Publish to npm
|
||||
run: pnpm --filter @huggingface/transformers publish --access public --no-git-checks
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
@@ -0,0 +1,45 @@
|
||||
name: Unit tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
types:
|
||||
- opened
|
||||
- reopened
|
||||
- synchronize
|
||||
- ready_for_review
|
||||
|
||||
env:
|
||||
TESTING_REMOTELY: true
|
||||
ONNXRUNTIME_NODE_INSTALL: skip
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: github.event.pull_request.draft == false
|
||||
runs-on:
|
||||
group: aws-general-8-plus
|
||||
strategy:
|
||||
matrix:
|
||||
node-version: [18, 20, 22]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
|
||||
- name: Use Node.js ${{ matrix.node-version }}
|
||||
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
|
||||
with:
|
||||
node-version: ${{ matrix.node-version }}
|
||||
cache: "pnpm"
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- run: pnpm build
|
||||
|
||||
# Setup the testing environment
|
||||
- run: git lfs install && GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/hf-internal-testing/tiny-random-T5ForConditionalGeneration ./packages/transformers/models/hf-internal-testing/tiny-random-T5ForConditionalGeneration
|
||||
|
||||
# Actually run tests
|
||||
- run: pnpm test
|
||||
@@ -0,0 +1,16 @@
|
||||
name: Upload PR Documentation
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["Build PR Documentation"]
|
||||
types:
|
||||
- completed
|
||||
|
||||
jobs:
|
||||
build:
|
||||
uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@9ad2de8582b56c017cb530c1165116d40433f1c6 # main
|
||||
with:
|
||||
package_name: transformers.js
|
||||
secrets:
|
||||
hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }}
|
||||
comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
__pycache__
|
||||
.vscode
|
||||
.cache
|
||||
.DS_STORE
|
||||
.idea
|
||||
node_modules
|
||||
deno.lock
|
||||
package-lock.json
|
||||
|
||||
# Do not track build artifacts/generated files
|
||||
packages/*/dist
|
||||
packages/*/types
|
||||
*.tsbuildinfo
|
||||
|
||||
# Do not track coverage reports
|
||||
packages/*/coverage
|
||||
@@ -0,0 +1,17 @@
|
||||
# Ignore artifacts and non-source files
|
||||
.github
|
||||
*.py
|
||||
*.md
|
||||
|
||||
# Build outputs & caches
|
||||
**/dist
|
||||
**/types
|
||||
**/node_modules
|
||||
**/coverage
|
||||
**/.cache
|
||||
**/docs/build
|
||||
*.tsbuildinfo
|
||||
|
||||
# Lock files
|
||||
pnpm-lock.yaml
|
||||
|
||||
+367
@@ -0,0 +1,367 @@
|
||||
<!---
|
||||
Copyright 2020 The HuggingFace Team. All rights reserved.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
|
||||
# Contribute to 🤗 Transformers.js
|
||||
|
||||
Everyone is welcome to contribute, and we value everybody's contribution. Code
|
||||
contributions are not the only way to help the community. Answering questions, helping
|
||||
others, and improving the documentation are also immensely valuable.
|
||||
|
||||
It also helps us if you spread the word! Reference the library in blog posts
|
||||
about the awesome projects it made possible, shout out on Twitter every time it has
|
||||
helped you, or simply ⭐️ the repository to say thank you.
|
||||
|
||||
**This guide was heavily inspired by the awesome [scikit-learn guide to contributing](https://github.com/scikit-learn/scikit-learn/blob/main/CONTRIBUTING.md) and our friends at [transformers](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md).**
|
||||
|
||||
## Ways to contribute
|
||||
|
||||
There are several ways you can contribute to 🤗 Transformers.js:
|
||||
|
||||
* Fix outstanding issues with the existing code.
|
||||
* Submit issues related to bugs or desired new features.
|
||||
* Implement new models.
|
||||
* Contribute to the examples or to the documentation.
|
||||
|
||||
## Fixing outstanding issues
|
||||
|
||||
If you notice an issue with the existing code and have a fix in mind, feel free to [start contributing](#create-a-pull-request) and open a Pull Request!
|
||||
|
||||
## Submitting a bug-related issue or feature request
|
||||
|
||||
Do your best to follow these guidelines when submitting a bug-related issue or a feature
|
||||
request. It will make it easier for us to come back to you quickly and with good
|
||||
feedback.
|
||||
|
||||
### Did you find a bug?
|
||||
|
||||
The 🤗 Transformers.js library is robust and reliable thanks to users who report the problems they encounter.
|
||||
|
||||
Before you report an issue, we would really appreciate it if you could **make sure the bug was not
|
||||
already reported** (use the search bar on GitHub under Issues). Your issue should also be related to bugs in the library itself, and not your code.
|
||||
|
||||
To create a new issue, please [use one of the templates](https://github.com/huggingface/transformers.js/issues/new/choose) we prepared for you. Most likely the [Bug Report](https://github.com/huggingface/transformers.js/issues/new?template=1_bug-report.yml).
|
||||
|
||||
### Do you want a new feature?
|
||||
|
||||
If there is a new feature you'd like to see in 🤗 Transformers.js, please open an issue and describe:
|
||||
|
||||
1. What is the *motivation* behind this feature? Is it related to a problem or frustration with the library? Is it a feature related to something you need for a project? Is it something you worked on and think it could benefit the community? Whatever it is, we'd love to hear about it!
|
||||
2. Describe your requested feature in as much detail as possible. The more you can tell us about it, the better we'll be able to help you.
|
||||
3. Provide a *code snippet* that demonstrates the feature's usage.
|
||||
4. If the feature is related to a paper, please include a link.
|
||||
|
||||
If your issue is well written we're already 80% of the way there by the time you create it.
|
||||
|
||||
We have added [a template](https://github.com/huggingface/transformers.js/issues/new?template=4_feature-request.yml) to help you get started with your issue.
|
||||
|
||||
## Do you want to implement a new model?
|
||||
|
||||
New models are constantly released and if you want to request support for a new model, please use the [template for new model requests](https://github.com/huggingface/transformers.js/issues/new?template=2_new_model.yml).
|
||||
|
||||
If you are willing to contribute the model yourself, let us know so we can help you add it to 🤗 Transformers.js! The process of adding support for a new model architecture has three main phases: **exporting the model to ONNX**, then **wiring it into the library**, and finally **adding tests**.
|
||||
|
||||
### 1. Export the Model to ONNX
|
||||
|
||||
Transformers.js runs models using ONNX Runtime. Before adding a model to the library, you need an ONNX export of it.
|
||||
|
||||
- For LLMs, we recommend exporting with [microsoft/onnxruntime-genai](https://github.com/microsoft/onnxruntime-genai)
|
||||
- For other models, we recommend exporting with [huggingface/optimum-onnx](https://github.com/huggingface/optimum-onnx)
|
||||
|
||||
Once exported, upload the ONNX files to the Hugging Face Hub alongside the model's original config, tokenizer, and other preprocessing files so they can be loaded with `from_pretrained`.
|
||||
|
||||
---
|
||||
|
||||
### 2. Add the Model to the Library
|
||||
|
||||
Every model in Transformers.js is built from the same pieces:
|
||||
|
||||
- **A model class**: extends `PreTrainedModel`, which handles all ONNX inference, generation, and KV-cache management
|
||||
- **Task head classes**: thin wrappers that wrap the output in the right output object (e.g. `MaskedLMOutput`)
|
||||
- **A tokenizer and/or processor**: only needed if the model requires a custom one; most models reuse an existing class
|
||||
|
||||
All model files live under `packages/transformers/src/models/<model_type>/`. Look at an existing model of the same type to understand what's needed; most are just a few lines.
|
||||
|
||||
#### Model class
|
||||
|
||||
Every model file exports a base class and one or more task heads. For the vast majority of models, these are empty subclasses. All the logic lives in `PreTrainedModel`.
|
||||
|
||||
**Decoder-only LLM:**
|
||||
|
||||
```js
|
||||
import { PreTrainedModel } from '../modeling_utils.js';
|
||||
|
||||
export class MyModelPreTrainedModel extends PreTrainedModel {}
|
||||
export class MyModelModel extends MyModelPreTrainedModel {}
|
||||
export class MyModelForCausalLM extends MyModelPreTrainedModel {}
|
||||
```
|
||||
|
||||
**Encoder-only model:**
|
||||
|
||||
```js
|
||||
import { PreTrainedModel } from '../modeling_utils.js';
|
||||
import { MaskedLMOutput, SequenceClassifierOutput } from '../modeling_outputs.js';
|
||||
|
||||
export class MyModelPreTrainedModel extends PreTrainedModel {}
|
||||
export class MyModelModel extends MyModelPreTrainedModel {}
|
||||
|
||||
export class MyModelForMaskedLM extends MyModelPreTrainedModel {
|
||||
async _call(model_inputs) {
|
||||
return new MaskedLMOutput(await super._call(model_inputs));
|
||||
}
|
||||
}
|
||||
|
||||
export class MyModelForSequenceClassification extends MyModelPreTrainedModel {
|
||||
async _call(model_inputs) {
|
||||
return new SequenceClassifierOutput(await super._call(model_inputs));
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Only add the task heads the model actually supports. The available output classes (`MaskedLMOutput`, `TokenClassifierOutput`, `Seq2SeqLMOutput`, etc.) are all in `modeling_outputs.js`.
|
||||
|
||||
#### Tokenizer and processor
|
||||
|
||||
Most models reuse an existing tokenizer (e.g. all Llama-family models use `LlamaTokenizer`). Only create a new one if the model genuinely needs custom tokenization or preprocessing logic.
|
||||
|
||||
| What | File | Barrel to update |
|
||||
| --- | --- | --- |
|
||||
| Custom tokenizer | `src/models/<name>/tokenization_<name>.js` | `src/models/tokenizers.js` |
|
||||
| Custom image processor | `src/models/<name>/image_processing_<name>.js` | `src/models/image_processors.js` |
|
||||
| Custom multimodal processor | `src/models/<name>/processing_<name>.js` | `src/models/processors.js` |
|
||||
| Custom audio/feature extractor | `src/models/<name>/feature_extraction_<name>.js` | `src/models/feature_extractors.js` |
|
||||
|
||||
The class name must match the `tokenizer_class` or `processor_class` field in the model's `tokenizer_config.json` / `preprocessor_config.json` on the Hub.
|
||||
|
||||
#### Wiring it up
|
||||
|
||||
Once the model file is written, three more files need updating:
|
||||
|
||||
1. **`src/models/models.js`**: add `export * from './<name>/modeling_<name>.js'`
|
||||
2. **`src/models/registry.js`**: map the `model_type` string (from `config.json`) to the class names, and set the correct loading category (`EncoderOnly`, `DecoderOnly`, `Seq2Seq`, etc.)
|
||||
3. **`src/configs.js`**: for generative models, add a `case` in `getNormalizedConfig()` to map the model's config field names to the normalized names the KV-cache runtime expects
|
||||
|
||||
Look at a similar existing model in each file to see exactly what to add.
|
||||
|
||||
---
|
||||
|
||||
### 3. Write Tests
|
||||
|
||||
Create `packages/transformers/tests/models/<model_type>/test_modeling_<model_type>.js`. The test runner auto-discovers files by this naming convention. No registration needed.
|
||||
|
||||
Use a small, fast model. The convention is to use a `tiny-random-*` model from `hf-internal-testing/` on the Hub. If one doesn't exist for your architecture, generate one with the `transformers` Python library:
|
||||
|
||||
```python
|
||||
from transformers import AutoConfig, AutoModelForCausalLM
|
||||
config = AutoConfig.for_model("my_model", num_hidden_layers=2, hidden_size=64, ...)
|
||||
model = AutoModelForCausalLM.from_config(config)
|
||||
model.push_to_hub("hf-internal-testing/tiny-random-MyModelForCausalLM")
|
||||
```
|
||||
|
||||
**Test file structure:**
|
||||
|
||||
```js
|
||||
import { MyModelForCausalLM, MyModelTokenizer } from "../../../src/transformers.js";
|
||||
import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js";
|
||||
|
||||
export default () => {
|
||||
describe("MyModelForCausalLM", () => {
|
||||
const model_id = "hf-internal-testing/tiny-random-MyModelForCausalLM";
|
||||
let model, tokenizer;
|
||||
|
||||
beforeAll(async () => {
|
||||
model = await MyModelForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS);
|
||||
tokenizer = await MyModelTokenizer.from_pretrained(model_id);
|
||||
}, MAX_MODEL_LOAD_TIME);
|
||||
|
||||
it("batch_size=1", async () => {
|
||||
const inputs = tokenizer("hello");
|
||||
const outputs = await model.generate({ ...inputs, max_length: 10 });
|
||||
expect(outputs.tolist()).toEqual([[/* expected token ids */]]);
|
||||
}, MAX_TEST_EXECUTION_TIME);
|
||||
|
||||
it("batch_size>1", async () => {
|
||||
const inputs = tokenizer(["hello", "hello world"], { padding: true });
|
||||
const outputs = await model.generate({ ...inputs, max_length: 10 });
|
||||
expect(outputs.tolist()).toEqual([[...], [...]]);
|
||||
}, MAX_TEST_EXECUTION_TIME);
|
||||
|
||||
afterAll(async () => { await model?.dispose(); }, MAX_MODEL_DISPOSE_TIME);
|
||||
});
|
||||
};
|
||||
```
|
||||
|
||||
Run your tests with:
|
||||
|
||||
```bash
|
||||
# All tests
|
||||
pnpm test
|
||||
|
||||
# Only your model's tests
|
||||
pnpm --filter @huggingface/transformers test -t "MyModelForCausalLM"
|
||||
```
|
||||
|
||||
|
||||
## Create a Pull Request
|
||||
|
||||
Before writing any code, we strongly advise you to search through the existing PRs or
|
||||
issues to make sure nobody is already working on the same thing. If you are
|
||||
unsure, it is always a good idea to open an issue to get some feedback.
|
||||
|
||||
You will need basic `git` proficiency to contribute to 🤗 Transformers.js.
|
||||
While `git` is not the easiest tool to use, it has the greatest
|
||||
manual. Type `git --help` in a shell and enjoy! If you prefer books, [Pro
|
||||
Git](https://git-scm.com/book/en/v2) is a very good reference.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
You'll need the following tools installed to contribute to 🤗 Transformers.js:
|
||||
|
||||
- **[Node.js v18](https://nodejs.org/)** or above
|
||||
- **[pnpm](https://pnpm.io/)** - Fast, disk space efficient package manager
|
||||
|
||||
To install pnpm:
|
||||
```bash
|
||||
npm install -g pnpm
|
||||
```
|
||||
|
||||
Follow the steps below to start contributing:
|
||||
|
||||
1. Fork the [repository](https://github.com/huggingface/transformers.js) by
|
||||
clicking on the **[Fork](https://github.com/huggingface/transformers.js/fork)** button on the repository's page. This creates a copy of the code
|
||||
under your GitHub user account.
|
||||
|
||||
2. Clone your fork to your local disk, and add the base repository as a remote:
|
||||
|
||||
```bash
|
||||
git clone git@github.com:<your Github handle>/transformers.js.git
|
||||
cd transformers.js
|
||||
```
|
||||
|
||||
3. Create a new branch to hold your development changes:
|
||||
|
||||
```bash
|
||||
git checkout -b a-descriptive-name-for-my-changes
|
||||
```
|
||||
|
||||
> 🚨 **Do not** work on the `main` branch!
|
||||
|
||||
4. Set up a development environment by running the following command:
|
||||
```bash
|
||||
pnpm install
|
||||
```
|
||||
5. Develop the features in your branch.
|
||||
6. Now you can go to your fork of the repository on GitHub and click on **Pull Request** to open a pull request. Make sure you tick off all the boxes on our [checklist](#pull-request-checklist) below. When you're ready, you can send your changes to the project maintainers for review.
|
||||
7. It's ok if maintainers request changes, it happens to our core contributors
|
||||
too! So everyone can see the changes in the pull request, work in your local
|
||||
branch and push the changes to your fork. They will automatically appear in
|
||||
the pull request.
|
||||
|
||||
### Pull request checklist
|
||||
☐ The pull request title should summarize your contribution.
|
||||
☐ If your pull request addresses an issue, please mention the issue number in the pull
|
||||
request description to make sure they are linked (and people viewing the issue know you
|
||||
are working on it).
|
||||
☐ To indicate a work in progress please prefix the title with `[WIP]`. These are
|
||||
useful to avoid duplicated work, and to differentiate it from PRs ready to be merged.
|
||||
☐ Make sure existing tests pass (`pnpm test`).
|
||||
☐ Make sure the build completes successfully (`pnpm build`).
|
||||
☐ Make sure your code is [formatted properly with Prettier](#code-formatting) (`pnpm format:check`).
|
||||
☐ If adding a new feature, also add tests for it.
|
||||
☐ If your changes affect user-facing functionality, update the relevant documentation.
|
||||
|
||||
### Tests
|
||||
We are using [Jest](https://jestjs.io/) to execute unit-tests. All tests can be found in `packages/transformers/tests` and have to end with `.test.js`
|
||||
|
||||
Execute all tests
|
||||
```bash
|
||||
pnpm test
|
||||
```
|
||||
|
||||
Execute tests for a specific package
|
||||
```bash
|
||||
pnpm --filter @huggingface/transformers test
|
||||
```
|
||||
|
||||
Execute a specific test file
|
||||
```bash
|
||||
cd packages/transformers
|
||||
pnpm test -- ./tests/models.test.js
|
||||
```
|
||||
|
||||
### Style guide
|
||||
|
||||
#### Code formatting
|
||||
We use [Prettier](https://prettier.io/) to maintain consistent code formatting across the project. Please ensure your code is formatted before submitting a pull request.
|
||||
|
||||
**Format all files:**
|
||||
```bash
|
||||
pnpm format
|
||||
```
|
||||
|
||||
**Check formatting without making changes:**
|
||||
```bash
|
||||
pnpm format:check
|
||||
```
|
||||
|
||||
**IDE Integration (recommended)**
|
||||
|
||||
We recommend setting up Prettier in your IDE to format on save:
|
||||
|
||||
**Visual Studio Code:**
|
||||
1. Install the [Prettier extension](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode)
|
||||
2. Open Settings (Ctrl+, or Cmd+,)
|
||||
3. Search for "format on save"
|
||||
4. Enable "Editor: Format On Save"
|
||||
5. Set Prettier as your default formatter: search for "default formatter" and select "Prettier - Code formatter"
|
||||
|
||||
**IntelliJ IDEA / WebStorm:**
|
||||
1. Go to `Settings` → `Languages & Frameworks` → `JavaScript` → `Prettier`
|
||||
2. Set the Prettier package path (usually `node_modules/prettier`)
|
||||
3. Check "On save" under "Run for files"
|
||||
4. Add file patterns: `{**/*,*}.{js,ts,jsx,tsx,json,css,scss,md}`
|
||||
5. Click "Apply" and "OK"
|
||||
|
||||
## Project Structure
|
||||
|
||||
This project uses **pnpm workspaces** to manage multiple packages in a monorepo. Currently, there is one workspace:
|
||||
|
||||
- `packages/transformers` - The main Transformers.js library
|
||||
|
||||
This structure allows for better organization and makes it easier to add framework-specific integrations in the future.
|
||||
|
||||
## How to make changes to transformers.js
|
||||
|
||||
### Development workflow
|
||||
|
||||
The recommended way to develop and test changes is to use the watch mode build and install from the local package:
|
||||
|
||||
1. Start the build in watch mode:
|
||||
```bash
|
||||
pnpm dev
|
||||
```
|
||||
This will automatically rebuild the library whenever you make changes to the source code.
|
||||
|
||||
2. Create a separate test project and install transformers.js from your local development directory:
|
||||
```bash
|
||||
mkdir my-test-project
|
||||
cd my-test-project
|
||||
npm init -y
|
||||
npm install file:/path/to/transformers.js/packages/transformers
|
||||
```
|
||||
Replace `/path/to/transformers.js` with the actual path to your cloned repository.
|
||||
|
||||
3. Make your changes to the transformers.js source code in the main repository. The watch mode will automatically rebuild the library.
|
||||
|
||||
4. Test your changes in your test project. The changes will be automatically reflected since the package is linked via the `file:` protocol.
|
||||
|
||||
This workflow allows for rapid iteration and testing during development.
|
||||
@@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -0,0 +1,456 @@
|
||||
|
||||
|
||||
<p align="center">
|
||||
<br/>
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/transformersjs-dark.svg" width="500" style="max-width: 100%;">
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/transformersjs-light.svg" width="500" style="max-width: 100%;">
|
||||
<img alt="transformers.js javascript library logo" src="https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/transformersjs-light.svg" width="500" style="max-width: 100%;">
|
||||
</picture>
|
||||
<br/>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://www.npmjs.com/package/@huggingface/transformers"><img alt="NPM" src="https://img.shields.io/npm/v/@huggingface/transformers"></a>
|
||||
<a href="https://www.npmjs.com/package/@huggingface/transformers"><img alt="NPM Downloads" src="https://img.shields.io/npm/dw/@huggingface/transformers"></a>
|
||||
<a href="https://www.jsdelivr.com/package/npm/@huggingface/transformers"><img alt="jsDelivr Hits" src="https://img.shields.io/jsdelivr/npm/hw/@huggingface/transformers"></a>
|
||||
<a href="https://github.com/huggingface/transformers.js/blob/main/LICENSE"><img alt="License" src="https://img.shields.io/github/license/huggingface/transformers.js?color=blue"></a>
|
||||
<a href="https://huggingface.co/docs/transformers.js/index"><img alt="Documentation" src="https://img.shields.io/website/http/huggingface.co/docs/transformers.js/index.svg?down_color=red&down_message=offline&up_message=online"></a>
|
||||
</p>
|
||||
|
||||
|
||||
<h3 align="center">
|
||||
<p>State-of-the-art Machine Learning for the Web</p>
|
||||
</h3>
|
||||
|
||||
Run 🤗 Transformers directly in your browser, with no need for a server!
|
||||
|
||||
Transformers.js is designed to be functionally equivalent to Hugging Face's [transformers](https://github.com/huggingface/transformers) python library, meaning you can run the same pretrained models using a very similar API. These models support common tasks in different modalities, such as:
|
||||
- 📝 **Natural Language Processing**: text classification, named entity recognition, question answering, language modeling, summarization, translation, multiple choice, and text generation.
|
||||
- 🖼️ **Computer Vision**: image classification, object detection, segmentation, and depth estimation.
|
||||
- 🗣️ **Audio**: automatic speech recognition, audio classification, and text-to-speech.
|
||||
- 🐙 **Multimodal**: embeddings, zero-shot audio classification, zero-shot image classification, and zero-shot object detection.
|
||||
|
||||
Transformers.js uses [ONNX Runtime](https://onnxruntime.ai/) to run models in the browser. The best part about it, is that you can easily [convert](#convert-your-models-to-onnx) your pretrained PyTorch, TensorFlow, or JAX models to ONNX using [🤗 Optimum](https://github.com/huggingface/optimum#onnx--onnx-runtime).
|
||||
|
||||
For more information, check out the full [documentation](https://huggingface.co/docs/transformers.js).
|
||||
|
||||
|
||||
## Installation
|
||||
|
||||
|
||||
To install via [NPM](https://www.npmjs.com/package/@huggingface/transformers), run:
|
||||
```bash
|
||||
npm i @huggingface/transformers
|
||||
```
|
||||
|
||||
Alternatively, you can use it in vanilla JS, without any bundler, by using a CDN or static hosting. For example, using [ES Modules](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Modules), you can import the library with:
|
||||
```html
|
||||
<script type="module">
|
||||
import { pipeline } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0';
|
||||
</script>
|
||||
```
|
||||
|
||||
|
||||
## Quick tour
|
||||
|
||||
|
||||
It's super simple to translate from existing code! Just like the python library, we support the `pipeline` API. Pipelines group together a pretrained model with preprocessing of inputs and postprocessing of outputs, making it the easiest way to run models with the library.
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<th width="440px" align="center"><b>Python (original)</b></th>
|
||||
<th width="440px" align="center"><b>Javascript (ours)</b></th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
```python
|
||||
from transformers import pipeline
|
||||
|
||||
# Allocate a pipeline for sentiment-analysis
|
||||
pipe = pipeline('sentiment-analysis')
|
||||
|
||||
out = pipe('I love transformers!')
|
||||
# [{'label': 'POSITIVE', 'score': 0.999806941}]
|
||||
```
|
||||
|
||||
</td>
|
||||
<td>
|
||||
|
||||
```javascript
|
||||
import { pipeline } from '@huggingface/transformers';
|
||||
|
||||
// Allocate a pipeline for sentiment-analysis
|
||||
const pipe = await pipeline('sentiment-analysis');
|
||||
|
||||
const out = await pipe('I love transformers!');
|
||||
// [{'label': 'POSITIVE', 'score': 0.999817686}]
|
||||
```
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
You can also use a different model by specifying the model id or path as the second argument to the `pipeline` function. For example:
|
||||
```javascript
|
||||
// Use a different model for sentiment-analysis
|
||||
const pipe = await pipeline('sentiment-analysis', 'Xenova/bert-base-multilingual-uncased-sentiment');
|
||||
```
|
||||
|
||||
By default, when running in the browser, the model will be run on your CPU (via WASM). If you would like
|
||||
to run the model on your GPU (via WebGPU), you can do this by setting `device: 'webgpu'`, for example:
|
||||
```javascript
|
||||
// Run the model on WebGPU
|
||||
const pipe = await pipeline('sentiment-analysis', 'Xenova/distilbert-base-uncased-finetuned-sst-2-english', {
|
||||
device: 'webgpu',
|
||||
});
|
||||
```
|
||||
|
||||
For more information, check out the [WebGPU guide](https://huggingface.co/docs/transformers.js/guides/webgpu).
|
||||
|
||||
> [!WARNING]
|
||||
> The WebGPU API is still experimental in many browsers, so if you run into any issues,
|
||||
> please file a [bug report](https://github.com/huggingface/transformers.js/issues/new?title=%5BWebGPU%5D%20Error%20running%20MODEL_ID_GOES_HERE&assignees=&labels=bug,webgpu&projects=&template=1_bug-report.yml).
|
||||
|
||||
In resource-constrained environments, such as web browsers, it is advisable to use a quantized version of
|
||||
the model to lower bandwidth and optimize performance. This can be achieved by adjusting the `dtype` option,
|
||||
which allows you to select the appropriate data type for your model. While the available options may vary
|
||||
depending on the specific model, typical choices include `"fp32"` (default for WebGPU), `"fp16"`, `"q8"`
|
||||
(default for WASM), and `"q4"`. For more information, check out the [quantization guide](https://huggingface.co/docs/transformers.js/guides/dtypes).
|
||||
```javascript
|
||||
// Run the model at 4-bit quantization
|
||||
const pipe = await pipeline('sentiment-analysis', 'Xenova/distilbert-base-uncased-finetuned-sst-2-english', {
|
||||
dtype: 'q4',
|
||||
});
|
||||
```
|
||||
|
||||
Ready to dive in? Explore our wide variety of demo applications and templates [here](https://github.com/huggingface/transformers.js-examples). You can also launch your own project instantly using the official Transformers.js [template](https://huggingface.co/new-space?template=static-templates%2Ftransformers.js) on Hugging Face!
|
||||
|
||||
|
||||
|
||||
## Custom usage
|
||||
|
||||
|
||||
|
||||
By default, Transformers.js uses [hosted pretrained models](https://huggingface.co/models?library=transformers.js) and [precompiled WASM binaries](https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0/dist/), which should work out-of-the-box. You can customize this as follows:
|
||||
|
||||
### Settings
|
||||
|
||||
```javascript
|
||||
import { env } from '@huggingface/transformers';
|
||||
|
||||
// Specify a custom location for models (defaults to '/models/').
|
||||
env.localModelPath = '/path/to/models/';
|
||||
|
||||
// Disable the loading of remote models from the Hugging Face Hub:
|
||||
env.allowRemoteModels = false;
|
||||
|
||||
// Set location of .wasm files. Defaults to use a CDN.
|
||||
env.backends.onnx.wasm.wasmPaths = '/path/to/files/';
|
||||
```
|
||||
|
||||
For a full list of available settings, check out the [API Reference](https://huggingface.co/docs/transformers.js/api/env).
|
||||
|
||||
### Convert your models to ONNX
|
||||
|
||||
We recommend using [Optimum](https://github.com/huggingface/optimum-onnx) to convert your PyTorch models to ONNX in a single command. For the full list of supported architectures, check out the [Optimum documentation](https://huggingface.co/docs/optimum-onnx/onnx/overview).
|
||||
|
||||
|
||||
## Supported tasks/models
|
||||
|
||||
Here is the list of all tasks and architectures currently supported by Transformers.js. If you don't see your task/model listed here or it is not yet supported, feel free to open up a feature request [here](https://github.com/huggingface/transformers.js/issues/new/choose).
|
||||
|
||||
To find compatible models on the Hub, select the "transformers.js" library tag in the filter menu (or visit [this link](https://huggingface.co/models?library=transformers.js)). You can refine your search by selecting the task you're interested in (e.g., [text-classification](https://huggingface.co/models?pipeline_tag=text-classification&library=transformers.js)).
|
||||
|
||||
|
||||
### Tasks
|
||||
|
||||
#### Natural Language Processing
|
||||
|
||||
| Task | ID | Description | Supported? |
|
||||
|--------------------------|----|-------------|------------|
|
||||
| [Fill-Mask](https://huggingface.co/tasks/fill-mask) | `fill-mask` | Masking some of the words in a sentence and predicting which words should replace those masks. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.FillMaskPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=fill-mask&library=transformers.js) |
|
||||
| [Question Answering](https://huggingface.co/tasks/question-answering) | `question-answering` | Retrieve the answer to a question from a given text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.QuestionAnsweringPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=question-answering&library=transformers.js) |
|
||||
| [Sentence Similarity](https://huggingface.co/tasks/sentence-similarity) | `sentence-similarity` | Determining how similar two texts are. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.FeatureExtractionPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=sentence-similarity&library=transformers.js) |
|
||||
| [Summarization](https://huggingface.co/tasks/summarization) | `summarization` | Producing a shorter version of a document while preserving its important information. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.SummarizationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=summarization&library=transformers.js) |
|
||||
| [Table Question Answering](https://huggingface.co/tasks/table-question-answering) | `table-question-answering` | Answering a question about information from a given table. | ❌ |
|
||||
| [Text Classification](https://huggingface.co/tasks/text-classification) | `text-classification` or `sentiment-analysis` | Assigning a label or class to a given text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TextClassificationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=text-classification&library=transformers.js) |
|
||||
| [Text Generation](https://huggingface.co/tasks/text-generation#completion-generation-models) | `text-generation` | Producing new text by predicting the next word in a sequence. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TextGenerationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=text-generation&library=transformers.js) |
|
||||
| [Text-to-text Generation](https://huggingface.co/tasks/text-generation#text-to-text-generation-models) | `text2text-generation` | Converting one text sequence into another text sequence. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.Text2TextGenerationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=text2text-generation&library=transformers.js) |
|
||||
| [Token Classification](https://huggingface.co/tasks/token-classification) | `token-classification` or `ner` | Assigning a label to each token in a text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TokenClassificationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=token-classification&library=transformers.js) |
|
||||
| [Translation](https://huggingface.co/tasks/translation) | `translation` | Converting text from one language to another. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TranslationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=translation&library=transformers.js) |
|
||||
| [Zero-Shot Classification](https://huggingface.co/tasks/zero-shot-classification) | `zero-shot-classification` | Classifying text into classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotClassificationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=zero-shot-classification&library=transformers.js) |
|
||||
| [Feature Extraction](https://huggingface.co/tasks/feature-extraction) | `feature-extraction` | Transforming raw data into numerical features that can be processed while preserving the information in the original dataset. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.FeatureExtractionPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=feature-extraction&library=transformers.js) |
|
||||
|
||||
#### Vision
|
||||
|
||||
| Task | ID | Description | Supported? |
|
||||
|--------------------------|----|-------------|------------|
|
||||
| [Background Removal](https://huggingface.co/tasks/image-segmentation#background-removal) | `background-removal` | Isolating the main subject of an image by removing or making the background transparent. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.BackgroundRemovalPipeline)<br>[(models)](https://huggingface.co/models?other=background-removal&library=transformers.js) |
|
||||
| [Depth Estimation](https://huggingface.co/tasks/depth-estimation) | `depth-estimation` | Predicting the depth of objects present in an image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.DepthEstimationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=depth-estimation&library=transformers.js) |
|
||||
| [Image Classification](https://huggingface.co/tasks/image-classification) | `image-classification` | Assigning a label or class to an entire image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageClassificationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=image-classification&library=transformers.js) |
|
||||
| [Image Segmentation](https://huggingface.co/tasks/image-segmentation) | `image-segmentation` | Divides an image into segments where each pixel is mapped to an object. This task has multiple variants such as instance segmentation, panoptic segmentation and semantic segmentation. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageSegmentationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=image-segmentation&library=transformers.js) |
|
||||
| [Image-to-Image](https://huggingface.co/tasks/image-to-image) | `image-to-image` | Transforming a source image to match the characteristics of a target image or a target image domain. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageToImagePipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=image-to-image&library=transformers.js) |
|
||||
| [Mask Generation](https://huggingface.co/tasks/mask-generation) | `mask-generation` | Generate masks for the objects in an image. | ❌ |
|
||||
| [Object Detection](https://huggingface.co/tasks/object-detection) | `object-detection` | Identify objects of certain defined classes within an image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ObjectDetectionPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=object-detection&library=transformers.js) |
|
||||
| [Video Classification](https://huggingface.co/tasks/video-classification) | n/a | Assigning a label or class to an entire video. | ❌ |
|
||||
| [Unconditional Image Generation](https://huggingface.co/tasks/unconditional-image-generation) | n/a | Generating images with no condition in any context (like a prompt text or another image). | ❌ |
|
||||
| [Image Feature Extraction](https://huggingface.co/tasks/image-feature-extraction) | `image-feature-extraction` | Transforming raw data into numerical features that can be processed while preserving the information in the original image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageFeatureExtractionPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=image-feature-extraction&library=transformers.js) |
|
||||
|
||||
#### Audio
|
||||
|
||||
| Task | ID | Description | Supported? |
|
||||
|--------------------------|----|-------------|------------|
|
||||
| [Audio Classification](https://huggingface.co/tasks/audio-classification) | `audio-classification` | Assigning a label or class to a given audio. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.AudioClassificationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=audio-classification&library=transformers.js) |
|
||||
| [Audio-to-Audio](https://huggingface.co/tasks/audio-to-audio) | n/a | Generating audio from an input audio source. | ❌ |
|
||||
| [Automatic Speech Recognition](https://huggingface.co/tasks/automatic-speech-recognition) | `automatic-speech-recognition` | Transcribing a given audio into text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.AutomaticSpeechRecognitionPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=automatic-speech-recognition&library=transformers.js) |
|
||||
| [Text-to-Speech](https://huggingface.co/tasks/text-to-speech) | `text-to-speech` or `text-to-audio` | Generating natural-sounding speech given text input. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TextToAudioPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=text-to-audio&library=transformers.js) |
|
||||
|
||||
|
||||
#### Tabular
|
||||
|
||||
| Task | ID | Description | Supported? |
|
||||
|--------------------------|----|-------------|------------|
|
||||
| [Tabular Classification](https://huggingface.co/tasks/tabular-classification) | n/a | Classifying a target category (a group) based on set of attributes. | ❌ |
|
||||
| [Tabular Regression](https://huggingface.co/tasks/tabular-regression) | n/a | Predicting a numerical value given a set of attributes. | ❌ |
|
||||
|
||||
|
||||
#### Multimodal
|
||||
|
||||
| Task | ID | Description | Supported? |
|
||||
|--------------------------|----|-------------|------------|
|
||||
| [Document Question Answering](https://huggingface.co/tasks/document-question-answering) | `document-question-answering` | Answering questions on document images. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.DocumentQuestionAnsweringPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=document-question-answering&library=transformers.js) |
|
||||
| [Image-to-Text](https://huggingface.co/tasks/image-to-text) | `image-to-text` | Output text from a given image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageToTextPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=image-to-text&library=transformers.js) |
|
||||
| [Text-to-Image](https://huggingface.co/tasks/text-to-image) | `text-to-image` | Generates images from input text. | ❌ |
|
||||
| [Visual Question Answering](https://huggingface.co/tasks/visual-question-answering) | `visual-question-answering` | Answering open-ended questions based on an image. | ❌ |
|
||||
| [Zero-Shot Audio Classification](https://huggingface.co/learn/audio-course/chapter4/classification_models#zero-shot-audio-classification) | `zero-shot-audio-classification` | Classifying audios into classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotAudioClassificationPipeline)<br>[(models)](https://huggingface.co/models?other=zero-shot-audio-classification&library=transformers.js) |
|
||||
| [Zero-Shot Image Classification](https://huggingface.co/tasks/zero-shot-image-classification) | `zero-shot-image-classification` | Classifying images into classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotImageClassificationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=zero-shot-image-classification&library=transformers.js) |
|
||||
| [Zero-Shot Object Detection](https://huggingface.co/tasks/zero-shot-object-detection) | `zero-shot-object-detection` | Identify objects of classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotObjectDetectionPipeline)<br>[(models)](https://huggingface.co/models?other=zero-shot-object-detection&library=transformers.js) |
|
||||
|
||||
|
||||
#### Reinforcement Learning
|
||||
|
||||
| Task | ID | Description | Supported? |
|
||||
|--------------------------|----|-------------|------------|
|
||||
| [Reinforcement Learning](https://huggingface.co/tasks/reinforcement-learning) | n/a | Learning from actions by interacting with an environment through trial and error and receiving rewards (negative or positive) as feedback. | ✅ |
|
||||
|
||||
|
||||
|
||||
### Models
|
||||
|
||||
1. **[AFMoE](https://huggingface.co/docs/transformers/model_doc/afmoe)** (from Arcee AI, Prime Intellect, and DatologyAI) released with the paper [Arcee Trinity Large Technical Report](https://huggingface.co/papers/2602.17004), by Varun Singh, Lucas Krauss, Sami Jaghouar, Matej Sirovatka, Charles Goddard, Fares Obied, Jack Min Ong, Jannik Straube, Fern, Aria Harley, Conner Stewart, Colin Kealty, Maziyar Panahi, Simon Kirsten, Anushka Deshpande, Anneketh Vij, Arthur Bresnu, Pranav Veldurthi, Raghav Ravishankar, Hardik Bishnoi, Mark McQuade, Johannes Hagemann, Lucas Atkins.
|
||||
1. **[ALBERT](https://huggingface.co/docs/transformers/model_doc/albert)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://huggingface.co/papers/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.
|
||||
1. **[Arcee](https://huggingface.co/docs/transformers/model_doc/arcee)** (from Arcee AI) released with the blog post [Announcing Arcee Foundation Models](https://www.arcee.ai/blog/announcing-the-arcee-foundation-model-family) by Fernando Fernandes, Varun Singh, Charles Goddard, Lucas Atkins, Mark McQuade, Maziyar Panahi, Conner Stewart, Colin Kealty, Raghav Ravishankar, Lucas Krauss, Anneketh Vij, Pranav Veldurthi, Abhishek Thakur, Julien Simon, Scott Zembsch, Benjamin Langer, Aleksiej Cecocho, Maitri Patel.
|
||||
1. **[Audio Spectrogram Transformer](https://huggingface.co/docs/transformers/model_doc/audio-spectrogram-transformer)** (from MIT) released with the paper [AST: Audio Spectrogram Transformer](https://huggingface.co/papers/2104.01778) by Yuan Gong, Yu-An Chung, James Glass.
|
||||
1. **[BART](https://huggingface.co/docs/transformers/model_doc/bart)** (from Facebook) released with the paper [BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension](https://huggingface.co/papers/1910.13461) by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer.
|
||||
1. **[BEiT](https://huggingface.co/docs/transformers/model_doc/beit)** (from Microsoft) released with the paper [BEiT: BERT Pre-Training of Image Transformers](https://huggingface.co/papers/2106.08254) by Hangbo Bao, Li Dong, Furu Wei.
|
||||
1. **[BERT](https://huggingface.co/docs/transformers/model_doc/bert)** (from Google) released with the paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://huggingface.co/papers/1810.04805) by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova.
|
||||
1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://huggingface.co/papers/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston.
|
||||
1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://huggingface.co/papers/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston.
|
||||
1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/).
|
||||
1. **[CamemBERT](https://huggingface.co/docs/transformers/model_doc/camembert)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://huggingface.co/papers/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot.
|
||||
1. **[CHMv2](https://huggingface.co/docs/transformers/main/model_doc/chmv2)** (from Meta) released with the paper [CHMv2: Improvements in Global Canopy Height Mapping using DINOv3](https://huggingface.co/papers/2603.06382) by John Brandt, Seungeun Yi, Jamie Tolan, Xinyuan Li, Peter Potapov, Jessica Ertel, Justine Spore, Huy V. Vo, Michaël Ramamonjisoa, Patrick Labatut, Piotr Bojanowski, Camille Couprie.
|
||||
1. **Chatterbox** (from Resemble AI) released with the repository [Chatterbox TTS](https://github.com/resemble-ai/chatterbox) by the Resemble AI team.
|
||||
1. **[Chinese-CLIP](https://huggingface.co/docs/transformers/model_doc/chinese_clip)** (from OFA-Sys) released with the paper [Chinese CLIP: Contrastive Vision-Language Pretraining in Chinese](https://huggingface.co/papers/2211.01335) by An Yang, Junshu Pan, Junyang Lin, Rui Men, Yichang Zhang, Jingren Zhou, Chang Zhou.
|
||||
1. **[CLAP](https://huggingface.co/docs/transformers/model_doc/clap)** (from LAION-AI) released with the paper [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation](https://huggingface.co/papers/2211.06687) by Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov.
|
||||
1. **[CLIP](https://huggingface.co/docs/transformers/model_doc/clip)** (from OpenAI) released with the paper [Learning Transferable Visual Models From Natural Language Supervision](https://huggingface.co/papers/2103.00020) by Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, Gretchen Krueger, Ilya Sutskever.
|
||||
1. **[CLIPSeg](https://huggingface.co/docs/transformers/model_doc/clipseg)** (from University of Göttingen) released with the paper [Image Segmentation Using Text and Image Prompts](https://huggingface.co/papers/2112.10003) by Timo Lüddecke and Alexander Ecker.
|
||||
1. **[CodeGen](https://huggingface.co/docs/transformers/model_doc/codegen)** (from Salesforce) released with the paper [A Conversational Paradigm for Program Synthesis](https://huggingface.co/papers/2203.13474) by Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, Caiming Xiong.
|
||||
1. **[CodeLlama](https://huggingface.co/docs/transformers/model_doc/llama_code)** (from MetaAI) released with the paper [Code Llama: Open Foundation Models for Code](https://ai.meta.com/research/publications/code-llama-open-foundation-models-for-code/) by Baptiste Rozière, Jonas Gehring, Fabian Gloeckle, Sten Sootla, Itai Gat, Xiaoqing Ellen Tan, Yossi Adi, Jingyu Liu, Tal Remez, Jérémy Rapin, Artyom Kozhevnikov, Ivan Evtimov, Joanna Bitton, Manish Bhatt, Cristian Canton Ferrer, Aaron Grattafiori, Wenhan Xiong, Alexandre Défossez, Jade Copet, Faisal Azhar, Hugo Touvron, Louis Martin, Nicolas Usunier, Thomas Scialom, Gabriel Synnaeve.
|
||||
1. **[Cohere](https://huggingface.co/docs/transformers/main/model_doc/cohere)** (from Cohere) released with the blog post [Command-R: Retrieval Augmented Generation at Production Scale](https://cohere.com/blog/command-r) by Cohere.
|
||||
1. **[Cohere2](https://huggingface.co/docs/transformers/main/model_doc/cohere2)** (from Cohere) released with the blog post [Introducing Command R7B: Fast and efficient generative AI](https://cohere.com/blog/command-r7b) by Cohere.
|
||||
1. **[Cohere ASR](https://huggingface.co/docs/transformers/main/model_doc/cohere_asr)** (from Cohere) released with the blog post [Introducing Cohere Transcribe: a new state-of-the-art in open-source speech recognition](https://cohere.com/blog/transcribe) by Cohere.
|
||||
1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://huggingface.co/papers/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan.
|
||||
1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://huggingface.co/papers/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie.
|
||||
1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://huggingface.co/papers/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie.
|
||||
1. **[D-FINE](https://huggingface.co/docs/transformers/model_doc/d_fine)** (from University of Science and Technology of China) released with the paper [D-FINE: Redefine Regression Task in DETRs as Fine-grained Distribution Refinement](https://huggingface.co/papers/2410.13842) by Yansong Peng, Hebei Li, Peixi Wu, Yueyi Zhang, Xiaoyan Sun, Feng Wu.
|
||||
1. **[DAC](https://huggingface.co/docs/transformers/model_doc/dac)** (from Descript) released with the paper [Descript Audio Codec: High-Fidelity Audio Compression with Improved RVQGAN](https://huggingface.co/papers/2306.06546) by Rithesh Kumar, Prem Seetharaman, Alejandro Luebs, Ishaan Kumar, Kundan Kumar.
|
||||
1. **[DeBERTa](https://huggingface.co/docs/transformers/model_doc/deberta)** (from Microsoft) released with the paper [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://huggingface.co/papers/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen.
|
||||
1. **[DeBERTa-v2](https://huggingface.co/docs/transformers/model_doc/deberta-v2)** (from Microsoft) released with the paper [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://huggingface.co/papers/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen.
|
||||
1. **[Decision Transformer](https://huggingface.co/docs/transformers/model_doc/decision_transformer)** (from Berkeley/Facebook/Google) released with the paper [Decision Transformer: Reinforcement Learning via Sequence Modeling](https://huggingface.co/papers/2106.01345) by Lili Chen, Kevin Lu, Aravind Rajeswaran, Kimin Lee, Aditya Grover, Michael Laskin, Pieter Abbeel, Aravind Srinivas, Igor Mordatch.
|
||||
1. **[DeepSeek-V3](https://huggingface.co/docs/transformers/model_doc/deepseek_v3)** (from DeepSeek-AI) released with the paper [DeepSeek-V3 Technical Report](https://huggingface.co/papers/2412.19437) by the DeepSeek-AI team.
|
||||
1. **[DeepSeek-V4](https://huggingface.co/docs/transformers/model_doc/deepseek_v4)** (from DeepSeek-AI) released with the paper [DeepSeek-V4: Towards Highly Efficient Million-Token Context Intelligence](https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf) by the DeepSeek-AI team.
|
||||
1. **[DeiT](https://huggingface.co/docs/transformers/model_doc/deit)** (from Facebook) released with the paper [Training data-efficient image transformers & distillation through attention](https://huggingface.co/papers/2012.12877) by Hugo Touvron, Matthieu Cord, Matthijs Douze, Francisco Massa, Alexandre Sablayrolles, Hervé Jégou.
|
||||
1. **[Depth Anything](https://huggingface.co/docs/transformers/main/model_doc/depth_anything)** (from University of Hong Kong and TikTok) released with the paper [Depth Anything: Unleashing the Power of Large-Scale Unlabeled Data](https://huggingface.co/papers/2401.10891) by Lihe Yang, Bingyi Kang, Zilong Huang, Xiaogang Xu, Jiashi Feng, Hengshuang Zhao.
|
||||
1. **Depth Pro** (from Apple) released with the paper [Depth Pro: Sharp Monocular Metric Depth in Less Than a Second](https://huggingface.co/papers/2410.02073) by Aleksei Bochkovskii, Amaël Delaunoy, Hugo Germain, Marcel Santos, Yichao Zhou, Stephan R. Richter, Vladlen Koltun.
|
||||
1. **[DETR](https://huggingface.co/docs/transformers/model_doc/detr)** (from Facebook) released with the paper [End-to-End Object Detection with Transformers](https://huggingface.co/papers/2005.12872) by Nicolas Carion, Francisco Massa, Gabriel Synnaeve, Nicolas Usunier, Alexander Kirillov, Sergey Zagoruyko.
|
||||
1. **[DINOv2](https://huggingface.co/docs/transformers/model_doc/dinov2)** (from Meta AI) released with the paper [DINOv2: Learning Robust Visual Features without Supervision](https://huggingface.co/papers/2304.07193) by Maxime Oquab, Timothée Darcet, Théo Moutakanni, Huy Vo, Marc Szafraniec, Vasil Khalidov, Pierre Fernandez, Daniel Haziza, Francisco Massa, Alaaeldin El-Nouby, Mahmoud Assran, Nicolas Ballas, Wojciech Galuba, Russell Howes, Po-Yao Huang, Shang-Wen Li, Ishan Misra, Michael Rabbat, Vasu Sharma, Gabriel Synnaeve, Hu Xu, Hervé Jegou, Julien Mairal, Patrick Labatut, Armand Joulin, Piotr Bojanowski.
|
||||
1. **[DINOv2 with Registers](https://huggingface.co/docs/transformers/model_doc/dinov2_with_registers)** (from Meta AI) released with the paper [DINOv2 with Registers](https://huggingface.co/papers/2309.16588) by Timothée Darcet, Maxime Oquab, Julien Mairal, Piotr Bojanowski.
|
||||
1. **[DINOv3](https://huggingface.co/docs/transformers/model_doc/dinov3)** (from Meta AI) released with the paper [DINOv3](https://huggingface.co/papers/2508.10104) by Oriane Siméoni, Huy V. Vo, Maximilian Seitzer, Federico Baldassarre, Maxime Oquab, Cijo Jose, Vasil Khalidov, Marc Szafraniec, Seungeun Yi, Michaël Ramamonjisoa, Francisco Massa, Daniel Haziza, Luca Wehrstedt, Jianyuan Wang, Timothée Darcet, Théo Moutakanni, Leonel Sentana, Claire Roberts, Andrea Vedaldi, Jamie Tolan, John Brandt, Camille Couprie, Julien Mairal, Hervé Jégou, Patrick Labatut, Piotr Bojanowski.
|
||||
1. **[DistilBERT](https://huggingface.co/docs/transformers/model_doc/distilbert)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://huggingface.co/papers/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/research_projects/distillation), RoBERTa into [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/research_projects/distillation), Multilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/research_projects/distillation) and a German version of DistilBERT.
|
||||
1. **[DiT](https://huggingface.co/docs/transformers/model_doc/dit)** (from Microsoft Research) released with the paper [DiT: Self-supervised Pre-training for Document Image Transformer](https://huggingface.co/papers/2203.02378) by Junlong Li, Yiheng Xu, Tengchao Lv, Lei Cui, Cha Zhang, Furu Wei.
|
||||
1. **[Donut](https://huggingface.co/docs/transformers/model_doc/donut)** (from NAVER), released together with the paper [OCR-free Document Understanding Transformer](https://huggingface.co/papers/2111.15664) by Geewook Kim, Teakgyu Hong, Moonbin Yim, Jeongyeon Nam, Jinyoung Park, Jinyeong Yim, Wonseok Hwang, Sangdoo Yun, Dongyoon Han, Seunghyun Park.
|
||||
1. **[DPT](https://huggingface.co/docs/transformers/master/model_doc/dpt)** (from Intel Labs) released with the paper [Vision Transformers for Dense Prediction](https://huggingface.co/papers/2103.13413) by René Ranftl, Alexey Bochkovskiy, Vladlen Koltun.
|
||||
1. **[EdgeTAM](https://huggingface.co/docs/transformers/model_doc/edgetam)** (from Facebook) released with the paper [EdgeTAM: On-Device Track Anything Model](https://huggingface.co/papers/2501.07256) by Chong Zhou, Chenchen Zhu, Yunyang Xiong, Saksham Suri, Fanyi Xiao, Lemeng Wu, Raghuraman Krishnamoorthi, Bo Dai, Chen Change Loy, Vikas Chandra, Bilge Soran.
|
||||
1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://huggingface.co/papers/1905.11946) by Mingxing Tan, Quoc V. Le.
|
||||
1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://huggingface.co/papers/2003.10555) by Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning.
|
||||
1. **ERNIE-4.5** (from Baidu ERNIE Team) released with the blog post [Announcing the Open Source Release of the ERNIE 4.5 Model Family](https://ernie.baidu.com/blog/posts/ernie4.5/) by the Baidu ERNIE Team.
|
||||
1. **[ESM](https://huggingface.co/docs/transformers/model_doc/esm)** (from Meta AI) are transformer protein language models. **ESM-1b** was released with the paper [Biological structure and function emerge from scaling unsupervised learning to 250 million protein sequences](https://www.pnas.org/content/118/15/e2016239118) by Alexander Rives, Joshua Meier, Tom Sercu, Siddharth Goyal, Zeming Lin, Jason Liu, Demi Guo, Myle Ott, C. Lawrence Zitnick, Jerry Ma, and Rob Fergus. **ESM-1v** was released with the paper [Language models enable zero-shot prediction of the effects of mutations on protein function](https://doi.org/10.1101/2021.07.09.450648) by Joshua Meier, Roshan Rao, Robert Verkuil, Jason Liu, Tom Sercu and Alexander Rives. **ESM-2 and ESMFold** were released with the paper [Language models of protein sequences at the scale of evolution enable accurate structure prediction](https://doi.org/10.1101/2022.07.20.500902) by Zeming Lin, Halil Akin, Roshan Rao, Brian Hie, Zhongkai Zhu, Wenting Lu, Allan dos Santos Costa, Maryam Fazel-Zarandi, Tom Sercu, Sal Candido, Alexander Rives.
|
||||
1. **[EuroBERT](https://huggingface.co/docs/transformers/model_doc/eurobert)** (from the EuroBERT team) released with the paper [EuroBERT: Scaling Multilingual Encoders for European Languages](https://huggingface.co/papers/2503.05500) by Nicolas Boizard, Hippolyte Gisserot-Boukhlef, Duarte M. Alves, André Martins, Ayoub Hammal, Caio Corro, Céline Hudelot, Emmanuel Malherbe, Etienne Malaboeuf, Fanny Jourdan, Gabriel Hautreux, João Alves, Kevin El-Haddad, Manuel Faysse, Maxime Peyrard, Nuno M. Guerreiro, Patrick Fernandes, Ricardo Rei, Pierre Colombo.
|
||||
1. **EXAONE** (from LG AI Research) released with the papers [EXAONE 3.0 7.8B Instruction Tuned Language Model](https://huggingface.co/papers/2408.03541) and [EXAONE 3.5: Series of Large Language Models for Real-world Use Cases](https://huggingface.co/papers/2412.04862) by the LG AI Research team.
|
||||
1. **[Falcon](https://huggingface.co/docs/transformers/model_doc/falcon)** (from Technology Innovation Institute) by Almazrouei, Ebtesam and Alobeidli, Hamza and Alshamsi, Abdulaziz and Cappelli, Alessandro and Cojocaru, Ruxandra and Debbah, Merouane and Goffinet, Etienne and Heslow, Daniel and Launay, Julien and Malartic, Quentin and Noune, Badreddine and Pannier, Baptiste and Penedo, Guilherme.
|
||||
1. **[FalconH1](https://huggingface.co/docs/transformers/model_doc/falcon_h1)** (from Technology Innovation Institute) released with the blog post [Falcon-H1-Tiny: A series of extremely small, yet powerful language models redefining capabilities at small scale](https://huggingface.co/spaces/tiiuae/tiny-h1-blogpost) by the Falcon-LLM team.
|
||||
1. **FastViT** (from Apple) released with the paper [FastViT: A Fast Hybrid Vision Transformer using Structural Reparameterization](https://huggingface.co/papers/2303.14189) by Pavan Kumar Anasosalu Vasu, James Gabriel, Jeff Zhu, Oncel Tuzel and Anurag Ranjan.
|
||||
1. **[FLAN-T5](https://huggingface.co/docs/transformers/model_doc/flan-t5)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-t5-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei
|
||||
1. **Florence2** (from Microsoft) released with the paper [Florence-2: Advancing a Unified Representation for a Variety of Vision Tasks](https://huggingface.co/papers/2311.06242) by Bin Xiao, Haiping Wu, Weijian Xu, Xiyang Dai, Houdong Hu, Yumao Lu, Michael Zeng, Ce Liu, Lu Yuan.
|
||||
1. **[Gemma](https://huggingface.co/docs/transformers/main/model_doc/gemma)** (from Google) released with the blog post [Gemma: Open Models Based on Gemini Technology and Research](https://blog.google/technology/developers/gemma-open-models/) by the Gemma Google team.
|
||||
1. **[Gemma2](https://huggingface.co/docs/transformers/main/model_doc/gemma2)** (from Google) released with the blog post [Gemma2: Open Models Based on Gemini Technology and Research](https://blog.google/technology/developers/google-gemma-2/) by the Gemma Google team.
|
||||
1. **[Gemma3](https://huggingface.co/docs/transformers/main/model_doc/gemma3)** (from Google) released with the blog post [Introducing Gemma 3: The most capable model you can run on a single GPU or TPU](https://blog.google/technology/developers/gemma-3/) by the Gemma Google team.
|
||||
1. **[Gemma3n](https://huggingface.co/docs/transformers/main/model_doc/gemma3n)** (from Google) released with the blog post [Announcing Gemma 3n preview: powerful, efficient, mobile-first AI](https://developers.googleblog.com/en/introducing-gemma-3n/) by the Gemma Google team.
|
||||
1. **[Gemma4](https://huggingface.co/docs/transformers/main/model_doc/gemma4)** (from Google) released with the blog post [Gemma 4](https://blog.google/innovation-and-ai/technology/developers-tools/gemma-4/) by the Gemma Google team.
|
||||
1. **[GLM](https://huggingface.co/docs/transformers/main/model_doc/glm)** (from the GLM Team, THUDM & ZhipuAI) released with the paper [ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools](https://huggingface.co/papers/2406.12793v2) by Team GLM: Aohan Zeng, Bin Xu, Bowen Wang, Chenhui Zhang, Da Yin, Dan Zhang, Diego Rojas, Guanyu Feng, Hanlin Zhao, Hanyu Lai, Hao Yu, Hongning Wang, Jiadai Sun, Jiajie Zhang, Jiale Cheng, Jiayi Gui, Jie Tang, Jing Zhang, Jingyu Sun, Juanzi Li, Lei Zhao, Lindong Wu, Lucen Zhong, Mingdao Liu, Minlie Huang, Peng Zhang, Qinkai Zheng, Rui Lu, Shuaiqi Duan, Shudan Zhang, Shulin Cao, Shuxun Yang, Weng Lam Tam, Wenyi Zhao, Xiao Liu, Xiao Xia, Xiaohan Zhang, Xiaotao Gu, Xin Lv, Xinghan Liu, Xinyi Liu, Xinyue Yang, Xixuan Song, Xunkai Zhang, Yifan An, Yifan Xu, Yilin Niu, Yuantao Yang, Yueyan Li, Yushi Bai, Yuxiao Dong, Zehan Qi, Zhaoyu Wang, Zhen Yang, Zhengxiao Du, Zhenyu Hou, Zihan Wang.
|
||||
1. **[GLM-MoE-DSA](https://huggingface.co/docs/transformers/main/model_doc/glm_moe_dsa)** (from the GLM Team, ZhipuAI & Tsinghua University) released with the paper [GLM-5: from Vibe Coding to Agentic Engineering](https://huggingface.co/papers/2602.15763) by Team GLM.
|
||||
1. **[GLM-OCR](https://huggingface.co/docs/transformers/main/model_doc/glm_ocr)** (from the GLM Team, ZhipuAI & Tsinghua University) released with the paper [GLM-OCR Technical Report](https://huggingface.co/papers/2603.10910) by Team GLM: Shuaiqi Duan, Yadong Xue, Weihan Wang, Zhe Su, Huan Liu, Sheng Yang, Guobing Gan, Guo Wang, Zihan Wang, Shengdong Yan, Dexin Jin, Yuxuan Zhang, Guohong Wen, Yanfeng Wang, Yutao Zhang, Xiaohan Zhang, Wenyi Hong, Yukuo Cen, Da Yin, Bin Chen, Wenmeng Yu, Xiaotao Gu, Jie Tang.
|
||||
1. **[GLPN](https://huggingface.co/docs/transformers/model_doc/glpn)** (from KAIST) released with the paper [Global-Local Path Networks for Monocular Depth Estimation with Vertical CutDepth](https://huggingface.co/papers/2201.07436) by Doyeon Kim, Woonghyun Ga, Pyungwhan Ahn, Donggyu Joo, Sehwan Chun, Junmo Kim.
|
||||
1. **[GPT Neo](https://huggingface.co/docs/transformers/model_doc/gpt_neo)** (from EleutherAI) released in the repository [EleutherAI/gpt-neo](https://github.com/EleutherAI/gpt-neo) by Sid Black, Stella Biderman, Leo Gao, Phil Wang and Connor Leahy.
|
||||
1. **[GPT NeoX](https://huggingface.co/docs/transformers/model_doc/gpt_neox)** (from EleutherAI) released with the paper [GPT-NeoX-20B: An Open-Source Autoregressive Language Model](https://huggingface.co/papers/2204.06745) by Sid Black, Stella Biderman, Eric Hallahan, Quentin Anthony, Leo Gao, Laurence Golding, Horace He, Connor Leahy, Kyle McDonell, Jason Phang, Michael Pieler, USVSN Sai Prashanth, Shivanshu Purohit, Laria Reynolds, Jonathan Tow, Ben Wang, Samuel Weinbach
|
||||
1. **[GPT OSS](https://huggingface.co/docs/transformers/model_doc/gpt_oss)** (from OpenAI) released with the blog [Introducing gpt-oss](https://openai.com/index/introducing-gpt-oss/) by Sandhini Agarwal, Lama Ahmad, Jason Ai, Sam Altman, Andy Applebaum, Edwin Arbus, Rahul K. Arora, Yu Bai, Bowen Baker, Haiming Bao, Boaz Barak, Ally Bennett, Tyler Bertao, Nivedita Brett, Eugene Brevdo, Greg Brockman, Sebastien Bubeck, Che Chang, Kai Chen, Mark Chen, Enoch Cheung, Aidan Clark, Dan Cook, Marat Dukhan, Casey Dvorak, Kevin Fives, Vlad Fomenko, Timur Garipov, Kristian Georgiev, Mia Glaese, Tarun Gogineni, Adam Goucher, Lukas Gross, Katia Gil Guzman, John Hallman, Jackie Hehir, Johannes Heidecke, Alec Helyar, Haitang Hu, Romain Huet, Jacob Huh, Saachi Jain, Zach Johnson, Chris Koch, Irina Kofman, Dominik Kundel, Jason Kwon, Volodymyr Kyrylov, Elaine Ya Le, Guillaume Leclerc, James Park Lennon, Scott Lessans, Mario Lezcano-Casado, Yuanzhi Li, Zhuohan Li, Ji Lin, Jordan Liss, Lily (Xiaoxuan) Liu, Jiancheng Liu, Kevin Lu, Chris Lu, Zoran Martinovic, Lindsay McCallum, Josh McGrath, Scott McKinney, Aidan McLaughlin, Song Mei, Steve Mostovoy, Tong Mu, Gideon Myles, Alexander Neitz, Alex Nichol, Jakub Pachocki, Alex Paino, Dana Palmie, Ashley Pantuliano, Giambattista Parascandolo, Jongsoo Park, Leher Pathak, Carolina Paz, Ludovic Peran, Dmitry Pimenov, Michelle Pokrass, Elizabeth Proehl, Huida Qiu, Gaby Raila, Filippo Raso, Hongyu Ren, Kimmy Richardson, David Robinson, Bob Rotsted, Hadi Salman, Suvansh Sanjeev, Max Schwarzer, D. Sculley, Harshit Sikchi, Kendal Simon, Karan Singhal, Yang Song, Dane Stuckey, Zhiqing Sun, Philippe Tillet, Sam Toizer, Foivos Tsimpourlas, Nikhil Vyas, Eric Wallace, Xin Wang, Miles Wang, Olivia Watkins, Kevin Weil, Amy Wendling, Kevin Whinnery, Cedric Whitney, Hannah Wong, Lin Yang, Yu Yang, Michihiro Yasunaga, Kristen Ying, Wojciech Zaremba, Wenting Zhan, Cyril Zhang, Brian Zhang, Eddie Zhang, Shengjia Zhao.
|
||||
1. **[GPT-2](https://huggingface.co/docs/transformers/model_doc/gpt2)** (from OpenAI) released with the paper [Language Models are Unsupervised Multitask Learners](https://blog.openai.com/better-language-models/) by Alec Radford*, Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever**.
|
||||
1. **[GPT-J](https://huggingface.co/docs/transformers/model_doc/gptj)** (from EleutherAI) released in the repository [kingoflolz/mesh-transformer-jax](https://github.com/kingoflolz/mesh-transformer-jax/) by Ben Wang and Aran Komatsuzaki.
|
||||
1. **[GPTBigCode](https://huggingface.co/docs/transformers/model_doc/gpt_bigcode)** (from BigCode) released with the paper [SantaCoder: don't reach for the stars!](https://huggingface.co/papers/2301.03988) by Loubna Ben Allal, Raymond Li, Denis Kocetkov, Chenghao Mou, Christopher Akiki, Carlos Munoz Ferrandis, Niklas Muennighoff, Mayank Mishra, Alex Gu, Manan Dey, Logesh Kumar Umapathi, Carolyn Jane Anderson, Yangtian Zi, Joel Lamy Poirier, Hailey Schoelkopf, Sergey Troshin, Dmitry Abulkhanov, Manuel Romero, Michael Lappert, Francesco De Toni, Bernardo García del Río, Qian Liu, Shamik Bose, Urvashi Bhattacharyya, Terry Yue Zhuo, Ian Yu, Paulo Villegas, Marco Zocca, Sourab Mangrulkar, David Lansky, Huu Nguyen, Danish Contractor, Luis Villa, Jia Li, Dzmitry Bahdanau, Yacine Jernite, Sean Hughes, Daniel Fried, Arjun Guha, Harm de Vries, Leandro von Werra.
|
||||
1. **[Granite](https://huggingface.co/docs/transformers/main/model_doc/granite)** (from IBM) released with the paper [Power Scheduler: A Batch Size and Token Number Agnostic Learning Rate Scheduler](https://huggingface.co/papers/2408.13359) by Yikang Shen, Matthew Stallone, Mayank Mishra, Gaoyuan Zhang, Shawn Tan, Aditya Prasad, Adriana Meza Soria, David D. Cox, Rameswar Panda.
|
||||
1. **[GraniteMoeHybrid](https://huggingface.co/docs/transformers/main/model_doc/granitemoehybrid)** (from IBM) released with the blog post [IBM Granite 4.0: hyper-efficient, high performance hybrid models for enterprise](https://www.ibm.com/new/announcements/ibm-granite-4-0-hyper-efficient-high-performance-hybrid-models) by the IBM Granite team.
|
||||
1. **[Granite Speech](https://huggingface.co/docs/transformers/main/model_doc/granite_speech)** (from IBM) released with the paper [Granite-speech: open-source speech-aware LLMs with strong English ASR capabilities](https://huggingface.co/papers/2505.08699) by the IBM Granite team.
|
||||
1. **[Grounding DINO](https://huggingface.co/docs/transformers/model_doc/grounding-dino)** (from IDEA-Research) released with the paper [Grounding DINO: Marrying DINO with Grounded Pre-Training for Open-Set Object Detection](https://huggingface.co/papers/2303.05499) by Shilong Liu, Zhaoyang Zeng, Tianhe Ren, Feng Li, Hao Zhang, Jie Yang, Qing Jiang, Chunyuan Li, Jianwei Yang, Hang Su, Jun Zhu, Lei Zhang.
|
||||
1. **[GroupViT](https://huggingface.co/docs/transformers/model_doc/groupvit)** (from UCSD, NVIDIA) released with the paper [GroupViT: Semantic Segmentation Emerges from Text Supervision](https://huggingface.co/papers/2202.11094) by Jiarui Xu, Shalini De Mello, Sifei Liu, Wonmin Byeon, Thomas Breuel, Jan Kautz, Xiaolong Wang.
|
||||
1. **[Helium](https://huggingface.co/docs/transformers/main/model_doc/helium)** (from the Kyutai Team) released with the blog post [Announcing Helium-1 Preview](https://kyutai.org/2025/01/13/helium.html) by the Kyutai Team.
|
||||
1. **[HerBERT](https://huggingface.co/docs/transformers/model_doc/herbert)** (from Allegro.pl, AGH University of Science and Technology) released with the paper [KLEJ: Comprehensive Benchmark for Polish Language Understanding](https://www.aclweb.org/anthology/2020.acl-main.111.pdf) by Piotr Rybak, Robert Mroczkowski, Janusz Tracz, Ireneusz Gawlik.
|
||||
1. **[HRM-Text](https://huggingface.co/docs/transformers/model_doc/hrm_text)** (from Sapient Intelligence) released with the paper [Hierarchical Reasoning Model](https://huggingface.co/papers/2506.21734) by Guan Wang, Jin Li, Yuhao Sun, Xing Chen, Changling Liu, Yue Wu, Meng Lu, Sen Song, Yasin Abbasi Yadkori.
|
||||
1. **[Hiera](https://huggingface.co/docs/transformers/model_doc/hiera)** (from Meta) released with the paper [Hiera: A Hierarchical Vision Transformer without the Bells-and-Whistles](https://huggingface.co/papers/2306.00989) by Chaitanya Ryali, Yuan-Ting Hu, Daniel Bolya, Chen Wei, Haoqi Fan, Po-Yao Huang, Vaibhav Aggarwal, Arkabandhu Chowdhury, Omid Poursaeed, Judy Hoffman, Jitendra Malik, Yanghao Li, Christoph Feichtenhofer.
|
||||
1. **[Hubert](https://huggingface.co/docs/transformers/model_doc/hubert)** (from Facebook) released with the paper [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units](https://huggingface.co/papers/2106.07447) by Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia, Ruslan Salakhutdinov, Abdelrahman Mohamed.
|
||||
1. **[HunYuanDenseV1](https://huggingface.co/docs/transformers/model_doc/hunyuan_v1_dense)** (from Tencent) released with the paper [Hunyuan-Large: An Open-Source MoE Model with 52 Billion Activated Parameters by Tencent](https://huggingface.co/papers/2411.02265) by the Tencent team.
|
||||
1. **[I-JEPA](https://huggingface.co/docs/transformers/model_doc/ijepa)** (from Meta) released with the paper [Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture](https://huggingface.co/papers/2301.08243) by Mahmoud Assran, Quentin Duval, Ishan Misra, Piotr Bojanowski, Pascal Vincent, Michael Rabbat, Yann LeCun, Nicolas Ballas.
|
||||
1. **[Idefics3](https://huggingface.co/docs/transformers/model_doc/idefics3)** (from Hugging Face) released with the paper [Building and better understanding vision-language models: insights and future directions](https://huggingface.co/papers/2408.12637) by Hugo Laurençon, Andrés Marafioti, Victor Sanh, Léo Tronchon.
|
||||
1. **JAIS** (from Core42) released with the paper [Jais and Jais-chat: Arabic-Centric Foundation and Instruction-Tuned Open Generative Large Language Models](https://huggingface.co/papers/2308.16149) by Neha Sengupta, Sunil Kumar Sahu, Bokang Jia, Satheesh Katipomu, Haonan Li, Fajri Koto, William Marshall, Gurpreet Gosal, Cynthia Liu, Zhiming Chen, Osama Mohammed Afzal, Samta Kamboj, Onkar Pandit, Rahul Pal, Lalit Pradhan, Zain Muhammad Mujahid, Massa Baali, Xudong Han, Sondos Mahmoud Bsharat, Alham Fikri Aji, Zhiqiang Shen, Zhengzhong Liu, Natalia Vassilieva, Joel Hestness, Andy Hock, Andrew Feldman, Jonathan Lee, Andrew Jackson, Hector Xuguang Ren, Preslav Nakov, Timothy Baldwin, Eric Xing.
|
||||
1. **Janus** (from DeepSeek) released with the paper [Janus: Decoupling Visual Encoding for Unified Multimodal Understanding and Generation](https://huggingface.co/papers/2410.13848) Chengyue Wu, Xiaokang Chen, Zhiyu Wu, Yiyang Ma, Xingchao Liu, Zizheng Pan, Wen Liu, Zhenda Xie, Xingkai Yu, Chong Ruan, Ping Luo.
|
||||
1. **JinaCLIP** (from Jina AI) released with the paper [Jina CLIP: Your CLIP Model Is Also Your Text Retriever](https://huggingface.co/papers/2405.20204) by Andreas Koukounas, Georgios Mastrapas, Michael Günther, Bo Wang, Scott Martens, Isabelle Mohr, Saba Sturua, Mohammad Kalim Akram, Joan Fontanals Martínez, Saahil Ognawala, Susana Guzman, Maximilian Werk, Nan Wang, Han Xiao.
|
||||
1. **LiteWhisper** (from University of Washington, Kotoba Technologies) released with the paper [LiteASR: Efficient Automatic Speech Recognition with Low-Rank Approximation](https://huggingface.co/papers/2502.20583) by Keisuke Kamahori, Jungo Kasai, Noriyuki Kojima, Baris Kasikci.
|
||||
1. **[LongT5](https://huggingface.co/docs/transformers/model_doc/longt5)** (from Google AI) released with the paper [LongT5: Efficient Text-To-Text Transformer for Long Sequences](https://huggingface.co/papers/2112.07916) by Mandy Guo, Joshua Ainslie, David Uthus, Santiago Ontanon, Jianmo Ni, Yun-Hsuan Sung, Yinfei Yang.
|
||||
1. **[LFM2](https://huggingface.co/docs/transformers/model_doc/lfm2)** (from Liquid AI) released with the blog post [Introducing LFM2: The Fastest On-Device Foundation Models on the Market](https://www.liquid.ai/blog/liquid-foundation-models-v2-our-second-series-of-generative-ai-models) by the Liquid AI Team.
|
||||
1. **[Lfm2Moe](https://huggingface.co/docs/transformers/model_doc/lfm2_moe)** (from Liquid AI) released with the blog post [LFM2-8B-A1B: An Efficient On-device Mixture-of-Experts](https://www.liquid.ai/blog/lfm2-8b-a1b-an-efficient-on-device-mixture-of-experts) by the Liquid AI Team.
|
||||
1. **[LFM2-VL](https://huggingface.co/docs/transformers/model_doc/lfm2_vl)** (from Liquid AI) released with the blog post [LFM2-VL: Efficient Vision-Language Models](https://www.liquid.ai/blog/lfm2-vl-efficient-vision-language-models) by the Liquid AI Team.
|
||||
1. **[LightOnOcr](https://huggingface.co/docs/transformers/model_doc/lighton_ocr)** (from LightOn) released with the blog post [LightOnOCR-1B: The Case for End-to-End and Efficient Domain-Specific Vision-Language Models for OCR](https://huggingface.co/blog/lightonai/lightonocr) by the LightOn Team.
|
||||
1. **[LLaMA](https://huggingface.co/docs/transformers/model_doc/llama)** (from The FAIR team of Meta AI) released with the paper [LLaMA: Open and Efficient Foundation Language Models](https://huggingface.co/papers/2302.13971) by Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, Guillaume Lample.
|
||||
1. **[Llama2](https://huggingface.co/docs/transformers/model_doc/llama2)** (from The FAIR team of Meta AI) released with the paper [Llama2: Open Foundation and Fine-Tuned Chat Models](https://huggingface.co/papers/2307.09288) by Hugo Touvron, Louis Martin, Kevin Stone, Peter Albert, Amjad Almahairi, Yasmine Babaei, Nikolay Bashlykov, Soumya Batra, Prajjwal Bhargava, Shruti Bhosale, Dan Bikel, Lukas Blecher, Cristian Canton Ferrer, Moya Chen, Guillem Cucurull, David Esiobu, Jude Fernandes, Jeremy Fu, Wenyin Fu, Brian Fuller, Cynthia Gao, Vedanuj Goswami, Naman Goyal, Anthony Hartshorn, Saghar Hosseini, Rui Hou, Hakan Inan, Marcin Kardas, Viktor Kerkez Madian Khabsa, Isabel Kloumann, Artem Korenev, Punit Singh Koura, Marie-Anne Lachaux, Thibaut Lavril, Jenya Lee, Diana Liskovich, Yinghai Lu, Yuning Mao, Xavier Martinet, Todor Mihaylov, Pushka rMishra, Igor Molybog, Yixin Nie, Andrew Poulton, Jeremy Reizenstein, Rashi Rungta, Kalyan Saladi, Alan Schelten, Ruan Silva, Eric Michael Smith, Ranjan Subramanian, Xiaoqing EllenTan, Binh Tang, Ross Taylor, Adina Williams, Jian Xiang Kuan, Puxin Xu, Zheng Yan, Iliyan Zarov, Yuchen Zhang, Angela Fan, Melanie Kambadur, Sharan Narang, Aurelien Rodriguez, Robert Stojnic, Sergey Edunov, Thomas Scialom.
|
||||
1. **[Llama3](https://huggingface.co/docs/transformers/model_doc/llama3)** (from The FAIR team of Meta AI) released with the paper [The Llama 3 Herd of Models](https://huggingface.co/papers/2407.21783) by the Llama Team at Meta.
|
||||
1. **[Llama4](https://huggingface.co/docs/transformers/model_doc/llama4)** (from The FAIR team of Meta AI) released with the blog post [The Llama 4 herd: The beginning of a new era of natively multimodal AI innovation](https://ai.meta.com/blog/llama-4-multimodal-intelligence/) by the Llama Team at Meta.
|
||||
1. **[LLaVa](https://huggingface.co/docs/transformers/model_doc/llava)** (from Microsoft Research & University of Wisconsin-Madison) released with the paper [Visual Instruction Tuning](https://huggingface.co/papers/2304.08485) by Haotian Liu, Chunyuan Li, Yuheng Li and Yong Jae Lee.
|
||||
1. **[LLaVA-OneVision](https://huggingface.co/docs/transformers/model_doc/llava_onevision)** (from ByteDance & NTU & CUHK & HKUST) released with the paper [LLaVA-OneVision: Easy Visual Task Transfer](https://huggingface.co/papers/2408.03326) by Bo Li, Yuanhan Zhang, Dong Guo, Renrui Zhang, Feng Li, Hao Zhang, Kaichen Zhang, Yanwei Li, Ziwei Liu, Chunyuan Li
|
||||
1. **[M2M100](https://huggingface.co/docs/transformers/model_doc/m2m_100)** (from Facebook) released with the paper [Beyond English-Centric Multilingual Machine Translation](https://huggingface.co/papers/2010.11125) by Angela Fan, Shruti Bhosale, Holger Schwenk, Zhiyi Ma, Ahmed El-Kishky, Siddharth Goyal, Mandeep Baines, Onur Celebi, Guillaume Wenzek, Vishrav Chaudhary, Naman Goyal, Tom Birch, Vitaliy Liptchinsky, Sergey Edunov, Edouard Grave, Michael Auli, Armand Joulin.
|
||||
1. **[MarianMT](https://huggingface.co/docs/transformers/model_doc/marian)** Machine translation models trained using [OPUS](http://opus.nlpl.eu/) data by Jörg Tiedemann. The [Marian Framework](https://marian-nmt.github.io/) is being developed by the Microsoft Translator Team.
|
||||
1. **[MaskFormer](https://huggingface.co/docs/transformers/model_doc/maskformer)** (from Meta and UIUC) released with the paper [Per-Pixel Classification is Not All You Need for Semantic Segmentation](https://huggingface.co/papers/2107.06278) by Bowen Cheng, Alexander G. Schwing, Alexander Kirillov.
|
||||
1. **[mBART](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Denoising Pre-training for Neural Machine Translation](https://huggingface.co/papers/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
|
||||
1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://huggingface.co/papers/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan.
|
||||
1. **Metric3D** released with the paper [Metric3D: Towards Zero-shot Metric 3D Prediction from A Single Image](https://huggingface.co/papers/2307.10984) by Wei Yin, Chi Zhang, Hao Chen, Zhipeng Cai, Gang Yu, Kaixuan Wang, Xiaozhi Chen, Chunhua Shen.
|
||||
1. **Metric3Dv2** released with the paper [Metric3Dv2: A Versatile Monocular Geometric Foundation Model for Zero-shot Metric Depth and Surface Normal Estimation](https://huggingface.co/papers/2404.15506) by Mu Hu, Wei Yin, Chi Zhang, Zhipeng Cai, Xiaoxiao Long, Kaixuan Wang, Hao Chen, Gang Yu, Chunhua Shen, Shaojie Shen.
|
||||
1. **[MusicGen](https://huggingface.co/docs/transformers/model_doc/musicgen)** (from Meta) released with the paper [Simple and Controllable Music Generation](https://huggingface.co/papers/2306.05284) by Jade Copet, Felix Kreuk, Itai Gat, Tal Remez, David Kant, Gabriel Synnaeve, Yossi Adi and Alexandre Défossez.
|
||||
1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://huggingface.co/papers/2209.03592) by Peng Wang, Cheng Da, and Cong Yao.
|
||||
1. **[Mimi](https://huggingface.co/docs/transformers/model_doc/mimi)** (from Kyutai) released with the paper [Moshi: a speech-text foundation model for real-time dialogue](https://huggingface.co/papers/2410.00037) by Alexandre Défossez, Laurent Mazaré, Manu Orsini, Amélie Royer, Patrick Pérez, Hervé Jégou, Edouard Grave and Neil Zeghidour.
|
||||
1. **[Ministral](https://huggingface.co/docs/transformers/model_doc/ministral)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team.
|
||||
1. **[Ministral3](https://huggingface.co/docs/transformers/model_doc/ministral3)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team.
|
||||
1. **[Mistral](https://huggingface.co/docs/transformers/model_doc/mistral)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team: Albert Jiang, Alexandre Sablayrolles, Arthur Mensch, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Florian Bressand, Gianna Lengyel, Guillaume Lample, Lélio Renard Lavaud, Lucile Saulnier, Marie-Anne Lachaux, Pierre Stock, Teven Le Scao, Thibaut Lavril, Thomas Wang, Timothée Lacroix, William El Sayed.
|
||||
1. **[Mistral3](https://huggingface.co/docs/transformers/model_doc/mistral3)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team.
|
||||
1. **[Mistral4](https://huggingface.co/docs/transformers/model_doc/mistral4)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team.
|
||||
1. **[MMS](https://huggingface.co/docs/transformers/model_doc/mms)** (from Facebook) released with the paper [Scaling Speech Technology to 1,000+ Languages](https://huggingface.co/papers/2305.13516) by Vineel Pratap, Andros Tjandra, Bowen Shi, Paden Tomasello, Arun Babu, Sayani Kundu, Ali Elkahky, Zhaoheng Ni, Apoorv Vyas, Maryam Fazel-Zarandi, Alexei Baevski, Yossi Adi, Xiaohui Zhang, Wei-Ning Hsu, Alexis Conneau, Michael Auli.
|
||||
1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (from CMU/Google Brain) released with the paper [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://huggingface.co/papers/2004.02984) by Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou.
|
||||
1. **MobileCLIP** (from Apple) released with the paper [MobileCLIP: Fast Image-Text Models through Multi-Modal Reinforced Training](https://huggingface.co/papers/2311.17049) by Pavan Kumar Anasosalu Vasu, Hadi Pouransari, Fartash Faghri, Raviteja Vemulapalli, Oncel Tuzel.
|
||||
1. **MobileLLM** (from Meta) released with the paper [MobileLLM: Optimizing Sub-billion Parameter Language Models for On-Device Use Cases](https://huggingface.co/papers/2402.14905) by Zechun Liu, Changsheng Zhao, Forrest Iandola, Chen Lai, Yuandong Tian, Igor Fedorov, Yunyang Xiong, Ernie Chang, Yangyang Shi, Raghuraman Krishnamoorthi, Liangzhen Lai, Vikas Chandra.
|
||||
1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://huggingface.co/papers/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam.
|
||||
1. **[MobileNetV2](https://huggingface.co/docs/transformers/model_doc/mobilenet_v2)** (from Google Inc.) released with the paper [MobileNetV2: Inverted Residuals and Linear Bottlenecks](https://huggingface.co/papers/1801.04381) by Mark Sandler, Andrew Howard, Menglong Zhu, Andrey Zhmoginov, Liang-Chieh Chen.
|
||||
1. **MobileNetV3** (from Google Inc.) released with the paper [Searching for MobileNetV3](https://huggingface.co/papers/1905.02244) by Andrew Howard, Mark Sandler, Grace Chu, Liang-Chieh Chen, Bo Chen, Mingxing Tan, Weijun Wang, Yukun Zhu, Ruoming Pang, Vijay Vasudevan, Quoc V. Le, Hartwig Adam.
|
||||
1. **MobileNetV4** (from Google Inc.) released with the paper [MobileNetV4 - Universal Models for the Mobile Ecosystem](https://huggingface.co/papers/2404.10518) by Danfeng Qin, Chas Leichner, Manolis Delakis, Marco Fornoni, Shixin Luo, Fan Yang, Weijun Wang, Colby Banbury, Chengxi Ye, Berkin Akin, Vaibhav Aggarwal, Tenghui Zhu, Daniele Moro, Andrew Howard.
|
||||
1. **[MobileViT](https://huggingface.co/docs/transformers/model_doc/mobilevit)** (from Apple) released with the paper [MobileViT: Light-weight, General-purpose, and Mobile-friendly Vision Transformer](https://huggingface.co/papers/2110.02178) by Sachin Mehta and Mohammad Rastegari.
|
||||
1. **[MobileViTV2](https://huggingface.co/docs/transformers/model_doc/mobilevitv2)** (from Apple) released with the paper [Separable Self-attention for Mobile Vision Transformers](https://huggingface.co/papers/2206.02680) by Sachin Mehta and Mohammad Rastegari.
|
||||
1. **[ModernBERT](https://huggingface.co/docs/transformers/model_doc/modernbert)** (from Answer.AI and LightOn) released with the paper [Smarter, Better, Faster, Longer: A Modern Bidirectional Encoder for Fast, Memory Efficient, and Long Context Finetuning and Inference](https://huggingface.co/papers/2412.13663) by Benjamin Warner, Antoine Chaffin, Benjamin Clavié, Orion Weller, Oskar Hallström, Said Taghadouini, Alexis Gallagher, Raja Biswas, Faisal Ladhak, Tom Aarsen, Nathan Cooper, Griffin Adams, Jeremy Howard, Iacopo Poli.
|
||||
1. **[ModernBERT Decoder](https://huggingface.co/docs/transformers/model_doc/modernbert-decoder)** (from Johns Hopkins University and LightOn) released with the paper [Seq vs Seq: An Open Suite of Paired Encoders and Decoders](https://huggingface.co/papers/2507.11412) by Orion Weller, Kathryn Ricci, Marc Marone, Antoine Chaffin, Dawn Lawrie, Benjamin Van Durme.
|
||||
1. **Moondream1** released in the repository [moondream](https://github.com/vikhyat/moondream) by vikhyat.
|
||||
1. **[Moonshine](https://huggingface.co/docs/transformers/model_doc/moonshine)** (from Useful Sensors) released with the paper [Moonshine: Speech Recognition for Live Transcription and Voice Commands](https://huggingface.co/papers/2410.15608) by Nat Jeffries, Evan King, Manjunath Kudlur, Guy Nicholson, James Wang, Pete Warden.
|
||||
1. **[MPNet](https://huggingface.co/docs/transformers/model_doc/mpnet)** (from Microsoft Research) released with the paper [MPNet: Masked and Permuted Pre-training for Language Understanding](https://huggingface.co/papers/2004.09297) by Kaitao Song, Xu Tan, Tao Qin, Jianfeng Lu, Tie-Yan Liu.
|
||||
1. **[MPT](https://huggingface.co/docs/transformers/model_doc/mpt)** (from MosaicML) released with the repository [llm-foundry](https://github.com/mosaicml/llm-foundry/) by the MosaicML NLP Team.
|
||||
1. **[MT5](https://huggingface.co/docs/transformers/model_doc/mt5)** (from Google AI) released with the paper [mT5: A massively multilingual pre-trained text-to-text transformer](https://huggingface.co/papers/2010.11934) by Linting Xue, Noah Constant, Adam Roberts, Mihir Kale, Rami Al-Rfou, Aditya Siddhant, Aditya Barua, Colin Raffel.
|
||||
1. **[NanoChat](https://huggingface.co/docs/transformers/model_doc/nanochat)** released with the repository [nanochat: The best ChatGPT that $100 can buy](https://github.com/karpathy/nanochat) by Andrej Karpathy.
|
||||
1. **[Nemotron](https://huggingface.co/docs/transformers/model_doc/nemotron)** (from NVIDIA) released with the blog post [NVIDIA AI Foundation Models: Build Custom Enterprise Chatbots and Co-Pilots with Production-Ready LLMs](https://developer.nvidia.com/blog/nvidia-ai-foundation-models-build-custom-enterprise-chatbots-and-co-pilots-with-production-ready-llms/) by the Nemotron team.
|
||||
1. **[Nemotron-H](https://huggingface.co/docs/transformers/model_doc/nemotron_h)** (from NVIDIA) released with the papers [Nemotron-H: A Family of Accurate and Efficient Hybrid Mamba-Transformer Models](https://huggingface.co/papers/2504.03624), [NVIDIA Nemotron 3: Efficient and Open Intelligence](https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-White-Paper.pdf), [Nemotron 3 Nano: Open, Efficient Mixture-of-Experts Hybrid Mamba-Transformer Model for Agentic Reasoning](https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Nano-Technical-Report.pdf), and [Nemotron 3 Super: Open, Efficient Mixture-of-Experts Hybrid Mamba-Transformer Model for Agentic Reasoning](https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Super-Technical-Report.pdf) by the Nemotron team.
|
||||
1. **NeoBERT** (from Chandar Research Lab) released with the paper [NeoBERT: A Next-Generation BERT](https://huggingface.co/papers/2502.19587) by Lola Le Breton, Quentin Fournier, Mariam El Mezouar, John X. Morris, Sarath Chandar.
|
||||
1. **[NLLB](https://huggingface.co/docs/transformers/model_doc/nllb)** (from Meta) released with the paper [No Language Left Behind: Scaling Human-Centered Machine Translation](https://huggingface.co/papers/2207.04672) by the NLLB team.
|
||||
1. **[Nougat](https://huggingface.co/docs/transformers/model_doc/nougat)** (from Meta AI) released with the paper [Nougat: Neural Optical Understanding for Academic Documents](https://huggingface.co/papers/2308.13418) by Lukas Blecher, Guillem Cucurull, Thomas Scialom, Robert Stojnic.
|
||||
1. **[OLMo](https://huggingface.co/docs/transformers/master/model_doc/olmo)** (from Ai2) released with the paper [OLMo: Accelerating the Science of Language Models](https://huggingface.co/papers/2402.00838) by Dirk Groeneveld, Iz Beltagy, Pete Walsh, Akshita Bhagia, Rodney Kinney, Oyvind Tafjord, Ananya Harsh Jha, Hamish Ivison, Ian Magnusson, Yizhong Wang, Shane Arora, David Atkinson, Russell Authur, Khyathi Raghavi Chandu, Arman Cohan, Jennifer Dumas, Yanai Elazar, Yuling Gu, Jack Hessel, Tushar Khot, William Merrill, Jacob Morrison, Niklas Muennighoff, Aakanksha Naik, Crystal Nam, Matthew E. Peters, Valentina Pyatkin, Abhilasha Ravichander, Dustin Schwenk, Saurabh Shah, Will Smith, Emma Strubell, Nishant Subramani, Mitchell Wortsman, Pradeep Dasigi, Nathan Lambert, Kyle Richardson, Luke Zettlemoyer, Jesse Dodge, Kyle Lo, Luca Soldaini, Noah A. Smith, Hannaneh Hajishirzi.
|
||||
1. **[OLMo2](https://huggingface.co/docs/transformers/master/model_doc/olmo2)** (from Ai2) released with the blog [OLMo 2: The best fully open language model to date](https://allenai.org/blog/olmo2) by the Ai2 OLMo team.
|
||||
1. **[OLMo3](https://huggingface.co/docs/transformers/master/model_doc/olmo3)** (from Ai2) released with the blog [Olmo 3: Charting a path through the model flow to lead open-source AI](https://allenai.org/blog/olmo3) by the Ai2 OLMo team.
|
||||
1. **[Olmo Hybrid](https://huggingface.co/docs/transformers/master/model_doc/olmo_hybrid)** (from Ai2) released with the blog [Introducing Olmo Hybrid: Combining transformers and linear RNNs for superior scaling](https://allenai.org/blog/olmohybrid) by the Ai2 OLMo team.
|
||||
1. **OpenELM** (from Apple) released with the paper [OpenELM: An Efficient Language Model Family with Open-source Training and Inference Framework](https://huggingface.co/papers/2404.14619) by Sachin Mehta, Mohammad Hossein Sekhavat, Qingqing Cao, Maxwell Horton, Yanzi Jin, Chenfan Sun, Iman Mirzadeh, Mahyar Najibi, Dmitry Belenko, Peter Zatloukal, Mohammad Rastegari.
|
||||
1. **[OPT](https://huggingface.co/docs/transformers/master/model_doc/opt)** (from Meta AI) released with the paper [OPT: Open Pre-trained Transformer Language Models](https://huggingface.co/papers/2205.01068) by Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen et al.
|
||||
1. **[OWL-ViT](https://huggingface.co/docs/transformers/model_doc/owlvit)** (from Google AI) released with the paper [Simple Open-Vocabulary Object Detection with Vision Transformers](https://huggingface.co/papers/2205.06230) by Matthias Minderer, Alexey Gritsenko, Austin Stone, Maxim Neumann, Dirk Weissenborn, Alexey Dosovitskiy, Aravindh Mahendran, Anurag Arnab, Mostafa Dehghani, Zhuoran Shen, Xiao Wang, Xiaohua Zhai, Thomas Kipf, and Neil Houlsby.
|
||||
1. **[OWLv2](https://huggingface.co/docs/transformers/model_doc/owlv2)** (from Google AI) released with the paper [Scaling Open-Vocabulary Object Detection](https://huggingface.co/papers/2306.09683) by Matthias Minderer, Alexey Gritsenko, Neil Houlsby.
|
||||
1. **[PaliGemma](https://huggingface.co/docs/transformers/main/model_doc/paligemma)** (from Google) released with the papers [PaliGemma: A versatile 3B VLM for transfer](https://huggingface.co/papers/2407.07726) and [PaliGemma 2: A Family of Versatile VLMs for Transfer](https://huggingface.co/papers/2412.03555) by the PaliGemma Google team.
|
||||
1. **[Parakeet](https://huggingface.co/docs/transformers/main/model_doc/parakeet)** (from NVIDIA) released with the blog post [Introducing the Parakeet ASR family](https://developer.nvidia.com/blog/pushing-the-boundaries-of-speech-recognition-with-nemo-parakeet-asr-models/) by the NVIDIA NeMo team.
|
||||
1. **[PatchTSMixer](https://huggingface.co/docs/transformers/main/model_doc/patchtsmixer)** (from IBM) released with the paper [TSMixer: Lightweight MLP-Mixer Model for Multivariate Time Series Forecasting](https://huggingface.co/papers/2306.09364) by Vijay Ekambaram, Arindam Jati, Nam Nguyen, Phanwadee Sinthong, Jayant Kalagnanam.
|
||||
1. **[PatchTST](https://huggingface.co/docs/transformers/main/model_doc/patchtst)** (from Princeton University, IBM) released with the paper [A Time Series is Worth 64 Words: Long-term Forecasting with Transformers](https://huggingface.co/papers/2211.14730) by Yuqi Nie, Nam H. Nguyen, Phanwadee Sinthong, Jayant Kalagnanam.
|
||||
1. **[Phi](https://huggingface.co/docs/transformers/main/model_doc/phi)** (from Microsoft) released with the papers - [Textbooks Are All You Need](https://huggingface.co/papers/2306.11644) by Suriya Gunasekar, Yi Zhang, Jyoti Aneja, Caio César Teodoro Mendes, Allie Del Giorno, Sivakanth Gopi, Mojan Javaheripi, Piero Kauffmann, Gustavo de Rosa, Olli Saarikivi, Adil Salim, Shital Shah, Harkirat Singh Behl, Xin Wang, Sébastien Bubeck, Ronen Eldan, Adam Tauman Kalai, Yin Tat Lee and Yuanzhi Li, [Textbooks Are All You Need II: phi-1.5 technical report](https://huggingface.co/papers/2309.05463) by Yuanzhi Li, Sébastien Bubeck, Ronen Eldan, Allie Del Giorno, Suriya Gunasekar and Yin Tat Lee.
|
||||
1. **[Phi3](https://huggingface.co/docs/transformers/main/model_doc/phi3)** (from Microsoft) released with the paper [Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone](https://huggingface.co/papers/2404.14219v2) by Marah Abdin, Sam Ade Jacobs, Ammar Ahmad Awan, Jyoti Aneja, Ahmed Awadallah, Hany Awadalla, Nguyen Bach, Amit Bahree, Arash Bakhtiari, Harkirat Behl, Alon Benhaim, Misha Bilenko, Johan Bjorck, Sébastien Bubeck, Martin Cai, Caio César Teodoro Mendes, Weizhu Chen, Vishrav Chaudhary, Parul Chopra, Allie Del Giorno, Gustavo de Rosa, Matthew Dixon, Ronen Eldan, Dan Iter, Amit Garg, Abhishek Goswami, Suriya Gunasekar, Emman Haider, Junheng Hao, Russell J. Hewett, Jamie Huynh, Mojan Javaheripi, Xin Jin, Piero Kauffmann, Nikos Karampatziakis, Dongwoo Kim, Mahoud Khademi, Lev Kurilenko, James R. Lee, Yin Tat Lee, Yuanzhi Li, Chen Liang, Weishung Liu, Eric Lin, Zeqi Lin, Piyush Madan, Arindam Mitra, Hardik Modi, Anh Nguyen, Brandon Norick, Barun Patra, Daniel Perez-Becker, Thomas Portet, Reid Pryzant, Heyang Qin, Marko Radmilac, Corby Rosset, Sambudha Roy, Olatunji Ruwase, Olli Saarikivi, Amin Saied, Adil Salim, Michael Santacroce, Shital Shah, Ning Shang, Hiteshi Sharma, Xia Song, Masahiro Tanaka, Xin Wang, Rachel Ward, Guanhua Wang, Philipp Witte, Michael Wyatt, Can Xu, Jiahang Xu, Sonali Yadav, Fan Yang, Ziyi Yang, Donghan Yu, Chengruidong Zhang, Cyril Zhang, Jianwen Zhang, Li Lyna Zhang, Yi Zhang, Yue Zhang, Yunan Zhang, Xiren Zhou.
|
||||
1. **Phi3V** (from Microsoft) released with the paper [Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone](https://huggingface.co/papers/2404.14219v4) by Marah Abdin, Jyoti Aneja, Hany Awadalla, Ahmed Awadallah, Ammar Ahmad Awan, Nguyen Bach, Amit Bahree, Arash Bakhtiari, Jianmin Bao, Harkirat Behl, Alon Benhaim, Misha Bilenko, Johan Bjorck, Sébastien Bubeck, Martin Cai, Qin Cai, Vishrav Chaudhary, Dong Chen, Dongdong Chen, Weizhu Chen, Yen-Chun Chen, Yi-Ling Chen, Hao Cheng, Parul Chopra, Xiyang Dai, Matthew Dixon, Ronen Eldan, Victor Fragoso, Jianfeng Gao, Mei Gao, Min Gao, Amit Garg, Allie Del Giorno, Abhishek Goswami, Suriya Gunasekar, Emman Haider, Junheng Hao, Russell J. Hewett, Wenxiang Hu, Jamie Huynh, Dan Iter, Sam Ade Jacobs, Mojan Javaheripi, Xin Jin, Nikos Karampatziakis, Piero Kauffmann, Mahoud Khademi, Dongwoo Kim, Young Jin Kim, Lev Kurilenko, James R. Lee, Yin Tat Lee, Yuanzhi Li, Yunsheng Li, Chen Liang, Lars Liden, Xihui Lin, Zeqi Lin, Ce Liu, Liyuan Liu, Mengchen Liu, Weishung Liu, Xiaodong Liu, Chong Luo, Piyush Madan, Ali Mahmoudzadeh, David Majercak, Matt Mazzola, Caio César Teodoro Mendes, Arindam Mitra, Hardik Modi, Anh Nguyen, Brandon Norick, Barun Patra, Daniel Perez-Becker, Thomas Portet, Reid Pryzant, Heyang Qin, Marko Radmilac, Liliang Ren, Gustavo de Rosa, Corby Rosset, Sambudha Roy, Olatunji Ruwase, Olli Saarikivi, Amin Saied, Adil Salim, Michael Santacroce, Shital Shah, Ning Shang, Hiteshi Sharma, Yelong Shen, Swadheen Shukla, Xia Song, Masahiro Tanaka, Andrea Tupini, Praneetha Vaddamanu, Chunyu Wang, Guanhua Wang, Lijuan Wang , Shuohang Wang, Xin Wang, Yu Wang, Rachel Ward, Wen Wen, Philipp Witte, Haiping Wu, Xiaoxia Wu, Michael Wyatt, Bin Xiao, Can Xu, Jiahang Xu, Weijian Xu, Jilong Xue, Sonali Yadav, Fan Yang, Jianwei Yang, Yifan Yang, Ziyi Yang, Donghan Yu, Lu Yuan, Chenruidong Zhang, Cyril Zhang, Jianwen Zhang, Li Lyna Zhang, Yi Zhang, Yue Zhang, Yunan Zhang, Xiren Zhou.
|
||||
1. **[PVT](https://huggingface.co/docs/transformers/main/model_doc/pvt)** (from Nanjing University, The University of Hong Kong etc.) released with the paper [Pyramid Vision Transformer: A Versatile Backbone for Dense Prediction without Convolutions](https://huggingface.co/papers/2102.12122) by Wenhai Wang, Enze Xie, Xiang Li, Deng-Ping Fan, Kaitao Song, Ding Liang, Tong Lu, Ping Luo, Ling Shao.
|
||||
1. **PyAnnote** released in the repository [pyannote/pyannote-audio](https://github.com/pyannote/pyannote-audio) by Hervé Bredin.
|
||||
1. **[Qwen2](https://huggingface.co/docs/transformers/model_doc/qwen2)** (from the Qwen team, Alibaba Group) released with the papers [Qwen Technical Report](https://huggingface.co/papers/2309.16609) and [Qwen2 Technical Report](https://huggingface.co/papers/2407.10671) by Jinze Bai, Shuai Bai, Yunfei Chu, Zeyu Cui, Kai Dang, Xiaodong Deng, Yang Fan, Wenbin Ge, Yu Han, Fei Huang, Binyuan Hui, Luo Ji, Mei Li, Junyang Lin, Runji Lin, Dayiheng Liu, Gao Liu, Chengqiang Lu, Keming Lu, Jianxin Ma, Rui Men, Xingzhang Ren, Xuancheng Ren, Chuanqi Tan, Sinan Tan, Jianhong Tu, Peng Wang, Shijie Wang, Wei Wang, Shengguang Wu, Benfeng Xu, Jin Xu, An Yang, Hao Yang, Jian Yang, Shusheng Yang, Yang Yao, Bowen Yu, Hongyi Yuan, Zheng Yuan, Jianwei Zhang, Xingxuan Zhang, Yichang Zhang, Zhenru Zhang, Chang Zhou, Jingren Zhou, Xiaohuan Zhou and Tianhang Zhu.
|
||||
1. **[Qwen2 MoE](https://huggingface.co/docs/transformers/model_doc/qwen2_moe)** (from the Qwen team, Alibaba Group) released with the paper [Qwen2 Technical Report](https://huggingface.co/papers/2407.10671) by Jinze Bai, Shuai Bai, Yunfei Chu, Zeyu Cui, Kai Dang, Xiaodong Deng, Yang Fan, Wenbin Ge, Yu Han, Fei Huang, Binyuan Hui, Luo Ji, Mei Li, Junyang Lin, Runji Lin, Dayiheng Liu, Gao Liu, Chengqiang Lu, Keming Lu, Jianxin Ma, Rui Men, Xingzhang Ren, Xuancheng Ren, Chuanqi Tan, Sinan Tan, Jianhong Tu, Peng Wang, Shijie Wang, Wei Wang, Shengguang Wu, Benfeng Xu, Jin Xu, An Yang, Hao Yang, Jian Yang, Shusheng Yang, Yang Yao, Bowen Yu, Hongyi Yuan, Zheng Yuan, Jianwei Zhang, Xingxuan Zhang, Yichang Zhang, Zhenru Zhang, Chang Zhou, Jingren Zhou, Xiaohuan Zhou and Tianhang Zhu.
|
||||
1. **[Qwen2-VL](https://huggingface.co/docs/transformers/model_doc/qwen2_vl)** (from the Qwen team, Alibaba Group) released with the papers [Qwen-VL: A Versatile Vision-Language Model for Understanding, Localization, Text Reading, and Beyond](https://huggingface.co/papers/2308.12966) and [Qwen2-VL: Enhancing Vision-Language Model's Perception of the World at Any Resolution](https://huggingface.co/papers/2409.12191) by Peng Wang, Shuai Bai, Sinan Tan, Shijie Wang, Zhihao Fan, Jinze Bai, Keqin Chen, Xuejing Liu, Jialin Wang, Wenbin Ge, Yang Fan, Kai Dang, Mengfei Du, Xuancheng Ren, Rui Men, Dayiheng Liu, Chang Zhou, Jingren Zhou, Junyang Lin.
|
||||
1. **[Qwen2.5-VL](https://huggingface.co/docs/transformers/model_doc/qwen2_5_vl)** (from the Qwen team, Alibaba Group) released with the paper [Qwen2.5-VL Technical Report](https://huggingface.co/papers/2502.13923) by Shuai Bai, Keqin Chen, Xuejing Liu, Jialin Wang, Wenbin Ge, Sibo Song, Kai Dang, Peng Wang, Shijie Wang, Jun Tang, Humen Zhong, Yuanzhi Zhu, Mingkun Yang, Zhaohai Li, Jianqiang Wan, Pengfei Wang, Wei Ding, Zheren Fu, Yiheng Xu, Jiabo Ye, Xi Zhang, Tianbao Xie, Zesen Cheng, Hang Zhang, Zhibo Yang, Haiyang Xu, Junyang Lin.
|
||||
1. **[Qwen3](https://huggingface.co/docs/transformers/en/model_doc/qwen3)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3: Think Deeper, Act Faster](https://qwen.ai/blog?id=qwen3) by the Qwen team.
|
||||
1. **[Qwen3 MoE](https://huggingface.co/docs/transformers/en/model_doc/qwen3_moe)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3: Think Deeper, Act Faster](https://qwen.ai/blog?id=qwen3) by the Qwen team.
|
||||
1. **[Qwen3 Next](https://huggingface.co/docs/transformers/en/model_doc/qwen3_next)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3-Next: Towards Ultimate Training & Inference Efficiency](https://qwen.ai/blog?id=qwen3-next) by the Qwen team.
|
||||
1. **[Qwen3-VL](https://huggingface.co/docs/transformers/en/model_doc/qwen3_vl)** (from the Qwen team, Alibaba Group) released with the paper [Qwen3-VL Technical Report](https://huggingface.co/papers/2511.21631) by Shuai Bai, Yuxuan Cai, Ruizhe Chen, Keqin Chen, Xionghui Chen, Zesen Cheng, Lianghao Deng, Wei Ding, Chang Gao, Chunjiang Ge, Wenbin Ge, Zhifang Guo, Qidong Huang, Jie Huang, Fei Huang, Binyuan Hui, Shutong Jiang, Zhaohai Li, Mingsheng Li, Mei Li, Kaixin Li, Zicheng Lin, Junyang Lin, Xuejing Liu, Jiawei Liu, Chenglong Liu, Yang Liu, Dayiheng Liu, Shixuan Liu, Dunjie Lu, Ruilin Luo, Chenxu Lv, Rui Men, Lingchen Meng, Xuancheng Ren, Xingzhang Ren, Sibo Song, Yuchong Sun, Jun Tang, Jianhong Tu, Jianqiang Wan, Peng Wang, Pengfei Wang, Qiuyue Wang, Yuxuan Wang, Tianbao Xie, Yiheng Xu, Haiyang Xu, Jin Xu, Zhibo Yang, Mingkun Yang, Jianxin Yang, An Yang, Bowen Yu, Fei Zhang, Hang Zhang, Xi Zhang, Bo Zheng, Humen Zhong, Jingren Zhou, Fan Zhou, Jing Zhou, Yuanzhi Zhu, Ke Zhu.
|
||||
1. **[Qwen3-VL MoE](https://huggingface.co/docs/transformers/en/model_doc/qwen3_vl_moe)** (from the Qwen team, Alibaba Group) released with the paper [Qwen3-VL Technical Report](https://huggingface.co/papers/2511.21631) by Shuai Bai, Yuxuan Cai, Ruizhe Chen, Keqin Chen, Xionghui Chen, Zesen Cheng, Lianghao Deng, Wei Ding, Chang Gao, Chunjiang Ge, Wenbin Ge, Zhifang Guo, Qidong Huang, Jie Huang, Fei Huang, Binyuan Hui, Shutong Jiang, Zhaohai Li, Mingsheng Li, Mei Li, Kaixin Li, Zicheng Lin, Junyang Lin, Xuejing Liu, Jiawei Liu, Chenglong Liu, Yang Liu, Dayiheng Liu, Shixuan Liu, Dunjie Lu, Ruilin Luo, Chenxu Lv, Rui Men, Lingchen Meng, Xuancheng Ren, Xingzhang Ren, Sibo Song, Yuchong Sun, Jun Tang, Jianhong Tu, Jianqiang Wan, Peng Wang, Pengfei Wang, Qiuyue Wang, Yuxuan Wang, Tianbao Xie, Yiheng Xu, Haiyang Xu, Jin Xu, Zhibo Yang, Mingkun Yang, Jianxin Yang, An Yang, Bowen Yu, Fei Zhang, Hang Zhang, Xi Zhang, Bo Zheng, Humen Zhong, Jingren Zhou, Fan Zhou, Jing Zhou, Yuanzhi Zhu, Ke Zhu.
|
||||
1. **[Qwen3.5](https://huggingface.co/docs/transformers/en/model_doc/qwen3_5)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3.5: Towards Native Multimodal Agents](https://qwen.ai/blog?id=qwen3.5) by the Qwen team.
|
||||
1. **[Qwen3.5 Moe](https://huggingface.co/docs/transformers/en/model_doc/qwen3_5_moe)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3.5: Towards Native Multimodal Agents](https://qwen.ai/blog?id=qwen3.5) by the Qwen team.
|
||||
1. **[ResNet](https://huggingface.co/docs/transformers/model_doc/resnet)** (from Microsoft Research) released with the paper [Deep Residual Learning for Image Recognition](https://huggingface.co/papers/1512.03385) by Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun.
|
||||
1. **[RF-DETR](https://huggingface.co/docs/transformers/model_doc/rf_detr)** (from Roboflow) released with the blog post [RF-DETR: A SOTA Real-Time Object Detection Model](https://blog.roboflow.com/rf-detr/) by Peter Robicheaux, James Gallagher, Joseph Nelson, Isaac Robinson.
|
||||
1. **[RoBERTa](https://huggingface.co/docs/transformers/model_doc/roberta)** (from Facebook), released together with the paper [RoBERTa: A Robustly Optimized BERT Pretraining Approach](https://huggingface.co/papers/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov.
|
||||
1. **[RoFormer](https://huggingface.co/docs/transformers/model_doc/roformer)** (from ZhuiyiTechnology), released together with the paper [RoFormer: Enhanced Transformer with Rotary Position Embedding](https://huggingface.co/papers/2104.09864) by Jianlin Su and Yu Lu and Shengfeng Pan and Bo Wen and Yunfeng Liu.
|
||||
1. **[RT-DETR](https://huggingface.co/docs/transformers/model_doc/rt_detr)** (from Baidu), released together with the paper [DETRs Beat YOLOs on Real-time Object Detection](https://huggingface.co/papers/2304.08069) by Yian Zhao, Wenyu Lv, Shangliang Xu, Jinman Wei, Guanzhong Wang, Qingqing Dang, Yi Liu, Jie Chen.
|
||||
1. **[RT-DETRv2](https://huggingface.co/docs/transformers/model_doc/rt_detr_v2)** (from Baidu), released together with the paper [RT-DETRv2: Improved Baseline with Bag-of-Freebies for Real-Time Detection Transformer](https://huggingface.co/papers/2407.17140) by Wenyu Lv, Yian Zhao, Qinyao Chang, Kui Huang, Guanzhong Wang, Yi Liu.
|
||||
1. **Sapiens** (from Meta AI) released with the paper [Sapiens: Foundation for Human Vision Models](https://huggingface.co/papers/2408.12569) by Rawal Khirodkar, Timur Bagautdinov, Julieta Martinez, Su Zhaoen, Austin James, Peter Selednik, Stuart Anderson, Shunsuke Saito.
|
||||
1. **[SegFormer](https://huggingface.co/docs/transformers/model_doc/segformer)** (from NVIDIA) released with the paper [SegFormer: Simple and Efficient Design for Semantic Segmentation with Transformers](https://huggingface.co/papers/2105.15203) by Enze Xie, Wenhai Wang, Zhiding Yu, Anima Anandkumar, Jose M. Alvarez, Ping Luo.
|
||||
1. **[Segment Anything](https://huggingface.co/docs/transformers/model_doc/sam)** (from Meta AI) released with the paper [Segment Anything](https://huggingface.co/papers/2304.02643v1.pdf) by Alexander Kirillov, Eric Mintun, Nikhila Ravi, Hanzi Mao, Chloe Rolland, Laura Gustafson, Tete Xiao, Spencer Whitehead, Alex Berg, Wan-Yen Lo, Piotr Dollar, Ross Girshick.
|
||||
1. **[Segment Anything 2](https://huggingface.co/docs/transformers/model_doc/sam2)** (from Meta AI) released with the paper [SAM 2: Segment Anything in Images and Videos](https://huggingface.co/papers/2408.00714) by Nikhila Ravi, Valentin Gabeur, Yuan-Ting Hu, Ronghang Hu, Chaitanya Ryali, Tengyu Ma, Haitham Khedr, Roman Rädle, Chloe Rolland, Laura Gustafson, Eric Mintun, Junting Pan, Kalyan Vasudev Alwala, Nicolas Carion, Chao-Yuan Wu, Ross Girshick, Piotr Dollár, Christoph Feichtenhofer.
|
||||
1. **[Segment Anything 3](https://huggingface.co/docs/transformers/model_doc/sam3)** (from Meta Superintelligence Labs) released with the paper [SAM 3: Segment Anything with Concepts](https://ai.meta.com/research/publications/sam-3-segment-anything-with-concepts/) by SAM 3D Team, Xingyu Chen, Fu-Jen Chu, Pierre Gleize, Kevin J Liang, Alexander Sax, Hao Tang, Weiyao Wang, Michelle Guo, Thibaut Hardin, Xiang Li, Aohan Lin, Jiawei Liu, Ziqi Ma, Anushka Sagar, Bowen Song, Xiaodong Wang, Jianing Yang, Bowen Zhang, Piotr Dollar, Georgia Gkioxari, Matt Feiszli, Jitendra Malik, Nicolas Carion, Laura Gustafson, Yuan-Ting Hu, Shoubhik Debnath, Ronghang Hu, Didac Suris Coll-Vinent, Chaitanya Ryali, Kalyan Vasudev Alwala, Haitham Khedr, Andrew Huang, Jie Lei, Tengyu Ma, Baishan Guo, Arpit Kalla, Markus Marks, Joseph Greer, Meng Wang, Peize Sun, Roman Rädle, Triantafyllos Afouras, Effrosyni Mavroudi, Katherine Xu, Tsung-Han Wu, Yu Zhou, Liliane Momeni, Rishi Hazra, Shuangrui Ding, Sagar Vaze, Francois Porcher, Feng Li, Siyuan Li, Aishwarya Kamath, Ho Kei Cheng, Piotr Dollar, Nikhila Ravi, Kate Saenko, Pengchuan Zhang, Christoph Feichtenhofer.
|
||||
1. **[SigLIP](https://huggingface.co/docs/transformers/main/model_doc/siglip)** (from Google AI) released with the paper [Sigmoid Loss for Language Image Pre-Training](https://huggingface.co/papers/2303.15343) by Xiaohua Zhai, Basil Mustafa, Alexander Kolesnikov, Lucas Beyer.
|
||||
1. **[SmolLM3](https://huggingface.co/docs/transformers/main/model_doc/smollm3) (from Hugging Face) released with the blog post [SmolLM3: smol, multilingual, long-context reasoner](https://huggingface.co/blog/smollm3) by the Hugging Face TB Research team.
|
||||
1. **[SmolVLM](https://huggingface.co/docs/transformers/main/model_doc/smolvlm) (from Hugging Face) released with the blog posts [SmolVLM - small yet mighty Vision Language Model](https://huggingface.co/blog/smolvlm) and [SmolVLM Grows Smaller – Introducing the 250M & 500M Models!](https://huggingface.co/blog/smolervlm) by the Hugging Face TB Research team.
|
||||
1. **[SolarOpen](https://huggingface.co/docs/transformers/main/model_doc/solar_open)** (from Upstage) released with the paper [Solar Open Technical Report](https://huggingface.co/papers/2601.07022) by Sungrae Park, Sanghoon Kim, Jungho Cho, Gyoungjin Gim, Dawoon Jung, Mikyoung Cha, Eunhae Choo, Taekgyu Hong, Minbyul Jeong, SeHwan Joo, Minsoo Khang, Eunwon Kim, Minjeong Kim, Sujeong Kim, Yunsu Kim, Hyeonju Lee, Seunghyun Lee, Sukyung Lee, Siyoung Park, Gyungin Shin, Inseo Song, Wonho Song, Seonghoon Yang, Seungyoun Yi, Sanghoon Yoon, Jeonghyun Ko, Seyoung Song, Keunwoo Choi, Hwalsuk Lee, Sunghun Kim, Du-Seong Chang, Kyunghyun Cho, Junsuk Choe, Hwaran Lee, Jae-Gil Lee, KyungTae Lim, Alice Oh.
|
||||
1. **SNAC** (from Papla Media, ETH Zurich) released with the paper [SNAC: Multi-Scale Neural Audio Codec](https://huggingface.co/papers/2410.14411) by Hubert Siuzdak, Florian Grötschla, Luca A. Lanzendörfer.
|
||||
1. **[SpeechT5](https://huggingface.co/docs/transformers/model_doc/speecht5)** (from Microsoft Research) released with the paper [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://huggingface.co/papers/2110.07205) by Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei.
|
||||
1. **[SqueezeBERT](https://huggingface.co/docs/transformers/model_doc/squeezebert)** (from Berkeley) released with the paper [SqueezeBERT: What can computer vision teach NLP about efficient neural networks?](https://huggingface.co/papers/2006.11316) by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W. Keutzer.
|
||||
1. **[StableLm](https://huggingface.co/docs/transformers/model_doc/stablelm)** (from Stability AI) released with the paper [StableLM 3B 4E1T (Technical Report)](https://stability.wandb.io/stability-llm/stable-lm/reports/StableLM-3B-4E1T--VmlldzoyMjU4?accessToken=u3zujipenkx5g7rtcj9qojjgxpconyjktjkli2po09nffrffdhhchq045vp0wyfo) by Jonathan Tow, Marco Bellagente, Dakota Mahan, Carlos Riquelme Ruiz, Duy Phung, Maksym Zhuravinskyi, Nathan Cooper, Nikhil Pinnaparaju, Reshinth Adithyan, and James Baicoianu.
|
||||
1. **[Starcoder2](https://huggingface.co/docs/transformers/main/model_doc/starcoder2)** (from BigCode team) released with the paper [StarCoder 2 and The Stack v2: The Next Generation](https://huggingface.co/papers/2402.19173) by Anton Lozhkov, Raymond Li, Loubna Ben Allal, Federico Cassano, Joel Lamy-Poirier, Nouamane Tazi, Ao Tang, Dmytro Pykhtar, Jiawei Liu, Yuxiang Wei, Tianyang Liu, Max Tian, Denis Kocetkov, Arthur Zucker, Younes Belkada, Zijian Wang, Qian Liu, Dmitry Abulkhanov, Indraneil Paul, Zhuang Li, Wen-Ding Li, Megan Risdal, Jia Li, Jian Zhu, Terry Yue Zhuo, Evgenii Zheltonozhskii, Nii Osae Osae Dade, Wenhao Yu, Lucas Krauß, Naman Jain, Yixuan Su, Xuanli He, Manan Dey, Edoardo Abati, Yekun Chai, Niklas Muennighoff, Xiangru Tang, Muhtasham Oblokulov, Christopher Akiki, Marc Marone, Chenghao Mou, Mayank Mishra, Alex Gu, Binyuan Hui, Tri Dao, Armel Zebaze, Olivier Dehaene, Nicolas Patry, Canwen Xu, Julian McAuley, Han Hu, Torsten Scholak, Sebastien Paquet, Jennifer Robinson, Carolyn Jane Anderson, Nicolas Chapados, Mostofa Patwary, Nima Tajbakhsh, Yacine Jernite, Carlos Muñoz Ferrandis, Lingming Zhang, Sean Hughes, Thomas Wolf, Arjun Guha, Leandro von Werra, and Harm de Vries.
|
||||
1. **StyleTTS 2** (from Columbia University) released with the paper [StyleTTS 2: Towards Human-Level Text-to-Speech through Style Diffusion and Adversarial Training with Large Speech Language Models](https://huggingface.co/papers/2306.07691) by Yinghao Aaron Li, Cong Han, Vinay S. Raghavan, Gavin Mischler, Nima Mesgarani.
|
||||
1. **Supertonic** (from Supertone) released with the paper [SupertonicTTS: Towards Highly Efficient and Streamlined Text-to-Speech System](https://huggingface.co/papers/2503.23108) by Hyeongju Kim, Jinhyeok Yang, Yechan Yu, Seunghun Ji, Jacob Morton, Frederik Bous, Joon Byun, Juheon Lee.
|
||||
1. **[Swin Transformer](https://huggingface.co/docs/transformers/model_doc/swin)** (from Microsoft) released with the paper [Swin Transformer: Hierarchical Vision Transformer using Shifted Windows](https://huggingface.co/papers/2103.14030) by Ze Liu, Yutong Lin, Yue Cao, Han Hu, Yixuan Wei, Zheng Zhang, Stephen Lin, Baining Guo.
|
||||
1. **[Swin2SR](https://huggingface.co/docs/transformers/model_doc/swin2sr)** (from University of Würzburg) released with the paper [Swin2SR: SwinV2 Transformer for Compressed Image Super-Resolution and Restoration](https://huggingface.co/papers/2209.11345) by Marcos V. Conde, Ui-Jin Choi, Maxime Burchi, Radu Timofte.
|
||||
1. **[T5](https://huggingface.co/docs/transformers/model_doc/t5)** (from Google AI) released with the paper [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://huggingface.co/papers/1910.10683) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
|
||||
1. **[T5v1.1](https://huggingface.co/docs/transformers/model_doc/t5v1.1)** (from Google AI) released in the repository [google-research/text-to-text-transfer-transformer](https://github.com/google-research/text-to-text-transfer-transformer/blob/main/released_checkpoints.md#t511) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
|
||||
1. **[Table Transformer](https://huggingface.co/docs/transformers/model_doc/table-transformer)** (from Microsoft Research) released with the paper [PubTables-1M: Towards Comprehensive Table Extraction From Unstructured Documents](https://huggingface.co/papers/2110.00061) by Brandon Smock, Rohith Pesala, Robin Abraham.
|
||||
1. **[TrOCR](https://huggingface.co/docs/transformers/model_doc/trocr)** (from Microsoft), released together with the paper [TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models](https://huggingface.co/papers/2109.10282) by Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei.
|
||||
1. **Ultravox** (from Fixie.ai) released with the repository [fixie-ai/ultravox](https://github.com/fixie-ai/ultravox) by the Fixie.ai team.
|
||||
1. **[UniSpeech](https://huggingface.co/docs/transformers/model_doc/unispeech)** (from Microsoft Research) released with the paper [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled Data](https://huggingface.co/papers/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei, Michael Zeng, Xuedong Huang.
|
||||
1. **[UniSpeechSat](https://huggingface.co/docs/transformers/model_doc/unispeech-sat)** (from Microsoft Research) released with the paper [UNISPEECH-SAT: UNIVERSAL SPEECH REPRESENTATION LEARNING WITH SPEAKER AWARE PRE-TRAINING](https://huggingface.co/papers/2110.05752) by Sanyuan Chen, Yu Wu, Chengyi Wang, Zhengyang Chen, Zhuo Chen, Shujie Liu, Jian Wu, Yao Qian, Furu Wei, Jinyu Li, Xiangzhan Yu.
|
||||
1. **[VaultGemma](https://huggingface.co/docs/transformers/main/model_doc/vaultgemma)** (from Google) released with the technical report [VaultGemma: A Differentially Private Gemma Model](https://services.google.com/fh/files/blogs/vaultgemma_tech_report.pdf) by the VaultGemma Google team.
|
||||
1. **[Vision Transformer (ViT)](https://huggingface.co/docs/transformers/model_doc/vit)** (from Google AI) released with the paper [An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale](https://huggingface.co/papers/2010.11929) by Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, Georg Heigold, Sylvain Gelly, Jakob Uszkoreit, Neil Houlsby.
|
||||
1. **[ViTMAE](https://huggingface.co/docs/transformers/model_doc/vit_mae)** (from Meta AI) released with the paper [Masked Autoencoders Are Scalable Vision Learners](https://huggingface.co/papers/2111.06377) by Kaiming He, Xinlei Chen, Saining Xie, Yanghao Li, Piotr Dollár, Ross Girshick.
|
||||
1. **[ViTMatte](https://huggingface.co/docs/transformers/model_doc/vitmatte)** (from HUST-VL) released with the paper [ViTMatte: Boosting Image Matting with Pretrained Plain Vision Transformers](https://huggingface.co/papers/2305.15272) by Jingfeng Yao, Xinggang Wang, Shusheng Yang, Baoyuan Wang.
|
||||
1. **[ViTMSN](https://huggingface.co/docs/transformers/model_doc/vit_msn)** (from Meta AI) released with the paper [Masked Siamese Networks for Label-Efficient Learning](https://huggingface.co/papers/2204.07141) by Mahmoud Assran, Mathilde Caron, Ishan Misra, Piotr Bojanowski, Florian Bordes, Pascal Vincent, Armand Joulin, Michael Rabbat, Nicolas Ballas.
|
||||
1. **[ViTPose](https://huggingface.co/docs/transformers/model_doc/vitpose)** (from The University of Sydney) released with the paper [ViTPose: Simple Vision Transformer Baselines for Human Pose Estimation](https://huggingface.co/papers/2204.12484) by Yufei Xu, Jing Zhang, Qiming Zhang, Dacheng Tao.
|
||||
1. **[VITS](https://huggingface.co/docs/transformers/model_doc/vits)** (from Kakao Enterprise) released with the paper [Conditional Variational Autoencoder with Adversarial Learning for End-to-End Text-to-Speech](https://huggingface.co/papers/2106.06103) by Jaehyeon Kim, Jungil Kong, Juhee Son.
|
||||
1. **[Voxtral](https://huggingface.co/docs/transformers/model_doc/voxtral)** (from Mistral AI) released with the paper [Voxtral](https://huggingface.co/papers/2507.13264) by Alexander H. Liu, Andy Ehrenberg, Andy Lo, Clément Denoix, Corentin Barreau, Guillaume Lample, Jean-Malo Delignon, Khyathi Raghavi Chandu, Patrick von Platen, Pavankumar Reddy Muddireddy, Sanchit Gandhi, Soham Ghosh, Srijan Mishra, Thomas Foubert, Abhinav Rastogi, Adam Yang, Albert Q. Jiang, Alexandre Sablayrolles, Amélie Héliou, Amélie Martin, Anmol Agarwal, Antoine Roux, Arthur Darcet, Arthur Mensch, Baptiste Bout, Baptiste Rozière, Baudouin De Monicault, Chris Bamford, Christian Wallenwein, Christophe Renaudin, Clémence Lanfranchi, Darius Dabert, Devendra Singh Chaplot, Devon Mizelle, Diego de las Casas, Elliot Chane-Sane, Emilien Fugier, Emma Bou Hanna, Gabrielle Berrada, Gauthier Delerce, Gauthier Guinet, Georgii Novikov, Guillaume Martin, Himanshu Jaju, Jan Ludziejewski, Jason Rute, Jean-Hadrien Chabran, Jessica Chudnovsky, Joachim Studnia, Joep Barmentlo, Jonas Amar, Josselin Somerville Roberts, Julien Denize, Karan Saxena, Karmesh Yadav, Kartik Khandelwal, Kush Jain, Lélio Renard Lavaud, Léonard Blier, Lingxiao Zhao, Louis Martin, Lucile Saulnier, Luyu Gao, Marie Pellat, Mathilde Guillaumin, Mathis Felardos, Matthieu Dinot, Maxime Darrin, Maximilian Augustin, Mickaël Seznec, Neha Gupta, Nikhil Raghuraman, Olivier Duchenne, Patricia Wang, Patryk Saffer, Paul Jacob, Paul Wambergue, Paula Kurylowicz, Philomène Chagniot, Pierre Stock, Pravesh Agrawal, Rémi Delacourt, Romain Sauvestre, Roman Soletskyi, Sagar Vaze, Sandeep Subramanian, Saurabh Garg, Shashwat Dalal, Siddharth Gandhi, Sumukh Aithal, Szymon Antoniak, Teven Le Scao, Thibault Schueller, Thibaut Lavril, Thomas Robert, Thomas Wang, Timothée Lacroix, Tom Bewley, Valeriia Nemychnikova, Victor Paltz , Virgile Richard, Wen-Ding Li, William Marshall, Xuanyu Zhang, Yihan Wan, Yunhao Tang.
|
||||
1. **[Voxtral Realtime](https://huggingface.co/docs/transformers/model_doc/voxtral_realtime)** (from Mistral AI) released with the paper [Voxtral Realtime](https://huggingface.co/papers/2602.11298) by Alexander H. Liu, Andy Ehrenberg, Andy Lo, Chen-Yo Sun, Guillaume Lample, Jean-Malo Delignon, Khyathi Raghavi Chandu, Patrick von Platen, Pavankumar Reddy Muddireddy, Rohin Arora, Sanchit Gandhi, Sandeep Subramanian, Soham Ghosh, Srijan Mishra, Abhinav Rastogi, Adrien Sadé, Alan Jeffares, Albert Jiang, Alexandre Cahill, Alexandre Gavaudan, Alexandre Sablayrolles, Amélie Héliou, Amos You, Andrew Bai, Angele Lenglemetz, Anmol Agarwal, Anton Eliseev, Antonia Calvi, Arjun Majumdar, Avi Sooriyarachchi, Baptiste Bout, Baptiste Rozière, Baudouin De Monicault, Benjamin Tibi, Charlotte Cronjäger, Clémence Lanfranchi, Connor Chen, Corentin Barreau, Corentin Sautier, Cyprien Courtot, Darius Dabert, Diego de las Casas, Elizaveta Demyanenko, Elliot Chane-Sane, Enguerrand Paquin, Etienne Goffinet, Fabien Niel, Faruk Ahmed, Federico Baldassarre, Gabrielle Berrada, Gaëtan Ecrepont, Gauthier Guinet, Genevieve Hayes, Georgii Novikov, Giada Pistilli, Guillaume Kunsch, Guillaume Martin, Guillaume Raille, Gunjan Dhanuka, Gunshi Gupta, Han Zhou, Harshil Shah, Hope McGovern, Hugo Thimonier, Indraneel Mukherjee, Irene Zhang, Jaeyoung Kim, Jan Ludziejewski, Jason Rute, Joachim Studnia, John Harvill, Jonas Amar, Joséphine Delas, Josselin Somerville Roberts, Julien Tauran, Karmesh Yadav, Kartik Khandelwal, Kilian Tep, Kush Jain, Laurence Aitchison, Laurent Fainsin, Léonard Blier, Lingxiao Zhao, Louis Martin, Lucile Saulnier, Luyu Gao, Maarten Buyl, Manan Sharma, Margaret Jennings, Marie Pellat, Mark Prins, Martin Alexandre, Mathieu Poirée, Mathilde Guillaumin, Matthieu Dinot, Matthieu Futeral, Maxime Darrin, Maximilian Augustin, Mert Unsal, Mia Chiquier , Minh-Quang Pham, Nathan Grinsztajn, Neha Gupta, Olivier Bousquet, Olivier Duchenne, Patricia Wang, Paul Jacob, Paul Wambergue, Paula Kurylowicz, Philippe Pinel, Philomène Chagniot, Pierre Stock, Piotr Miłoś, Prateek Gupta, Pravesh Agrawal, Quentin Torroba, Ram Ramrakhya, Rishi Shah, Romain Sauvestre, Roman Soletskyi, Rosalie Millner, Rupert Menneer, Sagar Vaze, Samuel Barry, Samuel Humeau, Sean Cha, Shashwat Verma, Siddhant Waghjale, Siddharth Gandhi, Simon Lepage, Sumukh Aithal, Szymon Antoniak, Teven Le Scao, Théo Cachet, Theo Simon Sorg, Thibaut Lavril, Thomas Chabal, Thomas Foubert, Thomas Robert, Thomas Wang, Tim Lawson, Tom Bewley, Tom Edwards, Tyler Wang, Umar Jamil, Umberto Tomasini, Valeriia Nemychnikova, Van Phung, Vedant Nanda, Victor Jouault, Vincent Maladière, Virgile Richard, Vladislav Bataev, Wassim Bouaziz, Wen-Ding Li, William Havard, William Marshall, Xinghui Li, Xingran Guo, Xinyu Yang, Yannic Neuhaus, Yassine El Ouahidi, Yassir Bendou, Yihan Wang, Yimu Pan, Zaccharie Ramzi, Zhenlin Xu.
|
||||
1. **[Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2)** (from Facebook AI) released with the paper [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations](https://huggingface.co/papers/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael Auli.
|
||||
1. **[Wav2Vec2-BERT](https://huggingface.co/docs/transformers/main/model_doc/wav2vec2-bert)** (from Meta AI) released with the paper [Seamless: Multilingual Expressive and Streaming Speech Translation](https://ai.meta.com/research/publications/seamless-multilingual-expressive-and-streaming-speech-translation/) by the Seamless Communication team.
|
||||
1. **[WavLM](https://huggingface.co/docs/transformers/model_doc/wavlm)** (from Microsoft Research) released with the paper [WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://huggingface.co/papers/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei.
|
||||
1. **[Whisper](https://huggingface.co/docs/transformers/model_doc/whisper)** (from OpenAI) released with the paper [Robust Speech Recognition via Large-Scale Weak Supervision](https://cdn.openai.com/papers/whisper.pdf) by Alec Radford, Jong Wook Kim, Tao Xu, Greg Brockman, Christine McLeavey, Ilya Sutskever.
|
||||
1. **[XLM](https://huggingface.co/docs/transformers/model_doc/xlm)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://huggingface.co/papers/1901.07291) by Guillaume Lample and Alexis Conneau.
|
||||
1. **[XLM-RoBERTa](https://huggingface.co/docs/transformers/model_doc/xlm-roberta)** (from Facebook AI), released together with the paper [Unsupervised Cross-lingual Representation Learning at Scale](https://huggingface.co/papers/1911.02116) by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov.
|
||||
1. **[YOLOS](https://huggingface.co/docs/transformers/model_doc/yolos)** (from Huazhong University of Science & Technology) released with the paper [You Only Look at One Sequence: Rethinking Transformer in Vision through Object Detection](https://huggingface.co/papers/2106.00666) by Yuxin Fang, Bencheng Liao, Xinggang Wang, Jiemin Fang, Jiyang Qi, Rui Wu, Jianwei Niu, Wenyu Liu.
|
||||
1. **[Youtu-LLM](https://huggingface.co/docs/transformers/model_doc/youtu)** (from the Tencent Youtu Team) released with the paper [Youtu-LLM: Unlocking the Native Agentic Potential for Lightweight Large Language Models](https://huggingface.co/papers/2512.24618) by Junru Lu, Jiarui Qin, Lingfeng Qiao, Yinghui Li, Xinyi Dai, Bo Ke, Jianfeng He, Ruizhi Qiao, Di Yin, Xing Sun, Yunsheng Wu, Yinsong Liu, Shuangyin Liu, Mingkong Tang, Haodong Lin, Jiayi Kuang, Fanxu Meng, Xiaojuan Tang, Yunjia Xi, Junjie Huang, Haotong Yang, Zhenyi Shen, Yangning Li, Qianwen Zhang, Yifei Yu, Siyu An, Junnan Dong, Qiufeng Wang, Jie Wang, Keyu Chen, Wei Wen, Taian Guo, Zhifeng Shen, Daohai Yu, Jiahao Li, Ke Li, Zongyi Li, Xiaoyu Tan.
|
||||
1. **Zaya** (from Zyphra) released with the paper [ZAYA1-8B Technical Report](https://huggingface.co/papers/2605.05365) by Robert Washbourne, Rishi Iyer, Tomas Figliolia, Henry Zheng, Ryan Lorig-Roach, Sungyeon Yang, Pritish Yuvraj, Quentin Anthony, Yury Tokpanov, Xiao Yang, Ganesh Nanduru, Stephen Ebert, Praneeth Medepalli, Skyler Szot, Srivatsan Rajagopal, Alex Ong, Bhavana Mehta, Beren Millidge.
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`huggingface/transformers.js`
|
||||
- 原始仓库：https://github.com/huggingface/transformers.js
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"name": "@huggingface/transformers-monorepo",
|
||||
"version": "0.0.1",
|
||||
"private": true,
|
||||
"description": "Monorepo for Hugging Face Transformers.js and framework integrations",
|
||||
"type": "module",
|
||||
"packageManager": "pnpm@10.28.1",
|
||||
"scripts": {
|
||||
"build": "pnpm -r build",
|
||||
"test": "pnpm -r test",
|
||||
"format": "prettier --write .",
|
||||
"format:check": "prettier --check .",
|
||||
"dev": "node scripts/dev.mjs"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/huggingface/transformers.js.git"
|
||||
},
|
||||
"author": "Hugging Face",
|
||||
"license": "Apache-2.0",
|
||||
"bugs": {
|
||||
"url": "https://github.com/huggingface/transformers.js/issues"
|
||||
},
|
||||
"homepage": "https://github.com/huggingface/transformers.js#readme",
|
||||
"devDependencies": {
|
||||
"prettier": "3.8.1",
|
||||
"typescript": "5.9.3"
|
||||
},
|
||||
"prettier": {
|
||||
"overrides": [
|
||||
{
|
||||
"files": [
|
||||
"./packages/*/src/**/*.{js,mjs,cjs,ts}"
|
||||
],
|
||||
"options": {
|
||||
"singleQuote": true,
|
||||
"tabWidth": 4,
|
||||
"printWidth": 120
|
||||
}
|
||||
},
|
||||
{
|
||||
"files": [
|
||||
"./packages/*/tests/**/*.{js,mjs,cjs,ts}"
|
||||
],
|
||||
"options": {
|
||||
"tabWidth": 2,
|
||||
"printWidth": 10000000
|
||||
}
|
||||
},
|
||||
{
|
||||
"files": [
|
||||
"./scripts/**/*.{js,mjs,cjs,ts}",
|
||||
"./packages/*/scripts/**/*.{js,mjs,cjs,ts}",
|
||||
"./packages/*/docs/**/*.{js,mjs,cjs,ts}"
|
||||
],
|
||||
"options": {
|
||||
"tabWidth": 2,
|
||||
"printWidth": 160
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
# This folder contains the build files for the documentation. Do not commit these files.
|
||||
*
|
||||
!.gitignore
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"source": {
|
||||
"excludePattern": ""
|
||||
},
|
||||
"plugins": ["./plugins/preprocess.js"]
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
/**
|
||||
* JSDoc plugin to transform TypeScript-style type expressions into JSDoc-compatible syntax.
|
||||
*/
|
||||
|
||||
function extractBalancedBraces(text, start) {
|
||||
if (text[start] !== "{") return null;
|
||||
let depth = 1,
|
||||
i = start + 1;
|
||||
while (i < text.length && depth > 0) {
|
||||
if (text[i] === "{") depth++;
|
||||
else if (text[i] === "}") depth--;
|
||||
i++;
|
||||
}
|
||||
return depth === 0 ? { content: text.slice(start + 1, i - 1), endIndex: i } : null;
|
||||
}
|
||||
|
||||
function stripLeadingGenericParams(expr) {
|
||||
if (expr[0] !== "<") return expr;
|
||||
let depth = 1,
|
||||
i = 1;
|
||||
while (i < expr.length && depth > 0) {
|
||||
if (expr[i] === "<") depth++;
|
||||
else if (expr[i] === ">") depth--;
|
||||
i++;
|
||||
}
|
||||
return depth === 0 ? expr.slice(i) : expr;
|
||||
}
|
||||
|
||||
function hasTopLevelConditional(expr) {
|
||||
let angle = 0,
|
||||
paren = 0,
|
||||
brace = 0,
|
||||
bracket = 0;
|
||||
|
||||
for (let i = 0; i < expr.length; ++i) {
|
||||
const ch = expr[i];
|
||||
if (ch === "<") angle++;
|
||||
else if (ch === ">") angle = Math.max(0, angle - 1);
|
||||
else if (ch === "(") paren++;
|
||||
else if (ch === ")") paren = Math.max(0, paren - 1);
|
||||
else if (ch === "{") brace++;
|
||||
else if (ch === "}") brace = Math.max(0, brace - 1);
|
||||
else if (ch === "[") bracket++;
|
||||
else if (ch === "]") bracket = Math.max(0, bracket - 1);
|
||||
else if (ch === "?" && angle === 0 && paren === 0 && brace === 0 && bracket === 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function transformType(expr) {
|
||||
let result = expr,
|
||||
prev = "";
|
||||
while (result !== prev) {
|
||||
prev = result;
|
||||
result = stripLeadingGenericParams(result); // <T extends X>(...) -> (...)
|
||||
if (/^Promise<function\s*\(/.test(result)) {
|
||||
result = "Promise.<Function>";
|
||||
continue;
|
||||
}
|
||||
if (/^function\s*\(/.test(result)) {
|
||||
result = "Function";
|
||||
continue;
|
||||
}
|
||||
if (hasTopLevelConditional(result)) {
|
||||
result = "any";
|
||||
continue;
|
||||
}
|
||||
result = result
|
||||
.replace(/import\([^)]+\)((?:\.\w+)*)/g, (_, p) => p?.slice(1) || "any") // import() -> Type
|
||||
.replace(/\w+\[['"][^\]]+['"]\]\s+extends\s+[^}]+/g, "any") // X['p'] extends ... -> any
|
||||
.replace(/(\w+)(?:<[^>]+>)?\[[^\]]+\]/g, "$1") // Type[x] or Type<T>[x] -> Type
|
||||
.replace(/keyof\s+typeof\s+\w+/g, "string") // keyof typeof X -> string
|
||||
.replace(/typeof\s+\w+/g, "Object") // typeof X -> Object
|
||||
.replace(/\binfer\s+\w+/g, "any") // infer K -> any
|
||||
.replace(/\breadonly\s+/g, "") // readonly T -> T
|
||||
.replace(/\(\s*\w[\w<>, ]*\s+extends\s+\w+\s*\?[^)]*\)/g, "any") // (X extends Y ? A : B) -> any
|
||||
.replace(/(?<!\w)\(\s*(\w+)\s*\)/g, "$1") // (any) -> any (unwrap parens around simple types, not after words like "function")
|
||||
.replace(/(\w+)\?\s*:/g, "$1:") // x?: T -> x: T
|
||||
.replace(/;\s*([}\n])/g, " $1")
|
||||
.replace(/;\s+/g, ", ") // semicolons -> commas
|
||||
.replace(/\{\s*\[\w+\s+in\s+[^\]]+\][^}]*\}/g, "Object") // mapped types -> Object
|
||||
.replace(/new\s*\([^)]*\)\s*=>\s*\{[^}]*\}/g, "Function") // new () => {...} -> Function
|
||||
.replace(/new\s*\([^)]*\)\s*=>\s*\w+/g, "Function") // new () => T -> Function
|
||||
.replace(/\([^()]*\)\s*=>\s*\w[\w<>|[\], ]*/g, "Function") // () => T -> Function
|
||||
.replace(/\{[^{}]*\}\[\]/g, "Array") // {x:T}[] -> Array
|
||||
.replace(/(\w+)<[^>]+>\[\]/g, "Array.<$1>") // T<U>[] -> Array.<T>
|
||||
.replace(/\([^()]+\)\[\]/g, "Array") // (A|B)[] -> Array
|
||||
.replace(/(\w+)\[\]/g, "Array") // T[] -> Array (simple)
|
||||
.replace(/\[\w+\]/g, "Array") // [T] single-element tuple -> Array
|
||||
.replace(/\[[^\[\]]*,[^\[\]]*\]/g, "Array") // tuples with commas -> Array
|
||||
.replace(/\bnew\s+([A-Z]\w*)\b/g, "$1") // new Type -> Type
|
||||
.replace(/,?\s*\[\s*\w+\s*:\s*\w+\s*\]\s*:\s*\w+/g, "") // [key: string]: any -> (removed)
|
||||
.replace(/\(\s*(\w+)\s*&\s*\{\s*\}\s*\)/g, "$1") // (string & {}) -> string
|
||||
.replace(/\s*&\s*\{\s*\}/g, ""); // string & {} -> string
|
||||
if (!result.includes("=>")) result = result.replace(/\s*&\s*/g, "|"); // A & B -> A|B
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function transformComment(comment) {
|
||||
const tagRegex =
|
||||
/@(?:type|param|returns?|typedef|property|prop|callback|template|augments|extends|class|constructor|member|var|const|enum|throws?)[^\S\n]*\{/g;
|
||||
let result = "",
|
||||
lastIndex = 0,
|
||||
match;
|
||||
while ((match = tagRegex.exec(comment)) !== null) {
|
||||
const braceStart = match.index + match[0].length - 1;
|
||||
result += comment.slice(lastIndex, braceStart);
|
||||
const extracted = extractBalancedBraces(comment, braceStart);
|
||||
if (extracted) {
|
||||
result += "{" + transformType(extracted.content) + "}";
|
||||
lastIndex = tagRegex.lastIndex = extracted.endIndex;
|
||||
} else {
|
||||
result += "{";
|
||||
lastIndex = braceStart + 1;
|
||||
}
|
||||
}
|
||||
return result + comment.slice(lastIndex);
|
||||
}
|
||||
|
||||
export const handlers = {
|
||||
beforeParse: (e) => {
|
||||
e.source = e.source
|
||||
.replace(/\/\*\*[\s\S]*?\*\//g, transformComment) // transform JSDoc comments
|
||||
.replace(/\b(\d+)n\b/g, "BigInt($1)"); // BigInt literals
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,108 @@
|
||||
|
||||
import re
|
||||
README_TEMPLATE = """
|
||||
|
||||
<p align="center">
|
||||
<br/>
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/transformersjs-dark.svg" width="500" style="max-width: 100%;">
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/transformersjs-light.svg" width="500" style="max-width: 100%;">
|
||||
<img alt="transformers.js javascript library logo" src="https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/transformersjs-light.svg" width="500" style="max-width: 100%;">
|
||||
</picture>
|
||||
<br/>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://www.npmjs.com/package/@huggingface/transformers"><img alt="NPM" src="https://img.shields.io/npm/v/@huggingface/transformers"></a>
|
||||
<a href="https://www.npmjs.com/package/@huggingface/transformers"><img alt="NPM Downloads" src="https://img.shields.io/npm/dw/@huggingface/transformers"></a>
|
||||
<a href="https://www.jsdelivr.com/package/npm/@huggingface/transformers"><img alt="jsDelivr Hits" src="https://img.shields.io/jsdelivr/npm/hw/@huggingface/transformers"></a>
|
||||
<a href="https://github.com/huggingface/transformers.js/blob/main/LICENSE"><img alt="License" src="https://img.shields.io/github/license/huggingface/transformers.js?color=blue"></a>
|
||||
<a href="https://huggingface.co/docs/transformers.js/index"><img alt="Documentation" src="https://img.shields.io/website/http/huggingface.co/docs/transformers.js/index.svg?down_color=red&down_message=offline&up_message=online"></a>
|
||||
</p>
|
||||
|
||||
{intro}
|
||||
|
||||
## Installation
|
||||
|
||||
{installation}
|
||||
|
||||
## Quick tour
|
||||
|
||||
{quick_tour}
|
||||
|
||||
## Custom usage
|
||||
|
||||
{custom_usage}
|
||||
|
||||
## Supported tasks/models
|
||||
|
||||
Here is the list of all tasks and architectures currently supported by Transformers.js. If you don't see your task/model listed here or it is not yet supported, feel free to open up a feature request [here](https://github.com/huggingface/transformers.js/issues/new/choose).
|
||||
|
||||
To find compatible models on the Hub, select the "transformers.js" library tag in the filter menu (or visit [this link](https://huggingface.co/models?library=transformers.js)). You can refine your search by selecting the task you're interested in (e.g., [text-classification](https://huggingface.co/models?pipeline_tag=text-classification&library=transformers.js)).
|
||||
|
||||
{tasks}
|
||||
|
||||
{models}
|
||||
"""
|
||||
|
||||
|
||||
FILES_TO_INCLUDE = dict(
|
||||
intro='./docs/snippets/0_introduction.snippet',
|
||||
quick_tour='./docs/snippets/1_quick-tour.snippet',
|
||||
installation='./docs/snippets/2_installation.snippet',
|
||||
custom_usage='./docs/snippets/3_custom-usage.snippet',
|
||||
tasks='./docs/snippets/4_supported-tasks.snippet',
|
||||
models='./docs/snippets/5_supported-models.snippet',
|
||||
)
|
||||
|
||||
DOCS_BASE_URL = 'https://huggingface.co/docs/transformers.js'
|
||||
|
||||
# Map of custom links to replace, typically used for links to other sections of the README.
|
||||
CUSTOM_LINK_MAP = {
|
||||
'/custom_usage#convert-your-models-to-onnx': '#convert-your-models-to-onnx',
|
||||
'./api/env': DOCS_BASE_URL + '/api/env',
|
||||
'./guides/webgpu': DOCS_BASE_URL + '/guides/webgpu',
|
||||
'./guides/dtypes': DOCS_BASE_URL + '/guides/dtypes',
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
file_data = {}
|
||||
for key, file_path in FILES_TO_INCLUDE.items():
|
||||
with open(file_path, encoding='utf-8') as f:
|
||||
file_data[key] = f.read()
|
||||
|
||||
# Fix links:
|
||||
# NOTE: This regex does not match all markdown links, but works for the ones we need to replace.
|
||||
LINK_RE = r'(?<=\])\((.+?)\)'
|
||||
|
||||
def replace_fn(match):
|
||||
link = match.group(1)
|
||||
|
||||
if link in CUSTOM_LINK_MAP:
|
||||
link = CUSTOM_LINK_MAP[link]
|
||||
|
||||
elif link.startswith('/'):
|
||||
# Link to docs
|
||||
link = DOCS_BASE_URL + link
|
||||
|
||||
elif link.startswith('./'):
|
||||
# Relative link to file
|
||||
pass
|
||||
|
||||
elif link.startswith('http'):
|
||||
# Link to external site
|
||||
pass
|
||||
|
||||
return f'({link})'
|
||||
|
||||
result = README_TEMPLATE.format(**file_data)
|
||||
result = re.sub(LINK_RE, replace_fn, result, count=0, flags=re.MULTILINE)
|
||||
|
||||
with open('README.md', 'w', encoding='utf-8') as f:
|
||||
f.write(result)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,79 @@
|
||||
// Based on [this tutorial](https://github.com/jsdoc2md/jsdoc-to-markdown/wiki/How-to-create-one-output-file-per-class).
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import url from "node:url";
|
||||
|
||||
import jsdoc2md from "jsdoc-to-markdown";
|
||||
|
||||
const docs = path.dirname(path.dirname(url.fileURLToPath(import.meta.url)));
|
||||
const root = path.dirname(docs);
|
||||
|
||||
// jsdoc config file
|
||||
const conf = path.join(docs, "jsdoc-conf.json");
|
||||
|
||||
// input and output paths
|
||||
const inputFile = path.join(root, "/src/**/*.js");
|
||||
const outputDir = path.join(root, "/docs/source/api/");
|
||||
|
||||
// get template data
|
||||
const templateData = await jsdoc2md.getTemplateData({
|
||||
files: inputFile,
|
||||
configure: conf,
|
||||
"no-cache": true,
|
||||
});
|
||||
|
||||
// reduce templateData to an array of module names
|
||||
const moduleNames = templateData.reduce((moduleNames, identifier) => {
|
||||
if (identifier.kind === "module") {
|
||||
moduleNames.push(identifier.name);
|
||||
}
|
||||
return moduleNames;
|
||||
}, []);
|
||||
|
||||
// Clear all existing .md files from output directory (recursively)
|
||||
if (fs.existsSync(outputDir)) {
|
||||
const existingFiles = fs.readdirSync(outputDir, { recursive: true });
|
||||
for (const file of existingFiles) {
|
||||
if (file.endsWith(".md")) {
|
||||
fs.unlinkSync(path.join(outputDir, file));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// create a documentation file for each module
|
||||
for (const moduleName of moduleNames) {
|
||||
const template = `{{#module name="${moduleName}"}}{{>docs}}{{/module}}`;
|
||||
console.log(`rendering ${moduleName}, template: ${template}`);
|
||||
let output = await jsdoc2md.render({
|
||||
data: templateData,
|
||||
template: template,
|
||||
"heading-depth": 1,
|
||||
"no-gfm": true,
|
||||
"name-format": "backticks",
|
||||
"no-cache": true,
|
||||
separators: true,
|
||||
configure: conf,
|
||||
});
|
||||
|
||||
// Post-processing
|
||||
output = output.replace(/(^#+\s.+)/gm, "$1\n"); // Add new line after each header
|
||||
|
||||
// Remove <code> tags from headers
|
||||
output = output.replace(/^#+\s.+$/gm, (match) => match.replace(/<\/?code>/g, ""));
|
||||
|
||||
// Replace all generated marker names with ids (for linking), and add group class
|
||||
output = output.replace(/<a name="(\S+)"><\/a>/g, '<a id="$1" class="group"></a>');
|
||||
|
||||
// Unescape some of the characters which jsdoc2md escapes:
|
||||
// TODO: May need to extend this list
|
||||
output = output.replace(/\\([|_&*])/gm, "$1");
|
||||
|
||||
output = output.replaceAll("new exports.", "new ");
|
||||
|
||||
const outputPath = path.resolve(outputDir, `${moduleName}.md`);
|
||||
|
||||
console.log(`Writing to ${outputPath}`);
|
||||
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
fs.writeFileSync(outputPath, output);
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
|
||||
<h3 align="center">
|
||||
<p>State-of-the-art Machine Learning for the Web</p>
|
||||
</h3>
|
||||
|
||||
Run 🤗 Transformers directly in your browser, with no need for a server!
|
||||
|
||||
Transformers.js is designed to be functionally equivalent to Hugging Face's [transformers](https://github.com/huggingface/transformers) python library, meaning you can run the same pretrained models using a very similar API. These models support common tasks in different modalities, such as:
|
||||
- 📝 **Natural Language Processing**: text classification, named entity recognition, question answering, language modeling, summarization, translation, multiple choice, and text generation.
|
||||
- 🖼️ **Computer Vision**: image classification, object detection, segmentation, and depth estimation.
|
||||
- 🗣️ **Audio**: automatic speech recognition, audio classification, and text-to-speech.
|
||||
- 🐙 **Multimodal**: embeddings, zero-shot audio classification, zero-shot image classification, and zero-shot object detection.
|
||||
|
||||
Transformers.js uses [ONNX Runtime](https://onnxruntime.ai/) to run models in the browser. The best part about it, is that you can easily [convert](#convert-your-models-to-onnx) your pretrained PyTorch, TensorFlow, or JAX models to ONNX using [🤗 Optimum](https://github.com/huggingface/optimum#onnx--onnx-runtime).
|
||||
|
||||
For more information, check out the full [documentation](https://huggingface.co/docs/transformers.js).
|
||||
@@ -0,0 +1,74 @@
|
||||
|
||||
It's super simple to translate from existing code! Just like the python library, we support the `pipeline` API. Pipelines group together a pretrained model with preprocessing of inputs and postprocessing of outputs, making it the easiest way to run models with the library.
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<th width="440px" align="center"><b>Python (original)</b></th>
|
||||
<th width="440px" align="center"><b>Javascript (ours)</b></th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
```python
|
||||
from transformers import pipeline
|
||||
|
||||
# Allocate a pipeline for sentiment-analysis
|
||||
pipe = pipeline('sentiment-analysis')
|
||||
|
||||
out = pipe('I love transformers!')
|
||||
# [{'label': 'POSITIVE', 'score': 0.999806941}]
|
||||
```
|
||||
|
||||
</td>
|
||||
<td>
|
||||
|
||||
```javascript
|
||||
import { pipeline } from '@huggingface/transformers';
|
||||
|
||||
// Allocate a pipeline for sentiment-analysis
|
||||
const pipe = await pipeline('sentiment-analysis');
|
||||
|
||||
const out = await pipe('I love transformers!');
|
||||
// [{'label': 'POSITIVE', 'score': 0.999817686}]
|
||||
```
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
You can also use a different model by specifying the model id or path as the second argument to the `pipeline` function. For example:
|
||||
```javascript
|
||||
// Use a different model for sentiment-analysis
|
||||
const pipe = await pipeline('sentiment-analysis', 'Xenova/bert-base-multilingual-uncased-sentiment');
|
||||
```
|
||||
|
||||
By default, when running in the browser, the model will be run on your CPU (via WASM). If you would like
|
||||
to run the model on your GPU (via WebGPU), you can do this by setting `device: 'webgpu'`, for example:
|
||||
```javascript
|
||||
// Run the model on WebGPU
|
||||
const pipe = await pipeline('sentiment-analysis', 'Xenova/distilbert-base-uncased-finetuned-sst-2-english', {
|
||||
device: 'webgpu',
|
||||
});
|
||||
```
|
||||
|
||||
For more information, check out the [WebGPU guide](./guides/webgpu).
|
||||
|
||||
> [!WARNING]
|
||||
> The WebGPU API is still experimental in many browsers, so if you run into any issues,
|
||||
> please file a [bug report](https://github.com/huggingface/transformers.js/issues/new?title=%5BWebGPU%5D%20Error%20running%20MODEL_ID_GOES_HERE&assignees=&labels=bug,webgpu&projects=&template=1_bug-report.yml).
|
||||
|
||||
In resource-constrained environments, such as web browsers, it is advisable to use a quantized version of
|
||||
the model to lower bandwidth and optimize performance. This can be achieved by adjusting the `dtype` option,
|
||||
which allows you to select the appropriate data type for your model. While the available options may vary
|
||||
depending on the specific model, typical choices include `"fp32"` (default for WebGPU), `"fp16"`, `"q8"`
|
||||
(default for WASM), and `"q4"`. For more information, check out the [quantization guide](./guides/dtypes).
|
||||
```javascript
|
||||
// Run the model at 4-bit quantization
|
||||
const pipe = await pipeline('sentiment-analysis', 'Xenova/distilbert-base-uncased-finetuned-sst-2-english', {
|
||||
dtype: 'q4',
|
||||
});
|
||||
```
|
||||
|
||||
Ready to dive in? Explore our wide variety of demo applications and templates [here](https://github.com/huggingface/transformers.js-examples). You can also launch your own project instantly using the official Transformers.js [template](https://huggingface.co/new-space?template=static-templates%2Ftransformers.js) on Hugging Face!
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
|
||||
To install via [NPM](https://www.npmjs.com/package/@huggingface/transformers), run:
|
||||
```bash
|
||||
npm i @huggingface/transformers
|
||||
```
|
||||
|
||||
Alternatively, you can use it in vanilla JS, without any bundler, by using a CDN or static hosting. For example, using [ES Modules](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Modules), you can import the library with:
|
||||
```html
|
||||
<script type="module">
|
||||
import { pipeline } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0';
|
||||
</script>
|
||||
```
|
||||
@@ -0,0 +1,24 @@
|
||||
|
||||
|
||||
By default, Transformers.js uses [hosted pretrained models](https://huggingface.co/models?library=transformers.js) and [precompiled WASM binaries](https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0/dist/), which should work out-of-the-box. You can customize this as follows:
|
||||
|
||||
### Settings
|
||||
|
||||
```javascript
|
||||
import { env } from '@huggingface/transformers';
|
||||
|
||||
// Specify a custom location for models (defaults to '/models/').
|
||||
env.localModelPath = '/path/to/models/';
|
||||
|
||||
// Disable the loading of remote models from the Hugging Face Hub:
|
||||
env.allowRemoteModels = false;
|
||||
|
||||
// Set location of .wasm files. Defaults to use a CDN.
|
||||
env.backends.onnx.wasm.wasmPaths = '/path/to/files/';
|
||||
```
|
||||
|
||||
For a full list of available settings, check out the [API Reference](./api/env).
|
||||
|
||||
### Convert your models to ONNX
|
||||
|
||||
We recommend using [Optimum](https://github.com/huggingface/optimum-onnx) to convert your PyTorch models to ONNX in a single command. For the full list of supported architectures, check out the [Optimum documentation](https://huggingface.co/docs/optimum-onnx/onnx/overview).
|
||||
@@ -0,0 +1,71 @@
|
||||
|
||||
### Tasks
|
||||
|
||||
#### Natural Language Processing
|
||||
|
||||
| Task | ID | Description | Supported? |
|
||||
|--------------------------|----|-------------|------------|
|
||||
| [Fill-Mask](https://huggingface.co/tasks/fill-mask) | `fill-mask` | Masking some of the words in a sentence and predicting which words should replace those masks. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.FillMaskPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=fill-mask&library=transformers.js) |
|
||||
| [Question Answering](https://huggingface.co/tasks/question-answering) | `question-answering` | Retrieve the answer to a question from a given text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.QuestionAnsweringPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=question-answering&library=transformers.js) |
|
||||
| [Sentence Similarity](https://huggingface.co/tasks/sentence-similarity) | `sentence-similarity` | Determining how similar two texts are. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.FeatureExtractionPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=sentence-similarity&library=transformers.js) |
|
||||
| [Summarization](https://huggingface.co/tasks/summarization) | `summarization` | Producing a shorter version of a document while preserving its important information. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.SummarizationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=summarization&library=transformers.js) |
|
||||
| [Table Question Answering](https://huggingface.co/tasks/table-question-answering) | `table-question-answering` | Answering a question about information from a given table. | ❌ |
|
||||
| [Text Classification](https://huggingface.co/tasks/text-classification) | `text-classification` or `sentiment-analysis` | Assigning a label or class to a given text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TextClassificationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=text-classification&library=transformers.js) |
|
||||
| [Text Generation](https://huggingface.co/tasks/text-generation#completion-generation-models) | `text-generation` | Producing new text by predicting the next word in a sequence. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TextGenerationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=text-generation&library=transformers.js) |
|
||||
| [Text-to-text Generation](https://huggingface.co/tasks/text-generation#text-to-text-generation-models) | `text2text-generation` | Converting one text sequence into another text sequence. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.Text2TextGenerationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=text2text-generation&library=transformers.js) |
|
||||
| [Token Classification](https://huggingface.co/tasks/token-classification) | `token-classification` or `ner` | Assigning a label to each token in a text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TokenClassificationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=token-classification&library=transformers.js) |
|
||||
| [Translation](https://huggingface.co/tasks/translation) | `translation` | Converting text from one language to another. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TranslationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=translation&library=transformers.js) |
|
||||
| [Zero-Shot Classification](https://huggingface.co/tasks/zero-shot-classification) | `zero-shot-classification` | Classifying text into classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotClassificationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=zero-shot-classification&library=transformers.js) |
|
||||
| [Feature Extraction](https://huggingface.co/tasks/feature-extraction) | `feature-extraction` | Transforming raw data into numerical features that can be processed while preserving the information in the original dataset. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.FeatureExtractionPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=feature-extraction&library=transformers.js) |
|
||||
|
||||
#### Vision
|
||||
|
||||
| Task | ID | Description | Supported? |
|
||||
|--------------------------|----|-------------|------------|
|
||||
| [Background Removal](https://huggingface.co/tasks/image-segmentation#background-removal) | `background-removal` | Isolating the main subject of an image by removing or making the background transparent. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.BackgroundRemovalPipeline)<br>[(models)](https://huggingface.co/models?other=background-removal&library=transformers.js) |
|
||||
| [Depth Estimation](https://huggingface.co/tasks/depth-estimation) | `depth-estimation` | Predicting the depth of objects present in an image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.DepthEstimationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=depth-estimation&library=transformers.js) |
|
||||
| [Image Classification](https://huggingface.co/tasks/image-classification) | `image-classification` | Assigning a label or class to an entire image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageClassificationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=image-classification&library=transformers.js) |
|
||||
| [Image Segmentation](https://huggingface.co/tasks/image-segmentation) | `image-segmentation` | Divides an image into segments where each pixel is mapped to an object. This task has multiple variants such as instance segmentation, panoptic segmentation and semantic segmentation. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageSegmentationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=image-segmentation&library=transformers.js) |
|
||||
| [Image-to-Image](https://huggingface.co/tasks/image-to-image) | `image-to-image` | Transforming a source image to match the characteristics of a target image or a target image domain. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageToImagePipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=image-to-image&library=transformers.js) |
|
||||
| [Mask Generation](https://huggingface.co/tasks/mask-generation) | `mask-generation` | Generate masks for the objects in an image. | ❌ |
|
||||
| [Object Detection](https://huggingface.co/tasks/object-detection) | `object-detection` | Identify objects of certain defined classes within an image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ObjectDetectionPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=object-detection&library=transformers.js) |
|
||||
| [Video Classification](https://huggingface.co/tasks/video-classification) | n/a | Assigning a label or class to an entire video. | ❌ |
|
||||
| [Unconditional Image Generation](https://huggingface.co/tasks/unconditional-image-generation) | n/a | Generating images with no condition in any context (like a prompt text or another image). | ❌ |
|
||||
| [Image Feature Extraction](https://huggingface.co/tasks/image-feature-extraction) | `image-feature-extraction` | Transforming raw data into numerical features that can be processed while preserving the information in the original image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageFeatureExtractionPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=image-feature-extraction&library=transformers.js) |
|
||||
|
||||
#### Audio
|
||||
|
||||
| Task | ID | Description | Supported? |
|
||||
|--------------------------|----|-------------|------------|
|
||||
| [Audio Classification](https://huggingface.co/tasks/audio-classification) | `audio-classification` | Assigning a label or class to a given audio. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.AudioClassificationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=audio-classification&library=transformers.js) |
|
||||
| [Audio-to-Audio](https://huggingface.co/tasks/audio-to-audio) | n/a | Generating audio from an input audio source. | ❌ |
|
||||
| [Automatic Speech Recognition](https://huggingface.co/tasks/automatic-speech-recognition) | `automatic-speech-recognition` | Transcribing a given audio into text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.AutomaticSpeechRecognitionPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=automatic-speech-recognition&library=transformers.js) |
|
||||
| [Text-to-Speech](https://huggingface.co/tasks/text-to-speech) | `text-to-speech` or `text-to-audio` | Generating natural-sounding speech given text input. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TextToAudioPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=text-to-audio&library=transformers.js) |
|
||||
|
||||
|
||||
#### Tabular
|
||||
|
||||
| Task | ID | Description | Supported? |
|
||||
|--------------------------|----|-------------|------------|
|
||||
| [Tabular Classification](https://huggingface.co/tasks/tabular-classification) | n/a | Classifying a target category (a group) based on set of attributes. | ❌ |
|
||||
| [Tabular Regression](https://huggingface.co/tasks/tabular-regression) | n/a | Predicting a numerical value given a set of attributes. | ❌ |
|
||||
|
||||
|
||||
#### Multimodal
|
||||
|
||||
| Task | ID | Description | Supported? |
|
||||
|--------------------------|----|-------------|------------|
|
||||
| [Document Question Answering](https://huggingface.co/tasks/document-question-answering) | `document-question-answering` | Answering questions on document images. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.DocumentQuestionAnsweringPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=document-question-answering&library=transformers.js) |
|
||||
| [Image-to-Text](https://huggingface.co/tasks/image-to-text) | `image-to-text` | Output text from a given image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageToTextPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=image-to-text&library=transformers.js) |
|
||||
| [Text-to-Image](https://huggingface.co/tasks/text-to-image) | `text-to-image` | Generates images from input text. | ❌ |
|
||||
| [Visual Question Answering](https://huggingface.co/tasks/visual-question-answering) | `visual-question-answering` | Answering open-ended questions based on an image. | ❌ |
|
||||
| [Zero-Shot Audio Classification](https://huggingface.co/learn/audio-course/chapter4/classification_models#zero-shot-audio-classification) | `zero-shot-audio-classification` | Classifying audios into classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotAudioClassificationPipeline)<br>[(models)](https://huggingface.co/models?other=zero-shot-audio-classification&library=transformers.js) |
|
||||
| [Zero-Shot Image Classification](https://huggingface.co/tasks/zero-shot-image-classification) | `zero-shot-image-classification` | Classifying images into classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotImageClassificationPipeline)<br>[(models)](https://huggingface.co/models?pipeline_tag=zero-shot-image-classification&library=transformers.js) |
|
||||
| [Zero-Shot Object Detection](https://huggingface.co/tasks/zero-shot-object-detection) | `zero-shot-object-detection` | Identify objects of classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotObjectDetectionPipeline)<br>[(models)](https://huggingface.co/models?other=zero-shot-object-detection&library=transformers.js) |
|
||||
|
||||
|
||||
#### Reinforcement Learning
|
||||
|
||||
| Task | ID | Description | Supported? |
|
||||
|--------------------------|----|-------------|------------|
|
||||
| [Reinforcement Learning](https://huggingface.co/tasks/reinforcement-learning) | n/a | Learning from actions by interacting with an environment through trial and error and receiving rewards (negative or positive) as feedback. | ✅ |
|
||||
@@ -0,0 +1,216 @@
|
||||
|
||||
### Models
|
||||
|
||||
1. **[AFMoE](https://huggingface.co/docs/transformers/model_doc/afmoe)** (from Arcee AI, Prime Intellect, and DatologyAI) released with the paper [Arcee Trinity Large Technical Report](https://huggingface.co/papers/2602.17004), by Varun Singh, Lucas Krauss, Sami Jaghouar, Matej Sirovatka, Charles Goddard, Fares Obied, Jack Min Ong, Jannik Straube, Fern, Aria Harley, Conner Stewart, Colin Kealty, Maziyar Panahi, Simon Kirsten, Anushka Deshpande, Anneketh Vij, Arthur Bresnu, Pranav Veldurthi, Raghav Ravishankar, Hardik Bishnoi, Mark McQuade, Johannes Hagemann, Lucas Atkins.
|
||||
1. **[ALBERT](https://huggingface.co/docs/transformers/model_doc/albert)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://huggingface.co/papers/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.
|
||||
1. **[Arcee](https://huggingface.co/docs/transformers/model_doc/arcee)** (from Arcee AI) released with the blog post [Announcing Arcee Foundation Models](https://www.arcee.ai/blog/announcing-the-arcee-foundation-model-family) by Fernando Fernandes, Varun Singh, Charles Goddard, Lucas Atkins, Mark McQuade, Maziyar Panahi, Conner Stewart, Colin Kealty, Raghav Ravishankar, Lucas Krauss, Anneketh Vij, Pranav Veldurthi, Abhishek Thakur, Julien Simon, Scott Zembsch, Benjamin Langer, Aleksiej Cecocho, Maitri Patel.
|
||||
1. **[Audio Spectrogram Transformer](https://huggingface.co/docs/transformers/model_doc/audio-spectrogram-transformer)** (from MIT) released with the paper [AST: Audio Spectrogram Transformer](https://huggingface.co/papers/2104.01778) by Yuan Gong, Yu-An Chung, James Glass.
|
||||
1. **[BART](https://huggingface.co/docs/transformers/model_doc/bart)** (from Facebook) released with the paper [BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension](https://huggingface.co/papers/1910.13461) by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer.
|
||||
1. **[BEiT](https://huggingface.co/docs/transformers/model_doc/beit)** (from Microsoft) released with the paper [BEiT: BERT Pre-Training of Image Transformers](https://huggingface.co/papers/2106.08254) by Hangbo Bao, Li Dong, Furu Wei.
|
||||
1. **[BERT](https://huggingface.co/docs/transformers/model_doc/bert)** (from Google) released with the paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://huggingface.co/papers/1810.04805) by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova.
|
||||
1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://huggingface.co/papers/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston.
|
||||
1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://huggingface.co/papers/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston.
|
||||
1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/).
|
||||
1. **[CamemBERT](https://huggingface.co/docs/transformers/model_doc/camembert)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://huggingface.co/papers/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot.
|
||||
1. **[CHMv2](https://huggingface.co/docs/transformers/main/model_doc/chmv2)** (from Meta) released with the paper [CHMv2: Improvements in Global Canopy Height Mapping using DINOv3](https://huggingface.co/papers/2603.06382) by John Brandt, Seungeun Yi, Jamie Tolan, Xinyuan Li, Peter Potapov, Jessica Ertel, Justine Spore, Huy V. Vo, Michaël Ramamonjisoa, Patrick Labatut, Piotr Bojanowski, Camille Couprie.
|
||||
1. **Chatterbox** (from Resemble AI) released with the repository [Chatterbox TTS](https://github.com/resemble-ai/chatterbox) by the Resemble AI team.
|
||||
1. **[Chinese-CLIP](https://huggingface.co/docs/transformers/model_doc/chinese_clip)** (from OFA-Sys) released with the paper [Chinese CLIP: Contrastive Vision-Language Pretraining in Chinese](https://huggingface.co/papers/2211.01335) by An Yang, Junshu Pan, Junyang Lin, Rui Men, Yichang Zhang, Jingren Zhou, Chang Zhou.
|
||||
1. **[CLAP](https://huggingface.co/docs/transformers/model_doc/clap)** (from LAION-AI) released with the paper [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation](https://huggingface.co/papers/2211.06687) by Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov.
|
||||
1. **[CLIP](https://huggingface.co/docs/transformers/model_doc/clip)** (from OpenAI) released with the paper [Learning Transferable Visual Models From Natural Language Supervision](https://huggingface.co/papers/2103.00020) by Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, Gretchen Krueger, Ilya Sutskever.
|
||||
1. **[CLIPSeg](https://huggingface.co/docs/transformers/model_doc/clipseg)** (from University of Göttingen) released with the paper [Image Segmentation Using Text and Image Prompts](https://huggingface.co/papers/2112.10003) by Timo Lüddecke and Alexander Ecker.
|
||||
1. **[CodeGen](https://huggingface.co/docs/transformers/model_doc/codegen)** (from Salesforce) released with the paper [A Conversational Paradigm for Program Synthesis](https://huggingface.co/papers/2203.13474) by Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, Caiming Xiong.
|
||||
1. **[CodeLlama](https://huggingface.co/docs/transformers/model_doc/llama_code)** (from MetaAI) released with the paper [Code Llama: Open Foundation Models for Code](https://ai.meta.com/research/publications/code-llama-open-foundation-models-for-code/) by Baptiste Rozière, Jonas Gehring, Fabian Gloeckle, Sten Sootla, Itai Gat, Xiaoqing Ellen Tan, Yossi Adi, Jingyu Liu, Tal Remez, Jérémy Rapin, Artyom Kozhevnikov, Ivan Evtimov, Joanna Bitton, Manish Bhatt, Cristian Canton Ferrer, Aaron Grattafiori, Wenhan Xiong, Alexandre Défossez, Jade Copet, Faisal Azhar, Hugo Touvron, Louis Martin, Nicolas Usunier, Thomas Scialom, Gabriel Synnaeve.
|
||||
1. **[Cohere](https://huggingface.co/docs/transformers/main/model_doc/cohere)** (from Cohere) released with the blog post [Command-R: Retrieval Augmented Generation at Production Scale](https://cohere.com/blog/command-r) by Cohere.
|
||||
1. **[Cohere2](https://huggingface.co/docs/transformers/main/model_doc/cohere2)** (from Cohere) released with the blog post [Introducing Command R7B: Fast and efficient generative AI](https://cohere.com/blog/command-r7b) by Cohere.
|
||||
1. **[Cohere ASR](https://huggingface.co/docs/transformers/main/model_doc/cohere_asr)** (from Cohere) released with the blog post [Introducing Cohere Transcribe: a new state-of-the-art in open-source speech recognition](https://cohere.com/blog/transcribe) by Cohere.
|
||||
1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://huggingface.co/papers/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan.
|
||||
1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://huggingface.co/papers/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie.
|
||||
1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://huggingface.co/papers/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie.
|
||||
1. **[D-FINE](https://huggingface.co/docs/transformers/model_doc/d_fine)** (from University of Science and Technology of China) released with the paper [D-FINE: Redefine Regression Task in DETRs as Fine-grained Distribution Refinement](https://huggingface.co/papers/2410.13842) by Yansong Peng, Hebei Li, Peixi Wu, Yueyi Zhang, Xiaoyan Sun, Feng Wu.
|
||||
1. **[DAC](https://huggingface.co/docs/transformers/model_doc/dac)** (from Descript) released with the paper [Descript Audio Codec: High-Fidelity Audio Compression with Improved RVQGAN](https://huggingface.co/papers/2306.06546) by Rithesh Kumar, Prem Seetharaman, Alejandro Luebs, Ishaan Kumar, Kundan Kumar.
|
||||
1. **[DeBERTa](https://huggingface.co/docs/transformers/model_doc/deberta)** (from Microsoft) released with the paper [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://huggingface.co/papers/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen.
|
||||
1. **[DeBERTa-v2](https://huggingface.co/docs/transformers/model_doc/deberta-v2)** (from Microsoft) released with the paper [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://huggingface.co/papers/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen.
|
||||
1. **[Decision Transformer](https://huggingface.co/docs/transformers/model_doc/decision_transformer)** (from Berkeley/Facebook/Google) released with the paper [Decision Transformer: Reinforcement Learning via Sequence Modeling](https://huggingface.co/papers/2106.01345) by Lili Chen, Kevin Lu, Aravind Rajeswaran, Kimin Lee, Aditya Grover, Michael Laskin, Pieter Abbeel, Aravind Srinivas, Igor Mordatch.
|
||||
1. **[DeepSeek-V3](https://huggingface.co/docs/transformers/model_doc/deepseek_v3)** (from DeepSeek-AI) released with the paper [DeepSeek-V3 Technical Report](https://huggingface.co/papers/2412.19437) by the DeepSeek-AI team.
|
||||
1. **[DeepSeek-V4](https://huggingface.co/docs/transformers/model_doc/deepseek_v4)** (from DeepSeek-AI) released with the paper [DeepSeek-V4: Towards Highly Efficient Million-Token Context Intelligence](https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf) by the DeepSeek-AI team.
|
||||
1. **[DeiT](https://huggingface.co/docs/transformers/model_doc/deit)** (from Facebook) released with the paper [Training data-efficient image transformers & distillation through attention](https://huggingface.co/papers/2012.12877) by Hugo Touvron, Matthieu Cord, Matthijs Douze, Francisco Massa, Alexandre Sablayrolles, Hervé Jégou.
|
||||
1. **[Depth Anything](https://huggingface.co/docs/transformers/main/model_doc/depth_anything)** (from University of Hong Kong and TikTok) released with the paper [Depth Anything: Unleashing the Power of Large-Scale Unlabeled Data](https://huggingface.co/papers/2401.10891) by Lihe Yang, Bingyi Kang, Zilong Huang, Xiaogang Xu, Jiashi Feng, Hengshuang Zhao.
|
||||
1. **Depth Pro** (from Apple) released with the paper [Depth Pro: Sharp Monocular Metric Depth in Less Than a Second](https://huggingface.co/papers/2410.02073) by Aleksei Bochkovskii, Amaël Delaunoy, Hugo Germain, Marcel Santos, Yichao Zhou, Stephan R. Richter, Vladlen Koltun.
|
||||
1. **[DETR](https://huggingface.co/docs/transformers/model_doc/detr)** (from Facebook) released with the paper [End-to-End Object Detection with Transformers](https://huggingface.co/papers/2005.12872) by Nicolas Carion, Francisco Massa, Gabriel Synnaeve, Nicolas Usunier, Alexander Kirillov, Sergey Zagoruyko.
|
||||
1. **[DINOv2](https://huggingface.co/docs/transformers/model_doc/dinov2)** (from Meta AI) released with the paper [DINOv2: Learning Robust Visual Features without Supervision](https://huggingface.co/papers/2304.07193) by Maxime Oquab, Timothée Darcet, Théo Moutakanni, Huy Vo, Marc Szafraniec, Vasil Khalidov, Pierre Fernandez, Daniel Haziza, Francisco Massa, Alaaeldin El-Nouby, Mahmoud Assran, Nicolas Ballas, Wojciech Galuba, Russell Howes, Po-Yao Huang, Shang-Wen Li, Ishan Misra, Michael Rabbat, Vasu Sharma, Gabriel Synnaeve, Hu Xu, Hervé Jegou, Julien Mairal, Patrick Labatut, Armand Joulin, Piotr Bojanowski.
|
||||
1. **[DINOv2 with Registers](https://huggingface.co/docs/transformers/model_doc/dinov2_with_registers)** (from Meta AI) released with the paper [DINOv2 with Registers](https://huggingface.co/papers/2309.16588) by Timothée Darcet, Maxime Oquab, Julien Mairal, Piotr Bojanowski.
|
||||
1. **[DINOv3](https://huggingface.co/docs/transformers/model_doc/dinov3)** (from Meta AI) released with the paper [DINOv3](https://huggingface.co/papers/2508.10104) by Oriane Siméoni, Huy V. Vo, Maximilian Seitzer, Federico Baldassarre, Maxime Oquab, Cijo Jose, Vasil Khalidov, Marc Szafraniec, Seungeun Yi, Michaël Ramamonjisoa, Francisco Massa, Daniel Haziza, Luca Wehrstedt, Jianyuan Wang, Timothée Darcet, Théo Moutakanni, Leonel Sentana, Claire Roberts, Andrea Vedaldi, Jamie Tolan, John Brandt, Camille Couprie, Julien Mairal, Hervé Jégou, Patrick Labatut, Piotr Bojanowski.
|
||||
1. **[DistilBERT](https://huggingface.co/docs/transformers/model_doc/distilbert)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://huggingface.co/papers/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/research_projects/distillation), RoBERTa into [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/research_projects/distillation), Multilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/research_projects/distillation) and a German version of DistilBERT.
|
||||
1. **[DiT](https://huggingface.co/docs/transformers/model_doc/dit)** (from Microsoft Research) released with the paper [DiT: Self-supervised Pre-training for Document Image Transformer](https://huggingface.co/papers/2203.02378) by Junlong Li, Yiheng Xu, Tengchao Lv, Lei Cui, Cha Zhang, Furu Wei.
|
||||
1. **[Donut](https://huggingface.co/docs/transformers/model_doc/donut)** (from NAVER), released together with the paper [OCR-free Document Understanding Transformer](https://huggingface.co/papers/2111.15664) by Geewook Kim, Teakgyu Hong, Moonbin Yim, Jeongyeon Nam, Jinyoung Park, Jinyeong Yim, Wonseok Hwang, Sangdoo Yun, Dongyoon Han, Seunghyun Park.
|
||||
1. **[DPT](https://huggingface.co/docs/transformers/master/model_doc/dpt)** (from Intel Labs) released with the paper [Vision Transformers for Dense Prediction](https://huggingface.co/papers/2103.13413) by René Ranftl, Alexey Bochkovskiy, Vladlen Koltun.
|
||||
1. **[EdgeTAM](https://huggingface.co/docs/transformers/model_doc/edgetam)** (from Facebook) released with the paper [EdgeTAM: On-Device Track Anything Model](https://huggingface.co/papers/2501.07256) by Chong Zhou, Chenchen Zhu, Yunyang Xiong, Saksham Suri, Fanyi Xiao, Lemeng Wu, Raghuraman Krishnamoorthi, Bo Dai, Chen Change Loy, Vikas Chandra, Bilge Soran.
|
||||
1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://huggingface.co/papers/1905.11946) by Mingxing Tan, Quoc V. Le.
|
||||
1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://huggingface.co/papers/2003.10555) by Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning.
|
||||
1. **ERNIE-4.5** (from Baidu ERNIE Team) released with the blog post [Announcing the Open Source Release of the ERNIE 4.5 Model Family](https://ernie.baidu.com/blog/posts/ernie4.5/) by the Baidu ERNIE Team.
|
||||
1. **[ESM](https://huggingface.co/docs/transformers/model_doc/esm)** (from Meta AI) are transformer protein language models. **ESM-1b** was released with the paper [Biological structure and function emerge from scaling unsupervised learning to 250 million protein sequences](https://www.pnas.org/content/118/15/e2016239118) by Alexander Rives, Joshua Meier, Tom Sercu, Siddharth Goyal, Zeming Lin, Jason Liu, Demi Guo, Myle Ott, C. Lawrence Zitnick, Jerry Ma, and Rob Fergus. **ESM-1v** was released with the paper [Language models enable zero-shot prediction of the effects of mutations on protein function](https://doi.org/10.1101/2021.07.09.450648) by Joshua Meier, Roshan Rao, Robert Verkuil, Jason Liu, Tom Sercu and Alexander Rives. **ESM-2 and ESMFold** were released with the paper [Language models of protein sequences at the scale of evolution enable accurate structure prediction](https://doi.org/10.1101/2022.07.20.500902) by Zeming Lin, Halil Akin, Roshan Rao, Brian Hie, Zhongkai Zhu, Wenting Lu, Allan dos Santos Costa, Maryam Fazel-Zarandi, Tom Sercu, Sal Candido, Alexander Rives.
|
||||
1. **[EuroBERT](https://huggingface.co/docs/transformers/model_doc/eurobert)** (from the EuroBERT team) released with the paper [EuroBERT: Scaling Multilingual Encoders for European Languages](https://huggingface.co/papers/2503.05500) by Nicolas Boizard, Hippolyte Gisserot-Boukhlef, Duarte M. Alves, André Martins, Ayoub Hammal, Caio Corro, Céline Hudelot, Emmanuel Malherbe, Etienne Malaboeuf, Fanny Jourdan, Gabriel Hautreux, João Alves, Kevin El-Haddad, Manuel Faysse, Maxime Peyrard, Nuno M. Guerreiro, Patrick Fernandes, Ricardo Rei, Pierre Colombo.
|
||||
1. **EXAONE** (from LG AI Research) released with the papers [EXAONE 3.0 7.8B Instruction Tuned Language Model](https://huggingface.co/papers/2408.03541) and [EXAONE 3.5: Series of Large Language Models for Real-world Use Cases](https://huggingface.co/papers/2412.04862) by the LG AI Research team.
|
||||
1. **[Falcon](https://huggingface.co/docs/transformers/model_doc/falcon)** (from Technology Innovation Institute) by Almazrouei, Ebtesam and Alobeidli, Hamza and Alshamsi, Abdulaziz and Cappelli, Alessandro and Cojocaru, Ruxandra and Debbah, Merouane and Goffinet, Etienne and Heslow, Daniel and Launay, Julien and Malartic, Quentin and Noune, Badreddine and Pannier, Baptiste and Penedo, Guilherme.
|
||||
1. **[FalconH1](https://huggingface.co/docs/transformers/model_doc/falcon_h1)** (from Technology Innovation Institute) released with the blog post [Falcon-H1-Tiny: A series of extremely small, yet powerful language models redefining capabilities at small scale](https://huggingface.co/spaces/tiiuae/tiny-h1-blogpost) by the Falcon-LLM team.
|
||||
1. **FastViT** (from Apple) released with the paper [FastViT: A Fast Hybrid Vision Transformer using Structural Reparameterization](https://huggingface.co/papers/2303.14189) by Pavan Kumar Anasosalu Vasu, James Gabriel, Jeff Zhu, Oncel Tuzel and Anurag Ranjan.
|
||||
1. **[FLAN-T5](https://huggingface.co/docs/transformers/model_doc/flan-t5)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-t5-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei
|
||||
1. **Florence2** (from Microsoft) released with the paper [Florence-2: Advancing a Unified Representation for a Variety of Vision Tasks](https://huggingface.co/papers/2311.06242) by Bin Xiao, Haiping Wu, Weijian Xu, Xiyang Dai, Houdong Hu, Yumao Lu, Michael Zeng, Ce Liu, Lu Yuan.
|
||||
1. **[Gemma](https://huggingface.co/docs/transformers/main/model_doc/gemma)** (from Google) released with the blog post [Gemma: Open Models Based on Gemini Technology and Research](https://blog.google/technology/developers/gemma-open-models/) by the Gemma Google team.
|
||||
1. **[Gemma2](https://huggingface.co/docs/transformers/main/model_doc/gemma2)** (from Google) released with the blog post [Gemma2: Open Models Based on Gemini Technology and Research](https://blog.google/technology/developers/google-gemma-2/) by the Gemma Google team.
|
||||
1. **[Gemma3](https://huggingface.co/docs/transformers/main/model_doc/gemma3)** (from Google) released with the blog post [Introducing Gemma 3: The most capable model you can run on a single GPU or TPU](https://blog.google/technology/developers/gemma-3/) by the Gemma Google team.
|
||||
1. **[Gemma3n](https://huggingface.co/docs/transformers/main/model_doc/gemma3n)** (from Google) released with the blog post [Announcing Gemma 3n preview: powerful, efficient, mobile-first AI](https://developers.googleblog.com/en/introducing-gemma-3n/) by the Gemma Google team.
|
||||
1. **[Gemma4](https://huggingface.co/docs/transformers/main/model_doc/gemma4)** (from Google) released with the blog post [Gemma 4](https://blog.google/innovation-and-ai/technology/developers-tools/gemma-4/) by the Gemma Google team.
|
||||
1. **[GLM](https://huggingface.co/docs/transformers/main/model_doc/glm)** (from the GLM Team, THUDM & ZhipuAI) released with the paper [ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools](https://huggingface.co/papers/2406.12793v2) by Team GLM: Aohan Zeng, Bin Xu, Bowen Wang, Chenhui Zhang, Da Yin, Dan Zhang, Diego Rojas, Guanyu Feng, Hanlin Zhao, Hanyu Lai, Hao Yu, Hongning Wang, Jiadai Sun, Jiajie Zhang, Jiale Cheng, Jiayi Gui, Jie Tang, Jing Zhang, Jingyu Sun, Juanzi Li, Lei Zhao, Lindong Wu, Lucen Zhong, Mingdao Liu, Minlie Huang, Peng Zhang, Qinkai Zheng, Rui Lu, Shuaiqi Duan, Shudan Zhang, Shulin Cao, Shuxun Yang, Weng Lam Tam, Wenyi Zhao, Xiao Liu, Xiao Xia, Xiaohan Zhang, Xiaotao Gu, Xin Lv, Xinghan Liu, Xinyi Liu, Xinyue Yang, Xixuan Song, Xunkai Zhang, Yifan An, Yifan Xu, Yilin Niu, Yuantao Yang, Yueyan Li, Yushi Bai, Yuxiao Dong, Zehan Qi, Zhaoyu Wang, Zhen Yang, Zhengxiao Du, Zhenyu Hou, Zihan Wang.
|
||||
1. **[GLM-MoE-DSA](https://huggingface.co/docs/transformers/main/model_doc/glm_moe_dsa)** (from the GLM Team, ZhipuAI & Tsinghua University) released with the paper [GLM-5: from Vibe Coding to Agentic Engineering](https://huggingface.co/papers/2602.15763) by Team GLM.
|
||||
1. **[GLM-OCR](https://huggingface.co/docs/transformers/main/model_doc/glm_ocr)** (from the GLM Team, ZhipuAI & Tsinghua University) released with the paper [GLM-OCR Technical Report](https://huggingface.co/papers/2603.10910) by Team GLM: Shuaiqi Duan, Yadong Xue, Weihan Wang, Zhe Su, Huan Liu, Sheng Yang, Guobing Gan, Guo Wang, Zihan Wang, Shengdong Yan, Dexin Jin, Yuxuan Zhang, Guohong Wen, Yanfeng Wang, Yutao Zhang, Xiaohan Zhang, Wenyi Hong, Yukuo Cen, Da Yin, Bin Chen, Wenmeng Yu, Xiaotao Gu, Jie Tang.
|
||||
1. **[GLPN](https://huggingface.co/docs/transformers/model_doc/glpn)** (from KAIST) released with the paper [Global-Local Path Networks for Monocular Depth Estimation with Vertical CutDepth](https://huggingface.co/papers/2201.07436) by Doyeon Kim, Woonghyun Ga, Pyungwhan Ahn, Donggyu Joo, Sehwan Chun, Junmo Kim.
|
||||
1. **[GPT Neo](https://huggingface.co/docs/transformers/model_doc/gpt_neo)** (from EleutherAI) released in the repository [EleutherAI/gpt-neo](https://github.com/EleutherAI/gpt-neo) by Sid Black, Stella Biderman, Leo Gao, Phil Wang and Connor Leahy.
|
||||
1. **[GPT NeoX](https://huggingface.co/docs/transformers/model_doc/gpt_neox)** (from EleutherAI) released with the paper [GPT-NeoX-20B: An Open-Source Autoregressive Language Model](https://huggingface.co/papers/2204.06745) by Sid Black, Stella Biderman, Eric Hallahan, Quentin Anthony, Leo Gao, Laurence Golding, Horace He, Connor Leahy, Kyle McDonell, Jason Phang, Michael Pieler, USVSN Sai Prashanth, Shivanshu Purohit, Laria Reynolds, Jonathan Tow, Ben Wang, Samuel Weinbach
|
||||
1. **[GPT OSS](https://huggingface.co/docs/transformers/model_doc/gpt_oss)** (from OpenAI) released with the blog [Introducing gpt-oss](https://openai.com/index/introducing-gpt-oss/) by Sandhini Agarwal, Lama Ahmad, Jason Ai, Sam Altman, Andy Applebaum, Edwin Arbus, Rahul K. Arora, Yu Bai, Bowen Baker, Haiming Bao, Boaz Barak, Ally Bennett, Tyler Bertao, Nivedita Brett, Eugene Brevdo, Greg Brockman, Sebastien Bubeck, Che Chang, Kai Chen, Mark Chen, Enoch Cheung, Aidan Clark, Dan Cook, Marat Dukhan, Casey Dvorak, Kevin Fives, Vlad Fomenko, Timur Garipov, Kristian Georgiev, Mia Glaese, Tarun Gogineni, Adam Goucher, Lukas Gross, Katia Gil Guzman, John Hallman, Jackie Hehir, Johannes Heidecke, Alec Helyar, Haitang Hu, Romain Huet, Jacob Huh, Saachi Jain, Zach Johnson, Chris Koch, Irina Kofman, Dominik Kundel, Jason Kwon, Volodymyr Kyrylov, Elaine Ya Le, Guillaume Leclerc, James Park Lennon, Scott Lessans, Mario Lezcano-Casado, Yuanzhi Li, Zhuohan Li, Ji Lin, Jordan Liss, Lily (Xiaoxuan) Liu, Jiancheng Liu, Kevin Lu, Chris Lu, Zoran Martinovic, Lindsay McCallum, Josh McGrath, Scott McKinney, Aidan McLaughlin, Song Mei, Steve Mostovoy, Tong Mu, Gideon Myles, Alexander Neitz, Alex Nichol, Jakub Pachocki, Alex Paino, Dana Palmie, Ashley Pantuliano, Giambattista Parascandolo, Jongsoo Park, Leher Pathak, Carolina Paz, Ludovic Peran, Dmitry Pimenov, Michelle Pokrass, Elizabeth Proehl, Huida Qiu, Gaby Raila, Filippo Raso, Hongyu Ren, Kimmy Richardson, David Robinson, Bob Rotsted, Hadi Salman, Suvansh Sanjeev, Max Schwarzer, D. Sculley, Harshit Sikchi, Kendal Simon, Karan Singhal, Yang Song, Dane Stuckey, Zhiqing Sun, Philippe Tillet, Sam Toizer, Foivos Tsimpourlas, Nikhil Vyas, Eric Wallace, Xin Wang, Miles Wang, Olivia Watkins, Kevin Weil, Amy Wendling, Kevin Whinnery, Cedric Whitney, Hannah Wong, Lin Yang, Yu Yang, Michihiro Yasunaga, Kristen Ying, Wojciech Zaremba, Wenting Zhan, Cyril Zhang, Brian Zhang, Eddie Zhang, Shengjia Zhao.
|
||||
1. **[GPT-2](https://huggingface.co/docs/transformers/model_doc/gpt2)** (from OpenAI) released with the paper [Language Models are Unsupervised Multitask Learners](https://blog.openai.com/better-language-models/) by Alec Radford*, Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever**.
|
||||
1. **[GPT-J](https://huggingface.co/docs/transformers/model_doc/gptj)** (from EleutherAI) released in the repository [kingoflolz/mesh-transformer-jax](https://github.com/kingoflolz/mesh-transformer-jax/) by Ben Wang and Aran Komatsuzaki.
|
||||
1. **[GPTBigCode](https://huggingface.co/docs/transformers/model_doc/gpt_bigcode)** (from BigCode) released with the paper [SantaCoder: don't reach for the stars!](https://huggingface.co/papers/2301.03988) by Loubna Ben Allal, Raymond Li, Denis Kocetkov, Chenghao Mou, Christopher Akiki, Carlos Munoz Ferrandis, Niklas Muennighoff, Mayank Mishra, Alex Gu, Manan Dey, Logesh Kumar Umapathi, Carolyn Jane Anderson, Yangtian Zi, Joel Lamy Poirier, Hailey Schoelkopf, Sergey Troshin, Dmitry Abulkhanov, Manuel Romero, Michael Lappert, Francesco De Toni, Bernardo García del Río, Qian Liu, Shamik Bose, Urvashi Bhattacharyya, Terry Yue Zhuo, Ian Yu, Paulo Villegas, Marco Zocca, Sourab Mangrulkar, David Lansky, Huu Nguyen, Danish Contractor, Luis Villa, Jia Li, Dzmitry Bahdanau, Yacine Jernite, Sean Hughes, Daniel Fried, Arjun Guha, Harm de Vries, Leandro von Werra.
|
||||
1. **[Granite](https://huggingface.co/docs/transformers/main/model_doc/granite)** (from IBM) released with the paper [Power Scheduler: A Batch Size and Token Number Agnostic Learning Rate Scheduler](https://huggingface.co/papers/2408.13359) by Yikang Shen, Matthew Stallone, Mayank Mishra, Gaoyuan Zhang, Shawn Tan, Aditya Prasad, Adriana Meza Soria, David D. Cox, Rameswar Panda.
|
||||
1. **[GraniteMoeHybrid](https://huggingface.co/docs/transformers/main/model_doc/granitemoehybrid)** (from IBM) released with the blog post [IBM Granite 4.0: hyper-efficient, high performance hybrid models for enterprise](https://www.ibm.com/new/announcements/ibm-granite-4-0-hyper-efficient-high-performance-hybrid-models) by the IBM Granite team.
|
||||
1. **[Granite Speech](https://huggingface.co/docs/transformers/main/model_doc/granite_speech)** (from IBM) released with the paper [Granite-speech: open-source speech-aware LLMs with strong English ASR capabilities](https://huggingface.co/papers/2505.08699) by the IBM Granite team.
|
||||
1. **[Grounding DINO](https://huggingface.co/docs/transformers/model_doc/grounding-dino)** (from IDEA-Research) released with the paper [Grounding DINO: Marrying DINO with Grounded Pre-Training for Open-Set Object Detection](https://huggingface.co/papers/2303.05499) by Shilong Liu, Zhaoyang Zeng, Tianhe Ren, Feng Li, Hao Zhang, Jie Yang, Qing Jiang, Chunyuan Li, Jianwei Yang, Hang Su, Jun Zhu, Lei Zhang.
|
||||
1. **[GroupViT](https://huggingface.co/docs/transformers/model_doc/groupvit)** (from UCSD, NVIDIA) released with the paper [GroupViT: Semantic Segmentation Emerges from Text Supervision](https://huggingface.co/papers/2202.11094) by Jiarui Xu, Shalini De Mello, Sifei Liu, Wonmin Byeon, Thomas Breuel, Jan Kautz, Xiaolong Wang.
|
||||
1. **[Helium](https://huggingface.co/docs/transformers/main/model_doc/helium)** (from the Kyutai Team) released with the blog post [Announcing Helium-1 Preview](https://kyutai.org/2025/01/13/helium.html) by the Kyutai Team.
|
||||
1. **[HerBERT](https://huggingface.co/docs/transformers/model_doc/herbert)** (from Allegro.pl, AGH University of Science and Technology) released with the paper [KLEJ: Comprehensive Benchmark for Polish Language Understanding](https://www.aclweb.org/anthology/2020.acl-main.111.pdf) by Piotr Rybak, Robert Mroczkowski, Janusz Tracz, Ireneusz Gawlik.
|
||||
1. **[HRM-Text](https://huggingface.co/docs/transformers/model_doc/hrm_text)** (from Sapient Intelligence) released with the paper [Hierarchical Reasoning Model](https://huggingface.co/papers/2506.21734) by Guan Wang, Jin Li, Yuhao Sun, Xing Chen, Changling Liu, Yue Wu, Meng Lu, Sen Song, Yasin Abbasi Yadkori.
|
||||
1. **[Hiera](https://huggingface.co/docs/transformers/model_doc/hiera)** (from Meta) released with the paper [Hiera: A Hierarchical Vision Transformer without the Bells-and-Whistles](https://huggingface.co/papers/2306.00989) by Chaitanya Ryali, Yuan-Ting Hu, Daniel Bolya, Chen Wei, Haoqi Fan, Po-Yao Huang, Vaibhav Aggarwal, Arkabandhu Chowdhury, Omid Poursaeed, Judy Hoffman, Jitendra Malik, Yanghao Li, Christoph Feichtenhofer.
|
||||
1. **[Hubert](https://huggingface.co/docs/transformers/model_doc/hubert)** (from Facebook) released with the paper [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units](https://huggingface.co/papers/2106.07447) by Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia, Ruslan Salakhutdinov, Abdelrahman Mohamed.
|
||||
1. **[HunYuanDenseV1](https://huggingface.co/docs/transformers/model_doc/hunyuan_v1_dense)** (from Tencent) released with the paper [Hunyuan-Large: An Open-Source MoE Model with 52 Billion Activated Parameters by Tencent](https://huggingface.co/papers/2411.02265) by the Tencent team.
|
||||
1. **[I-JEPA](https://huggingface.co/docs/transformers/model_doc/ijepa)** (from Meta) released with the paper [Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture](https://huggingface.co/papers/2301.08243) by Mahmoud Assran, Quentin Duval, Ishan Misra, Piotr Bojanowski, Pascal Vincent, Michael Rabbat, Yann LeCun, Nicolas Ballas.
|
||||
1. **[Idefics3](https://huggingface.co/docs/transformers/model_doc/idefics3)** (from Hugging Face) released with the paper [Building and better understanding vision-language models: insights and future directions](https://huggingface.co/papers/2408.12637) by Hugo Laurençon, Andrés Marafioti, Victor Sanh, Léo Tronchon.
|
||||
1. **JAIS** (from Core42) released with the paper [Jais and Jais-chat: Arabic-Centric Foundation and Instruction-Tuned Open Generative Large Language Models](https://huggingface.co/papers/2308.16149) by Neha Sengupta, Sunil Kumar Sahu, Bokang Jia, Satheesh Katipomu, Haonan Li, Fajri Koto, William Marshall, Gurpreet Gosal, Cynthia Liu, Zhiming Chen, Osama Mohammed Afzal, Samta Kamboj, Onkar Pandit, Rahul Pal, Lalit Pradhan, Zain Muhammad Mujahid, Massa Baali, Xudong Han, Sondos Mahmoud Bsharat, Alham Fikri Aji, Zhiqiang Shen, Zhengzhong Liu, Natalia Vassilieva, Joel Hestness, Andy Hock, Andrew Feldman, Jonathan Lee, Andrew Jackson, Hector Xuguang Ren, Preslav Nakov, Timothy Baldwin, Eric Xing.
|
||||
1. **Janus** (from DeepSeek) released with the paper [Janus: Decoupling Visual Encoding for Unified Multimodal Understanding and Generation](https://huggingface.co/papers/2410.13848) Chengyue Wu, Xiaokang Chen, Zhiyu Wu, Yiyang Ma, Xingchao Liu, Zizheng Pan, Wen Liu, Zhenda Xie, Xingkai Yu, Chong Ruan, Ping Luo.
|
||||
1. **JinaCLIP** (from Jina AI) released with the paper [Jina CLIP: Your CLIP Model Is Also Your Text Retriever](https://huggingface.co/papers/2405.20204) by Andreas Koukounas, Georgios Mastrapas, Michael Günther, Bo Wang, Scott Martens, Isabelle Mohr, Saba Sturua, Mohammad Kalim Akram, Joan Fontanals Martínez, Saahil Ognawala, Susana Guzman, Maximilian Werk, Nan Wang, Han Xiao.
|
||||
1. **LiteWhisper** (from University of Washington, Kotoba Technologies) released with the paper [LiteASR: Efficient Automatic Speech Recognition with Low-Rank Approximation](https://huggingface.co/papers/2502.20583) by Keisuke Kamahori, Jungo Kasai, Noriyuki Kojima, Baris Kasikci.
|
||||
1. **[LongT5](https://huggingface.co/docs/transformers/model_doc/longt5)** (from Google AI) released with the paper [LongT5: Efficient Text-To-Text Transformer for Long Sequences](https://huggingface.co/papers/2112.07916) by Mandy Guo, Joshua Ainslie, David Uthus, Santiago Ontanon, Jianmo Ni, Yun-Hsuan Sung, Yinfei Yang.
|
||||
1. **[LFM2](https://huggingface.co/docs/transformers/model_doc/lfm2)** (from Liquid AI) released with the blog post [Introducing LFM2: The Fastest On-Device Foundation Models on the Market](https://www.liquid.ai/blog/liquid-foundation-models-v2-our-second-series-of-generative-ai-models) by the Liquid AI Team.
|
||||
1. **[Lfm2Moe](https://huggingface.co/docs/transformers/model_doc/lfm2_moe)** (from Liquid AI) released with the blog post [LFM2-8B-A1B: An Efficient On-device Mixture-of-Experts](https://www.liquid.ai/blog/lfm2-8b-a1b-an-efficient-on-device-mixture-of-experts) by the Liquid AI Team.
|
||||
1. **[LFM2-VL](https://huggingface.co/docs/transformers/model_doc/lfm2_vl)** (from Liquid AI) released with the blog post [LFM2-VL: Efficient Vision-Language Models](https://www.liquid.ai/blog/lfm2-vl-efficient-vision-language-models) by the Liquid AI Team.
|
||||
1. **[LightOnOcr](https://huggingface.co/docs/transformers/model_doc/lighton_ocr)** (from LightOn) released with the blog post [LightOnOCR-1B: The Case for End-to-End and Efficient Domain-Specific Vision-Language Models for OCR](https://huggingface.co/blog/lightonai/lightonocr) by the LightOn Team.
|
||||
1. **[LLaMA](https://huggingface.co/docs/transformers/model_doc/llama)** (from The FAIR team of Meta AI) released with the paper [LLaMA: Open and Efficient Foundation Language Models](https://huggingface.co/papers/2302.13971) by Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, Guillaume Lample.
|
||||
1. **[Llama2](https://huggingface.co/docs/transformers/model_doc/llama2)** (from The FAIR team of Meta AI) released with the paper [Llama2: Open Foundation and Fine-Tuned Chat Models](https://huggingface.co/papers/2307.09288) by Hugo Touvron, Louis Martin, Kevin Stone, Peter Albert, Amjad Almahairi, Yasmine Babaei, Nikolay Bashlykov, Soumya Batra, Prajjwal Bhargava, Shruti Bhosale, Dan Bikel, Lukas Blecher, Cristian Canton Ferrer, Moya Chen, Guillem Cucurull, David Esiobu, Jude Fernandes, Jeremy Fu, Wenyin Fu, Brian Fuller, Cynthia Gao, Vedanuj Goswami, Naman Goyal, Anthony Hartshorn, Saghar Hosseini, Rui Hou, Hakan Inan, Marcin Kardas, Viktor Kerkez Madian Khabsa, Isabel Kloumann, Artem Korenev, Punit Singh Koura, Marie-Anne Lachaux, Thibaut Lavril, Jenya Lee, Diana Liskovich, Yinghai Lu, Yuning Mao, Xavier Martinet, Todor Mihaylov, Pushka rMishra, Igor Molybog, Yixin Nie, Andrew Poulton, Jeremy Reizenstein, Rashi Rungta, Kalyan Saladi, Alan Schelten, Ruan Silva, Eric Michael Smith, Ranjan Subramanian, Xiaoqing EllenTan, Binh Tang, Ross Taylor, Adina Williams, Jian Xiang Kuan, Puxin Xu, Zheng Yan, Iliyan Zarov, Yuchen Zhang, Angela Fan, Melanie Kambadur, Sharan Narang, Aurelien Rodriguez, Robert Stojnic, Sergey Edunov, Thomas Scialom.
|
||||
1. **[Llama3](https://huggingface.co/docs/transformers/model_doc/llama3)** (from The FAIR team of Meta AI) released with the paper [The Llama 3 Herd of Models](https://huggingface.co/papers/2407.21783) by the Llama Team at Meta.
|
||||
1. **[Llama4](https://huggingface.co/docs/transformers/model_doc/llama4)** (from The FAIR team of Meta AI) released with the blog post [The Llama 4 herd: The beginning of a new era of natively multimodal AI innovation](https://ai.meta.com/blog/llama-4-multimodal-intelligence/) by the Llama Team at Meta.
|
||||
1. **[LLaVa](https://huggingface.co/docs/transformers/model_doc/llava)** (from Microsoft Research & University of Wisconsin-Madison) released with the paper [Visual Instruction Tuning](https://huggingface.co/papers/2304.08485) by Haotian Liu, Chunyuan Li, Yuheng Li and Yong Jae Lee.
|
||||
1. **[LLaVA-OneVision](https://huggingface.co/docs/transformers/model_doc/llava_onevision)** (from ByteDance & NTU & CUHK & HKUST) released with the paper [LLaVA-OneVision: Easy Visual Task Transfer](https://huggingface.co/papers/2408.03326) by Bo Li, Yuanhan Zhang, Dong Guo, Renrui Zhang, Feng Li, Hao Zhang, Kaichen Zhang, Yanwei Li, Ziwei Liu, Chunyuan Li
|
||||
1. **[M2M100](https://huggingface.co/docs/transformers/model_doc/m2m_100)** (from Facebook) released with the paper [Beyond English-Centric Multilingual Machine Translation](https://huggingface.co/papers/2010.11125) by Angela Fan, Shruti Bhosale, Holger Schwenk, Zhiyi Ma, Ahmed El-Kishky, Siddharth Goyal, Mandeep Baines, Onur Celebi, Guillaume Wenzek, Vishrav Chaudhary, Naman Goyal, Tom Birch, Vitaliy Liptchinsky, Sergey Edunov, Edouard Grave, Michael Auli, Armand Joulin.
|
||||
1. **[MarianMT](https://huggingface.co/docs/transformers/model_doc/marian)** Machine translation models trained using [OPUS](http://opus.nlpl.eu/) data by Jörg Tiedemann. The [Marian Framework](https://marian-nmt.github.io/) is being developed by the Microsoft Translator Team.
|
||||
1. **[MaskFormer](https://huggingface.co/docs/transformers/model_doc/maskformer)** (from Meta and UIUC) released with the paper [Per-Pixel Classification is Not All You Need for Semantic Segmentation](https://huggingface.co/papers/2107.06278) by Bowen Cheng, Alexander G. Schwing, Alexander Kirillov.
|
||||
1. **[mBART](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Denoising Pre-training for Neural Machine Translation](https://huggingface.co/papers/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
|
||||
1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://huggingface.co/papers/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan.
|
||||
1. **Metric3D** released with the paper [Metric3D: Towards Zero-shot Metric 3D Prediction from A Single Image](https://huggingface.co/papers/2307.10984) by Wei Yin, Chi Zhang, Hao Chen, Zhipeng Cai, Gang Yu, Kaixuan Wang, Xiaozhi Chen, Chunhua Shen.
|
||||
1. **Metric3Dv2** released with the paper [Metric3Dv2: A Versatile Monocular Geometric Foundation Model for Zero-shot Metric Depth and Surface Normal Estimation](https://huggingface.co/papers/2404.15506) by Mu Hu, Wei Yin, Chi Zhang, Zhipeng Cai, Xiaoxiao Long, Kaixuan Wang, Hao Chen, Gang Yu, Chunhua Shen, Shaojie Shen.
|
||||
1. **[MusicGen](https://huggingface.co/docs/transformers/model_doc/musicgen)** (from Meta) released with the paper [Simple and Controllable Music Generation](https://huggingface.co/papers/2306.05284) by Jade Copet, Felix Kreuk, Itai Gat, Tal Remez, David Kant, Gabriel Synnaeve, Yossi Adi and Alexandre Défossez.
|
||||
1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://huggingface.co/papers/2209.03592) by Peng Wang, Cheng Da, and Cong Yao.
|
||||
1. **[Mimi](https://huggingface.co/docs/transformers/model_doc/mimi)** (from Kyutai) released with the paper [Moshi: a speech-text foundation model for real-time dialogue](https://huggingface.co/papers/2410.00037) by Alexandre Défossez, Laurent Mazaré, Manu Orsini, Amélie Royer, Patrick Pérez, Hervé Jégou, Edouard Grave and Neil Zeghidour.
|
||||
1. **[Ministral](https://huggingface.co/docs/transformers/model_doc/ministral)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team.
|
||||
1. **[Ministral3](https://huggingface.co/docs/transformers/model_doc/ministral3)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team.
|
||||
1. **[Mistral](https://huggingface.co/docs/transformers/model_doc/mistral)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team: Albert Jiang, Alexandre Sablayrolles, Arthur Mensch, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Florian Bressand, Gianna Lengyel, Guillaume Lample, Lélio Renard Lavaud, Lucile Saulnier, Marie-Anne Lachaux, Pierre Stock, Teven Le Scao, Thibaut Lavril, Thomas Wang, Timothée Lacroix, William El Sayed.
|
||||
1. **[Mistral3](https://huggingface.co/docs/transformers/model_doc/mistral3)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team.
|
||||
1. **[Mistral4](https://huggingface.co/docs/transformers/model_doc/mistral4)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team.
|
||||
1. **[MMS](https://huggingface.co/docs/transformers/model_doc/mms)** (from Facebook) released with the paper [Scaling Speech Technology to 1,000+ Languages](https://huggingface.co/papers/2305.13516) by Vineel Pratap, Andros Tjandra, Bowen Shi, Paden Tomasello, Arun Babu, Sayani Kundu, Ali Elkahky, Zhaoheng Ni, Apoorv Vyas, Maryam Fazel-Zarandi, Alexei Baevski, Yossi Adi, Xiaohui Zhang, Wei-Ning Hsu, Alexis Conneau, Michael Auli.
|
||||
1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (from CMU/Google Brain) released with the paper [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://huggingface.co/papers/2004.02984) by Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou.
|
||||
1. **MobileCLIP** (from Apple) released with the paper [MobileCLIP: Fast Image-Text Models through Multi-Modal Reinforced Training](https://huggingface.co/papers/2311.17049) by Pavan Kumar Anasosalu Vasu, Hadi Pouransari, Fartash Faghri, Raviteja Vemulapalli, Oncel Tuzel.
|
||||
1. **MobileLLM** (from Meta) released with the paper [MobileLLM: Optimizing Sub-billion Parameter Language Models for On-Device Use Cases](https://huggingface.co/papers/2402.14905) by Zechun Liu, Changsheng Zhao, Forrest Iandola, Chen Lai, Yuandong Tian, Igor Fedorov, Yunyang Xiong, Ernie Chang, Yangyang Shi, Raghuraman Krishnamoorthi, Liangzhen Lai, Vikas Chandra.
|
||||
1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://huggingface.co/papers/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam.
|
||||
1. **[MobileNetV2](https://huggingface.co/docs/transformers/model_doc/mobilenet_v2)** (from Google Inc.) released with the paper [MobileNetV2: Inverted Residuals and Linear Bottlenecks](https://huggingface.co/papers/1801.04381) by Mark Sandler, Andrew Howard, Menglong Zhu, Andrey Zhmoginov, Liang-Chieh Chen.
|
||||
1. **MobileNetV3** (from Google Inc.) released with the paper [Searching for MobileNetV3](https://huggingface.co/papers/1905.02244) by Andrew Howard, Mark Sandler, Grace Chu, Liang-Chieh Chen, Bo Chen, Mingxing Tan, Weijun Wang, Yukun Zhu, Ruoming Pang, Vijay Vasudevan, Quoc V. Le, Hartwig Adam.
|
||||
1. **MobileNetV4** (from Google Inc.) released with the paper [MobileNetV4 - Universal Models for the Mobile Ecosystem](https://huggingface.co/papers/2404.10518) by Danfeng Qin, Chas Leichner, Manolis Delakis, Marco Fornoni, Shixin Luo, Fan Yang, Weijun Wang, Colby Banbury, Chengxi Ye, Berkin Akin, Vaibhav Aggarwal, Tenghui Zhu, Daniele Moro, Andrew Howard.
|
||||
1. **[MobileViT](https://huggingface.co/docs/transformers/model_doc/mobilevit)** (from Apple) released with the paper [MobileViT: Light-weight, General-purpose, and Mobile-friendly Vision Transformer](https://huggingface.co/papers/2110.02178) by Sachin Mehta and Mohammad Rastegari.
|
||||
1. **[MobileViTV2](https://huggingface.co/docs/transformers/model_doc/mobilevitv2)** (from Apple) released with the paper [Separable Self-attention for Mobile Vision Transformers](https://huggingface.co/papers/2206.02680) by Sachin Mehta and Mohammad Rastegari.
|
||||
1. **[ModernBERT](https://huggingface.co/docs/transformers/model_doc/modernbert)** (from Answer.AI and LightOn) released with the paper [Smarter, Better, Faster, Longer: A Modern Bidirectional Encoder for Fast, Memory Efficient, and Long Context Finetuning and Inference](https://huggingface.co/papers/2412.13663) by Benjamin Warner, Antoine Chaffin, Benjamin Clavié, Orion Weller, Oskar Hallström, Said Taghadouini, Alexis Gallagher, Raja Biswas, Faisal Ladhak, Tom Aarsen, Nathan Cooper, Griffin Adams, Jeremy Howard, Iacopo Poli.
|
||||
1. **[ModernBERT Decoder](https://huggingface.co/docs/transformers/model_doc/modernbert-decoder)** (from Johns Hopkins University and LightOn) released with the paper [Seq vs Seq: An Open Suite of Paired Encoders and Decoders](https://huggingface.co/papers/2507.11412) by Orion Weller, Kathryn Ricci, Marc Marone, Antoine Chaffin, Dawn Lawrie, Benjamin Van Durme.
|
||||
1. **Moondream1** released in the repository [moondream](https://github.com/vikhyat/moondream) by vikhyat.
|
||||
1. **[Moonshine](https://huggingface.co/docs/transformers/model_doc/moonshine)** (from Useful Sensors) released with the paper [Moonshine: Speech Recognition for Live Transcription and Voice Commands](https://huggingface.co/papers/2410.15608) by Nat Jeffries, Evan King, Manjunath Kudlur, Guy Nicholson, James Wang, Pete Warden.
|
||||
1. **[MPNet](https://huggingface.co/docs/transformers/model_doc/mpnet)** (from Microsoft Research) released with the paper [MPNet: Masked and Permuted Pre-training for Language Understanding](https://huggingface.co/papers/2004.09297) by Kaitao Song, Xu Tan, Tao Qin, Jianfeng Lu, Tie-Yan Liu.
|
||||
1. **[MPT](https://huggingface.co/docs/transformers/model_doc/mpt)** (from MosaicML) released with the repository [llm-foundry](https://github.com/mosaicml/llm-foundry/) by the MosaicML NLP Team.
|
||||
1. **[MT5](https://huggingface.co/docs/transformers/model_doc/mt5)** (from Google AI) released with the paper [mT5: A massively multilingual pre-trained text-to-text transformer](https://huggingface.co/papers/2010.11934) by Linting Xue, Noah Constant, Adam Roberts, Mihir Kale, Rami Al-Rfou, Aditya Siddhant, Aditya Barua, Colin Raffel.
|
||||
1. **[NanoChat](https://huggingface.co/docs/transformers/model_doc/nanochat)** released with the repository [nanochat: The best ChatGPT that $100 can buy](https://github.com/karpathy/nanochat) by Andrej Karpathy.
|
||||
1. **[Nemotron](https://huggingface.co/docs/transformers/model_doc/nemotron)** (from NVIDIA) released with the blog post [NVIDIA AI Foundation Models: Build Custom Enterprise Chatbots and Co-Pilots with Production-Ready LLMs](https://developer.nvidia.com/blog/nvidia-ai-foundation-models-build-custom-enterprise-chatbots-and-co-pilots-with-production-ready-llms/) by the Nemotron team.
|
||||
1. **[Nemotron-H](https://huggingface.co/docs/transformers/model_doc/nemotron_h)** (from NVIDIA) released with the papers [Nemotron-H: A Family of Accurate and Efficient Hybrid Mamba-Transformer Models](https://huggingface.co/papers/2504.03624), [NVIDIA Nemotron 3: Efficient and Open Intelligence](https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-White-Paper.pdf), [Nemotron 3 Nano: Open, Efficient Mixture-of-Experts Hybrid Mamba-Transformer Model for Agentic Reasoning](https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Nano-Technical-Report.pdf), and [Nemotron 3 Super: Open, Efficient Mixture-of-Experts Hybrid Mamba-Transformer Model for Agentic Reasoning](https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Super-Technical-Report.pdf) by the Nemotron team.
|
||||
1. **NeoBERT** (from Chandar Research Lab) released with the paper [NeoBERT: A Next-Generation BERT](https://huggingface.co/papers/2502.19587) by Lola Le Breton, Quentin Fournier, Mariam El Mezouar, John X. Morris, Sarath Chandar.
|
||||
1. **[NLLB](https://huggingface.co/docs/transformers/model_doc/nllb)** (from Meta) released with the paper [No Language Left Behind: Scaling Human-Centered Machine Translation](https://huggingface.co/papers/2207.04672) by the NLLB team.
|
||||
1. **[Nougat](https://huggingface.co/docs/transformers/model_doc/nougat)** (from Meta AI) released with the paper [Nougat: Neural Optical Understanding for Academic Documents](https://huggingface.co/papers/2308.13418) by Lukas Blecher, Guillem Cucurull, Thomas Scialom, Robert Stojnic.
|
||||
1. **[OLMo](https://huggingface.co/docs/transformers/master/model_doc/olmo)** (from Ai2) released with the paper [OLMo: Accelerating the Science of Language Models](https://huggingface.co/papers/2402.00838) by Dirk Groeneveld, Iz Beltagy, Pete Walsh, Akshita Bhagia, Rodney Kinney, Oyvind Tafjord, Ananya Harsh Jha, Hamish Ivison, Ian Magnusson, Yizhong Wang, Shane Arora, David Atkinson, Russell Authur, Khyathi Raghavi Chandu, Arman Cohan, Jennifer Dumas, Yanai Elazar, Yuling Gu, Jack Hessel, Tushar Khot, William Merrill, Jacob Morrison, Niklas Muennighoff, Aakanksha Naik, Crystal Nam, Matthew E. Peters, Valentina Pyatkin, Abhilasha Ravichander, Dustin Schwenk, Saurabh Shah, Will Smith, Emma Strubell, Nishant Subramani, Mitchell Wortsman, Pradeep Dasigi, Nathan Lambert, Kyle Richardson, Luke Zettlemoyer, Jesse Dodge, Kyle Lo, Luca Soldaini, Noah A. Smith, Hannaneh Hajishirzi.
|
||||
1. **[OLMo2](https://huggingface.co/docs/transformers/master/model_doc/olmo2)** (from Ai2) released with the blog [OLMo 2: The best fully open language model to date](https://allenai.org/blog/olmo2) by the Ai2 OLMo team.
|
||||
1. **[OLMo3](https://huggingface.co/docs/transformers/master/model_doc/olmo3)** (from Ai2) released with the blog [Olmo 3: Charting a path through the model flow to lead open-source AI](https://allenai.org/blog/olmo3) by the Ai2 OLMo team.
|
||||
1. **[Olmo Hybrid](https://huggingface.co/docs/transformers/master/model_doc/olmo_hybrid)** (from Ai2) released with the blog [Introducing Olmo Hybrid: Combining transformers and linear RNNs for superior scaling](https://allenai.org/blog/olmohybrid) by the Ai2 OLMo team.
|
||||
1. **OpenELM** (from Apple) released with the paper [OpenELM: An Efficient Language Model Family with Open-source Training and Inference Framework](https://huggingface.co/papers/2404.14619) by Sachin Mehta, Mohammad Hossein Sekhavat, Qingqing Cao, Maxwell Horton, Yanzi Jin, Chenfan Sun, Iman Mirzadeh, Mahyar Najibi, Dmitry Belenko, Peter Zatloukal, Mohammad Rastegari.
|
||||
1. **[OPT](https://huggingface.co/docs/transformers/master/model_doc/opt)** (from Meta AI) released with the paper [OPT: Open Pre-trained Transformer Language Models](https://huggingface.co/papers/2205.01068) by Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen et al.
|
||||
1. **[OWL-ViT](https://huggingface.co/docs/transformers/model_doc/owlvit)** (from Google AI) released with the paper [Simple Open-Vocabulary Object Detection with Vision Transformers](https://huggingface.co/papers/2205.06230) by Matthias Minderer, Alexey Gritsenko, Austin Stone, Maxim Neumann, Dirk Weissenborn, Alexey Dosovitskiy, Aravindh Mahendran, Anurag Arnab, Mostafa Dehghani, Zhuoran Shen, Xiao Wang, Xiaohua Zhai, Thomas Kipf, and Neil Houlsby.
|
||||
1. **[OWLv2](https://huggingface.co/docs/transformers/model_doc/owlv2)** (from Google AI) released with the paper [Scaling Open-Vocabulary Object Detection](https://huggingface.co/papers/2306.09683) by Matthias Minderer, Alexey Gritsenko, Neil Houlsby.
|
||||
1. **[PaliGemma](https://huggingface.co/docs/transformers/main/model_doc/paligemma)** (from Google) released with the papers [PaliGemma: A versatile 3B VLM for transfer](https://huggingface.co/papers/2407.07726) and [PaliGemma 2: A Family of Versatile VLMs for Transfer](https://huggingface.co/papers/2412.03555) by the PaliGemma Google team.
|
||||
1. **[Parakeet](https://huggingface.co/docs/transformers/main/model_doc/parakeet)** (from NVIDIA) released with the blog post [Introducing the Parakeet ASR family](https://developer.nvidia.com/blog/pushing-the-boundaries-of-speech-recognition-with-nemo-parakeet-asr-models/) by the NVIDIA NeMo team.
|
||||
1. **[PatchTSMixer](https://huggingface.co/docs/transformers/main/model_doc/patchtsmixer)** (from IBM) released with the paper [TSMixer: Lightweight MLP-Mixer Model for Multivariate Time Series Forecasting](https://huggingface.co/papers/2306.09364) by Vijay Ekambaram, Arindam Jati, Nam Nguyen, Phanwadee Sinthong, Jayant Kalagnanam.
|
||||
1. **[PatchTST](https://huggingface.co/docs/transformers/main/model_doc/patchtst)** (from Princeton University, IBM) released with the paper [A Time Series is Worth 64 Words: Long-term Forecasting with Transformers](https://huggingface.co/papers/2211.14730) by Yuqi Nie, Nam H. Nguyen, Phanwadee Sinthong, Jayant Kalagnanam.
|
||||
1. **[Phi](https://huggingface.co/docs/transformers/main/model_doc/phi)** (from Microsoft) released with the papers - [Textbooks Are All You Need](https://huggingface.co/papers/2306.11644) by Suriya Gunasekar, Yi Zhang, Jyoti Aneja, Caio César Teodoro Mendes, Allie Del Giorno, Sivakanth Gopi, Mojan Javaheripi, Piero Kauffmann, Gustavo de Rosa, Olli Saarikivi, Adil Salim, Shital Shah, Harkirat Singh Behl, Xin Wang, Sébastien Bubeck, Ronen Eldan, Adam Tauman Kalai, Yin Tat Lee and Yuanzhi Li, [Textbooks Are All You Need II: phi-1.5 technical report](https://huggingface.co/papers/2309.05463) by Yuanzhi Li, Sébastien Bubeck, Ronen Eldan, Allie Del Giorno, Suriya Gunasekar and Yin Tat Lee.
|
||||
1. **[Phi3](https://huggingface.co/docs/transformers/main/model_doc/phi3)** (from Microsoft) released with the paper [Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone](https://huggingface.co/papers/2404.14219v2) by Marah Abdin, Sam Ade Jacobs, Ammar Ahmad Awan, Jyoti Aneja, Ahmed Awadallah, Hany Awadalla, Nguyen Bach, Amit Bahree, Arash Bakhtiari, Harkirat Behl, Alon Benhaim, Misha Bilenko, Johan Bjorck, Sébastien Bubeck, Martin Cai, Caio César Teodoro Mendes, Weizhu Chen, Vishrav Chaudhary, Parul Chopra, Allie Del Giorno, Gustavo de Rosa, Matthew Dixon, Ronen Eldan, Dan Iter, Amit Garg, Abhishek Goswami, Suriya Gunasekar, Emman Haider, Junheng Hao, Russell J. Hewett, Jamie Huynh, Mojan Javaheripi, Xin Jin, Piero Kauffmann, Nikos Karampatziakis, Dongwoo Kim, Mahoud Khademi, Lev Kurilenko, James R. Lee, Yin Tat Lee, Yuanzhi Li, Chen Liang, Weishung Liu, Eric Lin, Zeqi Lin, Piyush Madan, Arindam Mitra, Hardik Modi, Anh Nguyen, Brandon Norick, Barun Patra, Daniel Perez-Becker, Thomas Portet, Reid Pryzant, Heyang Qin, Marko Radmilac, Corby Rosset, Sambudha Roy, Olatunji Ruwase, Olli Saarikivi, Amin Saied, Adil Salim, Michael Santacroce, Shital Shah, Ning Shang, Hiteshi Sharma, Xia Song, Masahiro Tanaka, Xin Wang, Rachel Ward, Guanhua Wang, Philipp Witte, Michael Wyatt, Can Xu, Jiahang Xu, Sonali Yadav, Fan Yang, Ziyi Yang, Donghan Yu, Chengruidong Zhang, Cyril Zhang, Jianwen Zhang, Li Lyna Zhang, Yi Zhang, Yue Zhang, Yunan Zhang, Xiren Zhou.
|
||||
1. **Phi3V** (from Microsoft) released with the paper [Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone](https://huggingface.co/papers/2404.14219v4) by Marah Abdin, Jyoti Aneja, Hany Awadalla, Ahmed Awadallah, Ammar Ahmad Awan, Nguyen Bach, Amit Bahree, Arash Bakhtiari, Jianmin Bao, Harkirat Behl, Alon Benhaim, Misha Bilenko, Johan Bjorck, Sébastien Bubeck, Martin Cai, Qin Cai, Vishrav Chaudhary, Dong Chen, Dongdong Chen, Weizhu Chen, Yen-Chun Chen, Yi-Ling Chen, Hao Cheng, Parul Chopra, Xiyang Dai, Matthew Dixon, Ronen Eldan, Victor Fragoso, Jianfeng Gao, Mei Gao, Min Gao, Amit Garg, Allie Del Giorno, Abhishek Goswami, Suriya Gunasekar, Emman Haider, Junheng Hao, Russell J. Hewett, Wenxiang Hu, Jamie Huynh, Dan Iter, Sam Ade Jacobs, Mojan Javaheripi, Xin Jin, Nikos Karampatziakis, Piero Kauffmann, Mahoud Khademi, Dongwoo Kim, Young Jin Kim, Lev Kurilenko, James R. Lee, Yin Tat Lee, Yuanzhi Li, Yunsheng Li, Chen Liang, Lars Liden, Xihui Lin, Zeqi Lin, Ce Liu, Liyuan Liu, Mengchen Liu, Weishung Liu, Xiaodong Liu, Chong Luo, Piyush Madan, Ali Mahmoudzadeh, David Majercak, Matt Mazzola, Caio César Teodoro Mendes, Arindam Mitra, Hardik Modi, Anh Nguyen, Brandon Norick, Barun Patra, Daniel Perez-Becker, Thomas Portet, Reid Pryzant, Heyang Qin, Marko Radmilac, Liliang Ren, Gustavo de Rosa, Corby Rosset, Sambudha Roy, Olatunji Ruwase, Olli Saarikivi, Amin Saied, Adil Salim, Michael Santacroce, Shital Shah, Ning Shang, Hiteshi Sharma, Yelong Shen, Swadheen Shukla, Xia Song, Masahiro Tanaka, Andrea Tupini, Praneetha Vaddamanu, Chunyu Wang, Guanhua Wang, Lijuan Wang , Shuohang Wang, Xin Wang, Yu Wang, Rachel Ward, Wen Wen, Philipp Witte, Haiping Wu, Xiaoxia Wu, Michael Wyatt, Bin Xiao, Can Xu, Jiahang Xu, Weijian Xu, Jilong Xue, Sonali Yadav, Fan Yang, Jianwei Yang, Yifan Yang, Ziyi Yang, Donghan Yu, Lu Yuan, Chenruidong Zhang, Cyril Zhang, Jianwen Zhang, Li Lyna Zhang, Yi Zhang, Yue Zhang, Yunan Zhang, Xiren Zhou.
|
||||
1. **[PVT](https://huggingface.co/docs/transformers/main/model_doc/pvt)** (from Nanjing University, The University of Hong Kong etc.) released with the paper [Pyramid Vision Transformer: A Versatile Backbone for Dense Prediction without Convolutions](https://huggingface.co/papers/2102.12122) by Wenhai Wang, Enze Xie, Xiang Li, Deng-Ping Fan, Kaitao Song, Ding Liang, Tong Lu, Ping Luo, Ling Shao.
|
||||
1. **PyAnnote** released in the repository [pyannote/pyannote-audio](https://github.com/pyannote/pyannote-audio) by Hervé Bredin.
|
||||
1. **[Qwen2](https://huggingface.co/docs/transformers/model_doc/qwen2)** (from the Qwen team, Alibaba Group) released with the papers [Qwen Technical Report](https://huggingface.co/papers/2309.16609) and [Qwen2 Technical Report](https://huggingface.co/papers/2407.10671) by Jinze Bai, Shuai Bai, Yunfei Chu, Zeyu Cui, Kai Dang, Xiaodong Deng, Yang Fan, Wenbin Ge, Yu Han, Fei Huang, Binyuan Hui, Luo Ji, Mei Li, Junyang Lin, Runji Lin, Dayiheng Liu, Gao Liu, Chengqiang Lu, Keming Lu, Jianxin Ma, Rui Men, Xingzhang Ren, Xuancheng Ren, Chuanqi Tan, Sinan Tan, Jianhong Tu, Peng Wang, Shijie Wang, Wei Wang, Shengguang Wu, Benfeng Xu, Jin Xu, An Yang, Hao Yang, Jian Yang, Shusheng Yang, Yang Yao, Bowen Yu, Hongyi Yuan, Zheng Yuan, Jianwei Zhang, Xingxuan Zhang, Yichang Zhang, Zhenru Zhang, Chang Zhou, Jingren Zhou, Xiaohuan Zhou and Tianhang Zhu.
|
||||
1. **[Qwen2 MoE](https://huggingface.co/docs/transformers/model_doc/qwen2_moe)** (from the Qwen team, Alibaba Group) released with the paper [Qwen2 Technical Report](https://huggingface.co/papers/2407.10671) by Jinze Bai, Shuai Bai, Yunfei Chu, Zeyu Cui, Kai Dang, Xiaodong Deng, Yang Fan, Wenbin Ge, Yu Han, Fei Huang, Binyuan Hui, Luo Ji, Mei Li, Junyang Lin, Runji Lin, Dayiheng Liu, Gao Liu, Chengqiang Lu, Keming Lu, Jianxin Ma, Rui Men, Xingzhang Ren, Xuancheng Ren, Chuanqi Tan, Sinan Tan, Jianhong Tu, Peng Wang, Shijie Wang, Wei Wang, Shengguang Wu, Benfeng Xu, Jin Xu, An Yang, Hao Yang, Jian Yang, Shusheng Yang, Yang Yao, Bowen Yu, Hongyi Yuan, Zheng Yuan, Jianwei Zhang, Xingxuan Zhang, Yichang Zhang, Zhenru Zhang, Chang Zhou, Jingren Zhou, Xiaohuan Zhou and Tianhang Zhu.
|
||||
1. **[Qwen2-VL](https://huggingface.co/docs/transformers/model_doc/qwen2_vl)** (from the Qwen team, Alibaba Group) released with the papers [Qwen-VL: A Versatile Vision-Language Model for Understanding, Localization, Text Reading, and Beyond](https://huggingface.co/papers/2308.12966) and [Qwen2-VL: Enhancing Vision-Language Model's Perception of the World at Any Resolution](https://huggingface.co/papers/2409.12191) by Peng Wang, Shuai Bai, Sinan Tan, Shijie Wang, Zhihao Fan, Jinze Bai, Keqin Chen, Xuejing Liu, Jialin Wang, Wenbin Ge, Yang Fan, Kai Dang, Mengfei Du, Xuancheng Ren, Rui Men, Dayiheng Liu, Chang Zhou, Jingren Zhou, Junyang Lin.
|
||||
1. **[Qwen2.5-VL](https://huggingface.co/docs/transformers/model_doc/qwen2_5_vl)** (from the Qwen team, Alibaba Group) released with the paper [Qwen2.5-VL Technical Report](https://huggingface.co/papers/2502.13923) by Shuai Bai, Keqin Chen, Xuejing Liu, Jialin Wang, Wenbin Ge, Sibo Song, Kai Dang, Peng Wang, Shijie Wang, Jun Tang, Humen Zhong, Yuanzhi Zhu, Mingkun Yang, Zhaohai Li, Jianqiang Wan, Pengfei Wang, Wei Ding, Zheren Fu, Yiheng Xu, Jiabo Ye, Xi Zhang, Tianbao Xie, Zesen Cheng, Hang Zhang, Zhibo Yang, Haiyang Xu, Junyang Lin.
|
||||
1. **[Qwen3](https://huggingface.co/docs/transformers/en/model_doc/qwen3)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3: Think Deeper, Act Faster](https://qwen.ai/blog?id=qwen3) by the Qwen team.
|
||||
1. **[Qwen3 MoE](https://huggingface.co/docs/transformers/en/model_doc/qwen3_moe)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3: Think Deeper, Act Faster](https://qwen.ai/blog?id=qwen3) by the Qwen team.
|
||||
1. **[Qwen3 Next](https://huggingface.co/docs/transformers/en/model_doc/qwen3_next)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3-Next: Towards Ultimate Training & Inference Efficiency](https://qwen.ai/blog?id=qwen3-next) by the Qwen team.
|
||||
1. **[Qwen3-VL](https://huggingface.co/docs/transformers/en/model_doc/qwen3_vl)** (from the Qwen team, Alibaba Group) released with the paper [Qwen3-VL Technical Report](https://huggingface.co/papers/2511.21631) by Shuai Bai, Yuxuan Cai, Ruizhe Chen, Keqin Chen, Xionghui Chen, Zesen Cheng, Lianghao Deng, Wei Ding, Chang Gao, Chunjiang Ge, Wenbin Ge, Zhifang Guo, Qidong Huang, Jie Huang, Fei Huang, Binyuan Hui, Shutong Jiang, Zhaohai Li, Mingsheng Li, Mei Li, Kaixin Li, Zicheng Lin, Junyang Lin, Xuejing Liu, Jiawei Liu, Chenglong Liu, Yang Liu, Dayiheng Liu, Shixuan Liu, Dunjie Lu, Ruilin Luo, Chenxu Lv, Rui Men, Lingchen Meng, Xuancheng Ren, Xingzhang Ren, Sibo Song, Yuchong Sun, Jun Tang, Jianhong Tu, Jianqiang Wan, Peng Wang, Pengfei Wang, Qiuyue Wang, Yuxuan Wang, Tianbao Xie, Yiheng Xu, Haiyang Xu, Jin Xu, Zhibo Yang, Mingkun Yang, Jianxin Yang, An Yang, Bowen Yu, Fei Zhang, Hang Zhang, Xi Zhang, Bo Zheng, Humen Zhong, Jingren Zhou, Fan Zhou, Jing Zhou, Yuanzhi Zhu, Ke Zhu.
|
||||
1. **[Qwen3-VL MoE](https://huggingface.co/docs/transformers/en/model_doc/qwen3_vl_moe)** (from the Qwen team, Alibaba Group) released with the paper [Qwen3-VL Technical Report](https://huggingface.co/papers/2511.21631) by Shuai Bai, Yuxuan Cai, Ruizhe Chen, Keqin Chen, Xionghui Chen, Zesen Cheng, Lianghao Deng, Wei Ding, Chang Gao, Chunjiang Ge, Wenbin Ge, Zhifang Guo, Qidong Huang, Jie Huang, Fei Huang, Binyuan Hui, Shutong Jiang, Zhaohai Li, Mingsheng Li, Mei Li, Kaixin Li, Zicheng Lin, Junyang Lin, Xuejing Liu, Jiawei Liu, Chenglong Liu, Yang Liu, Dayiheng Liu, Shixuan Liu, Dunjie Lu, Ruilin Luo, Chenxu Lv, Rui Men, Lingchen Meng, Xuancheng Ren, Xingzhang Ren, Sibo Song, Yuchong Sun, Jun Tang, Jianhong Tu, Jianqiang Wan, Peng Wang, Pengfei Wang, Qiuyue Wang, Yuxuan Wang, Tianbao Xie, Yiheng Xu, Haiyang Xu, Jin Xu, Zhibo Yang, Mingkun Yang, Jianxin Yang, An Yang, Bowen Yu, Fei Zhang, Hang Zhang, Xi Zhang, Bo Zheng, Humen Zhong, Jingren Zhou, Fan Zhou, Jing Zhou, Yuanzhi Zhu, Ke Zhu.
|
||||
1. **[Qwen3.5](https://huggingface.co/docs/transformers/en/model_doc/qwen3_5)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3.5: Towards Native Multimodal Agents](https://qwen.ai/blog?id=qwen3.5) by the Qwen team.
|
||||
1. **[Qwen3.5 Moe](https://huggingface.co/docs/transformers/en/model_doc/qwen3_5_moe)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3.5: Towards Native Multimodal Agents](https://qwen.ai/blog?id=qwen3.5) by the Qwen team.
|
||||
1. **[ResNet](https://huggingface.co/docs/transformers/model_doc/resnet)** (from Microsoft Research) released with the paper [Deep Residual Learning for Image Recognition](https://huggingface.co/papers/1512.03385) by Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun.
|
||||
1. **[RF-DETR](https://huggingface.co/docs/transformers/model_doc/rf_detr)** (from Roboflow) released with the blog post [RF-DETR: A SOTA Real-Time Object Detection Model](https://blog.roboflow.com/rf-detr/) by Peter Robicheaux, James Gallagher, Joseph Nelson, Isaac Robinson.
|
||||
1. **[RoBERTa](https://huggingface.co/docs/transformers/model_doc/roberta)** (from Facebook), released together with the paper [RoBERTa: A Robustly Optimized BERT Pretraining Approach](https://huggingface.co/papers/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov.
|
||||
1. **[RoFormer](https://huggingface.co/docs/transformers/model_doc/roformer)** (from ZhuiyiTechnology), released together with the paper [RoFormer: Enhanced Transformer with Rotary Position Embedding](https://huggingface.co/papers/2104.09864) by Jianlin Su and Yu Lu and Shengfeng Pan and Bo Wen and Yunfeng Liu.
|
||||
1. **[RT-DETR](https://huggingface.co/docs/transformers/model_doc/rt_detr)** (from Baidu), released together with the paper [DETRs Beat YOLOs on Real-time Object Detection](https://huggingface.co/papers/2304.08069) by Yian Zhao, Wenyu Lv, Shangliang Xu, Jinman Wei, Guanzhong Wang, Qingqing Dang, Yi Liu, Jie Chen.
|
||||
1. **[RT-DETRv2](https://huggingface.co/docs/transformers/model_doc/rt_detr_v2)** (from Baidu), released together with the paper [RT-DETRv2: Improved Baseline with Bag-of-Freebies for Real-Time Detection Transformer](https://huggingface.co/papers/2407.17140) by Wenyu Lv, Yian Zhao, Qinyao Chang, Kui Huang, Guanzhong Wang, Yi Liu.
|
||||
1. **Sapiens** (from Meta AI) released with the paper [Sapiens: Foundation for Human Vision Models](https://huggingface.co/papers/2408.12569) by Rawal Khirodkar, Timur Bagautdinov, Julieta Martinez, Su Zhaoen, Austin James, Peter Selednik, Stuart Anderson, Shunsuke Saito.
|
||||
1. **[SegFormer](https://huggingface.co/docs/transformers/model_doc/segformer)** (from NVIDIA) released with the paper [SegFormer: Simple and Efficient Design for Semantic Segmentation with Transformers](https://huggingface.co/papers/2105.15203) by Enze Xie, Wenhai Wang, Zhiding Yu, Anima Anandkumar, Jose M. Alvarez, Ping Luo.
|
||||
1. **[Segment Anything](https://huggingface.co/docs/transformers/model_doc/sam)** (from Meta AI) released with the paper [Segment Anything](https://huggingface.co/papers/2304.02643v1.pdf) by Alexander Kirillov, Eric Mintun, Nikhila Ravi, Hanzi Mao, Chloe Rolland, Laura Gustafson, Tete Xiao, Spencer Whitehead, Alex Berg, Wan-Yen Lo, Piotr Dollar, Ross Girshick.
|
||||
1. **[Segment Anything 2](https://huggingface.co/docs/transformers/model_doc/sam2)** (from Meta AI) released with the paper [SAM 2: Segment Anything in Images and Videos](https://huggingface.co/papers/2408.00714) by Nikhila Ravi, Valentin Gabeur, Yuan-Ting Hu, Ronghang Hu, Chaitanya Ryali, Tengyu Ma, Haitham Khedr, Roman Rädle, Chloe Rolland, Laura Gustafson, Eric Mintun, Junting Pan, Kalyan Vasudev Alwala, Nicolas Carion, Chao-Yuan Wu, Ross Girshick, Piotr Dollár, Christoph Feichtenhofer.
|
||||
1. **[Segment Anything 3](https://huggingface.co/docs/transformers/model_doc/sam3)** (from Meta Superintelligence Labs) released with the paper [SAM 3: Segment Anything with Concepts](https://ai.meta.com/research/publications/sam-3-segment-anything-with-concepts/) by SAM 3D Team, Xingyu Chen, Fu-Jen Chu, Pierre Gleize, Kevin J Liang, Alexander Sax, Hao Tang, Weiyao Wang, Michelle Guo, Thibaut Hardin, Xiang Li, Aohan Lin, Jiawei Liu, Ziqi Ma, Anushka Sagar, Bowen Song, Xiaodong Wang, Jianing Yang, Bowen Zhang, Piotr Dollar, Georgia Gkioxari, Matt Feiszli, Jitendra Malik, Nicolas Carion, Laura Gustafson, Yuan-Ting Hu, Shoubhik Debnath, Ronghang Hu, Didac Suris Coll-Vinent, Chaitanya Ryali, Kalyan Vasudev Alwala, Haitham Khedr, Andrew Huang, Jie Lei, Tengyu Ma, Baishan Guo, Arpit Kalla, Markus Marks, Joseph Greer, Meng Wang, Peize Sun, Roman Rädle, Triantafyllos Afouras, Effrosyni Mavroudi, Katherine Xu, Tsung-Han Wu, Yu Zhou, Liliane Momeni, Rishi Hazra, Shuangrui Ding, Sagar Vaze, Francois Porcher, Feng Li, Siyuan Li, Aishwarya Kamath, Ho Kei Cheng, Piotr Dollar, Nikhila Ravi, Kate Saenko, Pengchuan Zhang, Christoph Feichtenhofer.
|
||||
1. **[SigLIP](https://huggingface.co/docs/transformers/main/model_doc/siglip)** (from Google AI) released with the paper [Sigmoid Loss for Language Image Pre-Training](https://huggingface.co/papers/2303.15343) by Xiaohua Zhai, Basil Mustafa, Alexander Kolesnikov, Lucas Beyer.
|
||||
1. **[SmolLM3](https://huggingface.co/docs/transformers/main/model_doc/smollm3) (from Hugging Face) released with the blog post [SmolLM3: smol, multilingual, long-context reasoner](https://huggingface.co/blog/smollm3) by the Hugging Face TB Research team.
|
||||
1. **[SmolVLM](https://huggingface.co/docs/transformers/main/model_doc/smolvlm) (from Hugging Face) released with the blog posts [SmolVLM - small yet mighty Vision Language Model](https://huggingface.co/blog/smolvlm) and [SmolVLM Grows Smaller – Introducing the 250M & 500M Models!](https://huggingface.co/blog/smolervlm) by the Hugging Face TB Research team.
|
||||
1. **[SolarOpen](https://huggingface.co/docs/transformers/main/model_doc/solar_open)** (from Upstage) released with the paper [Solar Open Technical Report](https://huggingface.co/papers/2601.07022) by Sungrae Park, Sanghoon Kim, Jungho Cho, Gyoungjin Gim, Dawoon Jung, Mikyoung Cha, Eunhae Choo, Taekgyu Hong, Minbyul Jeong, SeHwan Joo, Minsoo Khang, Eunwon Kim, Minjeong Kim, Sujeong Kim, Yunsu Kim, Hyeonju Lee, Seunghyun Lee, Sukyung Lee, Siyoung Park, Gyungin Shin, Inseo Song, Wonho Song, Seonghoon Yang, Seungyoun Yi, Sanghoon Yoon, Jeonghyun Ko, Seyoung Song, Keunwoo Choi, Hwalsuk Lee, Sunghun Kim, Du-Seong Chang, Kyunghyun Cho, Junsuk Choe, Hwaran Lee, Jae-Gil Lee, KyungTae Lim, Alice Oh.
|
||||
1. **SNAC** (from Papla Media, ETH Zurich) released with the paper [SNAC: Multi-Scale Neural Audio Codec](https://huggingface.co/papers/2410.14411) by Hubert Siuzdak, Florian Grötschla, Luca A. Lanzendörfer.
|
||||
1. **[SpeechT5](https://huggingface.co/docs/transformers/model_doc/speecht5)** (from Microsoft Research) released with the paper [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://huggingface.co/papers/2110.07205) by Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei.
|
||||
1. **[SqueezeBERT](https://huggingface.co/docs/transformers/model_doc/squeezebert)** (from Berkeley) released with the paper [SqueezeBERT: What can computer vision teach NLP about efficient neural networks?](https://huggingface.co/papers/2006.11316) by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W. Keutzer.
|
||||
1. **[StableLm](https://huggingface.co/docs/transformers/model_doc/stablelm)** (from Stability AI) released with the paper [StableLM 3B 4E1T (Technical Report)](https://stability.wandb.io/stability-llm/stable-lm/reports/StableLM-3B-4E1T--VmlldzoyMjU4?accessToken=u3zujipenkx5g7rtcj9qojjgxpconyjktjkli2po09nffrffdhhchq045vp0wyfo) by Jonathan Tow, Marco Bellagente, Dakota Mahan, Carlos Riquelme Ruiz, Duy Phung, Maksym Zhuravinskyi, Nathan Cooper, Nikhil Pinnaparaju, Reshinth Adithyan, and James Baicoianu.
|
||||
1. **[Starcoder2](https://huggingface.co/docs/transformers/main/model_doc/starcoder2)** (from BigCode team) released with the paper [StarCoder 2 and The Stack v2: The Next Generation](https://huggingface.co/papers/2402.19173) by Anton Lozhkov, Raymond Li, Loubna Ben Allal, Federico Cassano, Joel Lamy-Poirier, Nouamane Tazi, Ao Tang, Dmytro Pykhtar, Jiawei Liu, Yuxiang Wei, Tianyang Liu, Max Tian, Denis Kocetkov, Arthur Zucker, Younes Belkada, Zijian Wang, Qian Liu, Dmitry Abulkhanov, Indraneil Paul, Zhuang Li, Wen-Ding Li, Megan Risdal, Jia Li, Jian Zhu, Terry Yue Zhuo, Evgenii Zheltonozhskii, Nii Osae Osae Dade, Wenhao Yu, Lucas Krauß, Naman Jain, Yixuan Su, Xuanli He, Manan Dey, Edoardo Abati, Yekun Chai, Niklas Muennighoff, Xiangru Tang, Muhtasham Oblokulov, Christopher Akiki, Marc Marone, Chenghao Mou, Mayank Mishra, Alex Gu, Binyuan Hui, Tri Dao, Armel Zebaze, Olivier Dehaene, Nicolas Patry, Canwen Xu, Julian McAuley, Han Hu, Torsten Scholak, Sebastien Paquet, Jennifer Robinson, Carolyn Jane Anderson, Nicolas Chapados, Mostofa Patwary, Nima Tajbakhsh, Yacine Jernite, Carlos Muñoz Ferrandis, Lingming Zhang, Sean Hughes, Thomas Wolf, Arjun Guha, Leandro von Werra, and Harm de Vries.
|
||||
1. **StyleTTS 2** (from Columbia University) released with the paper [StyleTTS 2: Towards Human-Level Text-to-Speech through Style Diffusion and Adversarial Training with Large Speech Language Models](https://huggingface.co/papers/2306.07691) by Yinghao Aaron Li, Cong Han, Vinay S. Raghavan, Gavin Mischler, Nima Mesgarani.
|
||||
1. **Supertonic** (from Supertone) released with the paper [SupertonicTTS: Towards Highly Efficient and Streamlined Text-to-Speech System](https://huggingface.co/papers/2503.23108) by Hyeongju Kim, Jinhyeok Yang, Yechan Yu, Seunghun Ji, Jacob Morton, Frederik Bous, Joon Byun, Juheon Lee.
|
||||
1. **[Swin Transformer](https://huggingface.co/docs/transformers/model_doc/swin)** (from Microsoft) released with the paper [Swin Transformer: Hierarchical Vision Transformer using Shifted Windows](https://huggingface.co/papers/2103.14030) by Ze Liu, Yutong Lin, Yue Cao, Han Hu, Yixuan Wei, Zheng Zhang, Stephen Lin, Baining Guo.
|
||||
1. **[Swin2SR](https://huggingface.co/docs/transformers/model_doc/swin2sr)** (from University of Würzburg) released with the paper [Swin2SR: SwinV2 Transformer for Compressed Image Super-Resolution and Restoration](https://huggingface.co/papers/2209.11345) by Marcos V. Conde, Ui-Jin Choi, Maxime Burchi, Radu Timofte.
|
||||
1. **[T5](https://huggingface.co/docs/transformers/model_doc/t5)** (from Google AI) released with the paper [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://huggingface.co/papers/1910.10683) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
|
||||
1. **[T5v1.1](https://huggingface.co/docs/transformers/model_doc/t5v1.1)** (from Google AI) released in the repository [google-research/text-to-text-transfer-transformer](https://github.com/google-research/text-to-text-transfer-transformer/blob/main/released_checkpoints.md#t511) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
|
||||
1. **[Table Transformer](https://huggingface.co/docs/transformers/model_doc/table-transformer)** (from Microsoft Research) released with the paper [PubTables-1M: Towards Comprehensive Table Extraction From Unstructured Documents](https://huggingface.co/papers/2110.00061) by Brandon Smock, Rohith Pesala, Robin Abraham.
|
||||
1. **[TrOCR](https://huggingface.co/docs/transformers/model_doc/trocr)** (from Microsoft), released together with the paper [TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models](https://huggingface.co/papers/2109.10282) by Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei.
|
||||
1. **Ultravox** (from Fixie.ai) released with the repository [fixie-ai/ultravox](https://github.com/fixie-ai/ultravox) by the Fixie.ai team.
|
||||
1. **[UniSpeech](https://huggingface.co/docs/transformers/model_doc/unispeech)** (from Microsoft Research) released with the paper [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled Data](https://huggingface.co/papers/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei, Michael Zeng, Xuedong Huang.
|
||||
1. **[UniSpeechSat](https://huggingface.co/docs/transformers/model_doc/unispeech-sat)** (from Microsoft Research) released with the paper [UNISPEECH-SAT: UNIVERSAL SPEECH REPRESENTATION LEARNING WITH SPEAKER AWARE PRE-TRAINING](https://huggingface.co/papers/2110.05752) by Sanyuan Chen, Yu Wu, Chengyi Wang, Zhengyang Chen, Zhuo Chen, Shujie Liu, Jian Wu, Yao Qian, Furu Wei, Jinyu Li, Xiangzhan Yu.
|
||||
1. **[VaultGemma](https://huggingface.co/docs/transformers/main/model_doc/vaultgemma)** (from Google) released with the technical report [VaultGemma: A Differentially Private Gemma Model](https://services.google.com/fh/files/blogs/vaultgemma_tech_report.pdf) by the VaultGemma Google team.
|
||||
1. **[Vision Transformer (ViT)](https://huggingface.co/docs/transformers/model_doc/vit)** (from Google AI) released with the paper [An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale](https://huggingface.co/papers/2010.11929) by Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, Georg Heigold, Sylvain Gelly, Jakob Uszkoreit, Neil Houlsby.
|
||||
1. **[ViTMAE](https://huggingface.co/docs/transformers/model_doc/vit_mae)** (from Meta AI) released with the paper [Masked Autoencoders Are Scalable Vision Learners](https://huggingface.co/papers/2111.06377) by Kaiming He, Xinlei Chen, Saining Xie, Yanghao Li, Piotr Dollár, Ross Girshick.
|
||||
1. **[ViTMatte](https://huggingface.co/docs/transformers/model_doc/vitmatte)** (from HUST-VL) released with the paper [ViTMatte: Boosting Image Matting with Pretrained Plain Vision Transformers](https://huggingface.co/papers/2305.15272) by Jingfeng Yao, Xinggang Wang, Shusheng Yang, Baoyuan Wang.
|
||||
1. **[ViTMSN](https://huggingface.co/docs/transformers/model_doc/vit_msn)** (from Meta AI) released with the paper [Masked Siamese Networks for Label-Efficient Learning](https://huggingface.co/papers/2204.07141) by Mahmoud Assran, Mathilde Caron, Ishan Misra, Piotr Bojanowski, Florian Bordes, Pascal Vincent, Armand Joulin, Michael Rabbat, Nicolas Ballas.
|
||||
1. **[ViTPose](https://huggingface.co/docs/transformers/model_doc/vitpose)** (from The University of Sydney) released with the paper [ViTPose: Simple Vision Transformer Baselines for Human Pose Estimation](https://huggingface.co/papers/2204.12484) by Yufei Xu, Jing Zhang, Qiming Zhang, Dacheng Tao.
|
||||
1. **[VITS](https://huggingface.co/docs/transformers/model_doc/vits)** (from Kakao Enterprise) released with the paper [Conditional Variational Autoencoder with Adversarial Learning for End-to-End Text-to-Speech](https://huggingface.co/papers/2106.06103) by Jaehyeon Kim, Jungil Kong, Juhee Son.
|
||||
1. **[Voxtral](https://huggingface.co/docs/transformers/model_doc/voxtral)** (from Mistral AI) released with the paper [Voxtral](https://huggingface.co/papers/2507.13264) by Alexander H. Liu, Andy Ehrenberg, Andy Lo, Clément Denoix, Corentin Barreau, Guillaume Lample, Jean-Malo Delignon, Khyathi Raghavi Chandu, Patrick von Platen, Pavankumar Reddy Muddireddy, Sanchit Gandhi, Soham Ghosh, Srijan Mishra, Thomas Foubert, Abhinav Rastogi, Adam Yang, Albert Q. Jiang, Alexandre Sablayrolles, Amélie Héliou, Amélie Martin, Anmol Agarwal, Antoine Roux, Arthur Darcet, Arthur Mensch, Baptiste Bout, Baptiste Rozière, Baudouin De Monicault, Chris Bamford, Christian Wallenwein, Christophe Renaudin, Clémence Lanfranchi, Darius Dabert, Devendra Singh Chaplot, Devon Mizelle, Diego de las Casas, Elliot Chane-Sane, Emilien Fugier, Emma Bou Hanna, Gabrielle Berrada, Gauthier Delerce, Gauthier Guinet, Georgii Novikov, Guillaume Martin, Himanshu Jaju, Jan Ludziejewski, Jason Rute, Jean-Hadrien Chabran, Jessica Chudnovsky, Joachim Studnia, Joep Barmentlo, Jonas Amar, Josselin Somerville Roberts, Julien Denize, Karan Saxena, Karmesh Yadav, Kartik Khandelwal, Kush Jain, Lélio Renard Lavaud, Léonard Blier, Lingxiao Zhao, Louis Martin, Lucile Saulnier, Luyu Gao, Marie Pellat, Mathilde Guillaumin, Mathis Felardos, Matthieu Dinot, Maxime Darrin, Maximilian Augustin, Mickaël Seznec, Neha Gupta, Nikhil Raghuraman, Olivier Duchenne, Patricia Wang, Patryk Saffer, Paul Jacob, Paul Wambergue, Paula Kurylowicz, Philomène Chagniot, Pierre Stock, Pravesh Agrawal, Rémi Delacourt, Romain Sauvestre, Roman Soletskyi, Sagar Vaze, Sandeep Subramanian, Saurabh Garg, Shashwat Dalal, Siddharth Gandhi, Sumukh Aithal, Szymon Antoniak, Teven Le Scao, Thibault Schueller, Thibaut Lavril, Thomas Robert, Thomas Wang, Timothée Lacroix, Tom Bewley, Valeriia Nemychnikova, Victor Paltz , Virgile Richard, Wen-Ding Li, William Marshall, Xuanyu Zhang, Yihan Wan, Yunhao Tang.
|
||||
1. **[Voxtral Realtime](https://huggingface.co/docs/transformers/model_doc/voxtral_realtime)** (from Mistral AI) released with the paper [Voxtral Realtime](https://huggingface.co/papers/2602.11298) by Alexander H. Liu, Andy Ehrenberg, Andy Lo, Chen-Yo Sun, Guillaume Lample, Jean-Malo Delignon, Khyathi Raghavi Chandu, Patrick von Platen, Pavankumar Reddy Muddireddy, Rohin Arora, Sanchit Gandhi, Sandeep Subramanian, Soham Ghosh, Srijan Mishra, Abhinav Rastogi, Adrien Sadé, Alan Jeffares, Albert Jiang, Alexandre Cahill, Alexandre Gavaudan, Alexandre Sablayrolles, Amélie Héliou, Amos You, Andrew Bai, Angele Lenglemetz, Anmol Agarwal, Anton Eliseev, Antonia Calvi, Arjun Majumdar, Avi Sooriyarachchi, Baptiste Bout, Baptiste Rozière, Baudouin De Monicault, Benjamin Tibi, Charlotte Cronjäger, Clémence Lanfranchi, Connor Chen, Corentin Barreau, Corentin Sautier, Cyprien Courtot, Darius Dabert, Diego de las Casas, Elizaveta Demyanenko, Elliot Chane-Sane, Enguerrand Paquin, Etienne Goffinet, Fabien Niel, Faruk Ahmed, Federico Baldassarre, Gabrielle Berrada, Gaëtan Ecrepont, Gauthier Guinet, Genevieve Hayes, Georgii Novikov, Giada Pistilli, Guillaume Kunsch, Guillaume Martin, Guillaume Raille, Gunjan Dhanuka, Gunshi Gupta, Han Zhou, Harshil Shah, Hope McGovern, Hugo Thimonier, Indraneel Mukherjee, Irene Zhang, Jaeyoung Kim, Jan Ludziejewski, Jason Rute, Joachim Studnia, John Harvill, Jonas Amar, Joséphine Delas, Josselin Somerville Roberts, Julien Tauran, Karmesh Yadav, Kartik Khandelwal, Kilian Tep, Kush Jain, Laurence Aitchison, Laurent Fainsin, Léonard Blier, Lingxiao Zhao, Louis Martin, Lucile Saulnier, Luyu Gao, Maarten Buyl, Manan Sharma, Margaret Jennings, Marie Pellat, Mark Prins, Martin Alexandre, Mathieu Poirée, Mathilde Guillaumin, Matthieu Dinot, Matthieu Futeral, Maxime Darrin, Maximilian Augustin, Mert Unsal, Mia Chiquier , Minh-Quang Pham, Nathan Grinsztajn, Neha Gupta, Olivier Bousquet, Olivier Duchenne, Patricia Wang, Paul Jacob, Paul Wambergue, Paula Kurylowicz, Philippe Pinel, Philomène Chagniot, Pierre Stock, Piotr Miłoś, Prateek Gupta, Pravesh Agrawal, Quentin Torroba, Ram Ramrakhya, Rishi Shah, Romain Sauvestre, Roman Soletskyi, Rosalie Millner, Rupert Menneer, Sagar Vaze, Samuel Barry, Samuel Humeau, Sean Cha, Shashwat Verma, Siddhant Waghjale, Siddharth Gandhi, Simon Lepage, Sumukh Aithal, Szymon Antoniak, Teven Le Scao, Théo Cachet, Theo Simon Sorg, Thibaut Lavril, Thomas Chabal, Thomas Foubert, Thomas Robert, Thomas Wang, Tim Lawson, Tom Bewley, Tom Edwards, Tyler Wang, Umar Jamil, Umberto Tomasini, Valeriia Nemychnikova, Van Phung, Vedant Nanda, Victor Jouault, Vincent Maladière, Virgile Richard, Vladislav Bataev, Wassim Bouaziz, Wen-Ding Li, William Havard, William Marshall, Xinghui Li, Xingran Guo, Xinyu Yang, Yannic Neuhaus, Yassine El Ouahidi, Yassir Bendou, Yihan Wang, Yimu Pan, Zaccharie Ramzi, Zhenlin Xu.
|
||||
1. **[Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2)** (from Facebook AI) released with the paper [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations](https://huggingface.co/papers/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael Auli.
|
||||
1. **[Wav2Vec2-BERT](https://huggingface.co/docs/transformers/main/model_doc/wav2vec2-bert)** (from Meta AI) released with the paper [Seamless: Multilingual Expressive and Streaming Speech Translation](https://ai.meta.com/research/publications/seamless-multilingual-expressive-and-streaming-speech-translation/) by the Seamless Communication team.
|
||||
1. **[WavLM](https://huggingface.co/docs/transformers/model_doc/wavlm)** (from Microsoft Research) released with the paper [WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://huggingface.co/papers/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei.
|
||||
1. **[Whisper](https://huggingface.co/docs/transformers/model_doc/whisper)** (from OpenAI) released with the paper [Robust Speech Recognition via Large-Scale Weak Supervision](https://cdn.openai.com/papers/whisper.pdf) by Alec Radford, Jong Wook Kim, Tao Xu, Greg Brockman, Christine McLeavey, Ilya Sutskever.
|
||||
1. **[XLM](https://huggingface.co/docs/transformers/model_doc/xlm)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://huggingface.co/papers/1901.07291) by Guillaume Lample and Alexis Conneau.
|
||||
1. **[XLM-RoBERTa](https://huggingface.co/docs/transformers/model_doc/xlm-roberta)** (from Facebook AI), released together with the paper [Unsupervised Cross-lingual Representation Learning at Scale](https://huggingface.co/papers/1911.02116) by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov.
|
||||
1. **[YOLOS](https://huggingface.co/docs/transformers/model_doc/yolos)** (from Huazhong University of Science & Technology) released with the paper [You Only Look at One Sequence: Rethinking Transformer in Vision through Object Detection](https://huggingface.co/papers/2106.00666) by Yuxin Fang, Bencheng Liao, Xinggang Wang, Jiemin Fang, Jiyang Qi, Rui Wu, Jianwei Niu, Wenyu Liu.
|
||||
1. **[Youtu-LLM](https://huggingface.co/docs/transformers/model_doc/youtu)** (from the Tencent Youtu Team) released with the paper [Youtu-LLM: Unlocking the Native Agentic Potential for Lightweight Large Language Models](https://huggingface.co/papers/2512.24618) by Junru Lu, Jiarui Qin, Lingfeng Qiao, Yinghui Li, Xinyi Dai, Bo Ke, Jianfeng He, Ruizhi Qiao, Di Yin, Xing Sun, Yunsheng Wu, Yinsong Liu, Shuangyin Liu, Mingkong Tang, Haodong Lin, Jiayi Kuang, Fanxu Meng, Xiaojuan Tang, Yunjia Xi, Junjie Huang, Haotong Yang, Zhenyi Shen, Yangning Li, Qianwen Zhang, Yifei Yu, Siyu An, Junnan Dong, Qiufeng Wang, Jie Wang, Keyu Chen, Wei Wen, Taian Guo, Zhifeng Shen, Daohai Yu, Jiahao Li, Ke Li, Zongyi Li, Xiaoyu Tan.
|
||||
1. **Zaya** (from Zyphra) released with the paper [ZAYA1-8B Technical Report](https://huggingface.co/papers/2605.05365) by Robert Washbourne, Rishi Iyer, Tomas Figliolia, Henry Zheng, Ryan Lorig-Roach, Sungyeon Yang, Pritish Yuvraj, Quentin Anthony, Yury Tokpanov, Xiao Yang, Ganesh Nanduru, Stephen Ebert, Praneeth Medepalli, Skyler Szot, Srivatsan Rajagopal, Alex Ong, Bhavana Mehta, Beren Millidge.
|
||||
@@ -0,0 +1,97 @@
|
||||
- local: index
|
||||
title: 🤗 Transformers.js
|
||||
- sections:
|
||||
- local: installation
|
||||
title: Installation
|
||||
- local: pipelines
|
||||
title: The pipeline API
|
||||
- local: custom_usage
|
||||
title: Custom usage
|
||||
title: Get started
|
||||
- sections:
|
||||
- local: tutorials/vanilla-js
|
||||
title: Building a Vanilla JS Application
|
||||
- local: tutorials/react
|
||||
title: Building a React Application
|
||||
- local: tutorials/next
|
||||
title: Building a Next.js Application
|
||||
- local: tutorials/browser-extension
|
||||
title: Building a Browser Extension
|
||||
- local: tutorials/electron
|
||||
title: Building an Electron Application
|
||||
- local: tutorials/node
|
||||
title: Server-side Inference in Node.js
|
||||
- local: tutorials/next-ai-sdk
|
||||
title: Building a Next.js AI Chatbot with Vercel AI SDK
|
||||
title: Tutorials
|
||||
- sections:
|
||||
- local: guides/webgpu
|
||||
title: Running models on WebGPU
|
||||
- local: guides/dtypes
|
||||
title: Using quantized models (dtypes)
|
||||
- local: guides/private
|
||||
title: Accessing Private/Gated Models
|
||||
- local: guides/node-audio-processing
|
||||
title: Server-side Audio Processing
|
||||
title: Developer Guides
|
||||
- sections:
|
||||
- local: integrations/vercel-ai-sdk
|
||||
title: Vercel AI SDK
|
||||
title: Integrations
|
||||
- sections:
|
||||
- local: api/transformers
|
||||
title: Index
|
||||
- local: api/pipelines
|
||||
title: Pipelines
|
||||
- local: api/models
|
||||
title: Models
|
||||
- local: api/tokenizers
|
||||
title: Tokenizers
|
||||
- local: api/processors
|
||||
title: Processors
|
||||
- local: api/configs
|
||||
title: Configs
|
||||
- local: api/env
|
||||
title: Environment variables
|
||||
- sections:
|
||||
- local: api/backends/onnx
|
||||
title: ONNX
|
||||
title: Backends
|
||||
isExpanded: false
|
||||
- sections:
|
||||
- local: api/generation/parameters
|
||||
title: Parameters
|
||||
- local: api/generation/configuration_utils
|
||||
title: Configuration
|
||||
- local: api/generation/logits_process
|
||||
title: Logits Processors
|
||||
- local: api/generation/logits_sampler
|
||||
title: Logits Samplers
|
||||
- local: api/generation/stopping_criteria
|
||||
title: Stopping Criteria
|
||||
- local: api/generation/streamers
|
||||
title: Streamers
|
||||
title: Generation
|
||||
isExpanded: false
|
||||
- sections:
|
||||
- local: api/utils/core
|
||||
title: Core
|
||||
- local: api/utils/hub
|
||||
title: Hub
|
||||
- local: api/utils/model_registry
|
||||
title: Model Registry
|
||||
- local: api/utils/image
|
||||
title: Image
|
||||
- local: api/utils/audio
|
||||
title: Audio
|
||||
- local: api/utils/tensor
|
||||
title: Tensor
|
||||
- local: api/utils/maths
|
||||
title: Maths
|
||||
- local: api/utils/logger
|
||||
title: Logger
|
||||
- local: api/utils/random
|
||||
title: Random
|
||||
title: Utilities
|
||||
isExpanded: false
|
||||
title: API Reference
|
||||
@@ -0,0 +1,3 @@
|
||||
# These docs are automatically generated from the JSDoc in the source code.
|
||||
*
|
||||
!.gitignore
|
||||
@@ -0,0 +1,7 @@
|
||||
# Use custom models
|
||||
|
||||
<include>
|
||||
{
|
||||
"path": "../snippets/3_custom-usage.snippet"
|
||||
}
|
||||
</include>
|
||||
@@ -0,0 +1,156 @@
|
||||
# Using quantized models (dtypes)
|
||||
|
||||
Before Transformers.js v3, we used the `quantized` option to specify whether to use a quantized (q8) or full-precision (fp32) variant of the model by setting `quantized` to `true` or `false`, respectively. Now, we've added the ability to select from a much larger list with the `dtype` parameter.
|
||||
|
||||
The list of available quantizations depends on the model, but some common ones are: full-precision (`"fp32"`), half-precision (`"fp16"`), 8-bit (`"q8"`, `"int8"`, `"uint8"`), and 4-bit (`"q4"`, `"bnb4"`, `"q4f16"`).
|
||||
|
||||
<p align="center">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/transformersjs-v3/dtypes-dark.jpg" style="max-width: 100%;">
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/transformersjs-v3/dtypes-light.jpg" style="max-width: 100%;">
|
||||
<img alt="Available dtypes for mixedbread-ai/mxbai-embed-xsmall-v1" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/transformersjs-v3/dtypes-dark.jpg" style="max-width: 100%;">
|
||||
</picture>
|
||||
<a href="https://huggingface.co/mixedbread-ai/mxbai-embed-xsmall-v1/tree/main/onnx">(e.g., mixedbread-ai/mxbai-embed-xsmall-v1)</a>
|
||||
</p>
|
||||
|
||||
## Basic usage
|
||||
|
||||
**Example:** Run Qwen2.5-0.5B-Instruct in 4-bit quantization ([demo](https://v2.scrimba.com/s0dlcpv0ci))
|
||||
|
||||
```js
|
||||
import { pipeline } from "@huggingface/transformers";
|
||||
|
||||
// Create a text generation pipeline
|
||||
const generator = await pipeline(
|
||||
"text-generation",
|
||||
"onnx-community/Qwen2.5-0.5B-Instruct",
|
||||
{ dtype: "q4", device: "webgpu" },
|
||||
);
|
||||
|
||||
// Define the list of messages
|
||||
const messages = [
|
||||
{ role: "system", content: "You are a helpful assistant." },
|
||||
{ role: "user", content: "Tell me a funny joke." },
|
||||
];
|
||||
|
||||
// Generate a response
|
||||
const output = await generator(messages, { max_new_tokens: 128 });
|
||||
console.log(output[0].generated_text.at(-1).content);
|
||||
```
|
||||
|
||||
## Detecting available dtypes
|
||||
|
||||
Not sure which quantizations a model offers? Use `ModelRegistry.get_available_dtypes()` to probe the repository and find out:
|
||||
|
||||
```js
|
||||
import { ModelRegistry } from "@huggingface/transformers";
|
||||
|
||||
const dtypes = await ModelRegistry.get_available_dtypes("onnx-community/all-MiniLM-L6-v2-ONNX");
|
||||
console.log(dtypes); // e.g., [ 'fp32', 'fp16', 'int8', 'uint8', 'q8', 'q4' ]
|
||||
```
|
||||
|
||||
This checks which ONNX files exist on the Hugging Face Hub for each dtype. For multi-session models (e.g., encoder-decoder), a dtype is only listed if **all** required session files are present.
|
||||
|
||||
You can use this to build UIs that let users pick a quantization level, or to automatically select the smallest available dtype:
|
||||
|
||||
```js
|
||||
const dtypes = await ModelRegistry.get_available_dtypes("onnx-community/Qwen3-0.6B-ONNX");
|
||||
|
||||
// Pick the smallest available quantization, falling back to fp32
|
||||
const preferred = ["q4", "q8", "fp16", "fp32"];
|
||||
const dtype = preferred.find((d) => dtypes.includes(d)) ?? "fp32";
|
||||
|
||||
const generator = await pipeline("text-generation", "onnx-community/Qwen3-0.6B-ONNX", { dtype });
|
||||
```
|
||||
|
||||
## Per-module dtypes
|
||||
|
||||
Some encoder-decoder models, like Whisper or Florence-2, are extremely sensitive to quantization settings: especially of the encoder. For this reason, we added the ability to select per-module dtypes, which can be done by providing a mapping from module name to dtype.
|
||||
|
||||
**Example:** Run Florence-2 on WebGPU ([demo](https://v2.scrimba.com/s0pdm485fo))
|
||||
|
||||
```js
|
||||
import { Florence2ForConditionalGeneration } from "@huggingface/transformers";
|
||||
|
||||
const model = await Florence2ForConditionalGeneration.from_pretrained(
|
||||
"onnx-community/Florence-2-base-ft",
|
||||
{
|
||||
dtype: {
|
||||
embed_tokens: "fp16",
|
||||
vision_encoder: "fp16",
|
||||
encoder_model: "q4",
|
||||
decoder_model_merged: "q4",
|
||||
},
|
||||
device: "webgpu",
|
||||
},
|
||||
);
|
||||
```
|
||||
|
||||
<p align="middle">
|
||||
<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/transformersjs-v3/florence-2-webgpu.gif" alt="Florence-2 running on WebGPU" />
|
||||
</p>
|
||||
|
||||
<details>
|
||||
<summary>
|
||||
See full code example
|
||||
</summary>
|
||||
|
||||
```js
|
||||
import {
|
||||
Florence2ForConditionalGeneration,
|
||||
AutoProcessor,
|
||||
AutoTokenizer,
|
||||
RawImage,
|
||||
} from "@huggingface/transformers";
|
||||
|
||||
// Load model, processor, and tokenizer
|
||||
const model_id = "onnx-community/Florence-2-base-ft";
|
||||
const model = await Florence2ForConditionalGeneration.from_pretrained(
|
||||
model_id,
|
||||
{
|
||||
dtype: {
|
||||
embed_tokens: "fp16",
|
||||
vision_encoder: "fp16",
|
||||
encoder_model: "q4",
|
||||
decoder_model_merged: "q4",
|
||||
},
|
||||
device: "webgpu",
|
||||
},
|
||||
);
|
||||
const processor = await AutoProcessor.from_pretrained(model_id);
|
||||
const tokenizer = await AutoTokenizer.from_pretrained(model_id);
|
||||
|
||||
// Load image and prepare vision inputs
|
||||
const url =
|
||||
"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg";
|
||||
const image = await RawImage.fromURL(url);
|
||||
const vision_inputs = await processor(image);
|
||||
|
||||
// Specify task and prepare text inputs
|
||||
const task = "<MORE_DETAILED_CAPTION>";
|
||||
const prompts = processor.construct_prompts(task);
|
||||
const text_inputs = tokenizer(prompts);
|
||||
|
||||
// Generate text
|
||||
const generated_ids = await model.generate({
|
||||
...text_inputs,
|
||||
...vision_inputs,
|
||||
max_new_tokens: 100,
|
||||
});
|
||||
|
||||
// Decode generated text
|
||||
const generated_text = tokenizer.batch_decode(generated_ids, {
|
||||
skip_special_tokens: false,
|
||||
})[0];
|
||||
|
||||
// Post-process the generated text
|
||||
const result = processor.post_process_generation(
|
||||
generated_text,
|
||||
task,
|
||||
image.size,
|
||||
);
|
||||
console.log(result);
|
||||
// { '<MORE_DETAILED_CAPTION>': 'A green car is parked in front of a tan building. The building has a brown door and two brown windows. The car is a two door and the door is closed. The green car has black tires.' }
|
||||
```
|
||||
|
||||
</details>
|
||||
@@ -0,0 +1,112 @@
|
||||
# Server-side Audio Processing in Node.js
|
||||
|
||||
A major benefit of writing code for the web is that you can access the multitude of APIs that are available in modern browsers. Unfortunately, when writing server-side code, we are not afforded such luxury, so we have to find another way. In this tutorial, we will design a simple Node.js application that uses Transformers.js for speech recognition with [Whisper](https://huggingface.co/Xenova/whisper-tiny.en), and in the process, learn how to process audio on the server.
|
||||
|
||||
The main problem we need to solve is that the [Web Audio API](https://developer.mozilla.org/en-US/docs/Web/API/Web_Audio_API) is not available in Node.js, meaning we can't use the [`AudioContext`](https://developer.mozilla.org/en-US/docs/Web/API/AudioContext) class to process audio. So, we will need to install third-party libraries to obtain the raw audio data. For this example, we will only consider `.wav` files, but the same principles apply to other audio formats.
|
||||
|
||||
<Tip>
|
||||
|
||||
This tutorial will be written as an ES module, but you can easily adapt it to use CommonJS instead. For more information, see the [node tutorial](https://huggingface.co/docs/transformers.js/tutorials/node).
|
||||
|
||||
</Tip>
|
||||
|
||||
**Useful links:**
|
||||
|
||||
- [Source code](https://github.com/huggingface/transformers.js/tree/main/examples/node-audio-processing)
|
||||
- [Documentation](https://huggingface.co/docs/transformers.js)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [Node.js](https://nodejs.org/en/) version 18+
|
||||
- [npm](https://www.npmjs.com/) version 9+
|
||||
|
||||
## Getting started
|
||||
|
||||
Let's start by creating a new Node.js project and installing Transformers.js via [NPM](https://www.npmjs.com/package/@huggingface/transformers):
|
||||
|
||||
```bash
|
||||
npm init -y
|
||||
npm i @huggingface/transformers
|
||||
```
|
||||
|
||||
<Tip>
|
||||
|
||||
Remember to add `"type": "module"` to your `package.json` to indicate that your project uses ECMAScript modules.
|
||||
|
||||
</Tip>
|
||||
|
||||
Next, let's install the [`wavefile`](https://www.npmjs.com/package/wavefile) package, which we will use for loading `.wav` files:
|
||||
|
||||
```bash
|
||||
npm i wavefile
|
||||
```
|
||||
|
||||
## Creating the application
|
||||
|
||||
Start by creating a new file called `index.js`, which will be the entry point for our application. Let's also import the necessary modules:
|
||||
|
||||
```js
|
||||
import { pipeline } from "@huggingface/transformers";
|
||||
import wavefile from "wavefile";
|
||||
```
|
||||
|
||||
For this tutorial, we will use the `Xenova/whisper-tiny.en` model, but feel free to choose one of the other whisper models from the [Hugging Face Hub](https://huggingface.co/models?library=transformers.js&search=whisper). Let's create our pipeline with:
|
||||
|
||||
```js
|
||||
let transcriber = await pipeline(
|
||||
"automatic-speech-recognition",
|
||||
"Xenova/whisper-tiny.en",
|
||||
);
|
||||
```
|
||||
|
||||
Next, let's load an audio file and convert it to the format required by Transformers.js:
|
||||
|
||||
```js
|
||||
// Load audio data
|
||||
let url =
|
||||
"https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav";
|
||||
let buffer = Buffer.from(await fetch(url).then((x) => x.arrayBuffer()));
|
||||
|
||||
// Read .wav file and convert it to required format
|
||||
let wav = new wavefile.WaveFile(buffer);
|
||||
wav.toBitDepth("32f"); // Pipeline expects input as a Float32Array
|
||||
wav.toSampleRate(16000); // Whisper expects audio with a sampling rate of 16000
|
||||
let audioData = wav.getSamples();
|
||||
if (Array.isArray(audioData)) {
|
||||
if (audioData.length > 1) {
|
||||
const SCALING_FACTOR = Math.sqrt(2);
|
||||
|
||||
// Merge channels (into first channel to save memory)
|
||||
for (let i = 0; i < audioData[0].length; ++i) {
|
||||
audioData[0][i] =
|
||||
(SCALING_FACTOR * (audioData[0][i] + audioData[1][i])) / 2;
|
||||
}
|
||||
}
|
||||
|
||||
// Select first channel
|
||||
audioData = audioData[0];
|
||||
}
|
||||
```
|
||||
|
||||
Finally, let's run the model and measure execution duration.
|
||||
|
||||
```js
|
||||
let start = performance.now();
|
||||
let output = await transcriber(audioData);
|
||||
let end = performance.now();
|
||||
console.log(`Execution duration: ${(end - start) / 1000} seconds`);
|
||||
console.log(output);
|
||||
```
|
||||
|
||||
You can now run the application with `node index.js`. Note that when running the script for the first time, it may take a while to download and cache the model. Subsequent requests will use the cached model, and model loading will be much faster.
|
||||
|
||||
You should see output similar to:
|
||||
|
||||
```
|
||||
Execution duration: 0.6460317999720574 seconds
|
||||
{
|
||||
text: ' And so my fellow Americans ask not what your country can do for you. Ask what you can do for your country.'
|
||||
}
|
||||
```
|
||||
|
||||
That's it! You've successfully created a Node.js application that uses Transformers.js for speech recognition with Whisper. You can now use this as a starting point for your own applications.
|
||||
@@ -0,0 +1,63 @@
|
||||
# Accessing Private/Gated Models
|
||||
|
||||
<Tip>
|
||||
|
||||
Due to the possibility of leaking access tokens to users of your website or web application, we only support accessing private/gated models from server-side environments (e.g., Node.js) that have access to the process' environment variables.
|
||||
|
||||
</Tip>
|
||||
|
||||
## Step 1: Generating a User Access Token
|
||||
|
||||
[User Access Tokens](https://huggingface.co/docs/hub/security-tokens) are the preferred way to authenticate an application to Hugging Face services.
|
||||
|
||||
To generate an access token, navigate to the [Access Tokens tab](https://huggingface.co/settings/tokens) in your settings and click on the **New token** button. Choose a name for your token and click **Generate a token** (we recommend keeping the "Role" as read-only). You can then click the **Copy** button next to your newly-created token to copy it to your clipboard.
|
||||
|
||||
<div class="flex justify-center">
|
||||
<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/new-token.png"/>
|
||||
<img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/new-token-dark.png"/>
|
||||
</div>
|
||||
|
||||
To delete or refresh User Access Tokens, you can click the **Manage** button.
|
||||
|
||||
## Step 2: Using the access token in Transformers.js
|
||||
|
||||
Transformers.js will attach an Authorization header to requests made to the Hugging Face Hub when the `HF_TOKEN` environment variable is set and visible to the process.
|
||||
|
||||
One way to do this is to call your program with the environment variable set. For example, let's say you have a file called `llama.js` with the following code:
|
||||
|
||||
```js
|
||||
import { AutoTokenizer } from "@huggingface/transformers";
|
||||
|
||||
// Load tokenizer for a gated repository.
|
||||
const tokenizer = await AutoTokenizer.from_pretrained(
|
||||
"meta-llama/Llama-2-7b-hf",
|
||||
);
|
||||
|
||||
// Encode text.
|
||||
const text = "Hello world!";
|
||||
const encoded = tokenizer.encode(text);
|
||||
console.log(encoded);
|
||||
```
|
||||
|
||||
You can then use the following command to set the `HF_TOKEN` environment variable and run the file:
|
||||
|
||||
```bash
|
||||
HF_TOKEN=hf_... node tests/llama.js
|
||||
```
|
||||
|
||||
(remember to replace `hf_...` with your actual access token).
|
||||
|
||||
If done correctly, you should see the following output:
|
||||
|
||||
```bash
|
||||
[ 1, 15043, 3186, 29991 ]
|
||||
```
|
||||
|
||||
Alternatively, you can set the environment variable directly in your code:
|
||||
|
||||
```js
|
||||
// Set access token (NB: Keep this private!)
|
||||
process.env.HF_TOKEN = "hf_...";
|
||||
|
||||
// ... rest of your code
|
||||
```
|
||||
@@ -0,0 +1,88 @@
|
||||
# Running models on WebGPU
|
||||
|
||||
WebGPU is a new web standard for accelerated graphics and compute. The [API](https://developer.mozilla.org/en-US/docs/Web/API/WebGPU_API) enables web developers to use the underlying system's GPU to carry out high-performance computations directly in the browser. WebGPU is the successor to [WebGL](https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API) and provides significantly better performance, because it allows for more direct interaction with modern GPUs. Lastly, it supports general-purpose GPU computations, which makes it just perfect for machine learning!
|
||||
|
||||
> [!WARNING]
|
||||
> As of October 2024, global WebGPU support is around 70% (according to [caniuse.com](https://caniuse.com/webgpu)), meaning some users may not be able to use the API.
|
||||
>
|
||||
> If the following demos do not work in your browser, you may need to enable it using a feature flag:
|
||||
>
|
||||
> - Firefox: with the `dom.webgpu.enabled` flag (see [here](https://developer.mozilla.org/en-US/docs/Mozilla/Firefox/Experimental_features#:~:text=tested%20by%20Firefox.-,WebGPU%20API,-The%20WebGPU%20API)).
|
||||
> - Safari: with the `WebGPU` feature flag (see [here](https://webkit.org/blog/14879/webgpu-now-available-for-testing-in-safari-technology-preview/)).
|
||||
> - Older Chromium browsers (on Windows, macOS, Linux): with the `enable-unsafe-webgpu` flag (see [here](https://developer.chrome.com/docs/web-platform/webgpu/troubleshooting-tips)).
|
||||
|
||||
## Usage in Transformers.js v3
|
||||
|
||||
Thanks to our collaboration with [ONNX Runtime Web](https://www.npmjs.com/package/onnxruntime-web), enabling WebGPU acceleration is as simple as setting `device: 'webgpu'` when loading a model. Let's see some examples!
|
||||
|
||||
**Example:** Compute text embeddings on WebGPU ([demo](https://v2.scrimba.com/s06a2smeej))
|
||||
|
||||
```js
|
||||
import { pipeline } from "@huggingface/transformers";
|
||||
|
||||
// Create a feature-extraction pipeline
|
||||
const extractor = await pipeline(
|
||||
"feature-extraction",
|
||||
"mixedbread-ai/mxbai-embed-xsmall-v1",
|
||||
{ device: "webgpu" },
|
||||
);
|
||||
|
||||
// Compute embeddings
|
||||
const texts = ["Hello world!", "This is an example sentence."];
|
||||
const embeddings = await extractor(texts, { pooling: "mean", normalize: true });
|
||||
console.log(embeddings.tolist());
|
||||
// [
|
||||
// [-0.016986183822155, 0.03228696808218956, -0.0013630966423079371, ... ],
|
||||
// [0.09050482511520386, 0.07207386940717697, 0.05762749910354614, ... ],
|
||||
// ]
|
||||
```
|
||||
|
||||
**Example:** Perform automatic speech recognition with OpenAI whisper on WebGPU ([demo](https://v2.scrimba.com/s0oi76h82g))
|
||||
|
||||
```js
|
||||
import { pipeline } from "@huggingface/transformers";
|
||||
|
||||
// Create automatic speech recognition pipeline
|
||||
const transcriber = await pipeline(
|
||||
"automatic-speech-recognition",
|
||||
"onnx-community/whisper-tiny.en",
|
||||
{ device: "webgpu" },
|
||||
);
|
||||
|
||||
// Transcribe audio from a URL
|
||||
const url =
|
||||
"https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav";
|
||||
const output = await transcriber(url);
|
||||
console.log(output);
|
||||
// { text: ' And so my fellow Americans ask not what your country can do for you, ask what you can do for your country.' }
|
||||
```
|
||||
|
||||
**Example:** Perform image classification with MobileNetV4 on WebGPU ([demo](https://v2.scrimba.com/s0fv2uab1t))
|
||||
|
||||
```js
|
||||
import { pipeline } from "@huggingface/transformers";
|
||||
|
||||
// Create image classification pipeline
|
||||
const classifier = await pipeline(
|
||||
"image-classification",
|
||||
"onnx-community/mobilenetv4_conv_small.e2400_r224_in1k",
|
||||
{ device: "webgpu" },
|
||||
);
|
||||
|
||||
// Classify an image from a URL
|
||||
const url =
|
||||
"https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/tiger.jpg";
|
||||
const output = await classifier(url);
|
||||
console.log(output);
|
||||
// [
|
||||
// { label: 'tiger, Panthera tigris', score: 0.6149784922599792 },
|
||||
// { label: 'tiger cat', score: 0.30281734466552734 },
|
||||
// { label: 'tabby, tabby cat', score: 0.0019135422771796584 },
|
||||
// { label: 'lynx, catamount', score: 0.0012161266058683395 },
|
||||
// { label: 'Egyptian cat', score: 0.0011465961579233408 }
|
||||
// ]
|
||||
```
|
||||
|
||||
## Reporting bugs and providing feedback
|
||||
|
||||
Due to the experimental nature of WebGPU, especially in non-Chromium browsers, you may experience issues when trying to run a model (even if it can run in WASM). If you do, please open [an issue on GitHub](https://github.com/huggingface/transformers.js/issues/new?title=[WebGPU]%20Error%20running%20MODEL_GOES_HERE&assignees=&labels=bug,webgpu&projects=&template=1_bug-report.yml) and we'll do our best to address it. Thanks!
|
||||
@@ -0,0 +1,46 @@
|
||||
# Transformers.js
|
||||
|
||||
<include>
|
||||
{
|
||||
"path": "../snippets/0_introduction.snippet"
|
||||
}
|
||||
</include>
|
||||
|
||||
## Quick tour
|
||||
|
||||
<include>
|
||||
{
|
||||
"path": "../snippets/1_quick-tour.snippet"
|
||||
}
|
||||
</include>
|
||||
|
||||
## Contents
|
||||
|
||||
The documentation is organized into 5 sections:
|
||||
|
||||
1. **GET STARTED** provides a quick tour of the library and installation instructions to get up and running.
|
||||
2. **TUTORIALS** are a great place to start if you're a beginner! We also include sample applications for you to play around with!
|
||||
3. **DEVELOPER GUIDES** show you how to use the library to achieve a specific goal.
|
||||
4. **INTEGRATIONS** show you how to use Transformers.js with other popular libraries and frameworks.
|
||||
5. **API REFERENCE** describes all classes and functions, as well as their available parameters and types.
|
||||
|
||||
## Supported tasks/models
|
||||
|
||||
Here is the list of all tasks and architectures currently supported by Transformers.js.
|
||||
If you don't see your task/model listed here or it is not yet supported, feel free
|
||||
to open up a feature request [here](https://github.com/huggingface/transformers.js/issues/new/choose).
|
||||
|
||||
To find compatible models on the Hub, select the "transformers.js" library tag in the filter menu (or visit [this link](https://huggingface.co/models?library=transformers.js)).
|
||||
You can refine your search by selecting the task you're interested in (e.g., [text-classification](https://huggingface.co/models?pipeline_tag=text-classification&library=transformers.js)).
|
||||
|
||||
<include>
|
||||
{
|
||||
"path": "../snippets/4_supported-tasks.snippet"
|
||||
}
|
||||
</include>
|
||||
|
||||
<include>
|
||||
{
|
||||
"path": "../snippets/5_supported-models.snippet"
|
||||
}
|
||||
</include>
|
||||
@@ -0,0 +1,7 @@
|
||||
# Installation
|
||||
|
||||
<include>
|
||||
{
|
||||
"path": "../snippets/2_installation.snippet"
|
||||
}
|
||||
</include>
|
||||
@@ -0,0 +1,286 @@
|
||||
# Using Transformers.js with the Vercel AI SDK
|
||||
|
||||
[Vercel AI SDK](https://ai-sdk.dev/) is a popular toolkit for building AI-powered applications. With [`@browser-ai/transformers-js`](https://www.browser-ai.dev/docs/ai-sdk-v6/transformers-js), you can use Transformers.js as a model provider for the AI SDK, enabling in-browser (and server-side) inference with a clean, declarative API.
|
||||
|
||||
This guide covers the core concepts and API patterns. For a full step-by-step project walkthrough, see the [Building a Next.js AI Chatbot](../tutorials/next-ai-sdk) tutorial.
|
||||
|
||||
## Why use the Vercel AI SDK with Transformers.js?
|
||||
|
||||
The `@browser-ai/transformers-js` provider builds on top of `@huggingface/transformers` to give you a standard AI SDK interface — handling Web Worker setup, message passing, progress tracking, streaming, interrupt handling, and state management, so you can use the same `streamText`, `generateText`, and `useChat` APIs you'd use with any other AI SDK provider.
|
||||
Read more about this [here](https://www.browser-ai.dev/docs/ai-sdk-v6/transformers-js/why).
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
npm install @browser-ai/transformers-js @huggingface/transformers ai @ai-sdk/react
|
||||
```
|
||||
|
||||
| @browser-ai/transformers-js | AI SDK | Notes |
|
||||
|---|---|---|
|
||||
| v2.0.0+ | v6.x | Current stable |
|
||||
| v1.0.0 | v5.x | Legacy |
|
||||
|
||||
## Text generation
|
||||
|
||||
### Streaming text
|
||||
|
||||
```js
|
||||
import { streamText } from "ai";
|
||||
import { transformersJS } from "@browser-ai/transformers-js";
|
||||
|
||||
const result = streamText({
|
||||
model: transformersJS("HuggingFaceTB/SmolLM2-360M-Instruct"),
|
||||
prompt: "Invent a new holiday and describe its traditions.",
|
||||
});
|
||||
|
||||
for await (const textPart of result.textStream) {
|
||||
console.log(textPart);
|
||||
}
|
||||
```
|
||||
|
||||
### Non-streaming text
|
||||
|
||||
```js
|
||||
import { generateText } from "ai";
|
||||
import { transformersJS } from "@browser-ai/transformers-js";
|
||||
|
||||
const result = await generateText({
|
||||
model: transformersJS("HuggingFaceTB/SmolLM2-360M-Instruct"),
|
||||
prompt: "Invent a new holiday and describe its traditions.",
|
||||
});
|
||||
console.log(result.text);
|
||||
```
|
||||
|
||||
## Text embeddings
|
||||
|
||||
```js
|
||||
import { embed, embedMany } from "ai";
|
||||
import { transformersJS } from "@browser-ai/transformers-js";
|
||||
|
||||
// Single embedding
|
||||
const { embedding } = await embed({
|
||||
model: transformersJS.embedding("Supabase/gte-small"),
|
||||
value: "Hello, world!",
|
||||
});
|
||||
|
||||
// Multiple embeddings
|
||||
const { embeddings } = await embedMany({
|
||||
model: transformersJS.embedding("Supabase/gte-small"),
|
||||
values: ["Hello", "World", "AI"],
|
||||
});
|
||||
```
|
||||
|
||||
## Audio transcription
|
||||
|
||||
```js
|
||||
import { experimental_transcribe as transcribe } from "ai";
|
||||
import { transformersJS } from "@browser-ai/transformers-js";
|
||||
|
||||
const transcript = await transcribe({
|
||||
model: transformersJS.transcription("Xenova/whisper-base"),
|
||||
audio: audioFile,
|
||||
});
|
||||
console.log(transcript.text);
|
||||
console.log(transcript.segments); // segments with timestamps
|
||||
```
|
||||
|
||||
## Vision models
|
||||
|
||||
```js
|
||||
import { streamText } from "ai";
|
||||
import { transformersJS } from "@browser-ai/transformers-js";
|
||||
|
||||
const result = streamText({
|
||||
model: transformersJS("HuggingFaceTB/SmolVLM-256M-Instruct", {
|
||||
isVisionModel: true,
|
||||
device: "webgpu",
|
||||
}),
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "Describe this image" },
|
||||
{ type: "image", image: someImageBlobOrUrl },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
for await (const chunk of result.textStream) {
|
||||
console.log(chunk);
|
||||
}
|
||||
```
|
||||
|
||||
## Web Worker offloading
|
||||
|
||||
For better performance, run model inference off the main thread with a Web Worker.
|
||||
|
||||
**1. Create `worker.ts`:**
|
||||
|
||||
```typescript
|
||||
import { TransformersJSWorkerHandler } from "@browser-ai/transformers-js";
|
||||
|
||||
const handler = new TransformersJSWorkerHandler();
|
||||
self.onmessage = (msg: MessageEvent) => {
|
||||
handler.onmessage(msg);
|
||||
};
|
||||
```
|
||||
|
||||
**2. Pass the worker when creating the model:**
|
||||
|
||||
```js
|
||||
import { streamText } from "ai";
|
||||
import { transformersJS } from "@browser-ai/transformers-js";
|
||||
|
||||
const model = transformersJS("HuggingFaceTB/SmolLM2-360M-Instruct", {
|
||||
device: "webgpu",
|
||||
worker: new Worker(new URL("./worker.ts", import.meta.url), {
|
||||
type: "module",
|
||||
}),
|
||||
});
|
||||
|
||||
const result = streamText({
|
||||
model,
|
||||
messages: [{ role: "user", content: "Hello!" }],
|
||||
});
|
||||
```
|
||||
|
||||
## Download progress tracking
|
||||
|
||||
Models are downloaded on first use. Track progress to provide a better UX:
|
||||
|
||||
```js
|
||||
import { streamText } from "ai";
|
||||
import { transformersJS } from "@browser-ai/transformers-js";
|
||||
|
||||
const model = transformersJS("HuggingFaceTB/SmolLM2-360M-Instruct");
|
||||
const availability = await model.availability();
|
||||
|
||||
if (availability === "unavailable") {
|
||||
console.log("Browser doesn't support Transformers.js");
|
||||
} else if (availability === "downloadable") {
|
||||
await model.createSessionWithProgress(({ progress }) => {
|
||||
console.log(`Download progress: ${Math.round(progress * 100)}%`);
|
||||
});
|
||||
}
|
||||
|
||||
// Model is ready
|
||||
const result = streamText({ model, prompt: "Hello!" });
|
||||
```
|
||||
|
||||
## Tool calling
|
||||
|
||||
<Tip>
|
||||
|
||||
For best tool calling results, use reasoning models like Qwen3 which handle multi-step reasoning well.
|
||||
|
||||
</Tip>
|
||||
|
||||
```js
|
||||
import { streamText, tool, stepCountIs } from "ai";
|
||||
import { transformersJS } from "@browser-ai/transformers-js";
|
||||
import { z } from "zod";
|
||||
|
||||
const result = await streamText({
|
||||
model: transformersJS("onnx-community/Qwen3-0.6B-ONNX"),
|
||||
messages: [{ role: "user", content: "What's the weather in San Francisco?" }],
|
||||
tools: {
|
||||
weather: tool({
|
||||
description: "Get the weather in a location",
|
||||
inputSchema: z.object({
|
||||
location: z.string().describe("The location to get the weather for"),
|
||||
}),
|
||||
execute: async ({ location }) => ({
|
||||
location,
|
||||
temperature: 72 + Math.floor(Math.random() * 21) - 10,
|
||||
}),
|
||||
}),
|
||||
},
|
||||
stopWhen: stepCountIs(5),
|
||||
});
|
||||
```
|
||||
|
||||
Tool calling also supports [tool execution approval (`needsApproval`)](https://ai-sdk.dev/docs/ai-sdk-core/tools-and-tool-calling#tool-execution-approval) for human-in-the-loop workflows.
|
||||
|
||||
## `useChat` with custom transport
|
||||
|
||||
When using the `useChat` hook, you create a [custom transport](https://ai-sdk.dev/docs/ai-sdk-ui/transport) to handle client-side inference. Here's a minimal example:
|
||||
|
||||
```typescript
|
||||
import {
|
||||
ChatTransport, UIMessageChunk, streamText,
|
||||
convertToModelMessages, ChatRequestOptions,
|
||||
} from "ai";
|
||||
import {
|
||||
TransformersJSLanguageModel,
|
||||
TransformersUIMessage,
|
||||
} from "@browser-ai/transformers-js";
|
||||
|
||||
export class TransformersChatTransport
|
||||
implements ChatTransport<TransformersUIMessage>
|
||||
{
|
||||
constructor(private readonly model: TransformersJSLanguageModel) {}
|
||||
|
||||
async sendMessages(
|
||||
options: {
|
||||
chatId: string;
|
||||
messages: TransformersUIMessage[];
|
||||
abortSignal: AbortSignal | undefined;
|
||||
} & {
|
||||
trigger: "submit-message" | "submit-tool-result" | "regenerate-message";
|
||||
messageId: string | undefined;
|
||||
} & ChatRequestOptions,
|
||||
): Promise<ReadableStream<UIMessageChunk>> {
|
||||
const prompt = await convertToModelMessages(options.messages);
|
||||
const result = streamText({
|
||||
model: this.model,
|
||||
messages: prompt,
|
||||
abortSignal: options.abortSignal,
|
||||
});
|
||||
return result.toUIMessageStream();
|
||||
}
|
||||
|
||||
async reconnectToStream(): Promise<ReadableStream<UIMessageChunk> | null> {
|
||||
return null; // client-side AI doesn't support stream reconnection
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Then use it in your component:
|
||||
|
||||
```typescript
|
||||
import { useChat } from "@ai-sdk/react";
|
||||
import { transformersJS, TransformersUIMessage } from "@browser-ai/transformers-js";
|
||||
|
||||
const model = transformersJS("HuggingFaceTB/SmolLM2-360M-Instruct", {
|
||||
device: "webgpu",
|
||||
worker: new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }),
|
||||
});
|
||||
|
||||
const { sendMessage, messages, stop } = useChat<TransformersUIMessage>({
|
||||
transport: new TransformersChatTransport(model),
|
||||
});
|
||||
```
|
||||
|
||||
## Browser compatibility fallback
|
||||
|
||||
If the device doesn't support in-browser inference, you can fall back to a server-side model:
|
||||
|
||||
```typescript
|
||||
import {
|
||||
transformersJS, TransformersUIMessage,
|
||||
doesBrowserSupportTransformersJS,
|
||||
} from "@browser-ai/transformers-js";
|
||||
|
||||
const { sendMessage, messages, stop } = useChat<TransformersUIMessage>({
|
||||
transport: doesBrowserSupportTransformersJS()
|
||||
? new TransformersChatTransport(model)
|
||||
: new DefaultChatTransport({ api: "/api/chat" }),
|
||||
});
|
||||
```
|
||||
|
||||
## Further reading
|
||||
|
||||
- [Building a Next.js AI Chatbot](../tutorials/next-ai-sdk) — a step-by-step tutorial building a full chatbot with tool calling
|
||||
- [`@browser-ai/transformers-js` documentation](https://www.browser-ai.dev/docs/ai-sdk-v6/transformers-js)
|
||||
- [Vercel AI SDK documentation](https://ai-sdk.dev/)
|
||||
@@ -0,0 +1,252 @@
|
||||
# The `pipeline` API
|
||||
|
||||
Just like the [transformers Python library](https://github.com/huggingface/transformers), Transformers.js provides users with a simple way to leverage the power of transformers. The `pipeline()` function is the easiest and fastest way to use a pretrained model for inference.
|
||||
|
||||
<Tip>
|
||||
|
||||
For the full list of available tasks/pipelines, check out [this table](#available-tasks).
|
||||
|
||||
</Tip>
|
||||
|
||||
## The basics
|
||||
|
||||
Start by creating an instance of `pipeline()` and specifying a task you want to use it for. For example, to create a sentiment analysis pipeline, you can do:
|
||||
|
||||
```javascript
|
||||
import { pipeline } from "@huggingface/transformers";
|
||||
|
||||
const classifier = await pipeline("sentiment-analysis");
|
||||
```
|
||||
|
||||
When running for the first time, the `pipeline` will download and cache the default pretrained model associated with the task. This can take a while, but subsequent calls will be much faster.
|
||||
|
||||
<Tip>
|
||||
|
||||
By default, models will be downloaded from the [Hugging Face Hub](https://huggingface.co/models) and stored in [browser cache](https://developer.mozilla.org/en-US/docs/Web/API/Cache), but there are ways to specify custom models and cache locations. For more information see [here](./custom_usage).
|
||||
|
||||
</Tip>
|
||||
|
||||
You can now use the classifier on your target text by calling it as a function:
|
||||
|
||||
```javascript
|
||||
const result = await classifier("I love transformers!");
|
||||
// [{'label': 'POSITIVE', 'score': 0.9998}]
|
||||
```
|
||||
|
||||
If you have multiple inputs, you can pass them as an array:
|
||||
|
||||
```javascript
|
||||
const result = await classifier([
|
||||
"I love transformers!",
|
||||
"I hate transformers!",
|
||||
]);
|
||||
// [{'label': 'POSITIVE', 'score': 0.9998}, {'label': 'NEGATIVE', 'score': 0.9982}]
|
||||
```
|
||||
|
||||
You can also specify a different model to use for the pipeline by passing it as the second argument to the `pipeline()` function. For example, to use a different model for sentiment analysis (like one trained to predict sentiment of a review as a number of stars between 1 and 5), you can do:
|
||||
|
||||
<!-- TODO: REPLACE 'nlptown/bert-base-multilingual-uncased-sentiment' with 'nlptown/bert-base-multilingual-uncased-sentiment'-->
|
||||
|
||||
```javascript
|
||||
const reviewer = await pipeline(
|
||||
"sentiment-analysis",
|
||||
"Xenova/bert-base-multilingual-uncased-sentiment",
|
||||
);
|
||||
|
||||
const result = await reviewer(
|
||||
"The Shawshank Redemption is a true masterpiece of cinema.",
|
||||
);
|
||||
// [{label: '5 stars', score: 0.8167929649353027}]
|
||||
```
|
||||
|
||||
Transformers.js supports loading any model hosted on the Hugging Face Hub, provided it has ONNX weights (located in a subfolder called `onnx`). For more information on how to convert your PyTorch, TensorFlow, or JAX model to ONNX, see the [conversion section](./custom_usage#convert-your-models-to-onnx).
|
||||
|
||||
The `pipeline()` function is a great way to quickly use a pretrained model for inference, as it takes care of all the preprocessing and postprocessing for you. For example, if you want to perform Automatic Speech Recognition (ASR) using OpenAI's Whisper model, you can do:
|
||||
|
||||
<!-- TODO: Replace 'Xenova/whisper-small.en' with 'openai/whisper-small.en' -->
|
||||
|
||||
```javascript
|
||||
// Create a pipeline for Automatic Speech Recognition
|
||||
const transcriber = await pipeline(
|
||||
"automatic-speech-recognition",
|
||||
"Xenova/whisper-small.en",
|
||||
);
|
||||
|
||||
// Transcribe an audio file, loaded from a URL.
|
||||
const result = await transcriber(
|
||||
"https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/mlk.flac",
|
||||
);
|
||||
// {text: ' I have a dream that one day this nation will rise up and live out the true meaning of its creed.'}
|
||||
```
|
||||
|
||||
## Pipeline options
|
||||
|
||||
### Loading
|
||||
|
||||
We offer a variety of options to control how models are loaded from the Hugging Face Hub (or locally).
|
||||
By default, when running in-browser, a _quantized_ version of the model is used, which is smaller and faster,
|
||||
but usually less accurate. To override this behaviour (i.e., use the unquantized model), you can use a custom
|
||||
`PretrainedOptions` object as the third parameter to the `pipeline` function:
|
||||
|
||||
```javascript
|
||||
// Create a pipeline for feature extraction, using the full-precision model (fp32)
|
||||
const pipe = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2", {
|
||||
dtype: "fp32",
|
||||
});
|
||||
```
|
||||
|
||||
Check out the section on [quantization](./guides/dtypes) to learn more.
|
||||
|
||||
You can also specify which revision of the model to use, by passing a `revision` parameter.
|
||||
Since the Hugging Face Hub uses a git-based versioning system, you can use any valid git revision specifier (e.g., branch name or commit hash).
|
||||
|
||||
```javascript
|
||||
const transcriber = await pipeline(
|
||||
"automatic-speech-recognition",
|
||||
"Xenova/whisper-tiny.en",
|
||||
{
|
||||
revision: "output_attentions",
|
||||
},
|
||||
);
|
||||
```
|
||||
|
||||
For the full list of options, check out the [PretrainedOptions](./api/utils/hub#module_utils/hub..PretrainedOptions) documentation.
|
||||
|
||||
### Running
|
||||
|
||||
Many pipelines have additional options that you can specify. For example, when using a model that does multilingual translation, you can specify the source and target languages like this:
|
||||
|
||||
<!-- TODO: Replace 'Xenova/nllb-200-distilled-600M' with 'facebook/nllb-200-distilled-600M' -->
|
||||
|
||||
```javascript
|
||||
// Create a pipeline for translation
|
||||
const translator = await pipeline(
|
||||
"translation",
|
||||
"Xenova/nllb-200-distilled-600M",
|
||||
);
|
||||
|
||||
// Translate from English to Greek
|
||||
const result = await translator("I like to walk my dog.", {
|
||||
src_lang: "eng_Latn",
|
||||
tgt_lang: "ell_Grek",
|
||||
});
|
||||
// [ { translation_text: 'Μου αρέσει να περπατάω το σκυλί μου.' } ]
|
||||
|
||||
// Translate back to English
|
||||
const result2 = await translator(result[0].translation_text, {
|
||||
src_lang: "ell_Grek",
|
||||
tgt_lang: "eng_Latn",
|
||||
});
|
||||
// [ { translation_text: 'I like to walk my dog.' } ]
|
||||
```
|
||||
|
||||
When using models that support auto-regressive generation, you can specify generation parameters like the number of new tokens, sampling methods, temperature, repetition penalty, and much more. For a full list of available parameters, see to the [GenerationConfig](./api/utils/generation#module_utils/generation.GenerationConfig) class.
|
||||
|
||||
For example, to generate a poem using `LaMini-Flan-T5-783M`, you can do:
|
||||
|
||||
<!-- TODO: Replace 'Xenova/LaMini-Flan-T5-783M' with 'MBZUAI/LaMini-Flan-T5-783M' -->
|
||||
|
||||
```javascript
|
||||
// Create a pipeline for text2text-generation
|
||||
const poet = await pipeline(
|
||||
"text2text-generation",
|
||||
"Xenova/LaMini-Flan-T5-783M",
|
||||
);
|
||||
const result = await poet("Write me a love poem about cheese.", {
|
||||
max_new_tokens: 200,
|
||||
temperature: 0.9,
|
||||
repetition_penalty: 2.0,
|
||||
no_repeat_ngram_size: 3,
|
||||
});
|
||||
```
|
||||
|
||||
Logging `result[0].generated_text` to the console gives:
|
||||
|
||||
```
|
||||
Cheese, oh cheese! You're the perfect comfort food.
|
||||
Your texture so smooth and creamy you can never get old.
|
||||
With every bite it melts in your mouth like buttery delights
|
||||
that make me feel right at home with this sweet treat of mine.
|
||||
|
||||
From classic to bold flavor combinations,
|
||||
I love how versatile you are as an ingredient too?
|
||||
Cheddar is my go-to for any occasion or mood;
|
||||
It adds depth and richness without being overpowering its taste buds alone
|
||||
```
|
||||
|
||||
### Streaming
|
||||
|
||||
Some pipelines such as `text-generation` or `automatic-speech-recognition` support streaming output. This is achieved using the `TextStreamer` class. For example, when using a chat model like `Qwen2.5-Coder-0.5B-Instruct`, you can specify a callback function that will be called with each generated token text (if unset, new tokens will be printed to the console).
|
||||
|
||||
```js
|
||||
import { pipeline, TextStreamer } from "@huggingface/transformers";
|
||||
|
||||
// Create a text generation pipeline
|
||||
const generator = await pipeline(
|
||||
"text-generation",
|
||||
"onnx-community/Qwen2.5-Coder-0.5B-Instruct",
|
||||
{ dtype: "q4" },
|
||||
);
|
||||
|
||||
// Define the list of messages
|
||||
const messages = [
|
||||
{ role: "system", content: "You are a helpful assistant." },
|
||||
{ role: "user", content: "Write a quick sort algorithm." },
|
||||
];
|
||||
|
||||
// Create text streamer
|
||||
const streamer = new TextStreamer(generator.tokenizer, {
|
||||
skip_prompt: true,
|
||||
// Optionally, do something with the text (e.g., write to a textbox)
|
||||
// callback_function: (text) => { /* Do something with text */ },
|
||||
});
|
||||
|
||||
// Generate a response
|
||||
const result = await generator(messages, {
|
||||
max_new_tokens: 512,
|
||||
do_sample: false,
|
||||
streamer,
|
||||
});
|
||||
```
|
||||
|
||||
Logging `result[0].generated_text` to the console gives:
|
||||
|
||||
<details>
|
||||
<summary>Click to view the console output</summary>
|
||||
<pre>
|
||||
Here's a simple implementation of the quick sort algorithm in Python:
|
||||
```python
|
||||
def quick_sort(arr):
|
||||
if len(arr) <= 1:
|
||||
return arr
|
||||
pivot = arr[len(arr) // 2]
|
||||
left = [x for x in arr if x < pivot]
|
||||
middle = [x for x in arr if x == pivot]
|
||||
right = [x for x in arr if x > pivot]
|
||||
return quick_sort(left) + middle + quick_sort(right)
|
||||
# Example usage:
|
||||
arr = [3, 6, 8, 10, 1, 2]
|
||||
sorted_arr = quick_sort(arr)
|
||||
print(sorted_arr)
|
||||
```
|
||||
### Explanation:
|
||||
- **Base Case**: If the array has less than or equal to one element (i.e., `len(arr)` is less than or equal to `1`), it is already sorted and can be returned as is.
|
||||
- **Pivot Selection**: The pivot is chosen as the middle element of the array.
|
||||
- **Partitioning**: The array is partitioned into three parts: elements less than the pivot (`left`), elements equal to the pivot (`middle`), and elements greater than the pivot (`right`). These partitions are then recursively sorted.
|
||||
- **Recursive Sorting**: The subarrays are sorted recursively using `quick_sort`.
|
||||
This approach ensures that each recursive call reduces the problem size by half until it reaches a base case.
|
||||
</pre>
|
||||
</details>
|
||||
|
||||
This streaming feature allows you to process the output as it is generated, rather than waiting for the entire output to be generated before processing it.
|
||||
|
||||
For more information on the available options for each pipeline, refer to the [API Reference](./api/pipelines).
|
||||
If you would like more control over the inference process, you can use the [`AutoModel`](./api/models), [`AutoTokenizer`](./api/tokenizers), or [`AutoProcessor`](./api/processors) classes instead.
|
||||
|
||||
## Available tasks
|
||||
|
||||
<include>
|
||||
{
|
||||
"path": "../snippets/4_supported-tasks.snippet"
|
||||
}
|
||||
</include>
|
||||
@@ -0,0 +1,3 @@
|
||||
# Building a browser extension
|
||||
|
||||
_Full tutorial coming soon..._ In the meantime, check out the example application: https://github.com/huggingface/transformers.js-examples/tree/main/browser-extension
|
||||
@@ -0,0 +1,3 @@
|
||||
# Building an Electron application
|
||||
|
||||
_Full tutorial coming soon..._ In the meantime, check out the example application: https://github.com/huggingface/transformers.js-examples/tree/main/electron
|
||||
@@ -0,0 +1,437 @@
|
||||
# Building a Next.js AI Chatbot with Vercel AI SDK
|
||||
|
||||
In this tutorial, we'll build an in-browser AI chatbot using Next.js, Transformers.js, and the Vercel AI SDK v6. The chatbot runs entirely client-side with WebGPU acceleration — and supports tool calling with human approval.
|
||||
|
||||
Useful links:
|
||||
- [Source code](https://github.com/huggingface/transformers.js-examples/tree/main/next-vercel-ai-sdk-v6-tool-calling)
|
||||
- [`@browser-ai/transformers-js` docs](https://www.browser-ai.dev/docs/ai-sdk-v6/transformers-js)
|
||||
- [Vercel AI SDK docs](https://ai-sdk.dev/)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [Node.js](https://nodejs.org/en/) version 18+
|
||||
- [npm](https://www.npmjs.com/) version 9+
|
||||
- A browser with WebGPU support (Chrome 113+, Edge 113+, or Firefox/Safari with flags enabled)
|
||||
|
||||
## Step 1: Create the project
|
||||
|
||||
Create a new Next.js application:
|
||||
|
||||
```bash
|
||||
npx create-next-app@latest next-ai-chatbot
|
||||
cd next-ai-chatbot
|
||||
```
|
||||
|
||||
Install the AI and Transformers.js dependencies:
|
||||
|
||||
```bash
|
||||
npm install ai @ai-sdk/react @browser-ai/transformers-js @huggingface/transformers zod
|
||||
```
|
||||
|
||||
## Step 2: Configure Next.js for browser inference
|
||||
|
||||
Transformers.js uses ONNX Runtime under the hood for both browser and server-side (Node.js) inference. In our case we only need the browser runtime so we can tell Next.js to exclude the Node.js-specific packages when bundling for the browser. Update `next.config.ts`
|
||||
|
||||
```typescript
|
||||
import type { NextConfig } from "next";
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
output: "export", // optional: export as a static site
|
||||
turbopack: {},
|
||||
webpack: (config) => {
|
||||
config.resolve.alias = {
|
||||
...config.resolve.alias,
|
||||
sharp$: false,
|
||||
"onnxruntime-node$": false,
|
||||
};
|
||||
return config;
|
||||
},
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
```
|
||||
|
||||
## Step 3: Create the Web Worker
|
||||
|
||||
Running model inference on the main thread would block the UI. The `@browser-ai/transformers-js` package provides a ready-made worker handler that handles all the complexity for you.
|
||||
|
||||
Create `src/app/worker.ts`:
|
||||
|
||||
```typescript
|
||||
import { TransformersJSWorkerHandler } from "@browser-ai/transformers-js";
|
||||
|
||||
const handler = new TransformersJSWorkerHandler();
|
||||
self.onmessage = (msg: MessageEvent) => {
|
||||
handler.onmessage(msg);
|
||||
};
|
||||
```
|
||||
|
||||
That's it — the handler takes care of model loading, inference, streaming, and communication with the main thread.
|
||||
|
||||
## Step 4: Define the model configuration
|
||||
|
||||
Create `src/app/models.ts` to define which models are available. These are ONNX-format models from Hugging Face:
|
||||
|
||||
```typescript
|
||||
import { WorkerLoadOptions } from "@browser-ai/transformers-js";
|
||||
|
||||
export interface ModelConfig extends Omit<WorkerLoadOptions, "modelId"> {
|
||||
id: string;
|
||||
name: string;
|
||||
supportsWorker?: boolean;
|
||||
}
|
||||
|
||||
export const MODELS: ModelConfig[] = [
|
||||
{
|
||||
id: "onnx-community/Qwen3-0.6B-ONNX",
|
||||
name: "Qwen3 0.6B",
|
||||
device: "webgpu",
|
||||
dtype: "q4f16",
|
||||
supportsWorker: true,
|
||||
},
|
||||
{
|
||||
id: "onnx-community/granite-4.0-350m-ONNX-web",
|
||||
name: "Granite 4.0 350M",
|
||||
device: "webgpu",
|
||||
dtype: "fp16",
|
||||
supportsWorker: true,
|
||||
},
|
||||
];
|
||||
```
|
||||
|
||||
<Tip>
|
||||
|
||||
For tool calling, use reasoning models like Qwen3 which handle multi-step reasoning well, or fine-tuned model specifically for tool-calling capabilities. The `supportsWorker` flag controls whether the model is loaded in a Web Worker for better performance.
|
||||
|
||||
</Tip>
|
||||
|
||||
## Step 5: Define tools
|
||||
|
||||
Create `src/app/tools.ts` with tools the model can call. Each tool uses [Zod](https://zod.dev/) for input validation:
|
||||
|
||||
```typescript
|
||||
import { tool } from "ai";
|
||||
import z from "zod";
|
||||
|
||||
export const createTools = () => ({
|
||||
getCurrentTime: tool({
|
||||
description: "Get the current date and time.",
|
||||
inputSchema: z.object({}),
|
||||
execute: async () => {
|
||||
const now = new Date();
|
||||
return {
|
||||
timestamp: now.toISOString(),
|
||||
date: now.toLocaleDateString("en-US", {
|
||||
weekday: "long", year: "numeric", month: "long", day: "numeric",
|
||||
}),
|
||||
time: now.toLocaleTimeString("en-US", {
|
||||
hour: "2-digit", minute: "2-digit", second: "2-digit", hour12: true,
|
||||
}),
|
||||
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
||||
};
|
||||
},
|
||||
}),
|
||||
randomNumber: tool({
|
||||
description: "Generate a random integer between min and max (inclusive).",
|
||||
inputSchema: z.object({
|
||||
min: z.number().describe("The minimum value (inclusive)"),
|
||||
max: z.number().describe("The maximum value (inclusive)"),
|
||||
}),
|
||||
execute: async ({ min, max }) => {
|
||||
return Math.floor(Math.random() * (Math.floor(max) - Math.ceil(min) + 1)) + Math.ceil(min);
|
||||
},
|
||||
}),
|
||||
getLocation: tool({
|
||||
description: "Get the user's current geographic location.",
|
||||
inputSchema: z.object({}),
|
||||
needsApproval: true, // requires user confirmation before executing
|
||||
execute: async () => {
|
||||
return new Promise((resolve, reject) => {
|
||||
navigator.geolocation.getCurrentPosition(
|
||||
(pos) => resolve({
|
||||
latitude: pos.coords.latitude,
|
||||
longitude: pos.coords.longitude,
|
||||
}),
|
||||
(err) => reject(err.message),
|
||||
);
|
||||
});
|
||||
},
|
||||
}),
|
||||
});
|
||||
```
|
||||
|
||||
The `getLocation` tool uses `needsApproval: true`, which means the AI SDK will pause execution and wait for the user to approve or reject the tool call before running it.
|
||||
|
||||
## Step 6: Create the chat transport
|
||||
|
||||
The Vercel AI SDK's `useChat` hook needs a [transport](https://ai-sdk.dev/docs/ai-sdk-ui/transport) that handles communication with the model. For client-side inference, we implement a custom `ChatTransport`.
|
||||
|
||||
Create `src/app/chat-transport.ts`:
|
||||
|
||||
```typescript
|
||||
import {
|
||||
ChatTransport, UIMessageChunk, streamText,
|
||||
convertToModelMessages, ChatRequestOptions,
|
||||
createUIMessageStream, stepCountIs,
|
||||
} from "ai";
|
||||
import {
|
||||
TransformersJSLanguageModel,
|
||||
TransformersUIMessage,
|
||||
transformersJS,
|
||||
} from "@browser-ai/transformers-js";
|
||||
import { MODELS } from "./models";
|
||||
import { createTools } from "./tools";
|
||||
|
||||
export class TransformersChatTransport
|
||||
implements ChatTransport<TransformersUIMessage>
|
||||
{
|
||||
private model: TransformersJSLanguageModel;
|
||||
private tools: ReturnType<typeof createTools>;
|
||||
|
||||
constructor() {
|
||||
const config = MODELS[0];
|
||||
this.model = transformersJS(config.id, {
|
||||
device: config.device,
|
||||
dtype: config.dtype,
|
||||
...(config.supportsWorker
|
||||
? {
|
||||
worker: new Worker(new URL("./worker.ts", import.meta.url), {
|
||||
type: "module",
|
||||
}),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
this.tools = createTools();
|
||||
}
|
||||
|
||||
async sendMessages(
|
||||
options: {
|
||||
chatId: string;
|
||||
messages: TransformersUIMessage[];
|
||||
abortSignal: AbortSignal | undefined;
|
||||
} & {
|
||||
trigger: "submit-message" | "submit-tool-result" | "regenerate-message";
|
||||
messageId: string | undefined;
|
||||
} & ChatRequestOptions,
|
||||
): Promise<ReadableStream<UIMessageChunk>> {
|
||||
const { messages, abortSignal } = options;
|
||||
const prompt = await convertToModelMessages(messages);
|
||||
|
||||
return createUIMessageStream<TransformersUIMessage>({
|
||||
execute: async ({ writer }) => {
|
||||
// Track download progress if the model hasn't been downloaded yet
|
||||
let downloadProgressId: string | undefined;
|
||||
const availability = await this.model.availability();
|
||||
|
||||
if (availability !== "available") {
|
||||
await this.model.createSessionWithProgress(
|
||||
(progress: number) => {
|
||||
const percent = Math.round(progress * 100);
|
||||
|
||||
if (progress >= 1) {
|
||||
if (downloadProgressId) {
|
||||
writer.write({
|
||||
type: "data-modelDownloadProgress",
|
||||
id: downloadProgressId,
|
||||
data: {
|
||||
status: "complete", progress: 100,
|
||||
message: "Model ready!",
|
||||
},
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (!downloadProgressId) {
|
||||
downloadProgressId = `download-${Date.now()}`;
|
||||
}
|
||||
|
||||
writer.write({
|
||||
type: "data-modelDownloadProgress",
|
||||
id: downloadProgressId,
|
||||
data: {
|
||||
status: "downloading", progress: percent,
|
||||
message: `Downloading model... ${percent}%`,
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const result = streamText({
|
||||
model: this.model,
|
||||
tools: this.tools,
|
||||
stopWhen: stepCountIs(5),
|
||||
messages: prompt,
|
||||
abortSignal,
|
||||
});
|
||||
|
||||
writer.merge(result.toUIMessageStream({ sendStart: false }));
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async reconnectToStream(): Promise<ReadableStream<UIMessageChunk> | null> {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Key parts of the transport:
|
||||
- **Availability check**: Determines if the model needs downloading before inference.
|
||||
- **Progress streaming**: Sends download progress as custom data parts (`data-modelDownloadProgress`) that the UI can render as a progress bar.
|
||||
- **Tool support**: Passes the tools to `streamText()` so the model can call them.
|
||||
- **Step limiting**: `stopWhen: stepCountIs(5)` prevents infinite tool-calling loops.
|
||||
|
||||
## Step 7: Build the chat UI
|
||||
|
||||
Now wire everything together in your page component. Create `src/app/page.tsx`:
|
||||
|
||||
```tsx
|
||||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
import { useChat } from "@ai-sdk/react";
|
||||
import { TransformersUIMessage } from "@browser-ai/transformers-js";
|
||||
import { lastAssistantMessageIsCompleteWithApprovalResponses } from "ai";
|
||||
import { TransformersChatTransport } from "./chat-transport";
|
||||
|
||||
export default function ChatPage() {
|
||||
const [input, setInput] = useState("");
|
||||
|
||||
const {
|
||||
messages,
|
||||
sendMessage,
|
||||
status,
|
||||
stop,
|
||||
addToolApprovalResponse,
|
||||
} = useChat<TransformersUIMessage>({
|
||||
transport: new TransformersChatTransport(),
|
||||
experimental_throttle: 75,
|
||||
// Automatically resumes after tool approval responses are submitted
|
||||
sendAutomaticallyWhen: lastAssistantMessageIsCompleteWithApprovalResponses,
|
||||
});
|
||||
|
||||
const handleSubmit = (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
if (input.trim() && status === "ready") {
|
||||
sendMessage({ text: input });
|
||||
setInput("");
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={{ maxWidth: 600, margin: "0 auto", padding: 24 }}>
|
||||
<h1>AI Chatbot</h1>
|
||||
|
||||
<div>
|
||||
{messages.map((message) => (
|
||||
<div key={message.id} style={{ marginBottom: 16 }}>
|
||||
<strong>{message.role === "user" ? "You" : "Assistant"}:</strong>
|
||||
{message.parts.map((part, i) => {
|
||||
switch (part.type) {
|
||||
case "text":
|
||||
return <p key={i}>{part.text}</p>;
|
||||
|
||||
case "data-modelDownloadProgress":
|
||||
if (!part.data.message) return null;
|
||||
return (
|
||||
<div key={i}>
|
||||
<p>{part.data.message}</p>
|
||||
{part.data.status === "downloading" && (
|
||||
<progress value={part.data.progress} max={100} />
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
|
||||
default:
|
||||
// Handle tool parts
|
||||
if (part.type.startsWith("tool-") && "state" in part) {
|
||||
if (
|
||||
part.state === "approval-requested" &&
|
||||
"approval" in part
|
||||
) {
|
||||
return (
|
||||
<div key={i} style={{ border: "1px solid #ccc", padding: 8 }}>
|
||||
<p>Tool <strong>{part.type.replace("tool-", "")}</strong> wants to run.</p>
|
||||
<button onClick={() =>
|
||||
addToolApprovalResponse({ id: part.approval!.id, approved: true })
|
||||
}>
|
||||
Approve
|
||||
</button>
|
||||
<button onClick={() =>
|
||||
addToolApprovalResponse({
|
||||
id: part.approval!.id, approved: false,
|
||||
reason: "User denied",
|
||||
})
|
||||
}>
|
||||
Deny
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if ("output" in part && part.output) {
|
||||
return (
|
||||
<pre key={i} style={{ background: "#f5f5f5", padding: 8 }}>
|
||||
{JSON.stringify(part.output, null, 2)}
|
||||
</pre>
|
||||
);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
})}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{status === "submitted" && <p><em>Thinking...</em></p>}
|
||||
|
||||
<form onSubmit={handleSubmit}>
|
||||
<input
|
||||
value={input}
|
||||
onChange={(e) => setInput(e.target.value)}
|
||||
placeholder="Ask something..."
|
||||
style={{ width: "100%", padding: 8 }}
|
||||
/>
|
||||
<div style={{ marginTop: 8 }}>
|
||||
{status === "streaming" ? (
|
||||
<button type="button" onClick={stop}>Stop</button>
|
||||
) : (
|
||||
<button type="submit" disabled={!input.trim()}>Send</button>
|
||||
)}
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
The component renders message parts based on their `type`:
|
||||
- `text` — standard text output from the model.
|
||||
- `data-modelDownloadProgress` — custom data parts sent by the transport during model download.
|
||||
- `tool-*` — tool call parts with states like `approval-requested`, `output-available`, etc.
|
||||
|
||||
The `sendAutomaticallyWhen: lastAssistantMessageIsCompleteWithApprovalResponses` option tells `useChat` to automatically resume generation after the user responds to a tool approval request.
|
||||
|
||||
## Step 8: Run the application
|
||||
|
||||
Start the development server:
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
Open your browser and navigate to the URL shown in the terminal. The first time you send a message, the model will be downloaded and cached in the browser. Subsequent visits will load the cached model.
|
||||
|
||||
Try prompts like:
|
||||
- "What time is it?"
|
||||
- "Generate a random number between 1 and 100"
|
||||
- "Where am I located?" (this will trigger a tool approval prompt)
|
||||
|
||||
## Next steps
|
||||
|
||||
- Add more models and a model selector — see the [full example source](https://github.com/huggingface/transformers.js-examples/tree/main/next-vercel-ai-sdk-v6-tool-calling) for a multi-model implementation with Zustand state management.
|
||||
- Add a browser compatibility check with `doesBrowserSupportTransformersJS()` and fall back to a server-side route if WebGPU is unavailable.
|
||||
- Explore the [Vercel AI SDK agents documentation](https://ai-sdk.dev/docs/agents/overview) for more complex agent patterns.
|
||||
- See the [Vercel AI SDK guide](../integrations/vercel-ai-sdk) for a reference of all supported features (embeddings, vision, transcription, etc.).
|
||||
@@ -0,0 +1,432 @@
|
||||
# Building a Next.js application
|
||||
|
||||
In this tutorial, we'll build a simple Next.js application that performs sentiment analysis using Transformers.js!
|
||||
Since Transformers.js can run in the browser or in Node.js, you can choose whether you want to perform inference [client-side](#client-side-inference) or [server-side](#server-side-inference) (we'll show you how to do both). In either case, we will be developing with the new [App Router](https://nextjs.org/docs/app) paradigm.
|
||||
The final product will look something like this:
|
||||
|
||||
|
||||
|
||||
Useful links:
|
||||
|
||||
- Demo site: [client-side](https://huggingface.co/spaces/Xenova/next-example-app) or [server-side](https://huggingface.co/spaces/Xenova/next-server-example-app)
|
||||
- Source code: [client-side](https://github.com/huggingface/transformers.js/tree/main/examples/next-client) or [server-side](https://github.com/huggingface/transformers.js/tree/main/examples/next-server)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [Node.js](https://nodejs.org/en/) version 18+
|
||||
- [npm](https://www.npmjs.com/) version 9+
|
||||
|
||||
## Client-side inference
|
||||
|
||||
### Step 1: Initialise the project
|
||||
|
||||
Start by creating a new Next.js application using `create-next-app`:
|
||||
|
||||
```bash
|
||||
npx create-next-app@latest
|
||||
```
|
||||
|
||||
On installation, you'll see various prompts. For this demo, we'll be selecting those shown below in bold:
|
||||
|
||||
<pre>√ What is your project named? ... next
|
||||
√ Would you like to use TypeScript? ... <b>No</b> / Yes
|
||||
√ Would you like to use ESLint? ... No / <b>Yes</b>
|
||||
√ Would you like to use Tailwind CSS? ... No / <b>Yes</b>
|
||||
√ Would you like to use `src/` directory? ... No / <b>Yes</b>
|
||||
√ Would you like to use App Router? (recommended) ... No / <b>Yes</b>
|
||||
√ Would you like to customize the default import alias? ... <b>No</b> / Yes
|
||||
</pre>
|
||||
|
||||
### Step 2: Install and configure Transformers.js
|
||||
|
||||
You can install Transformers.js from [NPM](https://www.npmjs.com/package/@huggingface/transformers) with the following command:
|
||||
|
||||
```bash
|
||||
npm i @huggingface/transformers
|
||||
```
|
||||
|
||||
We also need to update the `next.config.js` file to ignore node-specific modules when bundling for the browser:
|
||||
|
||||
```js
|
||||
/** @type {import('next').NextConfig} */
|
||||
const nextConfig = {
|
||||
// (Optional) Export as a static site
|
||||
// See https://nextjs.org/docs/pages/building-your-application/deploying/static-exports#configuration
|
||||
output: "export", // Feel free to modify/remove this option
|
||||
|
||||
// Override the default webpack configuration
|
||||
webpack: (config) => {
|
||||
// See https://webpack.js.org/configuration/resolve/#resolvealias
|
||||
config.resolve.alias = {
|
||||
...config.resolve.alias,
|
||||
sharp$: false,
|
||||
"onnxruntime-node$": false,
|
||||
};
|
||||
return config;
|
||||
},
|
||||
};
|
||||
|
||||
module.exports = nextConfig;
|
||||
```
|
||||
|
||||
Next, we'll create a new [Web Worker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) script where we'll place all ML-related code. This is to ensure that the main thread is not blocked while the model is loading and performing inference. For this application, we'll be using [`Xenova/distilbert-base-uncased-finetuned-sst-2-english`](https://huggingface.co/Xenova/distilbert-base-uncased-finetuned-sst-2-english), a ~67M parameter model finetuned on the [Stanford Sentiment Treebank](https://huggingface.co/datasets/sst) dataset. Add the following code to `./src/app/worker.js`:
|
||||
|
||||
```js
|
||||
import { pipeline, env } from "@huggingface/transformers";
|
||||
|
||||
// Skip local model check
|
||||
env.allowLocalModels = false;
|
||||
|
||||
// Use the Singleton pattern to enable lazy construction of the pipeline.
|
||||
class PipelineSingleton {
|
||||
static task = "text-classification";
|
||||
static model = "Xenova/distilbert-base-uncased-finetuned-sst-2-english";
|
||||
static instance = null;
|
||||
|
||||
static async getInstance(progress_callback = null) {
|
||||
if (this.instance === null) {
|
||||
this.instance = pipeline(this.task, this.model, { progress_callback });
|
||||
}
|
||||
return this.instance;
|
||||
}
|
||||
}
|
||||
|
||||
// Listen for messages from the main thread
|
||||
self.addEventListener("message", async (event) => {
|
||||
// Retrieve the classification pipeline. When called for the first time,
|
||||
// this will load the pipeline and save it for future use.
|
||||
let classifier = await PipelineSingleton.getInstance((x) => {
|
||||
// We also add a progress callback to the pipeline so that we can
|
||||
// track model loading.
|
||||
self.postMessage(x);
|
||||
});
|
||||
|
||||
// Actually perform the classification
|
||||
let output = await classifier(event.data.text);
|
||||
|
||||
// Send the output back to the main thread
|
||||
self.postMessage({
|
||||
status: "complete",
|
||||
output: output,
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Step 3: Design the user interface
|
||||
|
||||
We'll now modify the default `./src/app/page.js` file so that it connects to our worker thread. Since we'll only be performing in-browser inference, we can opt-in to Client components using the [`'use client'` directive](https://nextjs.org/docs/getting-started/react-essentials#the-use-client-directive).
|
||||
|
||||
```jsx
|
||||
'use client'
|
||||
|
||||
import { useState, useEffect, useRef, useCallback } from 'react'
|
||||
|
||||
export default function Home() {
|
||||
/* TODO: Add state variables */
|
||||
|
||||
// Create a reference to the worker object.
|
||||
const worker = useRef(null);
|
||||
|
||||
// We use the `useEffect` hook to set up the worker as soon as the `App` component is mounted.
|
||||
useEffect(() => {
|
||||
if (!worker.current) {
|
||||
// Create the worker if it does not yet exist.
|
||||
worker.current = new Worker(new URL('./worker.js', import.meta.url), {
|
||||
type: 'module'
|
||||
});
|
||||
}
|
||||
|
||||
// Create a callback function for messages from the worker thread.
|
||||
const onMessageReceived = (e) => { /* TODO: See below */};
|
||||
|
||||
// Attach the callback function as an event listener.
|
||||
worker.current.addEventListener('message', onMessageReceived);
|
||||
|
||||
// Define a cleanup function for when the component is unmounted.
|
||||
return () => worker.current.removeEventListener('message', onMessageReceived);
|
||||
});
|
||||
|
||||
const classify = useCallback((text) => {
|
||||
if (worker.current) {
|
||||
worker.current.postMessage({ text });
|
||||
}
|
||||
}, []);
|
||||
|
||||
return ( /* TODO: See below */ )
|
||||
}
|
||||
```
|
||||
|
||||
Initialise the following state variables at the beginning of the `Home` component:
|
||||
|
||||
```jsx
|
||||
// Keep track of the classification result and the model loading status.
|
||||
const [result, setResult] = useState(null);
|
||||
const [ready, setReady] = useState(null);
|
||||
```
|
||||
|
||||
and fill in the `onMessageReceived` function to update these variables when the worker thread sends a message:
|
||||
|
||||
```js
|
||||
const onMessageReceived = (e) => {
|
||||
switch (e.data.status) {
|
||||
case "initiate":
|
||||
setReady(false);
|
||||
break;
|
||||
case "ready":
|
||||
setReady(true);
|
||||
break;
|
||||
case "complete":
|
||||
setResult(e.data.output[0]);
|
||||
break;
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
Finally, we can add a simple UI to the `Home` component, consisting of an input textbox and a preformatted text element to display the classification result:
|
||||
|
||||
```jsx
|
||||
<main className="flex min-h-screen flex-col items-center justify-center p-12">
|
||||
<h1 className="text-5xl font-bold mb-2 text-center">Transformers.js</h1>
|
||||
<h2 className="text-2xl mb-4 text-center">Next.js template</h2>
|
||||
|
||||
<input
|
||||
className="w-full max-w-xs p-2 border border-gray-300 rounded mb-4"
|
||||
type="text"
|
||||
placeholder="Enter text here"
|
||||
onInput={(e) => {
|
||||
classify(e.target.value);
|
||||
}}
|
||||
/>
|
||||
|
||||
{ready !== null && (
|
||||
<pre className="bg-gray-100 p-2 rounded">
|
||||
{!ready || !result ? "Loading..." : JSON.stringify(result, null, 2)}
|
||||
</pre>
|
||||
)}
|
||||
</main>
|
||||
```
|
||||
|
||||
You can now run your application using the following command:
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
Visit the URL shown in the terminal (e.g., [http://localhost:3000/](http://localhost:3000/)) to see your application in action!
|
||||
|
||||
### (Optional) Step 4: Build and deploy
|
||||
|
||||
To build your application, simply run:
|
||||
|
||||
```bash
|
||||
npm run build
|
||||
```
|
||||
|
||||
This will bundle your application and output the static files to the `out` folder.
|
||||
|
||||
For this demo, we will deploy our application as a static [Hugging Face Space](https://huggingface.co/docs/hub/spaces), but you can deploy it anywhere you like! If you haven't already, you can create a free Hugging Face account [here](https://huggingface.co/join).
|
||||
|
||||
1. Visit [https://huggingface.co/new-space](https://huggingface.co/new-space) and fill in the form. Remember to select "Static" as the space type.
|
||||
2. Click the "Create space" button at the bottom of the page.
|
||||
3. Go to "Files" → "Add file" → "Upload files". Drag the files from the `out` folder into the upload box and click "Upload". After they have uploaded, scroll down to the button and click "Commit changes to main".
|
||||
|
||||
**That's it!** Your application should now be live at `https://huggingface.co/spaces/<your-username>/<your-space-name>`!
|
||||
|
||||
## Server-side inference
|
||||
|
||||
While there are many different ways to perform server-side inference, the simplest (which we will discuss in this tutorial) is using the new [Route Handlers](https://nextjs.org/docs/app/building-your-application/routing/router-handlers) feature.
|
||||
|
||||
### Step 1: Initialise the project
|
||||
|
||||
Start by creating a new Next.js application using `create-next-app`:
|
||||
|
||||
```bash
|
||||
npx create-next-app@latest
|
||||
```
|
||||
|
||||
On installation, you'll see various prompts. For this demo, we'll be selecting those shown below in bold:
|
||||
|
||||
<pre>√ What is your project named? ... next
|
||||
√ Would you like to use TypeScript? ... <b>No</b> / Yes
|
||||
√ Would you like to use ESLint? ... No / <b>Yes</b>
|
||||
√ Would you like to use Tailwind CSS? ... No / <b>Yes</b>
|
||||
√ Would you like to use `src/` directory? ... No / <b>Yes</b>
|
||||
√ Would you like to use App Router? (recommended) ... No / <b>Yes</b>
|
||||
√ Would you like to customize the default import alias? ... <b>No</b> / Yes
|
||||
</pre>
|
||||
|
||||
### Step 2: Install and configure Transformers.js
|
||||
|
||||
You can install Transformers.js from [NPM](https://www.npmjs.com/package/@huggingface/transformers) with the following command:
|
||||
|
||||
```bash
|
||||
npm i @huggingface/transformers
|
||||
```
|
||||
|
||||
We also need to update the `next.config.js` file to prevent Webpack from bundling certain packages:
|
||||
|
||||
```js
|
||||
/** @type {import('next').NextConfig} */
|
||||
const nextConfig = {
|
||||
// (Optional) Export as a standalone site
|
||||
// See https://nextjs.org/docs/pages/api-reference/next-config-js/output#automatically-copying-traced-files
|
||||
output: "standalone", // Feel free to modify/remove this option
|
||||
|
||||
// Indicate that these packages should not be bundled by webpack
|
||||
experimental: {
|
||||
serverComponentsExternalPackages: ["sharp", "onnxruntime-node"],
|
||||
},
|
||||
};
|
||||
|
||||
module.exports = nextConfig;
|
||||
```
|
||||
|
||||
Next, let's set up our Route Handler. We can do this by creating two files in a new `./src/app/classify/` directory:
|
||||
|
||||
1. `pipeline.js` - to handle the construction of our pipeline.
|
||||
|
||||
```js
|
||||
import { pipeline } from "@huggingface/transformers";
|
||||
|
||||
// Use the Singleton pattern to enable lazy construction of the pipeline.
|
||||
// NOTE: We wrap the class in a function to prevent code duplication (see below).
|
||||
const P = () =>
|
||||
class PipelineSingleton {
|
||||
static task = "text-classification";
|
||||
static model = "Xenova/distilbert-base-uncased-finetuned-sst-2-english";
|
||||
static instance = null;
|
||||
|
||||
static async getInstance(progress_callback = null) {
|
||||
if (this.instance === null) {
|
||||
this.instance = pipeline(this.task, this.model, {
|
||||
progress_callback,
|
||||
});
|
||||
}
|
||||
return this.instance;
|
||||
}
|
||||
};
|
||||
|
||||
let PipelineSingleton;
|
||||
if (process.env.NODE_ENV !== "production") {
|
||||
// When running in development mode, attach the pipeline to the
|
||||
// global object so that it's preserved between hot reloads.
|
||||
// For more information, see https://vercel.com/guides/nextjs-prisma-postgres
|
||||
if (!global.PipelineSingleton) {
|
||||
global.PipelineSingleton = P();
|
||||
}
|
||||
PipelineSingleton = global.PipelineSingleton;
|
||||
} else {
|
||||
PipelineSingleton = P();
|
||||
}
|
||||
export default PipelineSingleton;
|
||||
```
|
||||
|
||||
2. `route.js` - to process requests made to the `/classify` route.
|
||||
|
||||
```js
|
||||
import { NextResponse } from "next/server";
|
||||
import PipelineSingleton from "./pipeline.js";
|
||||
|
||||
export async function GET(request) {
|
||||
const text = request.nextUrl.searchParams.get("text");
|
||||
if (!text) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: "Missing text parameter",
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
// Get the classification pipeline. When called for the first time,
|
||||
// this will load the pipeline and cache it for future use.
|
||||
const classifier = await PipelineSingleton.getInstance();
|
||||
|
||||
// Actually perform the classification
|
||||
const result = await classifier(text);
|
||||
|
||||
return NextResponse.json(result);
|
||||
}
|
||||
```
|
||||
|
||||
### Step 3: Design the user interface
|
||||
|
||||
We'll now modify the default `./src/app/page.js` file to make requests to our newly-created Route Handler.
|
||||
|
||||
```jsx
|
||||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
|
||||
export default function Home() {
|
||||
// Keep track of the classification result and the model loading status.
|
||||
const [result, setResult] = useState(null);
|
||||
const [ready, setReady] = useState(null);
|
||||
|
||||
const classify = async (text) => {
|
||||
if (!text) return;
|
||||
if (ready === null) setReady(false);
|
||||
|
||||
// Make a request to the /classify route on the server.
|
||||
const result = await fetch(`/classify?text=${encodeURIComponent(text)}`);
|
||||
|
||||
// If this is the first time we've made a request, set the ready flag.
|
||||
if (!ready) setReady(true);
|
||||
|
||||
const json = await result.json();
|
||||
setResult(json);
|
||||
};
|
||||
return (
|
||||
<main className="flex min-h-screen flex-col items-center justify-center p-12">
|
||||
<h1 className="text-5xl font-bold mb-2 text-center">Transformers.js</h1>
|
||||
<h2 className="text-2xl mb-4 text-center">
|
||||
Next.js template (server-side)
|
||||
</h2>
|
||||
<input
|
||||
type="text"
|
||||
className="w-full max-w-xs p-2 border border-gray-300 rounded mb-4"
|
||||
placeholder="Enter text here"
|
||||
onInput={(e) => {
|
||||
classify(e.target.value);
|
||||
}}
|
||||
/>
|
||||
|
||||
{ready !== null && (
|
||||
<pre className="bg-gray-100 p-2 rounded">
|
||||
{!ready || !result ? "Loading..." : JSON.stringify(result, null, 2)}
|
||||
</pre>
|
||||
)}
|
||||
</main>
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
You can now run your application using the following command:
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
Visit the URL shown in the terminal (e.g., [http://localhost:3000/](http://localhost:3000/)) to see your application in action!
|
||||
|
||||
### (Optional) Step 4: Build and deploy
|
||||
|
||||
For this demo, we will build and deploy our application to [Hugging Face Spaces](https://huggingface.co/docs/hub/spaces). If you haven't already, you can create a free Hugging Face account [here](https://huggingface.co/join).
|
||||
|
||||
1. Create a new `Dockerfile` in your project's root folder. You can use our [example Dockerfile](https://github.com/huggingface/transformers.js/blob/main/examples/next-server/Dockerfile) as a template.
|
||||
2. Visit [https://huggingface.co/new-space](https://huggingface.co/new-space) and fill in the form. Remember to select "Docker" as the space type (you can choose the "Blank" Docker template).
|
||||
3. Click the "Create space" button at the bottom of the page.
|
||||
4. Go to "Files" → "Add file" → "Upload files". Drag the files from your project folder (excluding `node_modules` and `.next`, if present) into the upload box and click "Upload". After they have uploaded, scroll down to the button and click "Commit changes to main".
|
||||
5. Add the following lines to the top of your `README.md`:
|
||||
```
|
||||
---
|
||||
title: Next Server Example App
|
||||
emoji: 🔥
|
||||
colorFrom: yellow
|
||||
colorTo: red
|
||||
sdk: docker
|
||||
pinned: false
|
||||
app_port: 3000
|
||||
---
|
||||
```
|
||||
|
||||
**That's it!** Your application should now be live at `https://huggingface.co/spaces/<your-username>/<your-space-name>`!
|
||||
@@ -0,0 +1,214 @@
|
||||
# Server-side Inference in Node.js
|
||||
|
||||
Although Transformers.js was originally designed to be used in the browser, it's also able to run inference on the server. In this tutorial, we will design a simple Node.js API that uses Transformers.js for sentiment analysis.
|
||||
|
||||
We'll also show you how to use the library in both CommonJS and ECMAScript modules, so you can choose the module system that works best for your project:
|
||||
|
||||
- [ECMAScript modules (ESM)](#ecmascript-modules-esm) - The official standard format
|
||||
to package JavaScript code for reuse. It's the default module system in modern
|
||||
browsers, with modules imported using `import` and exported using `export`.
|
||||
Fortunately, starting with version 13.2.0, Node.js has stable support of ES modules.
|
||||
- [CommonJS](#commonjs) - The default module system in Node.js. In this system,
|
||||
modules are imported using `require()` and exported using `module.exports`.
|
||||
|
||||
<Tip>
|
||||
|
||||
Although you can always use the [Python library](https://github.com/huggingface/transformers) for server-side inference, using Transformers.js means that you can write all of your code in JavaScript (instead of having to set up and communicate with a separate Python process).
|
||||
|
||||
</Tip>
|
||||
|
||||
**Useful links:**
|
||||
|
||||
- Source code ([ESM](https://github.com/huggingface/transformers.js/tree/main/examples/node/esm/app.js) or [CommonJS](https://github.com/huggingface/transformers.js/tree/main/examples/node/commonjs/app.js))
|
||||
- [Documentation](https://huggingface.co/docs/transformers.js)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [Node.js](https://nodejs.org/en/) version 18+
|
||||
- [npm](https://www.npmjs.com/) version 9+
|
||||
|
||||
## Getting started
|
||||
|
||||
Let's start by creating a new Node.js project and installing Transformers.js via [NPM](https://www.npmjs.com/package/@huggingface/transformers):
|
||||
|
||||
```bash
|
||||
npm init -y
|
||||
npm i @huggingface/transformers
|
||||
```
|
||||
|
||||
Next, create a new file called `app.js`, which will be the entry point for our application. Depending on whether you're using [ECMAScript modules](#ecmascript-modules-esm) or [CommonJS](#commonjs), you will need to do some things differently (see below).
|
||||
|
||||
We'll also create a helper class called `MyClassificationPipeline` control the loading of the pipeline. It uses the [singleton pattern](https://en.wikipedia.org/wiki/Singleton_pattern) to lazily create a single instance of the pipeline when `getInstance` is first called, and uses this pipeline for all subsequent calls:
|
||||
|
||||
### ECMAScript modules (ESM)
|
||||
|
||||
To indicate that your project uses ECMAScript modules, you need to add `"type": "module"` to your `package.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
...
|
||||
"type": "module",
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
Next, you will need to add the following imports to the top of `app.js`:
|
||||
|
||||
```javascript
|
||||
import http from "http";
|
||||
import querystring from "querystring";
|
||||
import url from "url";
|
||||
```
|
||||
|
||||
Following that, let's import Transformers.js and define the `MyClassificationPipeline` class.
|
||||
|
||||
```javascript
|
||||
import { pipeline, env } from "@huggingface/transformers";
|
||||
|
||||
class MyClassificationPipeline {
|
||||
static task = "text-classification";
|
||||
static model = "Xenova/distilbert-base-uncased-finetuned-sst-2-english";
|
||||
static instance = null;
|
||||
|
||||
static async getInstance(progress_callback = null) {
|
||||
if (this.instance === null) {
|
||||
// NOTE: Uncomment this to change the cache directory
|
||||
// env.cacheDir = './.cache';
|
||||
|
||||
this.instance = pipeline(this.task, this.model, { progress_callback });
|
||||
}
|
||||
|
||||
return this.instance;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### CommonJS
|
||||
|
||||
Start by adding the following imports to the top of `app.js`:
|
||||
|
||||
```javascript
|
||||
const http = require("http");
|
||||
const querystring = require("querystring");
|
||||
const url = require("url");
|
||||
```
|
||||
|
||||
Following that, let's import Transformers.js and define the `MyClassificationPipeline` class. Since Transformers.js is an ESM module, we will need to dynamically import the library using the [`import()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/import) function:
|
||||
|
||||
```javascript
|
||||
class MyClassificationPipeline {
|
||||
static task = "text-classification";
|
||||
static model = "Xenova/distilbert-base-uncased-finetuned-sst-2-english";
|
||||
static instance = null;
|
||||
|
||||
static async getInstance(progress_callback = null) {
|
||||
if (this.instance === null) {
|
||||
// Dynamically import the Transformers.js library
|
||||
let { pipeline, env } = await import("@huggingface/transformers");
|
||||
|
||||
// NOTE: Uncomment this to change the cache directory
|
||||
// env.cacheDir = './.cache';
|
||||
|
||||
this.instance = pipeline(this.task, this.model, { progress_callback });
|
||||
}
|
||||
|
||||
return this.instance;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Creating a basic HTTP server
|
||||
|
||||
Next, let's create a basic server with the built-in [HTTP](https://nodejs.org/api/http.html#http) module. We will listen for requests made to the server (using the `/classify` endpoint), extract the `text` query parameter, and run this through the pipeline.
|
||||
|
||||
```javascript
|
||||
// Define the HTTP server
|
||||
const server = http.createServer();
|
||||
const hostname = "127.0.0.1";
|
||||
const port = 3000;
|
||||
|
||||
// Listen for requests made to the server
|
||||
server.on("request", async (req, res) => {
|
||||
// Parse the request URL
|
||||
const parsedUrl = url.parse(req.url);
|
||||
|
||||
// Extract the query parameters
|
||||
const { text } = querystring.parse(parsedUrl.query);
|
||||
|
||||
// Set the response headers
|
||||
res.setHeader("Content-Type", "application/json");
|
||||
|
||||
let response;
|
||||
if (parsedUrl.pathname === "/classify" && text) {
|
||||
const classifier = await MyClassificationPipeline.getInstance();
|
||||
response = await classifier(text);
|
||||
res.statusCode = 200;
|
||||
} else {
|
||||
response = { error: "Bad request" };
|
||||
res.statusCode = 400;
|
||||
}
|
||||
|
||||
// Send the JSON response
|
||||
res.end(JSON.stringify(response));
|
||||
});
|
||||
|
||||
server.listen(port, hostname, () => {
|
||||
console.log(`Server running at http://${hostname}:${port}/`);
|
||||
});
|
||||
```
|
||||
|
||||
<Tip>
|
||||
|
||||
Since we use lazy loading, the first request made to the server will also be responsible for loading the pipeline. If you would like to begin loading the pipeline as soon as the server starts running, you can add the following line of code after defining `MyClassificationPipeline`:
|
||||
|
||||
```javascript
|
||||
MyClassificationPipeline.getInstance();
|
||||
```
|
||||
|
||||
</Tip>
|
||||
|
||||
To start the server, run the following command:
|
||||
|
||||
```bash
|
||||
node app.js
|
||||
```
|
||||
|
||||
The server should be live at http://127.0.0.1:3000/, which you can visit in your web browser. You should see the following message:
|
||||
|
||||
```json
|
||||
{ "error": "Bad request" }
|
||||
```
|
||||
|
||||
This is because we aren't targeting the `/classify` endpoint with a valid `text` query parameter. Let's try again, this time with a valid request. For example, you can visit http://127.0.0.1:3000/classify?text=I%20love%20Transformers.js and you should see:
|
||||
|
||||
```json
|
||||
[{ "label": "POSITIVE", "score": 0.9996721148490906 }]
|
||||
```
|
||||
|
||||
Great! We've successfully created a basic HTTP server that uses Transformers.js to classify text.
|
||||
|
||||
## (Optional) Customization
|
||||
|
||||
### Model caching
|
||||
|
||||
By default, the first time you run the application, it will download the model files and cache them on your file system (in `./node_modules/@huggingface/transformers/.cache/`). All subsequent requests will then use this model. You can change the location of the cache by setting `env.cacheDir`. For example, to cache the model in the `.cache` directory in the current working directory, you can add:
|
||||
|
||||
```javascript
|
||||
env.cacheDir = "./.cache";
|
||||
```
|
||||
|
||||
### Use local models
|
||||
|
||||
If you want to use local model files, you can set `env.localModelPath` as follows:
|
||||
|
||||
```javascript
|
||||
// Specify a custom location for models (defaults to '/models/').
|
||||
env.localModelPath = "/path/to/models/";
|
||||
```
|
||||
|
||||
You can also disable loading of remote models by setting `env.allowRemoteModels` to `false`:
|
||||
|
||||
```javascript
|
||||
// Disable the loading of remote models from the Hugging Face Hub:
|
||||
env.allowRemoteModels = false;
|
||||
```
|
||||
@@ -0,0 +1,524 @@
|
||||
# Building a React application
|
||||
|
||||
In this tutorial, we'll be building a simple React application that performs multilingual translation using Transformers.js! The final product will look something like this:
|
||||
|
||||
|
||||
|
||||
Useful links:
|
||||
|
||||
- [Demo site](https://huggingface.co/spaces/Xenova/react-translator)
|
||||
- [Source code](https://github.com/huggingface/transformers.js-examples/tree/main/react-translator)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [Node.js](https://nodejs.org/en/) version 18+
|
||||
- [npm](https://www.npmjs.com/) version 9+
|
||||
|
||||
## Step 1: Initialise the project
|
||||
|
||||
For this tutorial, we will use [Vite](https://vitejs.dev/) to initialise our project. Vite is a build tool that allows us to quickly set up a React application with minimal configuration. Run the following command in your terminal:
|
||||
|
||||
```bash
|
||||
npm create vite@latest react-translator -- --template react
|
||||
```
|
||||
|
||||
If prompted to install `create-vite`, type <kbd>y</kbd> and press <kbd>Enter</kbd>.
|
||||
|
||||
Next, enter the project directory and install the necessary development dependencies:
|
||||
|
||||
```bash
|
||||
cd react-translator
|
||||
npm install
|
||||
```
|
||||
|
||||
To test that our application is working, we can run the following command:
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
Visiting the URL shown in the terminal (e.g., [http://localhost:5173/](http://localhost:5173/)) should show the default "React + Vite" landing page.
|
||||
You can stop the development server by pressing <kbd>Ctrl</kbd> + <kbd>C</kbd> in the terminal.
|
||||
|
||||
## Step 2: Install and configure Transformers.js
|
||||
|
||||
Now we get to the fun part: adding machine learning to our application! First, install Transformers.js from [NPM](https://www.npmjs.com/package/@huggingface/transformers) with the following command:
|
||||
|
||||
```bash
|
||||
npm install @huggingface/transformers
|
||||
```
|
||||
|
||||
For this application, we will use the [Xenova/nllb-200-distilled-600M](https://huggingface.co/Xenova/nllb-200-distilled-600M) model, which can perform multilingual translation among 200 languages. Before we start, there are 2 things we need to take note of:
|
||||
|
||||
1. ML inference can be quite computationally intensive, so it's better to load and run the models in a separate thread from the main (UI) thread.
|
||||
2. Since the model is quite large (>1 GB), we don't want to download it until the user clicks the "Translate" button.
|
||||
|
||||
We can achieve both of these goals by using a [Web Worker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) and some [React hooks](https://react.dev/reference/react).
|
||||
|
||||
1. Create a file called `worker.js` in the `src` directory. This script will do all the heavy-lifting for us, including loading and running of the translation pipeline. To ensure the model is only loaded once, we will create the `MyTranslationPipeline` class which use the [singleton pattern](https://en.wikipedia.org/wiki/Singleton_pattern) to lazily create a single instance of the pipeline when `getInstance` is first called, and use this pipeline for all subsequent calls:
|
||||
|
||||
```javascript
|
||||
import { pipeline, TextStreamer } from "@huggingface/transformers";
|
||||
|
||||
class MyTranslationPipeline {
|
||||
static task = "translation";
|
||||
static model = "Xenova/nllb-200-distilled-600M";
|
||||
static instance = null;
|
||||
|
||||
static async getInstance(progress_callback = null) {
|
||||
this.instance ??= pipeline(this.task, this.model, { progress_callback });
|
||||
return this.instance;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
2. Modify `App.jsx` in the `src` directory. This file is automatically created when initializing our React project, and will contain some boilerplate code. Inside the `App` function, let's create the web worker and store a reference to it using the `useRef` hook:
|
||||
|
||||
```jsx
|
||||
// Remember to import the relevant hooks
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import './App.css'
|
||||
|
||||
function App() {
|
||||
// Create a reference to the worker object.
|
||||
const worker = useRef(null);
|
||||
|
||||
// We use the `useEffect` hook to setup the worker as soon as the `App` component is mounted.
|
||||
useEffect(() => {
|
||||
// Create the worker if it does not yet exist.
|
||||
worker.current ??= new Worker(new URL('./worker.js', import.meta.url), {
|
||||
type: 'module'
|
||||
});
|
||||
|
||||
// Create a callback function for messages from the worker thread.
|
||||
const onMessageReceived = (e) => {
|
||||
// TODO: Will fill in later
|
||||
};
|
||||
|
||||
// Attach the callback function as an event listener.
|
||||
worker.current.addEventListener('message', onMessageReceived);
|
||||
|
||||
// Define a cleanup function for when the component is unmounted.
|
||||
return () => worker.current.removeEventListener('message', onMessageReceived);
|
||||
});
|
||||
|
||||
return (
|
||||
// TODO: Rest of our app goes here...
|
||||
)
|
||||
}
|
||||
|
||||
export default App
|
||||
|
||||
```
|
||||
|
||||
## Step 3: Design the user interface
|
||||
|
||||
<Tip>
|
||||
|
||||
We recommend starting the development server again with `npm run dev`
|
||||
(if not already running) so that you can see your changes in real-time.
|
||||
|
||||
</Tip>
|
||||
|
||||
First, let's define our components. Create a folder called `components` in the `src` directory, and create the following files:
|
||||
|
||||
1. `LanguageSelector.jsx`: This component will allow the user to select the input and output languages. Check out the full list of languages [here](https://github.com/huggingface/transformers.js-examples/tree/main/react-translator/src/components/LanguageSelector.jsx).
|
||||
|
||||
```jsx
|
||||
const LANGUAGES = {
|
||||
"Acehnese (Arabic script)": "ace_Arab",
|
||||
"Acehnese (Latin script)": "ace_Latn",
|
||||
"Afrikaans": "afr_Latn",
|
||||
...
|
||||
"Zulu": "zul_Latn",
|
||||
}
|
||||
|
||||
export default function LanguageSelector({ type, onChange, defaultLanguage }) {
|
||||
return (
|
||||
<div className='language-selector'>
|
||||
<label>{type}: </label>
|
||||
<select onChange={onChange} defaultValue={defaultLanguage}>
|
||||
{Object.entries(LANGUAGES).map(([key, value]) => {
|
||||
return <option key={key} value={value}>{key}</option>
|
||||
})}
|
||||
</select>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
2. `Progress.jsx`: This component will display the progress for downloading each model file.
|
||||
```jsx
|
||||
export default function Progress({ text, percentage }) {
|
||||
percentage = percentage ?? 0;
|
||||
return (
|
||||
<div className="progress-container">
|
||||
<div className="progress-bar" style={{ width: `${percentage}%` }}>
|
||||
{text} ({`${percentage.toFixed(2)}%`})
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
We can now use these components in `App.jsx` by adding these imports to the top of the file:
|
||||
|
||||
```jsx
|
||||
import LanguageSelector from "./components/LanguageSelector";
|
||||
import Progress from "./components/Progress";
|
||||
```
|
||||
|
||||
Let's also add some state variables to keep track of a few things in our application, like model loading, languages, input text, and output text. Add the following code to the beginning of the `App` function in `src/App.jsx`:
|
||||
|
||||
```jsx
|
||||
function App() {
|
||||
// Model loading
|
||||
const [ready, setReady] = useState(null);
|
||||
const [disabled, setDisabled] = useState(false);
|
||||
const [progressItems, setProgressItems] = useState([]);
|
||||
|
||||
// Inputs and outputs
|
||||
const [input, setInput] = useState("I love walking my dog.");
|
||||
const [sourceLanguage, setSourceLanguage] = useState("eng_Latn");
|
||||
const [targetLanguage, setTargetLanguage] = useState("fra_Latn");
|
||||
const [output, setOutput] = useState("");
|
||||
|
||||
// rest of the code...
|
||||
}
|
||||
```
|
||||
|
||||
Next, we can add our custom components to the main `App` component. We will also add two `textarea` elements for input and output text, and a `button` to trigger the translation. Modify the `return` statement to look like this:
|
||||
|
||||
```jsx
|
||||
return (
|
||||
<>
|
||||
<h1>Transformers.js</h1>
|
||||
<h2>ML-powered multilingual translation in React!</h2>
|
||||
|
||||
<div className="container">
|
||||
<div className="language-container">
|
||||
<LanguageSelector
|
||||
type={"Source"}
|
||||
defaultLanguage={"eng_Latn"}
|
||||
onChange={(x) => setSourceLanguage(x.target.value)}
|
||||
/>
|
||||
<LanguageSelector
|
||||
type={"Target"}
|
||||
defaultLanguage={"fra_Latn"}
|
||||
onChange={(x) => setTargetLanguage(x.target.value)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="textbox-container">
|
||||
<textarea
|
||||
value={input}
|
||||
rows={3}
|
||||
onChange={(e) => setInput(e.target.value)}
|
||||
></textarea>
|
||||
<textarea value={output} rows={3} readOnly></textarea>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<button disabled={disabled} onClick={translate}>
|
||||
Translate
|
||||
</button>
|
||||
|
||||
<div className="progress-bars-container">
|
||||
{ready === false && <label>Loading models... (only run once)</label>}
|
||||
{progressItems.map((data) => (
|
||||
<div key={data.file}>
|
||||
<Progress text={data.file} percentage={data.progress} />
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
```
|
||||
|
||||
Don't worry about the `translate` function for now. We will define it in the next section.
|
||||
|
||||
Finally, we can add some CSS to make our app look a little nicer. Modify the following files in the `src` directory:
|
||||
|
||||
1. `index.css`:
|
||||
<details>
|
||||
<summary>View code</summary>
|
||||
|
||||
```css
|
||||
:root {
|
||||
font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
|
||||
line-height: 1.5;
|
||||
font-weight: 400;
|
||||
color: #213547;
|
||||
background-color: #ffffff;
|
||||
|
||||
font-synthesis: none;
|
||||
text-rendering: optimizeLegibility;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
-moz-osx-font-smoothing: grayscale;
|
||||
-webkit-text-size-adjust: 100%;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
display: flex;
|
||||
place-items: center;
|
||||
min-width: 320px;
|
||||
min-height: 100vh;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: 3.2em;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
h1,
|
||||
h2 {
|
||||
margin: 8px;
|
||||
}
|
||||
|
||||
select {
|
||||
padding: 0.3em;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
textarea {
|
||||
padding: 0.6em;
|
||||
}
|
||||
|
||||
button {
|
||||
padding: 0.6em 1.2em;
|
||||
cursor: pointer;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
button[disabled] {
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
select,
|
||||
textarea,
|
||||
button {
|
||||
border-radius: 8px;
|
||||
border: 1px solid transparent;
|
||||
font-size: 1em;
|
||||
font-family: inherit;
|
||||
background-color: #f9f9f9;
|
||||
transition: border-color 0.25s;
|
||||
}
|
||||
|
||||
select:hover,
|
||||
textarea:hover,
|
||||
button:not([disabled]):hover {
|
||||
border-color: #646cff;
|
||||
}
|
||||
|
||||
select:focus,
|
||||
select:focus-visible,
|
||||
textarea:focus,
|
||||
textarea:focus-visible,
|
||||
button:focus,
|
||||
button:focus-visible {
|
||||
outline: 4px auto -webkit-focus-ring-color;
|
||||
}
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
1. `App.css`
|
||||
<details>
|
||||
<summary>View code</summary>
|
||||
|
||||
```css
|
||||
#root {
|
||||
max-width: 1280px;
|
||||
margin: 0 auto;
|
||||
padding: 2rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.language-container {
|
||||
display: flex;
|
||||
gap: 20px;
|
||||
}
|
||||
|
||||
.textbox-container {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
gap: 20px;
|
||||
width: 800px;
|
||||
}
|
||||
|
||||
.textbox-container > textarea,
|
||||
.language-selector {
|
||||
width: 50%;
|
||||
}
|
||||
|
||||
.language-selector > select {
|
||||
width: 150px;
|
||||
}
|
||||
|
||||
.progress-container {
|
||||
position: relative;
|
||||
font-size: 14px;
|
||||
color: white;
|
||||
background-color: #e9ecef;
|
||||
border: solid 1px;
|
||||
border-radius: 8px;
|
||||
text-align: left;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.progress-bar {
|
||||
padding: 0 4px;
|
||||
z-index: 0;
|
||||
top: 0;
|
||||
width: 1%;
|
||||
overflow: hidden;
|
||||
background-color: #007bff;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.progress-text {
|
||||
z-index: 2;
|
||||
}
|
||||
|
||||
.selector-container {
|
||||
display: flex;
|
||||
gap: 20px;
|
||||
}
|
||||
|
||||
.progress-bars-container {
|
||||
padding: 8px;
|
||||
height: 140px;
|
||||
}
|
||||
|
||||
.container {
|
||||
margin: 25px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
}
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
## Step 4: Connecting everything together
|
||||
|
||||
Now that we have a basic user interface set up, we can finally connect everything together.
|
||||
|
||||
First, let's define the `translate` function, which will be called when the user clicks the `Translate` button. This sends a message (containing the input text, source language, and target language) to the worker thread for processing. We will also disable the button so the user doesn't click it multiple times. Add the following code just before the `return` statement in the `App` function:
|
||||
|
||||
```jsx
|
||||
const translate = () => {
|
||||
setDisabled(true);
|
||||
setOutput("");
|
||||
worker.current.postMessage({
|
||||
text: input,
|
||||
src_lang: sourceLanguage,
|
||||
tgt_lang: targetLanguage,
|
||||
});
|
||||
};
|
||||
```
|
||||
|
||||
Now, let's add an event listener in `src/worker.js` to listen for messages from the main thread. We will send back messages (e.g., for model loading progress and text streaming) to the main thread with `self.postMessage`.
|
||||
|
||||
```javascript
|
||||
// Listen for messages from the main thread
|
||||
self.addEventListener("message", async (event) => {
|
||||
// Retrieve the translation pipeline. When called for the first time,
|
||||
// this will load the pipeline and save it for future use.
|
||||
const translator = await MyTranslationPipeline.getInstance((x) => {
|
||||
// We also add a progress callback to the pipeline so that we can
|
||||
// track model loading.
|
||||
self.postMessage(x);
|
||||
});
|
||||
|
||||
// Capture partial output as it streams from the pipeline
|
||||
const streamer = new TextStreamer(translator.tokenizer, {
|
||||
skip_prompt: true,
|
||||
skip_special_tokens: true,
|
||||
callback_function: function (text) {
|
||||
self.postMessage({
|
||||
status: "update",
|
||||
output: text,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// Actually perform the translation
|
||||
const output = await translator(event.data.text, {
|
||||
tgt_lang: event.data.tgt_lang,
|
||||
src_lang: event.data.src_lang,
|
||||
|
||||
// Allows for partial output to be captured
|
||||
streamer,
|
||||
});
|
||||
|
||||
// Send the output back to the main thread
|
||||
self.postMessage({
|
||||
status: "complete",
|
||||
output,
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
Finally, let's fill in our `onMessageReceived` function in `src/App.jsx`, which will update the application state in response to messages from the worker thread. Add the following code inside the `useEffect` hook we defined earlier:
|
||||
|
||||
```jsx
|
||||
const onMessageReceived = (e) => {
|
||||
switch (e.data.status) {
|
||||
case "initiate":
|
||||
// Model file start load: add a new progress item to the list.
|
||||
setReady(false);
|
||||
setProgressItems((prev) => [...prev, e.data]);
|
||||
break;
|
||||
|
||||
case "progress":
|
||||
// Model file progress: update one of the progress items.
|
||||
setProgressItems((prev) =>
|
||||
prev.map((item) => {
|
||||
if (item.file === e.data.file) {
|
||||
return { ...item, progress: e.data.progress };
|
||||
}
|
||||
return item;
|
||||
}),
|
||||
);
|
||||
break;
|
||||
|
||||
case "done":
|
||||
// Model file loaded: remove the progress item from the list.
|
||||
setProgressItems((prev) =>
|
||||
prev.filter((item) => item.file !== e.data.file),
|
||||
);
|
||||
break;
|
||||
|
||||
case "ready":
|
||||
// Pipeline ready: the worker is ready to accept messages.
|
||||
setReady(true);
|
||||
break;
|
||||
|
||||
case "update":
|
||||
// Generation update: update the output text.
|
||||
setOutput((o) => o + e.data.output);
|
||||
break;
|
||||
|
||||
case "complete":
|
||||
// Generation complete: re-enable the "Translate" button
|
||||
setDisabled(false);
|
||||
break;
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
You can now run the application with `npm run dev` and perform multilingual translation directly in your browser!
|
||||
|
||||
## (Optional) Step 5: Build and deploy
|
||||
|
||||
To build your application, simply run `npm run build`. This will bundle your application and output the static files to the `dist` folder.
|
||||
|
||||
For this demo, we will deploy our application as a static [Hugging Face Space](https://huggingface.co/docs/hub/spaces), but you can deploy it anywhere you like! If you haven't already, you can create a free Hugging Face account [here](https://huggingface.co/join).
|
||||
|
||||
1. Visit [https://huggingface.co/new-space](https://huggingface.co/new-space) and fill in the form. Remember to select "Static" as the space type.
|
||||
2. Go to "Files" → "Add file" → "Upload files". Drag the `index.html` file and `public/` folder from the `dist` folder into the upload box and click "Upload". After they have uploaded, scroll down to the button and click "Commit changes to main".
|
||||
|
||||
**That's it!** Your application should now be live at `https://huggingface.co/spaces/<your-username>/<your-space-name>`!
|
||||
@@ -0,0 +1,302 @@
|
||||
# Building a Vanilla JavaScript Application
|
||||
|
||||
In this tutorial, you’ll build a simple web application that detects objects in images using Transformers.js! To follow along, all you need is a code editor, a browser, and a simple server (e.g., VS Code Live Server).
|
||||
|
||||
Here's how it works: the user clicks “Upload image” and selects an image using an input dialog. After analysing the image with an object detection model, the predicted bounding boxes are overlaid on top of the image, like this:
|
||||
|
||||
|
||||
|
||||
Useful links:
|
||||
|
||||
- [Demo site](https://huggingface.co/spaces/Scrimba/vanilla-js-object-detector)
|
||||
- [Source code](https://github.com/huggingface/transformers.js-examples/tree/main/vanilla-js)
|
||||
|
||||
## Step 1: HTML and CSS setup
|
||||
|
||||
Before we start building with Transformers.js, we first need to lay the groundwork with some markup and styling. Create an `index.html` file with a basic HTML skeleton, and add the following `<main>` tag to the `<body>`:
|
||||
|
||||
```html
|
||||
<main class="container">
|
||||
<label class="custom-file-upload">
|
||||
<input id="file-upload" type="file" accept="image/*" />
|
||||
<img
|
||||
class="upload-icon"
|
||||
src="https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/upload-icon.png"
|
||||
/>
|
||||
Upload image
|
||||
</label>
|
||||
<div id="image-container"></div>
|
||||
<p id="status"></p>
|
||||
</main>
|
||||
```
|
||||
|
||||
<details>
|
||||
|
||||
<summary>Click here to see a breakdown of this markup.</summary>
|
||||
|
||||
We’re adding an `<input>` element with `type="file"` that accepts images. This allows the user to select an image from their local file system using a popup dialog. The default styling for this element looks quite bad, so let's add some styling. The easiest way to achieve this is to wrap the `<input>` element in a `<label>`, hide the input, and then style the label as a button.
|
||||
|
||||
We’re also adding an empty `<div>` container for displaying the image, plus an empty `<p>` tag that we'll use to give status updates to the user while we download and run the model, since both of these operations take some time.
|
||||
|
||||
</details>
|
||||
|
||||
Next, add the following CSS rules in a `style.css` file and link it to the HTML:
|
||||
|
||||
```css
|
||||
html,
|
||||
body {
|
||||
font-family: Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
.container {
|
||||
margin: 40px auto;
|
||||
width: max(50vw, 400px);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.custom-file-upload {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
border: 2px solid black;
|
||||
padding: 8px 16px;
|
||||
cursor: pointer;
|
||||
border-radius: 6px;
|
||||
}
|
||||
|
||||
#file-upload {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.upload-icon {
|
||||
width: 30px;
|
||||
}
|
||||
|
||||
#image-container {
|
||||
width: 100%;
|
||||
margin-top: 20px;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
#image-container > img {
|
||||
width: 100%;
|
||||
}
|
||||
```
|
||||
|
||||
Here's how the UI looks at this point:
|
||||
|
||||
|
||||
|
||||
## Step 2: JavaScript setup
|
||||
|
||||
With the _boring_ part out of the way, let's start writing some JavaScript code! Create a file called `index.js` and link to it in `index.html` by adding the following to the end of the `<body>`:
|
||||
|
||||
```html
|
||||
<script src="./index.js" type="module"></script>
|
||||
```
|
||||
|
||||
<Tip>
|
||||
|
||||
The `type="module"` attribute is important, as it turns our file into a [JavaScript module](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Modules), meaning that we’ll be able to use imports and exports.
|
||||
|
||||
</Tip>
|
||||
|
||||
Moving into `index.js`, let's import Transformers.js by adding the following line to the top of the file:
|
||||
|
||||
```js
|
||||
import {
|
||||
pipeline,
|
||||
env,
|
||||
} from "https://cdn.jsdelivr.net/npm/@huggingface/transformers";
|
||||
```
|
||||
|
||||
Since we will be downloading the model from the Hugging Face Hub, we can skip the local model check by setting:
|
||||
|
||||
```js
|
||||
env.allowLocalModels = false;
|
||||
```
|
||||
|
||||
Next, let's create references to the various DOM elements we will access later:
|
||||
|
||||
```js
|
||||
const fileUpload = document.getElementById("file-upload");
|
||||
const imageContainer = document.getElementById("image-container");
|
||||
const status = document.getElementById("status");
|
||||
```
|
||||
|
||||
## Step 3: Create an object detection pipeline
|
||||
|
||||
We’re finally ready to create our object detection pipeline! As a reminder, a [pipeline](../pipelines). is a high-level interface provided by the library to perform a specific task. In our case, we will instantiate an object detection pipeline with the `pipeline()` helper function.
|
||||
|
||||
Since this can take some time (especially the first time when we have to download the ~40MB model), we first update the `status` paragraph so that the user knows that we’re about to load the model.
|
||||
|
||||
```js
|
||||
status.textContent = "Loading model...";
|
||||
```
|
||||
|
||||
<Tip>
|
||||
|
||||
To keep this tutorial simple, we'll be loading and running the model in the main (UI) thread. This is not recommended for production applications, since the UI will freeze when we're performing these actions. This is because JavaScript is a single-threaded language. To overcome this, you can use a [web worker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) to download and run the model in the background. However, we’re not going to do cover that in this tutorial...
|
||||
|
||||
</Tip>
|
||||
|
||||
We can now call the `pipeline()` function that we imported at the top of our file, to create our object detection pipeline:
|
||||
|
||||
```js
|
||||
const detector = await pipeline("object-detection", "Xenova/detr-resnet-50");
|
||||
```
|
||||
|
||||
We’re passing two arguments into the `pipeline()` function: (1) task and (2) model.
|
||||
|
||||
1. The first tells Transformers.js what kind of task we want to perform. In our case, that is `object-detection`, but there are many other tasks that the library supports, including `text-generation`, `sentiment-analysis`, `summarization`, or `automatic-speech-recognition`. See [here](https://huggingface.co/docs/transformers.js/pipelines#tasks) for the full list.
|
||||
|
||||
2. The second argument specifies which model we would like to use to solve the given task. We will use [`Xenova/detr-resnet-50`](https://huggingface.co/Xenova/detr-resnet-50), as it is a relatively small (~40MB) but powerful model for detecting objects in an image.
|
||||
|
||||
Once the function returns, we’ll tell the user that the app is ready to be used.
|
||||
|
||||
```js
|
||||
status.textContent = "Ready";
|
||||
```
|
||||
|
||||
## Step 4: Create the image uploader
|
||||
|
||||
The next step is to support uploading/selection of images. To achieve this, we will listen for "change" events from the `fileUpload` element. In the callback function, we use a `FileReader()` to read the contents of the image if one is selected (and nothing otherwise).
|
||||
|
||||
```js
|
||||
fileUpload.addEventListener("change", function (e) {
|
||||
const file = e.target.files[0];
|
||||
if (!file) {
|
||||
return;
|
||||
}
|
||||
|
||||
const reader = new FileReader();
|
||||
|
||||
// Set up a callback when the file is loaded
|
||||
reader.onload = function (e2) {
|
||||
imageContainer.innerHTML = "";
|
||||
const image = document.createElement("img");
|
||||
image.src = e2.target.result;
|
||||
imageContainer.appendChild(image);
|
||||
// detect(image); // Uncomment this line to run the model
|
||||
};
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
```
|
||||
|
||||
Once the image has been loaded into the browser, the `reader.onload` callback function will be invoked. In it, we append the new `<img>` element to the `imageContainer` to be displayed to the user.
|
||||
|
||||
Don’t worry about the `detect(image)` function call (which is commented out) - we will explain it later! For now, try to run the app and upload an image to the browser. You should see your image displayed under the button like this:
|
||||
|
||||
|
||||
|
||||
## Step 5: Run the model
|
||||
|
||||
We’re finally ready to start interacting with Transformers.js! Let’s uncomment the `detect(image)` function call from the snippet above. Then we’ll define the function itself:
|
||||
|
||||
```js
|
||||
async function detect(img) {
|
||||
status.textContent = "Analysing...";
|
||||
const output = await detector(img.src, {
|
||||
threshold: 0.5,
|
||||
percentage: true,
|
||||
});
|
||||
status.textContent = "";
|
||||
console.log("output", output);
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
<Tip>
|
||||
|
||||
NOTE: The `detect` function needs to be asynchronous, since we’ll `await` the result of the the model.
|
||||
|
||||
</Tip>
|
||||
|
||||
Once we’ve updated the `status` to "Analysing", we’re ready to perform _inference_, which simply means to run the model with some data. This is done via the `detector()` function that was returned from `pipeline()`. The first argument we’re passing is the image data (`img.src`).
|
||||
|
||||
The second argument is an options object:
|
||||
|
||||
- We set the `threshold` property to `0.5`. This means that we want the model to be at least 50% confident before claiming it has detected an object in the image. The lower the threshold, the more objects it'll detect (but may misidentify objects); the higher the threshold, the fewer objects it'll detect (but may miss objects in the scene).
|
||||
- We also specify `percentage: true`, which means that we want the bounding box for the objects to be returned as percentages (instead of pixels).
|
||||
|
||||
If you now try to run the app and upload an image, you should see the following output logged to the console:
|
||||
|
||||
|
||||
|
||||
In the example above, we uploaded an image of two elephants, so the `output` variable holds an array with two objects, each containing a `label` (the string “elephant”), a `score` (indicating the model's confidence in its prediction) and a `box` object (representing the bounding box of the detected entity).
|
||||
|
||||
## Step 6: Render the boxes
|
||||
|
||||
The final step is to display the `box` coordinates as rectangles around each of the elephants.
|
||||
|
||||
At the end of our `detect()` function, we’ll run the `renderBox` function on each object in the `output` array, using `.forEach()`.
|
||||
|
||||
```js
|
||||
output.forEach(renderBox);
|
||||
```
|
||||
|
||||
Here’s the code for the `renderBox()` function with comments to help you understand what’s going on:
|
||||
|
||||
```js
|
||||
// Render a bounding box and label on the image
|
||||
function renderBox({ box, label }) {
|
||||
const { xmax, xmin, ymax, ymin } = box;
|
||||
|
||||
// Generate a random color for the box
|
||||
const color =
|
||||
"#" +
|
||||
Math.floor(Math.random() * 0xffffff)
|
||||
.toString(16)
|
||||
.padStart(6, 0);
|
||||
|
||||
// Draw the box
|
||||
const boxElement = document.createElement("div");
|
||||
boxElement.className = "bounding-box";
|
||||
Object.assign(boxElement.style, {
|
||||
borderColor: color,
|
||||
left: 100 * xmin + "%",
|
||||
top: 100 * ymin + "%",
|
||||
width: 100 * (xmax - xmin) + "%",
|
||||
height: 100 * (ymax - ymin) + "%",
|
||||
});
|
||||
|
||||
// Draw the label
|
||||
const labelElement = document.createElement("span");
|
||||
labelElement.textContent = label;
|
||||
labelElement.className = "bounding-box-label";
|
||||
labelElement.style.backgroundColor = color;
|
||||
|
||||
boxElement.appendChild(labelElement);
|
||||
imageContainer.appendChild(boxElement);
|
||||
}
|
||||
```
|
||||
|
||||
The bounding box and label span also need some styling, so add the following to the `style.css` file:
|
||||
|
||||
```css
|
||||
.bounding-box {
|
||||
position: absolute;
|
||||
box-sizing: border-box;
|
||||
border-width: 2px;
|
||||
border-style: solid;
|
||||
}
|
||||
|
||||
.bounding-box-label {
|
||||
color: white;
|
||||
position: absolute;
|
||||
font-size: 12px;
|
||||
margin-top: -16px;
|
||||
margin-left: -2px;
|
||||
padding: 1px;
|
||||
}
|
||||
```
|
||||
|
||||
**And that’s it!**
|
||||
|
||||
You've now built your own fully-functional AI application that detects objects in images, which runs completely in your browser: no external server, APIs, or build tools. Pretty cool! 🥳
|
||||
|
||||
|
||||
|
||||
The app is live at the following URL: [https://huggingface.co/spaces/Scrimba/vanilla-js-object-detector](https://huggingface.co/spaces/Scrimba/vanilla-js-object-detector)
|
||||
@@ -0,0 +1,192 @@
|
||||
/*
|
||||
* For a detailed explanation regarding each configuration property, visit:
|
||||
* https://jestjs.io/docs/configuration
|
||||
*/
|
||||
|
||||
/** @type {import('jest').Config} */
|
||||
export default {
|
||||
// All imported modules in your tests should be mocked automatically
|
||||
// automock: false,
|
||||
|
||||
// Stop running tests after `n` failures
|
||||
// bail: 0,
|
||||
|
||||
// Automatically clear mock calls, instances, contexts and results before every test
|
||||
clearMocks: true,
|
||||
|
||||
// Indicates whether the coverage information should be collected while executing the test
|
||||
collectCoverage: true,
|
||||
|
||||
// An array of glob patterns indicating a set of files for which coverage information should be collected
|
||||
// collectCoverageFrom: undefined,
|
||||
|
||||
// The directory where Jest should output its coverage files
|
||||
coverageDirectory: "coverage",
|
||||
|
||||
// An array of regexp pattern strings used to skip coverage collection
|
||||
coveragePathIgnorePatterns: ["node_modules", "tests"],
|
||||
|
||||
// Indicates which provider should be used to instrument code for coverage
|
||||
coverageProvider: "v8",
|
||||
|
||||
// A list of reporter names that Jest uses when writing coverage reports
|
||||
// coverageReporters: [
|
||||
// "json",
|
||||
// "text",
|
||||
// "lcov",
|
||||
// "clover"
|
||||
// ],
|
||||
|
||||
// An object that configures minimum threshold enforcement for coverage results
|
||||
// coverageThreshold: undefined,
|
||||
|
||||
// A path to a custom dependency extractor
|
||||
// dependencyExtractor: undefined,
|
||||
|
||||
// Make calling deprecated APIs throw helpful error messages
|
||||
// errorOnDeprecated: false,
|
||||
|
||||
// The default configuration for fake timers
|
||||
// fakeTimers: {
|
||||
// "enableGlobally": false
|
||||
// },
|
||||
|
||||
// Force coverage collection from ignored files using an array of glob patterns
|
||||
// forceCoverageMatch: [],
|
||||
|
||||
// A path to a module which exports an async function that is triggered once before all test suites
|
||||
// globalSetup: undefined,
|
||||
|
||||
// A path to a module which exports an async function that is triggered once after all test suites
|
||||
// globalTeardown: undefined,
|
||||
|
||||
// A set of global variables that need to be available in all test environments
|
||||
// globals: {},
|
||||
|
||||
// The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
|
||||
// maxWorkers: "50%",
|
||||
|
||||
// An array of directory names to be searched recursively up from the requiring module's location
|
||||
// moduleDirectories: [
|
||||
// "node_modules"
|
||||
// ],
|
||||
|
||||
// An array of file extensions your modules use
|
||||
// moduleFileExtensions: [
|
||||
// "js",
|
||||
// "mjs",
|
||||
// "cjs",
|
||||
// "jsx",
|
||||
// "ts",
|
||||
// "tsx",
|
||||
// "json",
|
||||
// "node"
|
||||
// ],
|
||||
|
||||
// A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
|
||||
// moduleNameMapper: {},
|
||||
|
||||
// An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
|
||||
// modulePathIgnorePatterns: [],
|
||||
|
||||
// Activates notifications for test results
|
||||
// notify: false,
|
||||
|
||||
// An enum that specifies notification mode. Requires { notify: true }
|
||||
// notifyMode: "failure-change",
|
||||
|
||||
// A preset that is used as a base for Jest's configuration
|
||||
// preset: undefined,
|
||||
|
||||
// Run tests from one or more projects
|
||||
// projects: undefined,
|
||||
|
||||
// Use this configuration option to add custom reporters to Jest
|
||||
// reporters: undefined,
|
||||
|
||||
// Automatically reset mock state before every test
|
||||
// resetMocks: false,
|
||||
|
||||
// Reset the module registry before running each individual test
|
||||
// resetModules: false,
|
||||
|
||||
// A path to a custom resolver
|
||||
// resolver: undefined,
|
||||
|
||||
// Automatically restore mock state and implementation before every test
|
||||
// restoreMocks: false,
|
||||
|
||||
// The root directory that Jest should scan for tests and modules within
|
||||
// rootDir: undefined,
|
||||
|
||||
// A list of paths to directories that Jest should use to search for files in
|
||||
roots: ["./tests/"],
|
||||
|
||||
// Allows you to use a custom runner instead of Jest's default test runner
|
||||
// runner: "jest-runner",
|
||||
|
||||
// The paths to modules that run some code to configure or set up the testing environment before each test
|
||||
// setupFiles: [],
|
||||
|
||||
// A list of paths to modules that run some code to configure or set up the testing framework before each test
|
||||
// setupFilesAfterEnv: [],
|
||||
|
||||
// The number of seconds after which a test is considered as slow and reported as such in the results.
|
||||
// slowTestThreshold: 5,
|
||||
|
||||
// Timeout in milliseconds for each test.
|
||||
testTimeout: 32000,
|
||||
|
||||
// A list of paths to snapshot serializer modules Jest should use for snapshot testing
|
||||
// snapshotSerializers: [],
|
||||
|
||||
// The test environment that will be used for testing
|
||||
// testEnvironment: "jest-environment-node",
|
||||
|
||||
// Options that will be passed to the testEnvironment
|
||||
// testEnvironmentOptions: {},
|
||||
|
||||
// Adds a location field to test results
|
||||
// testLocationInResults: false,
|
||||
|
||||
// The glob patterns Jest uses to detect test files
|
||||
// testMatch: [
|
||||
// "**/__tests__/**/*.[jt]s?(x)",
|
||||
// "**/?(*.)+(spec|test).[tj]s?(x)"
|
||||
// ],
|
||||
|
||||
// An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
|
||||
// testPathIgnorePatterns: [
|
||||
// "\\\\node_modules\\\\"
|
||||
// ],
|
||||
|
||||
// The regexp pattern or array of patterns that Jest uses to detect test files
|
||||
// testRegex: [],
|
||||
|
||||
// This option allows the use of a custom results processor
|
||||
// testResultsProcessor: undefined,
|
||||
|
||||
// This option allows use of a custom test runner
|
||||
// testRunner: "jest-circus/runner",
|
||||
|
||||
// A map from regular expressions to paths to transformers
|
||||
transform: {},
|
||||
|
||||
// An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
|
||||
// transformIgnorePatterns: [
|
||||
// "\\\\node_modules\\\\",
|
||||
// "\\.pnp\\.[^\\\\]+$"
|
||||
// ],
|
||||
|
||||
// An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
|
||||
// unmockedModulePathPatterns: undefined,
|
||||
|
||||
// Indicates whether each individual test should be reported during the run
|
||||
// verbose: undefined,
|
||||
|
||||
// An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
|
||||
// watchPathIgnorePatterns: [],
|
||||
|
||||
// Whether to use watchman for file crawling
|
||||
// watchman: true,
|
||||
};
|
||||
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"name": "@huggingface/transformers",
|
||||
"version": "4.2.0",
|
||||
"description": "State-of-the-art Machine Learning for the web. Run 🤗 Transformers directly in your browser, with no need for a server!",
|
||||
"main": "./dist/transformers.node.cjs",
|
||||
"types": "./types/transformers.d.ts",
|
||||
"type": "module",
|
||||
"exports": {
|
||||
"node": {
|
||||
"import": {
|
||||
"types": "./types/transformers.d.ts",
|
||||
"default": "./dist/transformers.node.mjs"
|
||||
},
|
||||
"require": {
|
||||
"types": "./types/transformers.d.ts",
|
||||
"default": "./dist/transformers.node.cjs"
|
||||
}
|
||||
},
|
||||
"default": {
|
||||
"types": "./types/transformers.d.ts",
|
||||
"default": "./dist/transformers.web.js"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"format": "prettier --write . --ignore-path ../../.prettierignore",
|
||||
"format:check": "prettier --check . --ignore-path ../../.prettierignore",
|
||||
"typegen": "tsc --build",
|
||||
"dev": "node scripts/dev.mjs",
|
||||
"build": "node scripts/build.mjs && pnpm typegen",
|
||||
"test": "node --experimental-vm-modules --expose-gc node_modules/jest/bin/jest.js --verbose --logHeapUsage",
|
||||
"readme": "python ./docs/scripts/build_readme.py",
|
||||
"docs-api": "node ./docs/scripts/generate.js",
|
||||
"docs-build": "doc-builder build transformers.js ./docs/source/ --not_python_module --build_dir ./docs/build/",
|
||||
"docs-preview": "doc-builder preview transformers.js ./docs/source/ --not_python_module"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/huggingface/transformers.js.git"
|
||||
},
|
||||
"keywords": [
|
||||
"transformers",
|
||||
"transformers.js",
|
||||
"huggingface",
|
||||
"hugging face",
|
||||
"machine learning",
|
||||
"deep learning",
|
||||
"artificial intelligence",
|
||||
"AI",
|
||||
"ML"
|
||||
],
|
||||
"author": "Hugging Face",
|
||||
"license": "Apache-2.0",
|
||||
"bugs": {
|
||||
"url": "https://github.com/huggingface/transformers.js/issues"
|
||||
},
|
||||
"homepage": "https://github.com/huggingface/transformers.js#readme",
|
||||
"dependencies": {
|
||||
"@huggingface/jinja": "^0.5.6",
|
||||
"@huggingface/tokenizers": "^0.1.3",
|
||||
"onnxruntime-node": "1.24.3",
|
||||
"onnxruntime-web": "1.26.0-dev.20260416-b7804b056c",
|
||||
"sharp": "^0.34.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/jest": "^30.0.0",
|
||||
"@types/node": "^24.1.0",
|
||||
"@webgpu/types": "^0.1.69",
|
||||
"esbuild": "^0.27.2",
|
||||
"jest": "^30.2.0",
|
||||
"jest-environment-node": "^30.2.0",
|
||||
"jsdoc-to-markdown": "^9.1.3",
|
||||
"typescript": "5.9.3"
|
||||
},
|
||||
"files": [
|
||||
"src",
|
||||
"dist",
|
||||
"types",
|
||||
"README.md",
|
||||
"LICENSE",
|
||||
"!**/*.tsbuildinfo"
|
||||
],
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
"jsdelivr": "./dist/transformers.min.js",
|
||||
"unpkg": "./dist/transformers.min.js"
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import { OUT_DIR } from "./build/constants.mjs";
|
||||
import prepareOutDir from "../../../scripts/prepareOutDir.mjs";
|
||||
import { colors, createLogger } from "../../../scripts/logger.mjs";
|
||||
import { buildAll } from "./build/buildAll.mjs";
|
||||
|
||||
const log = createLogger("transformers");
|
||||
|
||||
log.section("BUILD");
|
||||
log.info("Building transformers.js with esbuild...\n");
|
||||
|
||||
const startTime = performance.now();
|
||||
|
||||
try {
|
||||
prepareOutDir(OUT_DIR);
|
||||
|
||||
await buildAll(log);
|
||||
|
||||
const endTime = performance.now();
|
||||
const duration = (endTime - startTime).toFixed(2);
|
||||
log.success(`All builds completed in ${colors.bright}${duration}ms${colors.reset}\n`);
|
||||
} catch (error) {
|
||||
log.error(`Build failed: ${error.message}`);
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import { build as esbuild } from "esbuild";
|
||||
import path from "node:path";
|
||||
import { stripNodePrefixPlugin } from "./plugins/stripNodePrefixPlugin.mjs";
|
||||
import { ignoreModulesPlugin } from "./plugins/ignoreModulesPlugin.mjs";
|
||||
import { postBuildPlugin } from "./plugins/postBuildPlugin.mjs";
|
||||
import { externalNodeBuiltinsPlugin } from "./plugins/externalNodeBuiltinsPlugin.mjs";
|
||||
import { OUT_DIR, ROOT_DIR, getEsbuildProdConfig } from "./constants.mjs";
|
||||
import { reportSize } from "../../../../scripts/reportSize.mjs";
|
||||
import { colors } from "../../../../scripts/logger.mjs";
|
||||
import { BUILD_TARGETS } from "./targets.mjs";
|
||||
|
||||
/**
|
||||
* Helper function to create build configurations.
|
||||
*/
|
||||
async function buildTarget(
|
||||
{
|
||||
name = "",
|
||||
suffix = ".js",
|
||||
format = "esm", // 'esm' | 'cjs'
|
||||
ignoreModules = [],
|
||||
externalModules = [],
|
||||
usePostBuild = false,
|
||||
},
|
||||
log,
|
||||
) {
|
||||
const platform = format === "cjs" ? "node" : "neutral";
|
||||
|
||||
const regularFile = `transformers${name}${suffix}`;
|
||||
const minFile = `transformers${name}.min${suffix}`;
|
||||
|
||||
const plugins = [];
|
||||
// Add ignoreModulesPlugin FIRST so it can catch modules before stripNodePrefixPlugin marks them as external
|
||||
if (ignoreModules.length > 0) {
|
||||
plugins.push(ignoreModulesPlugin(ignoreModules));
|
||||
}
|
||||
plugins.push(stripNodePrefixPlugin());
|
||||
plugins.push(externalNodeBuiltinsPlugin());
|
||||
if (usePostBuild) {
|
||||
plugins.push(postBuildPlugin(OUT_DIR, ROOT_DIR));
|
||||
}
|
||||
|
||||
log.build(`Building ${colors.bright}${regularFile}${colors.reset}...`);
|
||||
await esbuild({
|
||||
...getEsbuildProdConfig(ROOT_DIR),
|
||||
platform,
|
||||
format,
|
||||
outfile: path.join(OUT_DIR, regularFile),
|
||||
external: externalModules,
|
||||
plugins,
|
||||
});
|
||||
reportSize(path.join(OUT_DIR, regularFile), log);
|
||||
|
||||
log.build(`Building ${colors.bright}${minFile}${colors.reset}...`);
|
||||
await esbuild({
|
||||
...getEsbuildProdConfig(ROOT_DIR),
|
||||
platform,
|
||||
format,
|
||||
outfile: path.join(OUT_DIR, minFile),
|
||||
minify: true,
|
||||
external: externalModules,
|
||||
plugins,
|
||||
legalComments: "none",
|
||||
});
|
||||
reportSize(path.join(OUT_DIR, minFile), log);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build all targets for production
|
||||
*/
|
||||
export async function buildAll(log) {
|
||||
for (const target of BUILD_TARGETS) {
|
||||
log.section(target.name);
|
||||
await buildTarget(target.config, log);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import { context } from "esbuild";
|
||||
import path from "node:path";
|
||||
import { stripNodePrefixPlugin } from "./plugins/stripNodePrefixPlugin.mjs";
|
||||
import { ignoreModulesPlugin } from "./plugins/ignoreModulesPlugin.mjs";
|
||||
import { postBuildPlugin } from "./plugins/postBuildPlugin.mjs";
|
||||
import { externalNodeBuiltinsPlugin } from "./plugins/externalNodeBuiltinsPlugin.mjs";
|
||||
import { rebuildPlugin } from "../../../../scripts/rebuildPlugin.mjs";
|
||||
import { OUT_DIR, ROOT_DIR, getEsbuildDevConfig } from "./constants.mjs";
|
||||
import { BUILD_TARGETS } from "./targets.mjs";
|
||||
|
||||
/**
|
||||
* Create an esbuild context for a single build target
|
||||
*/
|
||||
async function createBuildContext(targetName, targetConfig, log) {
|
||||
const { name, suffix, format, ignoreModules, externalModules, usePostBuild } = targetConfig;
|
||||
const platform = format === "cjs" ? "node" : "neutral";
|
||||
const outputFile = `transformers${name}${suffix}`;
|
||||
|
||||
const plugins = [];
|
||||
// Add ignoreModulesPlugin FIRST so it can catch modules before stripNodePrefixPlugin marks them as external
|
||||
if (ignoreModules.length > 0) {
|
||||
plugins.push(ignoreModulesPlugin(ignoreModules));
|
||||
}
|
||||
plugins.push(stripNodePrefixPlugin());
|
||||
plugins.push(externalNodeBuiltinsPlugin());
|
||||
if (usePostBuild) {
|
||||
plugins.push(postBuildPlugin(OUT_DIR, ROOT_DIR));
|
||||
}
|
||||
plugins.push(rebuildPlugin(targetName, log));
|
||||
|
||||
return context({
|
||||
...getEsbuildDevConfig(ROOT_DIR),
|
||||
platform,
|
||||
format,
|
||||
outfile: path.join(OUT_DIR, outputFile),
|
||||
external: externalModules,
|
||||
plugins,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Build all targets in watch mode for development
|
||||
* @returns {Promise<Array>} Array of esbuild contexts that can be disposed later
|
||||
*/
|
||||
export async function buildAllWithWatch(log) {
|
||||
log.dim("Creating build contexts...\n");
|
||||
|
||||
// Create contexts for all targets
|
||||
const contexts = await Promise.all(BUILD_TARGETS.map((target) => createBuildContext(target.name, target.config, log)));
|
||||
|
||||
log.dim("Starting initial build...\n");
|
||||
|
||||
// Wait for the initial builds to complete
|
||||
await Promise.all(contexts.map((ctx) => ctx.rebuild()));
|
||||
|
||||
// Start watching all targets
|
||||
await Promise.all(contexts.map((ctx) => ctx.watch()));
|
||||
|
||||
return contexts;
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export const DIST_FOLDER = "dist";
|
||||
export const NODE_IGNORE_MODULES = ["onnxruntime-web"];
|
||||
export const NODE_EXTERNAL_MODULES = [
|
||||
"onnxruntime-common",
|
||||
"onnxruntime-node",
|
||||
"sharp",
|
||||
// node:* modules are handled by externalNodeBuiltinsPlugin
|
||||
];
|
||||
|
||||
export const WEB_IGNORE_MODULES = ["onnxruntime-node", "sharp", "fs", "path", "url", "stream", "stream/promises"];
|
||||
export const WEB_EXTERNAL_MODULES = ["onnxruntime-common", "onnxruntime-web"];
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
export const ROOT_DIR = path.join(__dirname, "../..");
|
||||
export const OUT_DIR = path.join(ROOT_DIR, DIST_FOLDER);
|
||||
|
||||
export const getEsbuildDevConfig = (rootDir) => ({
|
||||
bundle: true,
|
||||
treeShaking: true,
|
||||
logLevel: "info",
|
||||
entryPoints: [path.join(rootDir, "src/transformers.js")],
|
||||
platform: "neutral",
|
||||
format: "esm",
|
||||
sourcemap: true,
|
||||
logOverride: {
|
||||
// Suppress import.meta warning for CJS builds - it's handled gracefully in the code
|
||||
"empty-import-meta": "silent",
|
||||
},
|
||||
});
|
||||
|
||||
export const getEsbuildProdConfig = (rootDir) => ({
|
||||
...getEsbuildDevConfig(rootDir),
|
||||
logLevel: "warning",
|
||||
sourcemap: false,
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
/**
|
||||
* Plugin to automatically mark all node:* imports as external.
|
||||
* This prevents having to manually list all Node.js built-in modules.
|
||||
*/
|
||||
export const externalNodeBuiltinsPlugin = () => ({
|
||||
name: "external-node-builtins",
|
||||
setup(build) {
|
||||
// Mark all node:* imports as external
|
||||
build.onResolve({ filter: /^node:/ }, (args) => ({
|
||||
path: args.path,
|
||||
external: true,
|
||||
}));
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,49 @@
|
||||
/**
|
||||
* Plugin to ignore/exclude certain modules by returning an empty module.
|
||||
* Equivalent to webpack's resolve.alias with false value.
|
||||
*/
|
||||
export const ignoreModulesPlugin = (modules = []) => ({
|
||||
name: "ignore-modules",
|
||||
setup(build) {
|
||||
// Escape special regex characters in module names
|
||||
const escapeRegex = (str) => str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const escapedModules = modules.map(escapeRegex);
|
||||
|
||||
// Match both "module" and "node:module" patterns
|
||||
const patterns = escapedModules.flatMap((mod) => [mod, `node:${mod}`]);
|
||||
const filter = new RegExp(`^(${patterns.join("|")})$`);
|
||||
|
||||
build.onResolve({ filter }, (args) => {
|
||||
return { path: args.path, namespace: "ignore-modules" };
|
||||
});
|
||||
build.onLoad({ filter: /.*/, namespace: "ignore-modules" }, (args) => {
|
||||
switch (args.path) {
|
||||
case "node:stream":
|
||||
return {
|
||||
contents: `
|
||||
const noop = () => {};
|
||||
export default {};
|
||||
export const Readable = { fromWeb: noop };
|
||||
`,
|
||||
};
|
||||
case "node:stream/promises":
|
||||
return {
|
||||
contents: `
|
||||
const noop = () => {};
|
||||
export default {};
|
||||
export const pipeline = noop;
|
||||
`,
|
||||
};
|
||||
case "node:fs":
|
||||
case "node:path":
|
||||
case "node:url":
|
||||
case "sharp":
|
||||
case "onnxruntime-node":
|
||||
default:
|
||||
return {
|
||||
contents: `export default {};`,
|
||||
};
|
||||
}
|
||||
});
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
import path from "node:path";
|
||||
import { copyFileSync, unlinkSync, existsSync } from "node:fs";
|
||||
import { colors, createLogger } from "../../../../../scripts/logger.mjs";
|
||||
|
||||
const log = createLogger("transformers");
|
||||
|
||||
/**
|
||||
* Plugin to post-process build files.
|
||||
* Equivalent to webpack's PostBuildPlugin.
|
||||
*/
|
||||
export const postBuildPlugin = (distDir, rootDir) => {
|
||||
// it should copy the files only once. In watch mode for example it should not rerun every time
|
||||
let completed = false;
|
||||
|
||||
return {
|
||||
name: "post-build",
|
||||
setup(build) {
|
||||
build.onEnd(() => {
|
||||
if (completed) return;
|
||||
completed = true;
|
||||
|
||||
const ORT_JSEP_FILE = "ort-wasm-simd-threaded.jsep.mjs";
|
||||
const ORT_BUNDLE_FILE = "ort.bundle.min.mjs";
|
||||
|
||||
// 1. Remove unnecessary files
|
||||
const file = path.join(distDir, ORT_BUNDLE_FILE);
|
||||
if (existsSync(file)) unlinkSync(file);
|
||||
|
||||
// 2. Copy unbundled JSEP file
|
||||
const ORT_SOURCE_DIR = path.join(rootDir, "node_modules/onnxruntime-web/dist");
|
||||
const src = path.join(ORT_SOURCE_DIR, ORT_JSEP_FILE);
|
||||
|
||||
if (existsSync(src)) {
|
||||
const dest = path.join(distDir, ORT_JSEP_FILE);
|
||||
copyFileSync(src, dest);
|
||||
log.success(`${colors.gray}Copied ${ORT_JSEP_FILE}${colors.reset}`);
|
||||
} else {
|
||||
log.warning(`Could not find ${ORT_JSEP_FILE} in node_modules`);
|
||||
}
|
||||
});
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,15 @@
|
||||
/**
|
||||
* Plugin to strip the "node:" prefix from module requests.
|
||||
* Equivalent to webpack's StripNodePrefixPlugin.
|
||||
*/
|
||||
export const stripNodePrefixPlugin = () => ({
|
||||
name: "strip-node-prefix",
|
||||
setup(build) {
|
||||
build.onResolve({ filter: /^node:/ }, (args) => {
|
||||
return {
|
||||
path: args.path.replace(/^node:/, ""),
|
||||
external: true,
|
||||
};
|
||||
});
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
import { NODE_IGNORE_MODULES, NODE_EXTERNAL_MODULES, WEB_IGNORE_MODULES, WEB_EXTERNAL_MODULES } from "./constants.mjs";
|
||||
|
||||
/**
|
||||
* Build target configuration
|
||||
* Each target defines a specific build variant
|
||||
*/
|
||||
export const BUILD_TARGETS = [
|
||||
{
|
||||
name: "Bundle Build (ESM)",
|
||||
config: {
|
||||
name: "",
|
||||
suffix: ".js",
|
||||
format: "esm",
|
||||
ignoreModules: WEB_IGNORE_MODULES,
|
||||
usePostBuild: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Web Build (ESM)",
|
||||
config: {
|
||||
name: ".web",
|
||||
suffix: ".js",
|
||||
format: "esm",
|
||||
ignoreModules: WEB_IGNORE_MODULES,
|
||||
externalModules: WEB_EXTERNAL_MODULES,
|
||||
usePostBuild: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Node Build (ESM)",
|
||||
config: {
|
||||
name: ".node",
|
||||
suffix: ".mjs",
|
||||
format: "esm",
|
||||
ignoreModules: NODE_IGNORE_MODULES,
|
||||
externalModules: NODE_EXTERNAL_MODULES,
|
||||
usePostBuild: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Node Build (CJS)",
|
||||
config: {
|
||||
name: ".node",
|
||||
suffix: ".cjs",
|
||||
format: "cjs",
|
||||
ignoreModules: NODE_IGNORE_MODULES,
|
||||
externalModules: NODE_EXTERNAL_MODULES,
|
||||
usePostBuild: false,
|
||||
},
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,113 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { unlinkSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { OUT_DIR } from "./build/constants.mjs";
|
||||
import prepareOutDir from "../../../scripts/prepareOutDir.mjs";
|
||||
import { colors, createLogger } from "../../../scripts/logger.mjs";
|
||||
import { buildAllWithWatch } from "./build/buildAllWithWatch.mjs";
|
||||
|
||||
const log = createLogger("transformers");
|
||||
const startTime = performance.now();
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT_DIR = path.join(__dirname, "..");
|
||||
|
||||
prepareOutDir(OUT_DIR);
|
||||
|
||||
// Remove tsbuildinfo to force TypeScript to rebuild type declarations
|
||||
try {
|
||||
unlinkSync(`${ROOT_DIR}/types/tsconfig.tsbuildinfo`);
|
||||
} catch (err) {
|
||||
// File doesn't exist, that's fine
|
||||
}
|
||||
|
||||
log.section("BUILD");
|
||||
log.info("Building transformers.js with esbuild in watch mode...");
|
||||
|
||||
// Build all targets with watch mode
|
||||
const contexts = await buildAllWithWatch(log);
|
||||
|
||||
const endTime = performance.now();
|
||||
const duration = (endTime - startTime).toFixed(2);
|
||||
log.success(`All builds completed in ${colors.bright}${duration}ms${colors.reset}`);
|
||||
|
||||
// Generate initial TypeScript declarations, then start watch mode
|
||||
log.section("TYPES");
|
||||
log.info("Generating initial type declarations...");
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
const tscBuild = spawn("tsc", ["--build"], {
|
||||
cwd: ROOT_DIR,
|
||||
stdio: "pipe",
|
||||
shell: true,
|
||||
});
|
||||
|
||||
tscBuild.stdout.on("data", (data) => {
|
||||
const output = data.toString().trim();
|
||||
if (output && output.includes("error")) {
|
||||
log.dim(output);
|
||||
}
|
||||
});
|
||||
|
||||
tscBuild.stderr.on("data", (data) => {
|
||||
const output = data.toString().trim();
|
||||
if (output) {
|
||||
log.error(output);
|
||||
}
|
||||
});
|
||||
|
||||
tscBuild.on("exit", (code) => {
|
||||
if (code === 0) {
|
||||
log.done("Type declarations generated");
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`TypeScript build failed with code ${code}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
log.info("Starting TypeScript watch mode...\n");
|
||||
|
||||
const tscWatch = spawn("tsc", ["--build", "--watch", "--preserveWatchOutput"], {
|
||||
cwd: ROOT_DIR,
|
||||
stdio: "pipe",
|
||||
shell: true,
|
||||
});
|
||||
|
||||
tscWatch.stdout.on("data", (data) => {
|
||||
const output = data.toString().trim();
|
||||
if (output) {
|
||||
output.split("\n").forEach((line) => {
|
||||
// Filter out verbose output, only show important messages
|
||||
if (
|
||||
line.includes("error") ||
|
||||
line.includes("File change detected") ||
|
||||
line.includes("Found 0 errors") ||
|
||||
(line.includes("Found") && line.includes("error"))
|
||||
) {
|
||||
log.dim(line);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
tscWatch.stderr.on("data", (data) => {
|
||||
const output = data.toString().trim();
|
||||
if (output) {
|
||||
output.split("\n").forEach((line) => {
|
||||
log.error(line);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
log.dim(`Watching for changes...\n`);
|
||||
|
||||
// Keep process alive and cleanup
|
||||
process.on("SIGINT", async () => {
|
||||
log.warning(`\nStopping watch mode...`);
|
||||
tscWatch.kill();
|
||||
await Promise.all(contexts.map((ctx) => ctx.dispose()));
|
||||
log.dim("Goodbye!");
|
||||
process.exit(0);
|
||||
});
|
||||
@@ -0,0 +1,391 @@
|
||||
/**
|
||||
* @file Handler file for choosing the correct version of ONNX Runtime, based on the environment.
|
||||
* Ideally, we could import the `onnxruntime-web` and `onnxruntime-node` packages only when needed,
|
||||
* but dynamic imports don't seem to work with the current webpack version and/or configuration.
|
||||
* This is possibly due to the experimental nature of top-level await statements.
|
||||
* So, we just import both packages, and use the appropriate one based on the environment:
|
||||
* - When running in node, we use `onnxruntime-node`.
|
||||
* - When running in the browser, we use `onnxruntime-web` (`onnxruntime-node` is not bundled).
|
||||
*
|
||||
* This module is not directly exported, but can be accessed through the environment variables:
|
||||
* ```javascript
|
||||
* import { env } from '@huggingface/transformers';
|
||||
* console.log(env.backends.onnx);
|
||||
* ```
|
||||
*
|
||||
* @module backends/onnx
|
||||
*/
|
||||
|
||||
import { env, apis, LogLevel } from '../env.js';
|
||||
|
||||
// NOTE: Import order matters here. We need to import `onnxruntime-node` before `onnxruntime-web`.
|
||||
// In either case, we select the default export if it exists, otherwise we use the named export.
|
||||
import * as ONNX_NODE from 'onnxruntime-node';
|
||||
import * as ONNX_WEB from 'onnxruntime-web/webgpu';
|
||||
import { loadWasmBinary, loadWasmFactory } from './utils/cacheWasm.js';
|
||||
import { isBlobURL, toAbsoluteURL } from '../utils/hub/utils.js';
|
||||
import { logger } from '../utils/logger.js';
|
||||
export { Tensor } from 'onnxruntime-common';
|
||||
|
||||
/**
|
||||
* @typedef {import('onnxruntime-common').InferenceSession.ExecutionProviderConfig} ONNXExecutionProviders
|
||||
*/
|
||||
|
||||
/** @type {Record<import("../utils/devices.js").DeviceType, ONNXExecutionProviders>} */
|
||||
const DEVICE_TO_EXECUTION_PROVIDER_MAPPING = Object.freeze({
|
||||
auto: null, // Auto-detect based on device and environment
|
||||
gpu: null, // Auto-detect GPU
|
||||
cpu: 'cpu', // CPU
|
||||
wasm: 'wasm', // WebAssembly
|
||||
webgpu: 'webgpu', // WebGPU
|
||||
cuda: 'cuda', // CUDA
|
||||
dml: 'dml', // DirectML
|
||||
coreml: 'coreml', // CoreML
|
||||
|
||||
webnn: { name: 'webnn', deviceType: 'cpu' }, // WebNN (default)
|
||||
'webnn-npu': { name: 'webnn', deviceType: 'npu' }, // WebNN NPU
|
||||
'webnn-gpu': { name: 'webnn', deviceType: 'gpu' }, // WebNN GPU
|
||||
'webnn-cpu': { name: 'webnn', deviceType: 'cpu' }, // WebNN CPU
|
||||
});
|
||||
|
||||
/**
|
||||
* Converts any LogLevel value to ONNX Runtime's numeric severity level (0-4).
|
||||
* This handles both standard LogLevel values (10, 20, 30, 40, 50) and custom intermediate values.
|
||||
*
|
||||
* @param {number} logLevel - The LogLevel value to convert
|
||||
* @returns {number} ONNX Runtime severity level (0-4)
|
||||
*/
|
||||
function getOnnxLogSeverityLevel(logLevel) {
|
||||
// ONNX Runtime's log severity levels are defined as follows:
|
||||
// (0) ORT_LOGGING_LEVEL_VERBOSE: Print all log messages.
|
||||
// (1) ORT_LOGGING_LEVEL_INFO: Print info and higher level log messages.
|
||||
// (2) ORT_LOGGING_LEVEL_WARNING: Print warning and higher level log messages.
|
||||
// (3) ORT_LOGGING_LEVEL_ERROR: Print error log messages.
|
||||
// (4) ORT_LOGGING_LEVEL_FATAL: Print only fatal log messages.
|
||||
//
|
||||
// In practice, ONNX Runtime's logging is extremely verbose (especially on session creation).
|
||||
// For this reason, we map multiple LogLevel values to the same ONNX severity level to avoid
|
||||
// overwhelming users with logs.
|
||||
if (logLevel <= LogLevel.DEBUG) {
|
||||
return 0; // ORT_LOGGING_LEVEL_VERBOSE
|
||||
} else if (logLevel <= LogLevel.INFO) {
|
||||
return 2; // ORT_LOGGING_LEVEL_WARNING
|
||||
} else if (logLevel <= LogLevel.WARNING) {
|
||||
return 3; // ORT_LOGGING_LEVEL_ERROR
|
||||
} else if (logLevel <= LogLevel.ERROR) {
|
||||
return 3; // ORT_LOGGING_LEVEL_ERROR
|
||||
} else {
|
||||
return 4; // ORT_LOGGING_LEVEL_FATAL
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps ONNX Runtime numeric severity levels to string log levels.
|
||||
* @type {Record<0 | 1 | 2 | 3 | 4, 'verbose' | 'info' | 'warning' | 'error' | 'fatal'>}
|
||||
*/
|
||||
const ONNX_LOG_LEVEL_NAMES = {
|
||||
0: 'verbose',
|
||||
1: 'info',
|
||||
2: 'warning',
|
||||
3: 'error',
|
||||
4: 'fatal',
|
||||
};
|
||||
|
||||
/**
|
||||
* The list of supported devices, sorted by priority/performance.
|
||||
* @type {import("../utils/devices.js").DeviceType[]}
|
||||
*/
|
||||
const supportedDevices = [];
|
||||
|
||||
/** @type {ONNXExecutionProviders[]} */
|
||||
let defaultDevices;
|
||||
let ONNX;
|
||||
const ORT_SYMBOL = Symbol.for('onnxruntime');
|
||||
|
||||
if (ORT_SYMBOL in globalThis) {
|
||||
// If the JS runtime exposes their own ONNX runtime, use it
|
||||
ONNX = globalThis[ORT_SYMBOL];
|
||||
} else if (apis.IS_NODE_ENV) {
|
||||
ONNX = ONNX_NODE;
|
||||
|
||||
// Updated as of ONNX Runtime 1.23.0-dev.20250612-70f14d7670
|
||||
// The following table lists the supported versions of ONNX Runtime Node.js binding provided with pre-built binaries.
|
||||
// | EPs/Platforms | Windows x64 | Windows arm64 | Linux x64 | Linux arm64 | MacOS x64 | MacOS arm64 |
|
||||
// | --------------------- | ------------------ | ------------------ | ------------------ | ------------------ | ------------------ | ------------------ |
|
||||
// | CPU | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ |
|
||||
// | WebGPU (experimental) | ✔️ | ✔️ | ✔️ | ❌ | ✔️ | ✔️ |
|
||||
// | DirectML | ✔️ | ✔️ | ❌ | ❌ | ❌ | ❌ |
|
||||
// | CUDA | ❌ | ❌ | ✔️ (CUDA v12) | ❌ | ❌ | ❌ |
|
||||
// | CoreML | ❌ | ❌ | ❌ | ❌ | ✔️ | ✔️ |
|
||||
switch (process.platform) {
|
||||
case 'win32': // Windows x64 and Windows arm64
|
||||
supportedDevices.push('dml');
|
||||
break;
|
||||
case 'linux': // Linux x64 and Linux arm64
|
||||
if (process.arch === 'x64') {
|
||||
supportedDevices.push('cuda');
|
||||
}
|
||||
break;
|
||||
case 'darwin': // MacOS x64 and MacOS arm64
|
||||
supportedDevices.push('coreml');
|
||||
break;
|
||||
}
|
||||
|
||||
supportedDevices.push('webgpu');
|
||||
supportedDevices.push('cpu');
|
||||
defaultDevices = ['cpu'];
|
||||
} else {
|
||||
ONNX = ONNX_WEB;
|
||||
|
||||
if (apis.IS_WEBNN_AVAILABLE) {
|
||||
// TODO: Only push supported providers (depending on available hardware)
|
||||
supportedDevices.push('webnn-npu', 'webnn-gpu', 'webnn-cpu', 'webnn');
|
||||
}
|
||||
|
||||
if (apis.IS_WEBGPU_AVAILABLE) {
|
||||
supportedDevices.push('webgpu');
|
||||
}
|
||||
|
||||
supportedDevices.push('wasm');
|
||||
defaultDevices = ['wasm'];
|
||||
}
|
||||
|
||||
// @ts-ignore
|
||||
const InferenceSession = ONNX.InferenceSession;
|
||||
|
||||
/**
|
||||
* Map a device to the execution providers to use for the given device.
|
||||
* @param {import("../utils/devices.js").DeviceType|"auto"|null} [device=null] (Optional) The device to run the inference on.
|
||||
* @returns {ONNXExecutionProviders[]} The execution providers to use for the given device.
|
||||
*/
|
||||
export function deviceToExecutionProviders(device = null) {
|
||||
// Use the default execution providers if the user hasn't specified anything
|
||||
if (!device) return defaultDevices;
|
||||
|
||||
// Handle overloaded cases
|
||||
switch (device) {
|
||||
case 'auto':
|
||||
return supportedDevices;
|
||||
case 'gpu':
|
||||
return supportedDevices.filter((x) => ['webgpu', 'cuda', 'dml', 'webnn-gpu'].includes(x));
|
||||
}
|
||||
|
||||
if (supportedDevices.includes(device)) {
|
||||
return [DEVICE_TO_EXECUTION_PROVIDER_MAPPING[device] ?? device];
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported device: "${device}". Should be one of: ${supportedDevices.join(', ')}.`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Currently, Transformers.js doesn't support simultaneous loading of sessions in WASM/WebGPU.
|
||||
* For this reason, we need to chain the loading calls.
|
||||
* @type {Promise<any>}
|
||||
*/
|
||||
let webInitChain = Promise.resolve();
|
||||
|
||||
/**
|
||||
* Promise that resolves when WASM binary has been loaded (if caching is enabled).
|
||||
* This ensures we only attempt to load the WASM binary once.
|
||||
* @type {Promise<void>|null}
|
||||
*/
|
||||
let wasmLoadPromise = null;
|
||||
|
||||
/**
|
||||
* Ensures the WASM binary is loaded and cached before creating an inference session.
|
||||
* Only runs once, even if called multiple times.
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function ensureWasmLoaded() {
|
||||
// If already loading or loaded, return the existing promise
|
||||
if (wasmLoadPromise) {
|
||||
return wasmLoadPromise;
|
||||
}
|
||||
|
||||
// Check if we should load the WASM binary
|
||||
const shouldUseWasmCache =
|
||||
env.useWasmCache &&
|
||||
typeof ONNX_ENV?.wasm?.wasmPaths === 'object' &&
|
||||
ONNX_ENV?.wasm?.wasmPaths?.wasm &&
|
||||
ONNX_ENV?.wasm?.wasmPaths?.mjs;
|
||||
|
||||
if (!shouldUseWasmCache) {
|
||||
// In Deno's web runtime, the WASM factory must be loaded via blob URL so that Node.js detection
|
||||
// can be patched out (see loadWasmFactory). Without caching, the factory is imported directly
|
||||
// from its URL and Deno would crash trying to use Node.js APIs. useWasmCache defaults to true
|
||||
// in this environment, so this only happens if the user explicitly disables it.
|
||||
if (apis.IS_DENO_WEB_RUNTIME) {
|
||||
throw new Error(
|
||||
"env.useWasmCache=false is not supported in Deno's web runtime. Remove the useWasmCache override.",
|
||||
);
|
||||
}
|
||||
wasmLoadPromise = Promise.resolve();
|
||||
return wasmLoadPromise;
|
||||
}
|
||||
|
||||
// Start loading the WASM binary
|
||||
wasmLoadPromise = (async () => {
|
||||
// At this point, we know wasmPaths is an object (not a string) because
|
||||
// shouldUseWasmCache checks for wasmPaths.wasm and wasmPaths.mjs
|
||||
const urls = /** @type {{ wasm: string, mjs: string }} */ (ONNX_ENV.wasm.wasmPaths);
|
||||
|
||||
// Load both in parallel; the .mjs blob URL is only kept if wasmBinary succeeded.
|
||||
// ORT only sets locateFile when wasmBinary is provided (onnxruntime PR https://github.com/microsoft/onnxruntime/pull/27411), which
|
||||
// prevents new URL(fileName, import.meta.url) from failing inside a blob URL factory.
|
||||
let wasmBinaryLoaded = false;
|
||||
await Promise.all([
|
||||
// Load and cache the WASM binary
|
||||
urls.wasm && !isBlobURL(urls.wasm)
|
||||
? (async () => {
|
||||
try {
|
||||
const wasmBinary = await loadWasmBinary(toAbsoluteURL(urls.wasm));
|
||||
if (wasmBinary) {
|
||||
ONNX_ENV.wasm.wasmBinary = wasmBinary;
|
||||
wasmBinaryLoaded = true;
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn('Failed to pre-load WASM binary:', err);
|
||||
}
|
||||
})()
|
||||
: Promise.resolve(),
|
||||
|
||||
// Load and cache the WASM factory as a blob URL
|
||||
urls.mjs && !isBlobURL(urls.mjs)
|
||||
? (async () => {
|
||||
try {
|
||||
const wasmFactoryBlob = await loadWasmFactory(toAbsoluteURL(urls.mjs));
|
||||
if (wasmFactoryBlob) {
|
||||
// @ts-ignore
|
||||
ONNX_ENV.wasm.wasmPaths.mjs = wasmFactoryBlob;
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn('Failed to pre-load WASM factory:', err);
|
||||
}
|
||||
})()
|
||||
: Promise.resolve(),
|
||||
]);
|
||||
|
||||
// If wasmBinary failed to load, revert wasmPaths.mjs to the original URL (factory can only be loaded from blob if ONNX_ENV.wasm.wasmBinary is set. @see ORT PR #27411)
|
||||
if (!wasmBinaryLoaded) {
|
||||
// @ts-ignore
|
||||
ONNX_ENV.wasm.wasmPaths.mjs = urls.mjs;
|
||||
}
|
||||
})();
|
||||
|
||||
return wasmLoadPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an ONNX inference session.
|
||||
* @param {Uint8Array|string} buffer_or_path The ONNX model buffer or path.
|
||||
* @param {import('onnxruntime-common').InferenceSession.SessionOptions} session_options ONNX inference session options.
|
||||
* @param {Object} session_config ONNX inference session configuration.
|
||||
* @returns {Promise<import('onnxruntime-common').InferenceSession & { config: Object }>} The ONNX inference session.
|
||||
*/
|
||||
export async function createInferenceSession(buffer_or_path, session_options, session_config) {
|
||||
await ensureWasmLoaded();
|
||||
const logSeverityLevel = getOnnxLogSeverityLevel(env.logLevel ?? LogLevel.WARNING);
|
||||
const load = () =>
|
||||
InferenceSession.create(buffer_or_path, {
|
||||
// Set default log severity level, but allow overriding through session options
|
||||
logSeverityLevel,
|
||||
...session_options,
|
||||
});
|
||||
const session = await (apis.IS_WEB_ENV ? (webInitChain = webInitChain.then(load)) : load());
|
||||
session.config = session_config;
|
||||
return session;
|
||||
}
|
||||
|
||||
/**
|
||||
* Currently, Transformers.js doesn't support simultaneous execution of sessions in WASM/WebGPU.
|
||||
* For this reason, we need to chain the inference calls (otherwise we get "Error: Session already started").
|
||||
* @type {Promise<any>}
|
||||
*/
|
||||
let webInferenceChain = Promise.resolve();
|
||||
|
||||
/**
|
||||
* Run an inference session.
|
||||
* @param {import('onnxruntime-common').InferenceSession} session The ONNX inference session.
|
||||
* @param {Record<string, import('onnxruntime-common').Tensor>} ortFeed The input tensors.
|
||||
* @returns {Promise<Record<string, import('onnxruntime-common').Tensor>>} The output tensors.
|
||||
*/
|
||||
export async function runInferenceSession(session, ortFeed) {
|
||||
const run = () => session.run(ortFeed);
|
||||
return apis.IS_WEB_ENV ? (webInferenceChain = webInferenceChain.then(run)) : run();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an object is an ONNX tensor.
|
||||
* @param {any} x The object to check
|
||||
* @returns {boolean} Whether the object is an ONNX tensor.
|
||||
*/
|
||||
export function isONNXTensor(x) {
|
||||
return x instanceof ONNX.Tensor;
|
||||
}
|
||||
/** @type {import('onnxruntime-common').Env} */
|
||||
const ONNX_ENV = ONNX?.env;
|
||||
|
||||
/**
|
||||
* Check if ONNX's WASM backend is being proxied.
|
||||
* @returns {boolean} Whether ONNX's WASM backend is being proxied.
|
||||
*/
|
||||
export function isONNXProxy() {
|
||||
// TODO: Update this when allowing non-WASM backends.
|
||||
return ONNX_ENV?.wasm?.proxy;
|
||||
}
|
||||
|
||||
if (ONNX_ENV) {
|
||||
if (ONNX_ENV.wasm) {
|
||||
// Initialize wasm backend with suitable default settings.
|
||||
|
||||
// (Optional) Set path to wasm files. This will override the default path search behavior of onnxruntime-web.
|
||||
// By default, we only do this if we are not in a service worker and the wasmPaths are not already set.
|
||||
if (
|
||||
// @ts-ignore Cannot find name 'ServiceWorkerGlobalScope'.ts(2304)
|
||||
!(typeof ServiceWorkerGlobalScope !== 'undefined' && self instanceof ServiceWorkerGlobalScope) &&
|
||||
ONNX_ENV.versions?.web &&
|
||||
!ONNX_ENV.wasm.wasmPaths
|
||||
) {
|
||||
const wasmPathPrefix = `https://cdn.jsdelivr.net/npm/onnxruntime-web@${ONNX_ENV.versions.web}/dist/`;
|
||||
|
||||
let wasmPathSuffix = '.asyncify'; // Default to asyncify WASM build
|
||||
if (apis.IS_SAFARI_BELOW_26 && !apis.IS_WEBGPU_AVAILABLE) {
|
||||
// Disable asyncify for Safari below 26 when WebGPU is not available
|
||||
wasmPathSuffix = '';
|
||||
}
|
||||
|
||||
ONNX_ENV.wasm.wasmPaths = {
|
||||
mjs: `${wasmPathPrefix}ort-wasm-simd-threaded${wasmPathSuffix}.mjs`,
|
||||
wasm: `${wasmPathPrefix}ort-wasm-simd-threaded${wasmPathSuffix}.wasm`,
|
||||
};
|
||||
}
|
||||
|
||||
// Users may wish to proxy the WASM backend to prevent the UI from freezing,
|
||||
// However, this is not necessary when using WebGPU, so we default to false.
|
||||
ONNX_ENV.wasm.proxy = false;
|
||||
}
|
||||
|
||||
if (ONNX_ENV.webgpu) {
|
||||
ONNX_ENV.webgpu.powerPreference = 'high-performance';
|
||||
}
|
||||
|
||||
/**
|
||||
* A function to map Transformers.js log levels to ONNX Runtime log severity
|
||||
* levels, and set the log level environment variable in ONNX Runtime.
|
||||
* @param {number} logLevel The log level to set.
|
||||
*/
|
||||
function setLogLevel(logLevel) {
|
||||
const severityLevel = getOnnxLogSeverityLevel(logLevel);
|
||||
ONNX_ENV.logLevel = ONNX_LOG_LEVEL_NAMES[severityLevel];
|
||||
}
|
||||
|
||||
// Set the initial log level to be the default Transformers.js log level.
|
||||
setLogLevel(env.logLevel ?? LogLevel.WARNING);
|
||||
|
||||
// Expose ONNX environment variables to `env.backends.onnx`
|
||||
env.backends.onnx = {
|
||||
...ONNX_ENV,
|
||||
setLogLevel,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import { apis, env } from '../../env.js';
|
||||
import { getCache } from '../../utils/cache.js';
|
||||
import { logger } from '../../utils/logger.js';
|
||||
|
||||
/**
|
||||
* Loads and caches a file from the given URL.
|
||||
* @param {string} url The URL of the file to load.
|
||||
* @returns {Promise<Response|import('../../utils/hub/FileResponse.js').FileResponse|null|string>} The response object, or null if loading failed.
|
||||
*/
|
||||
async function loadAndCacheFile(url) {
|
||||
const fileName = url.split('/').pop();
|
||||
|
||||
/** @type {import('../../utils/cache.js').CacheInterface|undefined} */
|
||||
let cache;
|
||||
try {
|
||||
cache = await getCache();
|
||||
|
||||
// Try to get from cache first
|
||||
if (cache) {
|
||||
const result = await cache.match(url);
|
||||
if (result) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to load ${fileName} from cache:`, error);
|
||||
}
|
||||
|
||||
// If not in cache, fetch it
|
||||
const response = await env.fetch(url);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch ${fileName}: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
// Cache the response for future use
|
||||
if (cache) {
|
||||
try {
|
||||
await cache.put(url, response.clone());
|
||||
} catch (e) {
|
||||
logger.warn(`Failed to cache ${fileName}:`, e);
|
||||
}
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads and caches the WASM binary for ONNX Runtime.
|
||||
* @param {string} wasmURL The URL of the WASM file to load.
|
||||
* @returns {Promise<ArrayBuffer|null>} The WASM binary as an ArrayBuffer, or null if loading failed.
|
||||
*/
|
||||
|
||||
export async function loadWasmBinary(wasmURL) {
|
||||
const response = await loadAndCacheFile(wasmURL);
|
||||
if (!response || typeof response === 'string') return null;
|
||||
|
||||
try {
|
||||
return await response.arrayBuffer();
|
||||
} catch (error) {
|
||||
logger.warn('Failed to read WASM binary:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads and caches the WASM Factory (.mjs file) for ONNX Runtime.
|
||||
* Creates a blob URL from cached content (when safe) to bridge Cache API with dynamic imports used in ORT.
|
||||
* @param {string} libURL The URL of the WASM Factory to load.
|
||||
* @returns {Promise<string|null>} The blob URL (if enabled), original URL (if disabled), or null if loading failed.
|
||||
*/
|
||||
export async function loadWasmFactory(libURL) {
|
||||
// We can't use Blob URLs in some environments (Service Workers, Chrome extensions) due to security restrictions on dynamic import() of blob URLs.
|
||||
// In such cases, just return the original URL and don't bother caching since dynamic import() won't use the Cache API anyway.
|
||||
// See https://github.com/huggingface/transformers.js/issues/1532.
|
||||
if (apis.IS_SERVICE_WORKER_ENV || apis.IS_CHROME_AVAILABLE) {
|
||||
return libURL;
|
||||
}
|
||||
|
||||
// Fetch from cache or network, then create blob URL
|
||||
const response = await loadAndCacheFile(libURL);
|
||||
if (!response || typeof response === 'string') return null;
|
||||
|
||||
try {
|
||||
let code = await response.text();
|
||||
|
||||
// Handle the case where we are importing the bundled version of the library in Deno (e.g., via CDN or local file),
|
||||
// where we need to patch out Node.js detection in the factory. Without this, Deno (which exposes globalThis.process.versions.node)
|
||||
// would enter the Node.js branch and try to use Node.js APIs (worker_threads, fs, etc.) that aren't used in the bundled web version.
|
||||
// Only needed for the asyncify (single-threaded) variant loaded via blob URL. The module-level pthread auto-start code is unreachable since asyncify never spawns workers.
|
||||
// See https://github.com/huggingface/transformers.js/pull/1546/ for more information.
|
||||
//
|
||||
// NOTE: This does not affect default usage via Deno (i.e., imported via npm: prefix), since we'll be using onnxruntime-node (Native) instead of onnxruntime-web (WASM).
|
||||
code = code.replaceAll('globalThis.process?.versions?.node', 'false');
|
||||
const blob = new Blob([code], { type: 'text/javascript' });
|
||||
return URL.createObjectURL(blob);
|
||||
} catch (error) {
|
||||
logger.warn('Failed to read WASM factory:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import { Tensor } from './utils/tensor.js';
|
||||
|
||||
/**
|
||||
* A cache class that stores past key values as named tensors.
|
||||
*/
|
||||
class _DynamicCache {
|
||||
/**
|
||||
* Create a DynamicCache, optionally pre-populated with entries.
|
||||
* @param {Record<string, Tensor>} [entries] Initial name→Tensor mappings.
|
||||
*/
|
||||
constructor(entries) {
|
||||
if (!entries) return;
|
||||
for (const key in entries) {
|
||||
if (key in this) {
|
||||
throw new TypeError(`Key "${key}" conflicts with an existing property on DynamicCache`);
|
||||
}
|
||||
const value = entries[key];
|
||||
if (!(value instanceof Tensor)) {
|
||||
throw new TypeError(`Expected a Tensor for key "${key}", got ${typeof value}`);
|
||||
}
|
||||
this[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the cached sequence length. This requires at least one attention cache entry to be present.
|
||||
* @returns {number} The past sequence length.
|
||||
*/
|
||||
get_seq_length() {
|
||||
/** @type {Record<string, Tensor>} */
|
||||
const self = /** @type {any} */ (this);
|
||||
|
||||
if (Object.keys(self).length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (const name in self) {
|
||||
if (name.startsWith('past_key_values.')) {
|
||||
return self[name].dims.at(-2);
|
||||
}
|
||||
}
|
||||
throw new Error('Unable to determine sequence length from the cache.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the cache in-place with new entries, disposing replaced GPU tensors.
|
||||
* @param {Record<string, Tensor>} newEntries The new name → Tensor mappings.
|
||||
*/
|
||||
update(newEntries) {
|
||||
for (const key in newEntries) {
|
||||
const oldValue = this[key];
|
||||
const newValue = newEntries[key];
|
||||
if (oldValue && oldValue !== newValue && oldValue.location === 'gpu-buffer') {
|
||||
oldValue.dispose();
|
||||
}
|
||||
this[key] = newValue;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispose all contained tensors whose data resides on the GPU.
|
||||
* Returns a promise that resolves when all disposals are complete.
|
||||
* @returns {Promise<void>} Promise that resolves when all GPU tensors are disposed.
|
||||
*/
|
||||
async dispose() {
|
||||
const promises = [];
|
||||
for (const t of /** @type {Tensor[]} */ (Object.values(this))) {
|
||||
if (t.location === 'gpu-buffer') {
|
||||
promises.push(t.dispose());
|
||||
}
|
||||
}
|
||||
await Promise.all(promises);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {Record<string, Tensor> & _DynamicCache} DynamicCache
|
||||
*/
|
||||
|
||||
export const DynamicCache = /** @type {new (entries?: Record<string, Tensor>) => DynamicCache} */ (
|
||||
/** @type {unknown} */ (_DynamicCache)
|
||||
);
|
||||
@@ -0,0 +1,581 @@
|
||||
/**
|
||||
* @file Helper module for using model configs. For more information, see the corresponding
|
||||
* [Python documentation](https://huggingface.co/docs/transformers/main/en/model_doc/auto#transformers.AutoConfig).
|
||||
*
|
||||
* **Example:** Load an `AutoConfig`.
|
||||
*
|
||||
* ```javascript
|
||||
* import { AutoConfig } from '@huggingface/transformers';
|
||||
* const config = await AutoConfig.from_pretrained('bert-base-uncased');
|
||||
* console.log(config);
|
||||
* // PretrainedConfig {
|
||||
* // "model_type": "bert",
|
||||
* // "is_encoder_decoder": false,
|
||||
* // "architectures": [
|
||||
* // "BertForMaskedLM"
|
||||
* // ],
|
||||
* // "vocab_size": 30522
|
||||
* // "num_attention_heads": 12,
|
||||
* // "num_hidden_layers": 12,
|
||||
* // "hidden_size": 768,
|
||||
* // "max_position_embeddings": 512,
|
||||
* // ...
|
||||
* // }
|
||||
* ```
|
||||
*
|
||||
* @module configs
|
||||
*/
|
||||
|
||||
import { pick } from './utils/core.js';
|
||||
import { getModelJSON } from './utils/hub.js';
|
||||
|
||||
/**
|
||||
* @typedef {import('./utils/hub.js').PretrainedOptions} PretrainedOptions
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {import('./utils/core.js').ProgressCallback} ProgressCallback
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {import('./utils/core.js').ProgressInfo} ProgressInfo
|
||||
*/
|
||||
|
||||
/**
|
||||
* Loads a config from the specified path.
|
||||
* @param {string} pretrained_model_name_or_path The path to the config directory.
|
||||
* @param {PretrainedOptions} options Additional options for loading the config.
|
||||
* @returns {Promise<Object>} A promise that resolves with information about the loaded config.
|
||||
*/
|
||||
async function loadConfig(pretrained_model_name_or_path, options) {
|
||||
return await getModelJSON(pretrained_model_name_or_path, 'config.json', true, options);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {PretrainedConfig} config
|
||||
* @returns {Object} The normalized configuration.
|
||||
*/
|
||||
function getNormalizedConfig(config) {
|
||||
const mapping = {};
|
||||
|
||||
let init_normalized_config = {};
|
||||
switch (config.model_type) {
|
||||
// Sub-configs
|
||||
case 'llava':
|
||||
case 'paligemma':
|
||||
case 'gemma3':
|
||||
case 'florence2':
|
||||
case 'llava_onevision':
|
||||
case 'idefics3':
|
||||
case 'granite_speech':
|
||||
case 'ultravox':
|
||||
case 'voxtral':
|
||||
case 'voxtral_realtime':
|
||||
case 'smolvlm':
|
||||
case 'gemma3n':
|
||||
case 'gemma4':
|
||||
case 'lfm2_vl':
|
||||
case 'chatterbox':
|
||||
case 'lighton_ocr':
|
||||
case 'glm_ocr':
|
||||
case 'mistral3':
|
||||
case 'qwen2_5_vl':
|
||||
case 'qwen3_vl':
|
||||
case 'qwen3_vl_moe':
|
||||
// @ts-expect-error TS2339
|
||||
init_normalized_config = getNormalizedConfig(config.text_config);
|
||||
break;
|
||||
case 'moondream1':
|
||||
// @ts-expect-error TS2339
|
||||
init_normalized_config = getNormalizedConfig(config.phi_config);
|
||||
break;
|
||||
case 'musicgen':
|
||||
// @ts-expect-error TS2339
|
||||
init_normalized_config = getNormalizedConfig(config.decoder);
|
||||
break;
|
||||
case 'multi_modality':
|
||||
// @ts-expect-error TS2339
|
||||
init_normalized_config = getNormalizedConfig(config.language_config);
|
||||
break;
|
||||
|
||||
// Decoder-only models
|
||||
case 'gpt2':
|
||||
case 'gptj':
|
||||
case 'jais':
|
||||
case 'codegen':
|
||||
case 'gpt_bigcode':
|
||||
mapping['num_heads'] = 'n_head';
|
||||
mapping['num_layers'] = 'n_layer';
|
||||
mapping['hidden_size'] = 'n_embd';
|
||||
break;
|
||||
case 'gpt_neox':
|
||||
case 'stablelm':
|
||||
case 'opt':
|
||||
case 'falcon':
|
||||
case 'modernbert-decoder':
|
||||
mapping['num_heads'] = 'num_attention_heads';
|
||||
mapping['num_layers'] = 'num_hidden_layers';
|
||||
mapping['hidden_size'] = 'hidden_size';
|
||||
break;
|
||||
case 'gpt_oss':
|
||||
case 'llama':
|
||||
case 'llama4_text':
|
||||
case 'nanochat':
|
||||
case 'apertus':
|
||||
case 'arcee':
|
||||
case 'afmoe':
|
||||
case 'lfm2':
|
||||
case 'lfm2_moe':
|
||||
case 'smollm3':
|
||||
case 'olmo':
|
||||
case 'olmo2':
|
||||
case 'olmo3':
|
||||
case 'mobilellm':
|
||||
case 'granite':
|
||||
case 'granitemoehybrid':
|
||||
case 'cohere':
|
||||
case 'cohere2':
|
||||
case 'mistral':
|
||||
case 'voxtral_realtime_text':
|
||||
case 'voxtral_realtime_encoder':
|
||||
case 'starcoder2':
|
||||
case 'qwen2':
|
||||
case 'qwen2_moe':
|
||||
case 'qwen2_vl':
|
||||
case 'qwen2_vl_text':
|
||||
case 'qwen2_5_vl_text':
|
||||
case 'qwen3_moe':
|
||||
case 'qwen3_vl_text':
|
||||
case 'qwen3_vl_moe_text':
|
||||
case 'phi':
|
||||
case 'phi3':
|
||||
case 'phi3_v':
|
||||
case 'llava_qwen2':
|
||||
mapping['num_heads'] = 'num_key_value_heads';
|
||||
mapping['num_layers'] = 'num_hidden_layers';
|
||||
mapping['hidden_size'] = 'hidden_size';
|
||||
mapping['num_attention_heads'] = 'num_attention_heads';
|
||||
mapping['dim_kv'] = 'head_dim';
|
||||
break;
|
||||
case 'qwen3':
|
||||
case 'solar_open':
|
||||
case 'glm_ocr_text':
|
||||
case 'gemma':
|
||||
case 'gemma2':
|
||||
case 'vaultgemma':
|
||||
case 'gemma3_text':
|
||||
case 'gemma3n_text':
|
||||
case 'gemma4_text':
|
||||
case 'glm':
|
||||
case 'helium':
|
||||
case 'ernie4_5':
|
||||
case 'hunyuan_v1_dense':
|
||||
case 'falcon_h1':
|
||||
case 'nemotron_h':
|
||||
case 'ministral':
|
||||
case 'ministral3':
|
||||
mapping['num_heads'] = 'num_key_value_heads';
|
||||
mapping['num_layers'] = 'num_hidden_layers';
|
||||
mapping['dim_kv'] = 'head_dim';
|
||||
break;
|
||||
case 'openelm':
|
||||
mapping['num_heads'] = 'num_kv_heads';
|
||||
mapping['num_layers'] = 'num_transformer_layers';
|
||||
mapping['dim_kv'] = 'head_dim';
|
||||
break;
|
||||
case 'gpt_neo':
|
||||
case 'donut-swin':
|
||||
mapping['num_heads'] = 'num_heads';
|
||||
mapping['num_layers'] = 'num_layers';
|
||||
mapping['hidden_size'] = 'hidden_size';
|
||||
break;
|
||||
case 'bloom':
|
||||
mapping['num_heads'] = 'n_head';
|
||||
mapping['num_layers'] = 'n_layer';
|
||||
mapping['hidden_size'] = 'hidden_size';
|
||||
break;
|
||||
case 'mpt':
|
||||
mapping['num_heads'] = 'n_heads';
|
||||
mapping['num_layers'] = 'n_layers';
|
||||
mapping['hidden_size'] = 'd_model';
|
||||
break;
|
||||
case 'exaone':
|
||||
mapping['num_heads'] = 'num_key_value_heads';
|
||||
mapping['num_layers'] = 'num_layers';
|
||||
mapping['dim_kv'] = 'head_dim';
|
||||
mapping['num_attention_heads'] = 'num_attention_heads';
|
||||
break;
|
||||
case 'youtu':
|
||||
case 'deepseek_v3':
|
||||
case 'deepseek_v4':
|
||||
case 'glm_moe_dsa':
|
||||
case 'mistral4':
|
||||
mapping['num_heads'] = 'num_key_value_heads';
|
||||
mapping['num_layers'] = 'num_hidden_layers';
|
||||
mapping['dim_kv'] = config.model_type === 'deepseek_v4' ? 'head_dim' : 'qk_head_dim';
|
||||
mapping['num_attention_heads'] = 'num_attention_heads';
|
||||
break;
|
||||
case 'zaya':
|
||||
mapping['num_heads'] = 'num_key_value_heads';
|
||||
mapping['num_layers'] = 'num_hidden_layers';
|
||||
mapping['hidden_size'] = 'hidden_size';
|
||||
mapping['dim_kv'] = 'head_dim';
|
||||
mapping['num_attention_heads'] = 'num_attention_heads';
|
||||
break;
|
||||
case 'hrm_text':
|
||||
mapping['num_heads'] = 'num_key_value_heads';
|
||||
mapping['num_layers'] = 'num_hidden_layers';
|
||||
mapping['hidden_size'] = 'hidden_size';
|
||||
mapping['dim_kv'] = 'head_dim';
|
||||
mapping['num_attention_heads'] = 'num_attention_heads';
|
||||
break;
|
||||
|
||||
// Encoder-decoder models
|
||||
case 't5':
|
||||
case 'mt5':
|
||||
case 'longt5':
|
||||
mapping['num_decoder_layers'] = 'num_decoder_layers';
|
||||
mapping['num_decoder_heads'] = 'num_heads';
|
||||
mapping['decoder_dim_kv'] = 'd_kv';
|
||||
mapping['num_encoder_layers'] = 'num_layers';
|
||||
mapping['num_encoder_heads'] = 'num_heads';
|
||||
mapping['encoder_dim_kv'] = 'd_kv';
|
||||
break;
|
||||
case 'bart':
|
||||
case 'mbart':
|
||||
case 'marian':
|
||||
case 'whisper':
|
||||
case 'lite-whisper':
|
||||
case 'm2m_100':
|
||||
case 'blenderbot':
|
||||
case 'blenderbot-small':
|
||||
case 'florence2_language':
|
||||
mapping['num_decoder_layers'] = 'decoder_layers';
|
||||
mapping['num_decoder_heads'] = 'decoder_attention_heads';
|
||||
mapping['decoder_hidden_size'] = 'd_model';
|
||||
mapping['num_encoder_layers'] = 'encoder_layers';
|
||||
mapping['num_encoder_heads'] = 'encoder_attention_heads';
|
||||
mapping['encoder_hidden_size'] = 'd_model';
|
||||
break;
|
||||
case 'speecht5':
|
||||
mapping['num_decoder_layers'] = 'decoder_layers';
|
||||
mapping['num_decoder_heads'] = 'decoder_attention_heads';
|
||||
mapping['decoder_hidden_size'] = 'hidden_size';
|
||||
mapping['num_encoder_layers'] = 'encoder_layers';
|
||||
mapping['num_encoder_heads'] = 'encoder_attention_heads';
|
||||
mapping['encoder_hidden_size'] = 'hidden_size';
|
||||
break;
|
||||
case 'trocr':
|
||||
mapping['num_encoder_layers'] = mapping['num_decoder_layers'] = 'decoder_layers';
|
||||
mapping['num_encoder_heads'] = mapping['num_decoder_heads'] = 'decoder_attention_heads';
|
||||
mapping['encoder_hidden_size'] = mapping['decoder_hidden_size'] = 'd_model';
|
||||
break;
|
||||
case 'musicgen_decoder':
|
||||
mapping['num_encoder_layers'] = mapping['num_decoder_layers'] = 'num_hidden_layers';
|
||||
mapping['num_encoder_heads'] = mapping['num_decoder_heads'] = 'num_attention_heads';
|
||||
mapping['encoder_hidden_size'] = mapping['decoder_hidden_size'] = 'hidden_size';
|
||||
break;
|
||||
case 'moonshine':
|
||||
mapping['num_decoder_layers'] = 'decoder_num_hidden_layers';
|
||||
mapping['num_decoder_heads'] = 'decoder_num_key_value_heads';
|
||||
mapping['num_encoder_layers'] = 'encoder_num_hidden_layers';
|
||||
mapping['num_encoder_heads'] = 'encoder_num_key_value_heads';
|
||||
mapping['encoder_hidden_size'] = mapping['decoder_hidden_size'] = 'hidden_size';
|
||||
break;
|
||||
case 'cohere_asr':
|
||||
mapping['num_decoder_layers'] = 'num_hidden_layers';
|
||||
mapping['num_decoder_heads'] = 'num_key_value_heads';
|
||||
mapping['decoder_hidden_size'] = 'hidden_size';
|
||||
mapping['decoder_dim_kv'] = 'head_dim';
|
||||
const {
|
||||
num_hidden_layers: num_encoder_layers,
|
||||
num_attention_heads: num_encoder_heads,
|
||||
hidden_size: encoder_hidden_size,
|
||||
} = /** @type {any} */ (config).encoder_config;
|
||||
init_normalized_config = {
|
||||
num_encoder_layers,
|
||||
num_encoder_heads,
|
||||
encoder_hidden_size,
|
||||
// @ts-expect-error TS2339
|
||||
encoder_dim_kv: config.head_dim,
|
||||
};
|
||||
break;
|
||||
case 'vision-encoder-decoder':
|
||||
// @ts-expect-error TS2339
|
||||
const decoderConfig = getNormalizedConfig(config.decoder);
|
||||
|
||||
const add_encoder_pkv = 'num_decoder_layers' in decoderConfig;
|
||||
const result = pick(config, ['model_type', 'is_encoder_decoder']);
|
||||
if (add_encoder_pkv) {
|
||||
// Decoder is part of an encoder-decoder model
|
||||
result.num_decoder_layers = decoderConfig.num_decoder_layers;
|
||||
result.num_decoder_heads = decoderConfig.num_decoder_heads;
|
||||
result.decoder_hidden_size = decoderConfig.decoder_hidden_size;
|
||||
|
||||
result.num_encoder_layers = decoderConfig.num_encoder_layers;
|
||||
result.num_encoder_heads = decoderConfig.num_encoder_heads;
|
||||
result.encoder_hidden_size = decoderConfig.encoder_hidden_size;
|
||||
} else {
|
||||
// Decoder is a decoder-only model
|
||||
result.num_layers = decoderConfig.num_layers;
|
||||
result.num_heads = decoderConfig.num_heads;
|
||||
result.hidden_size = decoderConfig.hidden_size;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// NOTE: If `num_attention_heads` is not set, it is assumed to be equal to `num_heads`
|
||||
const normalized_config = {
|
||||
...init_normalized_config,
|
||||
...pick(config, ['model_type', 'multi_query', 'is_encoder_decoder']),
|
||||
};
|
||||
for (const key in mapping) {
|
||||
normalized_config[key] = config[mapping[key]];
|
||||
}
|
||||
return normalized_config;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {PretrainedConfig} config
|
||||
* @param {{ prefix?: string, session_name?: string }} [options]
|
||||
* @returns {Set<string>}
|
||||
*/
|
||||
export function getCacheNames(config, options) {
|
||||
if (!(config instanceof PretrainedConfig)) {
|
||||
config = new PretrainedConfig(config);
|
||||
}
|
||||
|
||||
const pkv_prefix = options?.prefix ?? 'past_key_values';
|
||||
const cache_prefix = pkv_prefix === 'present' ? 'present' : 'past';
|
||||
/** @type {Set<string>} */
|
||||
const names = new Set();
|
||||
|
||||
if (['lfm2', 'lfm2_moe'].includes(config.model_type)) {
|
||||
const { layer_types } = /** @type {any} */ (config);
|
||||
for (let i = 0; i < layer_types.length; ++i) {
|
||||
if (layer_types[i] === 'full_attention') {
|
||||
names.add(`${pkv_prefix}.${i}.key`);
|
||||
names.add(`${pkv_prefix}.${i}.value`);
|
||||
} else if (layer_types[i] === 'conv') {
|
||||
names.add(`${cache_prefix}_conv.${i}`);
|
||||
} else {
|
||||
throw new Error(`Unsupported layer type: ${layer_types[i]}`);
|
||||
}
|
||||
}
|
||||
return names;
|
||||
} else if (['granitemoehybrid', 'falcon_h1', 'nemotron_h'].includes(config.model_type)) {
|
||||
const c = /** @type {any} */ (config);
|
||||
const layer_types = c.layer_types ?? c.layers_block_type;
|
||||
const num_layers = c.num_hidden_layers ?? layer_types?.length;
|
||||
|
||||
for (let i = 0; i < num_layers; ++i) {
|
||||
if (!layer_types || layer_types[i] === 'mamba') {
|
||||
names.add(`${cache_prefix}_conv.${i}`);
|
||||
names.add(`${cache_prefix}_ssm.${i}`);
|
||||
}
|
||||
if (!layer_types || layer_types[i] === 'attention') {
|
||||
names.add(`${pkv_prefix}.${i}.key`);
|
||||
names.add(`${pkv_prefix}.${i}.value`);
|
||||
}
|
||||
}
|
||||
return names;
|
||||
} else if (['qwen3_next', 'qwen3_5_text', 'qwen3_5_moe_text', 'olmo_hybrid'].includes(config.model_type)) {
|
||||
const { layer_types } = /** @type {any} */ (config);
|
||||
for (let i = 0; i < layer_types.length; ++i) {
|
||||
if (layer_types[i] === 'full_attention') {
|
||||
names.add(`${pkv_prefix}.${i}.key`);
|
||||
names.add(`${pkv_prefix}.${i}.value`);
|
||||
} else if (layer_types[i] === 'linear_attention') {
|
||||
if (config.model_type === 'olmo_hybrid') {
|
||||
names.add(`${cache_prefix}_conv.${i}.key`);
|
||||
names.add(`${cache_prefix}_conv.${i}.value`);
|
||||
names.add(`${cache_prefix}_conv.${i}.query`);
|
||||
} else {
|
||||
names.add(`${cache_prefix}_conv.${i}`);
|
||||
}
|
||||
names.add(`${cache_prefix}_recurrent.${i}`);
|
||||
} else {
|
||||
throw new Error(`Unsupported layer type: ${layer_types[i]}`);
|
||||
}
|
||||
}
|
||||
return names;
|
||||
} else if (['gemma4', 'gemma4_text'].includes(config.model_type)) {
|
||||
const c = /** @type {any} */ (
|
||||
config.model_type === 'gemma4' ? /** @type {any} */ (config).text_config : config
|
||||
);
|
||||
const num_hidden_layers = c.num_hidden_layers;
|
||||
const num_kv_shared_layers = c.num_kv_shared_layers ?? 0;
|
||||
const num_kv_layers = num_hidden_layers - num_kv_shared_layers;
|
||||
|
||||
for (let i = 0; i < num_kv_layers; ++i) {
|
||||
names.add(`${pkv_prefix}.${i}.key`);
|
||||
names.add(`${pkv_prefix}.${i}.value`);
|
||||
}
|
||||
return names;
|
||||
} else if (config.model_type === 'deepseek_v4') {
|
||||
const { layer_types, num_hidden_layers } = /** @type {any} */ (config);
|
||||
|
||||
for (let i = 0; i < num_hidden_layers; ++i) {
|
||||
names.add(`${pkv_prefix}.${i}.key`);
|
||||
names.add(`${pkv_prefix}.${i}.value`);
|
||||
|
||||
const layer_type = layer_types[i];
|
||||
if (layer_type === 'compressed_sparse_attention') {
|
||||
names.add(`${cache_prefix}_compressor.${i}.kv`);
|
||||
names.add(`${cache_prefix}_compressor.${i}.gate`);
|
||||
names.add(`${cache_prefix}_indexer.${i}.kv`);
|
||||
names.add(`${cache_prefix}_indexer.${i}.gate`);
|
||||
} else if (layer_type === 'heavily_compressed_attention') {
|
||||
names.add(`${cache_prefix}_compressor.${i}.kv`);
|
||||
names.add(`${cache_prefix}_compressor.${i}.gate`);
|
||||
} else if (layer_type && layer_type !== 'sliding_attention') {
|
||||
throw new Error(`Unsupported layer type: ${layer_type}`);
|
||||
}
|
||||
}
|
||||
return names;
|
||||
} else if (config.model_type === 'zaya') {
|
||||
const { num_hidden_layers, cca_time1 } = /** @type {any} */ (config);
|
||||
const stride = cca_time1 ?? 1;
|
||||
for (let i = 0; i < num_hidden_layers; i += stride) {
|
||||
names.add(`${pkv_prefix}.${i}.key`);
|
||||
names.add(`${pkv_prefix}.${i}.value`);
|
||||
names.add(`${pkv_prefix}.${i}.conv_state`);
|
||||
names.add(`${pkv_prefix}.${i}.shift_state`);
|
||||
}
|
||||
return names;
|
||||
} else if (['lfm2_vl', 'qwen3_5', 'qwen3_5_moe', 'voxtral_realtime'].includes(config.model_type)) {
|
||||
let subConfig;
|
||||
if (config.model_type === 'voxtral_realtime' && options?.session_name === 'audio_encoder') {
|
||||
subConfig = /** @type {any} */ (config).audio_config;
|
||||
} else {
|
||||
subConfig = /** @type {any} */ (config).text_config;
|
||||
}
|
||||
return getCacheNames(subConfig, options);
|
||||
}
|
||||
|
||||
return getKeyValueNames(config, { prefix: pkv_prefix });
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {PretrainedConfig} config
|
||||
* @param {{ prefix?: string }} [options]
|
||||
* @returns {Set<string>}
|
||||
*/
|
||||
function getKeyValueNames(config, { prefix = 'past_key_values' } = {}) {
|
||||
/** @type {Set<string>} */
|
||||
const names = new Set();
|
||||
const normalized_config = config.normalized_config;
|
||||
|
||||
if (
|
||||
normalized_config.is_encoder_decoder &&
|
||||
'num_encoder_heads' in normalized_config &&
|
||||
'num_decoder_heads' in normalized_config
|
||||
) {
|
||||
for (let i = 0; i < normalized_config.num_decoder_layers; ++i) {
|
||||
names.add(`${prefix}.${i}.encoder.key`);
|
||||
names.add(`${prefix}.${i}.encoder.value`);
|
||||
names.add(`${prefix}.${i}.decoder.key`);
|
||||
names.add(`${prefix}.${i}.decoder.value`);
|
||||
}
|
||||
} else if (normalized_config.multi_query) {
|
||||
// e.g., for `gpt_bigcode`
|
||||
for (let i = 0; i < normalized_config.num_layers; ++i) {
|
||||
names.add(`${prefix}.${i}.key_value`);
|
||||
}
|
||||
} else {
|
||||
for (let i = 0; i < normalized_config.num_layers; ++i) {
|
||||
names.add(`${prefix}.${i}.key`);
|
||||
names.add(`${prefix}.${i}.value`);
|
||||
}
|
||||
}
|
||||
|
||||
return names;
|
||||
}
|
||||
/**
|
||||
* Base class for all configuration classes. For more information, see the corresponding
|
||||
* [Python documentation](https://huggingface.co/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig).
|
||||
*/
|
||||
export class PretrainedConfig {
|
||||
// NOTE: Typo in original
|
||||
|
||||
/** @type {string|null} */
|
||||
model_type = null;
|
||||
|
||||
/** @type {boolean} */
|
||||
is_encoder_decoder = false;
|
||||
|
||||
/** @type {number} */
|
||||
max_position_embeddings;
|
||||
|
||||
/** @type {TransformersJSConfig} */
|
||||
'transformers.js_config';
|
||||
|
||||
/**
|
||||
* Create a new PreTrainedTokenizer instance.
|
||||
* @param {Object} configJSON The JSON of the config.
|
||||
*/
|
||||
constructor(configJSON) {
|
||||
Object.assign(this, configJSON);
|
||||
this.normalized_config = getNormalizedConfig(this);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a pre-trained config from the given `pretrained_model_name_or_path`.
|
||||
*
|
||||
* @param {string} pretrained_model_name_or_path The path to the pre-trained config.
|
||||
* @param {PretrainedOptions} options Additional options for loading the config.
|
||||
* @throws {Error} Throws an error if the config.json is not found in the `pretrained_model_name_or_path`.
|
||||
*
|
||||
* @returns {Promise<PretrainedConfig>} A new instance of the `PretrainedConfig` class.
|
||||
*/
|
||||
static async from_pretrained(
|
||||
pretrained_model_name_or_path,
|
||||
{ progress_callback = null, config = null, cache_dir = null, local_files_only = false, revision = 'main' } = {},
|
||||
) {
|
||||
if (config && !(config instanceof PretrainedConfig)) {
|
||||
config = new PretrainedConfig(config);
|
||||
}
|
||||
|
||||
const data =
|
||||
config ??
|
||||
(await loadConfig(pretrained_model_name_or_path, {
|
||||
progress_callback,
|
||||
config,
|
||||
cache_dir,
|
||||
local_files_only,
|
||||
revision,
|
||||
}));
|
||||
return new this(data);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained configs with the `from_pretrained` function.
|
||||
*
|
||||
* @example
|
||||
* const config = await AutoConfig.from_pretrained('Xenova/bert-base-uncased');
|
||||
*/
|
||||
export class AutoConfig {
|
||||
/** @type {typeof PretrainedConfig.from_pretrained} */
|
||||
static async from_pretrained(...args) {
|
||||
return PretrainedConfig.from_pretrained(...args);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transformers.js-specific configuration, possibly present in config.json under the key `transformers.js_config`.
|
||||
* @typedef {Object} TransformersJSConfig
|
||||
* @property {Record<import('./utils/devices.js').DeviceType, DeviceConfig>} [device_config] Device-specific configurations.
|
||||
* @property {Record<string, number>} [free_dimension_overrides] Override the free dimensions of the model.
|
||||
* See https://onnxruntime.ai/docs/tutorials/web/env-flags-and-session-options.html#freedimensionoverrides
|
||||
* for more information.
|
||||
* @property {import('./utils/devices.js').DeviceType} [device] The default device to use for the model.
|
||||
* @property {import('./utils/dtypes.js').DataType|Record<string, import('./utils/dtypes.js').DataType>} [dtype] The default data type to use for the model.
|
||||
* @property {import('./utils/hub.js').ExternalData|Record<string, import('./utils/hub.js').ExternalData>} [use_external_data_format=false] Whether to load the model using the external data format (used for models >= 2GB in size).
|
||||
*/
|
||||
|
||||
/**
|
||||
* Device-specific configuration options.
|
||||
* @typedef {Omit<TransformersJSConfig, "device" | "device_config">} DeviceConfig
|
||||
*/
|
||||
@@ -0,0 +1,293 @@
|
||||
/**
|
||||
* @file Module used to configure Transformers.js.
|
||||
*
|
||||
* **Example:** Disable remote models.
|
||||
* ```javascript
|
||||
* import { env } from '@huggingface/transformers';
|
||||
* env.allowRemoteModels = false;
|
||||
* ```
|
||||
*
|
||||
* **Example:** Set local model path.
|
||||
* ```javascript
|
||||
* import { env } from '@huggingface/transformers';
|
||||
* env.localModelPath = '/path/to/local/models/';
|
||||
* ```
|
||||
*
|
||||
* **Example:** Set cache directory.
|
||||
* ```javascript
|
||||
* import { env } from '@huggingface/transformers';
|
||||
* env.cacheDir = '/path/to/cache/directory/';
|
||||
* ```
|
||||
*
|
||||
* @module env
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import url from 'node:url';
|
||||
|
||||
const VERSION = '4.2.0';
|
||||
|
||||
const HAS_SELF = typeof self !== 'undefined';
|
||||
|
||||
const IS_FS_AVAILABLE = !isEmpty(fs);
|
||||
const IS_PATH_AVAILABLE = !isEmpty(path);
|
||||
const IS_WEB_CACHE_AVAILABLE = HAS_SELF && 'caches' in self;
|
||||
|
||||
// Runtime detection
|
||||
const IS_DENO_RUNTIME = typeof globalThis.Deno !== 'undefined';
|
||||
const IS_BUN_RUNTIME = typeof globalThis.Bun !== 'undefined';
|
||||
|
||||
const IS_DENO_WEB_RUNTIME = IS_DENO_RUNTIME && IS_WEB_CACHE_AVAILABLE && !IS_FS_AVAILABLE;
|
||||
|
||||
const IS_PROCESS_AVAILABLE = typeof process !== 'undefined';
|
||||
const IS_NODE_ENV = IS_PROCESS_AVAILABLE && process?.release?.name === 'node' && !IS_DENO_WEB_RUNTIME;
|
||||
|
||||
// Check if various APIs are available (depends on environment)
|
||||
const IS_BROWSER_ENV = typeof window !== 'undefined' && typeof window.document !== 'undefined';
|
||||
const IS_WEBWORKER_ENV =
|
||||
HAS_SELF &&
|
||||
['DedicatedWorkerGlobalScope', 'ServiceWorkerGlobalScope', 'SharedWorkerGlobalScope'].includes(
|
||||
self.constructor?.name,
|
||||
);
|
||||
const IS_WEB_ENV = IS_BROWSER_ENV || IS_WEBWORKER_ENV || IS_DENO_WEB_RUNTIME;
|
||||
|
||||
const IS_WEBGPU_AVAILABLE = IS_NODE_ENV || (typeof navigator !== 'undefined' && 'gpu' in navigator);
|
||||
const IS_WEBNN_AVAILABLE = typeof navigator !== 'undefined' && 'ml' in navigator;
|
||||
const IS_CRYPTO_AVAILABLE = typeof crypto !== 'undefined' && typeof crypto.getRandomValues === 'function';
|
||||
|
||||
const IS_CHROME_AVAILABLE =
|
||||
// @ts-ignore - chrome may not exist in all environments
|
||||
typeof chrome !== 'undefined' && typeof chrome.runtime !== 'undefined' && typeof chrome.runtime.id === 'string';
|
||||
|
||||
const IS_SERVICE_WORKER_ENV =
|
||||
// @ts-ignore - ServiceWorkerGlobalScope may not exist in all environments
|
||||
typeof ServiceWorkerGlobalScope !== 'undefined' && HAS_SELF && self instanceof ServiceWorkerGlobalScope;
|
||||
|
||||
/**
|
||||
* Check whether the current environment is a Safari browser older than version 26 (Safari >= 26 enables WebGPU by default).
|
||||
* @returns {boolean} Whether the current environment is Safari older than version 26.
|
||||
*/
|
||||
const isSafariBelow26 = () => {
|
||||
if (typeof navigator === 'undefined') {
|
||||
return false;
|
||||
}
|
||||
|
||||
const userAgent = navigator.userAgent;
|
||||
|
||||
// Safari has "Apple" in its vendor string. Exclude Chrome on iOS (CriOS),
|
||||
// Firefox on iOS (FxiOS), Edge on iOS (EdgiOS), and other browsers.
|
||||
const isSafari =
|
||||
(navigator.vendor || '').indexOf('Apple') > -1 &&
|
||||
!userAgent.match(/CriOS|FxiOS|EdgiOS|OPiOS|mercury|brave/i) &&
|
||||
!userAgent.includes('Chrome') &&
|
||||
!userAgent.includes('Android');
|
||||
if (!isSafari) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Safari reports its version via the "Version/<major>.<minor>" token in the user agent.
|
||||
// If the version is missing/unparseable, assume a modern Safari (i.e. not below 26).
|
||||
const match = userAgent.match(/Version\/(\d+)/);
|
||||
return match ? parseInt(match[1], 10) < 26 : false;
|
||||
};
|
||||
const IS_SAFARI_BELOW_26 = isSafariBelow26();
|
||||
|
||||
/**
|
||||
* A read-only object containing information about the APIs available in the current environment.
|
||||
*/
|
||||
export const apis = Object.freeze({
|
||||
/** Whether we are running in a browser environment (and not a web worker) */
|
||||
IS_BROWSER_ENV,
|
||||
|
||||
/** Whether we are running in a web worker environment */
|
||||
IS_WEBWORKER_ENV,
|
||||
|
||||
/** Whether we are running in a web-like environment (browser, web worker, or Deno web runtime) */
|
||||
IS_WEB_ENV,
|
||||
|
||||
/** Whether we are running in a service worker environment */
|
||||
IS_SERVICE_WORKER_ENV,
|
||||
|
||||
/** Whether we are running in Deno's web runtime (CDN imports, Cache API available, no filesystem) */
|
||||
IS_DENO_WEB_RUNTIME,
|
||||
|
||||
/** Whether the Cache API is available */
|
||||
IS_WEB_CACHE_AVAILABLE,
|
||||
|
||||
/** Whether the WebGPU API is available */
|
||||
IS_WEBGPU_AVAILABLE,
|
||||
|
||||
/** Whether the WebNN API is available */
|
||||
IS_WEBNN_AVAILABLE,
|
||||
|
||||
/** Whether we are running in a Safari browser older than version 26. */
|
||||
IS_SAFARI_BELOW_26,
|
||||
|
||||
/** Whether the Node.js process API is available */
|
||||
IS_PROCESS_AVAILABLE,
|
||||
|
||||
/** Whether we are running in a Node.js-like environment (node, deno, bun) */
|
||||
IS_NODE_ENV,
|
||||
|
||||
/** Whether the filesystem API is available */
|
||||
IS_FS_AVAILABLE,
|
||||
|
||||
/** Whether the path API is available */
|
||||
IS_PATH_AVAILABLE,
|
||||
|
||||
/** Whether the crypto API is available */
|
||||
IS_CRYPTO_AVAILABLE,
|
||||
|
||||
/** Whether the Chrome runtime API is available */
|
||||
IS_CHROME_AVAILABLE,
|
||||
});
|
||||
|
||||
const RUNNING_LOCALLY = IS_FS_AVAILABLE && IS_PATH_AVAILABLE;
|
||||
|
||||
let dirname__ = './';
|
||||
if (RUNNING_LOCALLY) {
|
||||
// NOTE: We wrap `import.meta` in a call to `Object` to prevent Webpack from trying to bundle it in CommonJS.
|
||||
// Although we get the warning: "Accessing import.meta directly is unsupported (only property access or destructuring is supported)",
|
||||
// it is safe to ignore since the bundled value (`{}`) isn't used for CommonJS environments (we use __dirname instead).
|
||||
const _import_meta_url = Object(import.meta).url;
|
||||
|
||||
if (_import_meta_url) {
|
||||
dirname__ = path.dirname(path.dirname(url.fileURLToPath(_import_meta_url))); // ESM
|
||||
} else if (typeof __dirname !== 'undefined') {
|
||||
dirname__ = path.dirname(__dirname); // CommonJS
|
||||
}
|
||||
}
|
||||
|
||||
// Only used for environments with access to file system
|
||||
const DEFAULT_CACHE_DIR = RUNNING_LOCALLY ? path.join(dirname__, '/.cache/') : null;
|
||||
|
||||
// Set local model path, based on available APIs
|
||||
const DEFAULT_LOCAL_MODEL_PATH = '/models/';
|
||||
const localModelPath = RUNNING_LOCALLY ? path.join(dirname__, DEFAULT_LOCAL_MODEL_PATH) : DEFAULT_LOCAL_MODEL_PATH;
|
||||
|
||||
// Ensure default fetch is called with the correct receiver in browser environments.
|
||||
const DEFAULT_FETCH = typeof globalThis.fetch === 'function' ? globalThis.fetch.bind(globalThis) : undefined;
|
||||
|
||||
/**
|
||||
* Log levels for controlling output verbosity.
|
||||
*
|
||||
* Each level is represented by a number, where higher numbers include all lower level messages.
|
||||
* Use these values to set `env.logLevel`.
|
||||
*
|
||||
* @example
|
||||
* import { env, LogLevel } from '@huggingface/transformers';
|
||||
*
|
||||
* // Set log level to show only errors
|
||||
* env.logLevel = LogLevel.ERROR;
|
||||
*
|
||||
* // Set log level to show errors, warnings, and info
|
||||
* env.logLevel = LogLevel.INFO;
|
||||
*
|
||||
* // Disable all logging
|
||||
* env.logLevel = LogLevel.NONE;
|
||||
*
|
||||
*/
|
||||
export const LogLevel = Object.freeze({
|
||||
/** All messages including debug output (value: 10) */
|
||||
DEBUG: 10,
|
||||
/** Errors, warnings, and info messages (value: 20) */
|
||||
INFO: 20,
|
||||
/** Errors and warnings (value: 30) */
|
||||
WARNING: 30,
|
||||
/** Only error messages (value: 40) */
|
||||
ERROR: 40,
|
||||
/** No logging output (value: 50) */
|
||||
NONE: 50,
|
||||
});
|
||||
|
||||
/**
|
||||
* Global variable given visible to users to control execution. This provides users a simple way to configure Transformers.js.
|
||||
* @typedef {Object} TransformersEnvironment
|
||||
* @property {string} version This version of Transformers.js.
|
||||
* @property {{onnx: Partial<import('onnxruntime-common').Env> & { setLogLevel?: (logLevel: number) => void }}} backends Expose environment variables of different backends,
|
||||
* allowing users to set these variables if they want to.
|
||||
* @property {number} logLevel The logging level. Use LogLevel enum values. Defaults to LogLevel.ERROR.
|
||||
* @property {boolean} allowRemoteModels Whether to allow loading of remote files, defaults to `true`.
|
||||
* If set to `false`, it will have the same effect as setting `local_files_only=true` when loading pipelines, models, tokenizers, processors, etc.
|
||||
* @property {string} remoteHost Host URL to load models from. Defaults to the Hugging Face Hub.
|
||||
* @property {string} remotePathTemplate Path template to fill in and append to `remoteHost` when loading models.
|
||||
* @property {boolean} allowLocalModels Whether to allow loading of local files, defaults to `false` if running in-browser, and `true` otherwise.
|
||||
* If set to `false`, it will skip the local file check and try to load the model from the remote host.
|
||||
* @property {string} localModelPath Path to load local models from. Defaults to `/models/`.
|
||||
* @property {boolean} useFS Whether to use the file system to load files. By default, it is `true` if available.
|
||||
* @property {boolean} useBrowserCache Whether to use Cache API to cache models. By default, it is `true` if available.
|
||||
* @property {boolean} useFSCache Whether to use the file system to cache files. By default, it is `true` if available.
|
||||
* @property {string|null} cacheDir The directory to use for caching files with the file system. By default, it is `./.cache`.
|
||||
* @property {boolean} useCustomCache Whether to use a custom cache system (defined by `customCache`), defaults to `false`.
|
||||
* @property {import('./utils/cache.js').CacheInterface|null} customCache The custom cache to use. Defaults to `null`. Note: this must be an object which
|
||||
* implements the `match` and `put` functions of the Web Cache API. For more information, see https://developer.mozilla.org/en-US/docs/Web/API/Cache.
|
||||
* @property {boolean} useWasmCache Whether to pre-load and cache WASM binaries and the WASM factory (.mjs) for ONNX Runtime.
|
||||
* Defaults to `true` when cache is available. This can improve performance and enables offline usage by avoiding repeated downloads.
|
||||
* @property {string} cacheKey The cache key to use for storing models and WASM binaries. Defaults to 'transformers-cache'.
|
||||
* @property {boolean} experimental_useCrossOriginStorage Whether to use the Cross-Origin Storage API to cache model files
|
||||
* across origins, allowing different sites to share the same cached model weights. Defaults to `false`.
|
||||
* Requires the Cross-Origin Storage Chrome extension: {@link https://chromewebstore.google.com/detail/cross-origin-storage/denpnpcgjgikjpoglpjefakmdcbmlgih}.
|
||||
* The `experimental_` prefix indicates that the underlying browser API is not yet standardised and may change or be
|
||||
* removed without a major version bump. For more information, see {@link https://github.com/WICG/cross-origin-storage}.
|
||||
* @property {(input: string | URL, init?: any) => Promise<any>} fetch The fetch function to use. Defaults to `fetch`.
|
||||
*/
|
||||
|
||||
let logLevel = LogLevel.WARNING; // Default log level
|
||||
/** @type {TransformersEnvironment} */
|
||||
export const env = {
|
||||
version: VERSION,
|
||||
|
||||
/////////////////// Backends settings ///////////////////
|
||||
// NOTE: These will be populated later by the backends themselves.
|
||||
backends: {
|
||||
// onnxruntime-web/onnxruntime-node
|
||||
onnx: {},
|
||||
},
|
||||
|
||||
/////////////////// Logging settings ///////////////////
|
||||
get logLevel() {
|
||||
return logLevel;
|
||||
},
|
||||
set logLevel(level) {
|
||||
logLevel = level;
|
||||
|
||||
// invoke hook to set ONNX Runtime log level when Transformers.js log level changes
|
||||
env.backends.onnx?.setLogLevel?.(level);
|
||||
},
|
||||
/////////////////// Model settings ///////////////////
|
||||
allowRemoteModels: true,
|
||||
remoteHost: 'https://huggingface.co/',
|
||||
remotePathTemplate: '{model}/resolve/{revision}/',
|
||||
|
||||
allowLocalModels: !(IS_BROWSER_ENV || IS_WEBWORKER_ENV || IS_DENO_WEB_RUNTIME), // Default to true for non-web environments, false for web environments
|
||||
localModelPath: localModelPath,
|
||||
useFS: IS_FS_AVAILABLE,
|
||||
|
||||
/////////////////// Cache settings ///////////////////
|
||||
useBrowserCache: IS_WEB_CACHE_AVAILABLE,
|
||||
|
||||
useFSCache: IS_FS_AVAILABLE,
|
||||
cacheDir: DEFAULT_CACHE_DIR,
|
||||
|
||||
useCustomCache: false,
|
||||
customCache: null,
|
||||
|
||||
useWasmCache: IS_WEB_CACHE_AVAILABLE || IS_FS_AVAILABLE,
|
||||
cacheKey: 'transformers-cache',
|
||||
|
||||
experimental_useCrossOriginStorage: false,
|
||||
|
||||
/////////////////// Custom fetch /////////////////////
|
||||
fetch: DEFAULT_FETCH,
|
||||
|
||||
//////////////////////////////////////////////////////
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {Object} obj
|
||||
* @private
|
||||
*/
|
||||
function isEmpty(obj) {
|
||||
return Object.keys(obj).length === 0;
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { FEATURE_EXTRACTOR_NAME } from './utils/constants.js';
|
||||
import { Callable } from './utils/generic.js';
|
||||
import { getModelJSON } from './utils/hub.js';
|
||||
|
||||
/**
|
||||
* Base class for feature extractors.
|
||||
*/
|
||||
export class FeatureExtractor extends Callable {
|
||||
/**
|
||||
* Constructs a new FeatureExtractor instance.
|
||||
*
|
||||
* @param {Object} config The configuration for the feature extractor.
|
||||
*/
|
||||
constructor(config) {
|
||||
super();
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiate one of the feature extractor classes of the library from a pretrained model.
|
||||
*
|
||||
* The feature extractor class to instantiate is selected based on the `feature_extractor_type` property of
|
||||
* the config object (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible)
|
||||
*
|
||||
* @param {string} pretrained_model_name_or_path The name or path of the pretrained model. Can be either:
|
||||
* - A string, the *model id* of a pretrained feature_extractor hosted inside a model repo on huggingface.co.
|
||||
* Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
|
||||
* user or organization name, like `dbmdz/bert-base-german-cased`.
|
||||
* - A path to a *directory* containing feature_extractor files, e.g., `./my_model_directory/`.
|
||||
* @param {import('./utils/hub.js').PretrainedOptions} options Additional options for loading the feature_extractor.
|
||||
*
|
||||
* @returns {Promise<FeatureExtractor>} A new instance of the Feature Extractor class.
|
||||
*/
|
||||
static async from_pretrained(pretrained_model_name_or_path, options = {}) {
|
||||
const config = await getModelJSON(pretrained_model_name_or_path, FEATURE_EXTRACTOR_NAME, true, options);
|
||||
return new this(config);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to validate audio inputs.
|
||||
* @param {any} audio The audio data.
|
||||
* @param {string} feature_extractor The name of the feature extractor.
|
||||
* @private
|
||||
*/
|
||||
export function validate_audio_inputs(audio, feature_extractor) {
|
||||
if (!(audio instanceof Float32Array || audio instanceof Float64Array)) {
|
||||
throw new Error(
|
||||
`${feature_extractor} expects input to be a Float32Array or a Float64Array, but got ${audio?.constructor?.name ?? typeof audio} instead. ` +
|
||||
`If using the feature extractor directly, remember to use \`read_audio(url, sampling_rate)\` to obtain the raw audio data of the file/url.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
/**
|
||||
* @module generation/configuration_utils
|
||||
*/
|
||||
|
||||
import { pick } from '../utils/core.js';
|
||||
|
||||
/**
|
||||
* Class that holds a configuration for a generation task.
|
||||
*/
|
||||
export class GenerationConfig {
|
||||
// Parameters that control the length of the output
|
||||
/**
|
||||
* The maximum length the generated tokens can have.
|
||||
* Corresponds to the length of the input prompt + `max_new_tokens`.
|
||||
* Its effect is overridden by `max_new_tokens`, if also set.
|
||||
* @type {number}
|
||||
* @default 20
|
||||
*/
|
||||
max_length = 20;
|
||||
|
||||
/**
|
||||
* The maximum numbers of tokens to generate, ignoring the number of tokens in the prompt.
|
||||
* @type {number}
|
||||
* @default null
|
||||
*/
|
||||
max_new_tokens = null;
|
||||
|
||||
/**
|
||||
* The minimum length of the sequence to be generated.
|
||||
* Corresponds to the length of the input prompt + `min_new_tokens`.
|
||||
* Its effect is overridden by `min_new_tokens`, if also set.
|
||||
* @type {number}
|
||||
* @default 0
|
||||
*/
|
||||
min_length = 0;
|
||||
|
||||
/**
|
||||
* The minimum numbers of tokens to generate, ignoring the number of tokens in the prompt.
|
||||
* @type {number}
|
||||
* @default null
|
||||
*/
|
||||
min_new_tokens = null;
|
||||
|
||||
/**
|
||||
* Controls the stopping condition for beam-based methods, like beam-search. It accepts the following values:
|
||||
* - `true`, where the generation stops as soon as there are `num_beams` complete candidates;
|
||||
* - `false`, where an heuristic is applied and the generation stops when is it very unlikely to find better candidates;
|
||||
* - `"never"`, where the beam search procedure only stops when there cannot be better candidates (canonical beam search algorithm).
|
||||
* @type {boolean|"never"}
|
||||
* @default false
|
||||
*/
|
||||
early_stopping = false;
|
||||
|
||||
/**
|
||||
* The maximum amount of time you allow the computation to run for in seconds.
|
||||
* Generation will still finish the current pass after allocated time has been passed.
|
||||
* @type {number}
|
||||
* @default null
|
||||
*/
|
||||
max_time = null;
|
||||
|
||||
// Parameters that control the generation strategy used
|
||||
/**
|
||||
* Whether or not to use sampling; use greedy decoding otherwise.
|
||||
* @type {boolean}
|
||||
* @default false
|
||||
*/
|
||||
do_sample = false;
|
||||
|
||||
/**
|
||||
* Number of beams for beam search. 1 means no beam search.
|
||||
* @type {number}
|
||||
* @default 1
|
||||
*/
|
||||
num_beams = 1;
|
||||
|
||||
/**
|
||||
* Number of groups to divide `num_beams` into in order to ensure diversity among different groups of beams.
|
||||
* See [this paper](https://huggingface.co/papers/1610.02424) for more details.
|
||||
* @type {number}
|
||||
* @default 1
|
||||
*/
|
||||
num_beam_groups = 1;
|
||||
|
||||
/**
|
||||
* The values balance the model confidence and the degeneration penalty in contrastive search decoding.
|
||||
* @type {number}
|
||||
* @default null
|
||||
*/
|
||||
penalty_alpha = null;
|
||||
|
||||
/**
|
||||
* Whether or not the model should use the past last key/values attentions (if applicable to the model) to speed up decoding.
|
||||
* @type {boolean}
|
||||
* @default true
|
||||
*/
|
||||
use_cache = true;
|
||||
|
||||
// Parameters for manipulation of the model output logits
|
||||
/**
|
||||
* The value used to modulate the next token probabilities.
|
||||
* @type {number}
|
||||
* @default 1.0
|
||||
*/
|
||||
temperature = 1.0;
|
||||
|
||||
/**
|
||||
* The number of highest probability vocabulary tokens to keep for top-k-filtering.
|
||||
* @type {number}
|
||||
* @default 50
|
||||
*/
|
||||
top_k = 50;
|
||||
|
||||
/**
|
||||
* If set to float < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or higher are kept for generation.
|
||||
* @type {number}
|
||||
* @default 1.0
|
||||
*/
|
||||
top_p = 1.0;
|
||||
|
||||
/**
|
||||
* Local typicality measures how similar the conditional probability of predicting a target token next is to the expected conditional probability of predicting a random token next, given the partial text already generated.
|
||||
* If set to float < 1, the smallest set of the most locally typical tokens with probabilities that add up to `typical_p` or higher are kept for generation.
|
||||
* See [this paper](https://huggingface.co/papers/2202.00666) for more details.
|
||||
* @type {number}
|
||||
* @default 1.0
|
||||
*/
|
||||
typical_p = 1.0;
|
||||
|
||||
/**
|
||||
* If set to float strictly between 0 and 1, only tokens with a conditional probability greater than `epsilon_cutoff` will be sampled.
|
||||
* In the paper, suggested values range from 3e-4 to 9e-4, depending on the size of the model.
|
||||
* See [Truncation Sampling as Language Model Desmoothing](https://huggingface.co/papers/2210.15191) for more details.
|
||||
* @type {number}
|
||||
* @default 0.0
|
||||
*/
|
||||
epsilon_cutoff = 0.0;
|
||||
|
||||
/**
|
||||
* Eta sampling is a hybrid of locally typical sampling and epsilon sampling.
|
||||
* If set to float strictly between 0 and 1, a token is only considered if it is greater than either `eta_cutoff` or `sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits)))`.
|
||||
* The latter term is intuitively the expected next token probability, scaled by `sqrt(eta_cutoff)`. In the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model.
|
||||
* See [Truncation Sampling as Language Model Desmoothing](https://huggingface.co/papers/2210.15191) for more details.
|
||||
* @type {number}
|
||||
* @default 0.0
|
||||
*/
|
||||
eta_cutoff = 0.0;
|
||||
|
||||
/**
|
||||
* This value is subtracted from a beam's score if it generates a token same as any beam from other group at a particular time.
|
||||
* Note that `diversity_penalty` is only effective if `group beam search` is enabled.
|
||||
* @type {number}
|
||||
* @default 0.0
|
||||
*/
|
||||
diversity_penalty = 0.0;
|
||||
|
||||
/**
|
||||
* The parameter for repetition penalty. 1.0 means no penalty.
|
||||
* See [this paper](https://huggingface.co/papers/1909.05858) for more details.
|
||||
* @type {number}
|
||||
* @default 1.0
|
||||
*/
|
||||
repetition_penalty = 1.0;
|
||||
|
||||
/**
|
||||
* The paramater for encoder_repetition_penalty.
|
||||
* An exponential penalty on sequences that are not in the original input.
|
||||
* 1.0 means no penalty.
|
||||
* @type {number}
|
||||
* @default 1.0
|
||||
*/
|
||||
encoder_repetition_penalty = 1.0;
|
||||
|
||||
/**
|
||||
* Exponential penalty to the length that is used with beam-based generation.
|
||||
* It is applied as an exponent to the sequence length, which in turn is used to divide the score of the sequence.
|
||||
* Since the score is the log likelihood of the sequence (i.e. negative), `length_penalty` > 0.0 promotes longer sequences, while `length_penalty` < 0.0 encourages shorter sequences.
|
||||
* @type {number}
|
||||
* @default 1.0
|
||||
*/
|
||||
length_penalty = 1.0;
|
||||
|
||||
/**
|
||||
* If set to int > 0, all ngrams of that size can only occur once.
|
||||
* @type {number}
|
||||
* @default 0
|
||||
*/
|
||||
no_repeat_ngram_size = 0;
|
||||
|
||||
/**
|
||||
* List of token ids that are not allowed to be generated.
|
||||
* In order to get the token ids of the words that should not appear in the generated text, use
|
||||
* `tokenizer(bad_words, { add_prefix_space: true, add_special_tokens: false }).input_ids`.
|
||||
* @type {number[][]}
|
||||
* @default null
|
||||
*/
|
||||
bad_words_ids = null;
|
||||
|
||||
/**
|
||||
* List of token ids that must be generated.
|
||||
* If given a `number[][]`, this is treated as a simple list of words that must be included, the opposite to `bad_words_ids`.
|
||||
* If given `number[][][]`, this triggers a [disjunctive constraint](https://github.com/huggingface/transformers/issues/14081), where one can allow different forms of each word.
|
||||
* @type {number[][]|number[][][]}
|
||||
* @default null
|
||||
*/
|
||||
force_words_ids = null;
|
||||
|
||||
/**
|
||||
* Whether to renormalize the logits after applying all the logits processors or warpers (including the custom ones).
|
||||
* It's highly recommended to set this flag to `true` as the search algorithms suppose the score logits are normalized but some logit processors or warpers break the normalization.
|
||||
* @type {boolean}
|
||||
* @default false
|
||||
*/
|
||||
renormalize_logits = false;
|
||||
|
||||
/**
|
||||
* Custom constraints that can be added to the generation to ensure that the output will contain the use of certain tokens as defined by `Constraint` objects, in the most sensible way possible.
|
||||
* @type {Object[]}
|
||||
* @default null
|
||||
*/
|
||||
constraints = null;
|
||||
|
||||
/**
|
||||
* The id of the token to force as the first generated token after the `decoder_start_token_id`.
|
||||
* Useful for multilingual models like mBART where the first generated token needs to be the target language token.
|
||||
* @type {number}
|
||||
* @default null
|
||||
*/
|
||||
forced_bos_token_id = null;
|
||||
|
||||
/**
|
||||
* The id of the token to force as the last generated token when `max_length` is reached.
|
||||
* Optionally, use a list to set multiple *end-of-sequence* tokens.
|
||||
* @type {number|number[]}
|
||||
* @default null
|
||||
*/
|
||||
forced_eos_token_id = null;
|
||||
|
||||
/**
|
||||
* Whether to remove possible *nan* and *inf* outputs of the model to prevent the generation method to crash. Note that using `remove_invalid_values` can slow down generation.
|
||||
* @type {boolean}
|
||||
*/
|
||||
remove_invalid_values = false;
|
||||
|
||||
/**
|
||||
* This Tuple adds an exponentially increasing length penalty, after a certain amount of tokens have been generated.
|
||||
* The tuple shall consist of: `(start_index, decay_factor)` where `start_index` indicates where penalty starts and `decay_factor` represents the factor of exponential decay.
|
||||
* @type {[number, number]}
|
||||
* @default null
|
||||
*/
|
||||
exponential_decay_length_penalty = null;
|
||||
|
||||
/**
|
||||
* A list of tokens that will be suppressed at generation.
|
||||
* The `SuppressTokens` logit processor will set their log probs to `-inf` so that they are not sampled.
|
||||
* @type {number[]}
|
||||
* @default null
|
||||
*/
|
||||
suppress_tokens = null;
|
||||
|
||||
/**
|
||||
* A streamer that will be used to stream the generation.
|
||||
* @type {import('./streamers.js').TextStreamer}
|
||||
* @default null
|
||||
*/
|
||||
streamer = null;
|
||||
|
||||
/**
|
||||
* A list of tokens that will be suppressed at the beginning of the generation.
|
||||
* The `SuppressBeginTokens` logit processor will set their log probs to `-inf` so that they are not sampled.
|
||||
* @type {number[]}
|
||||
* @default null
|
||||
*/
|
||||
begin_suppress_tokens = null;
|
||||
|
||||
/**
|
||||
* A list of pairs of integers which indicates a mapping from generation indices to token indices that will be forced before sampling.
|
||||
* For example, `[[1, 123]]` means the second generated token will always be a token of index 123.
|
||||
* @type {[number, number][]}
|
||||
* @default null
|
||||
*/
|
||||
forced_decoder_ids = null;
|
||||
|
||||
/**
|
||||
* The guidance scale for classifier free guidance (CFG). CFG is enabled by setting `guidance_scale > 1`.
|
||||
* Higher guidance scale encourages the model to generate samples that are more closely linked to the input
|
||||
* prompt, usually at the expense of poorer quality.
|
||||
* @type {number}
|
||||
* @default null
|
||||
*/
|
||||
guidance_scale = null;
|
||||
|
||||
// Parameters that define the output variables of `generate`
|
||||
/**
|
||||
* The number of independently computed returned sequences for each element in the batch.
|
||||
* @type {number}
|
||||
* @default 1
|
||||
*/
|
||||
num_return_sequences = 1;
|
||||
|
||||
/**
|
||||
* Whether or not to return the attentions tensors of all attention layers.
|
||||
* See `attentions` under returned tensors for more details.
|
||||
* @type {boolean}
|
||||
* @default false
|
||||
*/
|
||||
output_attentions = false;
|
||||
|
||||
/**
|
||||
* Whether or not to return the hidden states of all layers.
|
||||
* See `hidden_states` under returned tensors for more details.
|
||||
* @type {boolean}
|
||||
* @default false
|
||||
*/
|
||||
output_hidden_states = false;
|
||||
|
||||
/**
|
||||
* Whether or not to return the prediction scores.
|
||||
* See `scores` under returned tensors for more details.
|
||||
* @type {boolean}
|
||||
* @default false
|
||||
*/
|
||||
output_scores = false;
|
||||
|
||||
/**
|
||||
* Whether or not to return a `ModelOutput` instead of a plain tuple.
|
||||
* @type {boolean}
|
||||
* @default false
|
||||
*/
|
||||
return_dict_in_generate = false;
|
||||
|
||||
// Special tokens that can be used at generation time
|
||||
/**
|
||||
* The id of the *padding* token.
|
||||
* @type {number}
|
||||
* @default null
|
||||
*/
|
||||
pad_token_id = null;
|
||||
|
||||
/**
|
||||
* The id of the *beginning-of-sequence* token.
|
||||
* @type {number}
|
||||
* @default null
|
||||
*/
|
||||
bos_token_id = null;
|
||||
|
||||
/**
|
||||
* The id of the *end-of-sequence* token.
|
||||
* Optionally, use a list to set multiple *end-of-sequence* tokens.
|
||||
* @type {number|number[]}
|
||||
* @default null
|
||||
*/
|
||||
eos_token_id = null;
|
||||
|
||||
// Generation parameters exclusive to encoder-decoder models
|
||||
/**
|
||||
* If set to int > 0, all ngrams of that size that occur in the `encoder_input_ids` cannot occur in the `decoder_input_ids`.
|
||||
* @type {number}
|
||||
* @default 0
|
||||
*/
|
||||
encoder_no_repeat_ngram_size = 0;
|
||||
|
||||
/**
|
||||
* If an encoder-decoder model starts decoding with a different token than *bos*, the id of that token.
|
||||
* @type {number}
|
||||
* @default null
|
||||
*/
|
||||
decoder_start_token_id = null;
|
||||
|
||||
// Wild card
|
||||
/**
|
||||
* Additional generation kwargs will be forwarded to the `generate` function of the model.
|
||||
* Kwargs that are not present in `generate`'s signature will be used in the model forward pass.
|
||||
* @type {Object}
|
||||
* @default {}
|
||||
*/
|
||||
generation_kwargs = {};
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {GenerationConfig|import('../configs.js').PretrainedConfig} config
|
||||
*/
|
||||
constructor(config) {
|
||||
Object.assign(this, pick(config, Object.getOwnPropertyNames(this)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,751 @@
|
||||
/**
|
||||
* @module generation/logits_process
|
||||
*/
|
||||
|
||||
import { Callable } from '../utils/generic.js';
|
||||
import { Tensor } from '../utils/tensor.js';
|
||||
|
||||
import { max, log_softmax } from '../utils/maths.js';
|
||||
|
||||
/**
|
||||
* Abstract base class for all logit processors that can be applied during generation.
|
||||
*/
|
||||
export class LogitsProcessor extends Callable {
|
||||
/**
|
||||
* Apply the processor to the input logits.
|
||||
*
|
||||
* @abstract
|
||||
* @param {bigint[][]} input_ids The input ids.
|
||||
* @param {Tensor} logits The logits to process.
|
||||
* @throws {Error} Throws an error if `_call` is not implemented in the subclass.
|
||||
*/
|
||||
_call(input_ids, logits) {
|
||||
throw Error('`_call` should be implemented in a subclass');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract base class for all logit warpers that can be applied during generation with multinomial sampling.
|
||||
*/
|
||||
export class LogitsWarper extends Callable {
|
||||
/**
|
||||
* Apply the processor to the input logits.
|
||||
*
|
||||
* @abstract
|
||||
* @param {bigint[][]} input_ids The input ids.
|
||||
* @param {Tensor} logits The logits to process.
|
||||
* @throws {Error} Throws an error if `_call` is not implemented in the subclass.
|
||||
*/
|
||||
_call(input_ids, logits) {
|
||||
throw Error('`_call` should be implemented in a subclass');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A class representing a list of logits processors. A logits processor is a function that modifies the logits
|
||||
* output of a language model. This class provides methods for adding new processors and applying all processors to a
|
||||
* batch of logits.
|
||||
*/
|
||||
export class LogitsProcessorList extends Callable {
|
||||
/**
|
||||
* Constructs a new instance of `LogitsProcessorList`.
|
||||
*/
|
||||
constructor() {
|
||||
super();
|
||||
this.processors = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a new logits processor to the list.
|
||||
*
|
||||
* @param {LogitsProcessor} item The logits processor function to add.
|
||||
*/
|
||||
push(item) {
|
||||
this.processors.push(item);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds multiple logits processors to the list.
|
||||
*
|
||||
* @param {LogitsProcessor[]} items The logits processor functions to add.
|
||||
*/
|
||||
extend(items) {
|
||||
this.processors.push(...items);
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies all logits processors in the list to a batch of logits, modifying them in-place.
|
||||
*
|
||||
* @param {bigint[][]} input_ids The input IDs for the language model.
|
||||
* @param {Tensor} logits
|
||||
*/
|
||||
_call(input_ids, logits) {
|
||||
let toReturn = logits;
|
||||
// NOTE: Most processors modify logits inplace
|
||||
for (const processor of this.processors) {
|
||||
toReturn = processor(input_ids, toReturn);
|
||||
}
|
||||
return toReturn;
|
||||
}
|
||||
|
||||
[Symbol.iterator]() {
|
||||
return this.processors.values();
|
||||
}
|
||||
}
|
||||
|
||||
// DEPRECATED: https://github.com/huggingface/transformers/pull/29485
|
||||
// /**
|
||||
// * A logits processor that forces a specific token to be generated by the decoder.
|
||||
// */
|
||||
// export class ForceTokensLogitsProcessor extends LogitsProcessor {
|
||||
// /**
|
||||
// * Constructs a new instance of `ForceTokensLogitsProcessor`.
|
||||
// *
|
||||
// * @param {[number, number][]} forced_decoder_ids The ids of tokens that should be forced.
|
||||
// */
|
||||
// constructor(forced_decoder_ids) {
|
||||
// super();
|
||||
// // TODO: convert to `new Map(forced_decoder_ids)`
|
||||
// this.force_token_map = Object.fromEntries(forced_decoder_ids ?? []);
|
||||
// }
|
||||
|
||||
// /**
|
||||
// * Apply the processor to the input logits.
|
||||
// *
|
||||
// * @param {bigint[][]} input_ids The input ids.
|
||||
// * @param {Tensor} logits The logits to process.
|
||||
// * @returns {Tensor} The processed logits.
|
||||
// */
|
||||
// _call(input_ids, logits) {
|
||||
// console.log('this.force_token_map', this.force_token_map)
|
||||
// console.log('call ForceTokensLogitsProcessor', input_ids, logits)
|
||||
// console.log('input_ids.length', input_ids.length)
|
||||
// let map = this.force_token_map[input_ids.length];
|
||||
// if (map) { // There exists a mapping
|
||||
// logits.data.fill(-Infinity)
|
||||
// logits.data[map] = 0;
|
||||
// }
|
||||
// console.log('map', map)
|
||||
// // throw Error("Not implemented")
|
||||
// return logits;
|
||||
// }
|
||||
// }
|
||||
|
||||
/**
|
||||
* A LogitsProcessor that forces a BOS token at the beginning of the generated sequence.
|
||||
*/
|
||||
export class ForcedBOSTokenLogitsProcessor extends LogitsProcessor {
|
||||
/**
|
||||
* Create a ForcedBOSTokenLogitsProcessor.
|
||||
* @param {number} bos_token_id The ID of the beginning-of-sequence token to be forced.
|
||||
*/
|
||||
constructor(bos_token_id) {
|
||||
super();
|
||||
this.bos_token_id = bos_token_id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the BOS token forcing to the logits.
|
||||
* @param {bigint[][]} input_ids The input IDs.
|
||||
* @param {Tensor} logits The logits.
|
||||
* @returns {Tensor} The logits with BOS token forcing.
|
||||
*/
|
||||
_call(input_ids, logits) {
|
||||
for (let i = 0; i < input_ids.length; ++i) {
|
||||
if (input_ids[i].length === 1) {
|
||||
const batch_logits_data = /** @type {Float32Array} */ (logits[i].data);
|
||||
batch_logits_data.fill(-Infinity);
|
||||
batch_logits_data[this.bos_token_id] = 0;
|
||||
}
|
||||
}
|
||||
return logits;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A logits processor that enforces the specified token as the last generated token when `max_length` is reached.
|
||||
*/
|
||||
export class ForcedEOSTokenLogitsProcessor extends LogitsProcessor {
|
||||
/**
|
||||
* Create a ForcedEOSTokenLogitsProcessor.
|
||||
* @param {number} max_length The maximum length of the sequence to be generated.
|
||||
* @param {number|number[]} eos_token_id The id(s) of the *end-of-sequence* token.
|
||||
*/
|
||||
constructor(max_length, eos_token_id) {
|
||||
super();
|
||||
this.max_length = max_length;
|
||||
this.eos_token_id = Array.isArray(eos_token_id) ? eos_token_id : [eos_token_id];
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the processor to input_ids and logits.
|
||||
*
|
||||
* @param {bigint[][]} input_ids The input ids.
|
||||
* @param {Tensor} logits The logits tensor.
|
||||
*/
|
||||
_call(input_ids, logits) {
|
||||
for (let i = 0; i < input_ids.length; ++i) {
|
||||
if (input_ids[i].length === this.max_length - 1) {
|
||||
const batch_logits_data = /** @type {Float32Array} */ (logits[i].data);
|
||||
batch_logits_data.fill(-Infinity);
|
||||
for (const eos_token of this.eos_token_id) {
|
||||
batch_logits_data[eos_token] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
return logits;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A LogitsProcessor that suppresses a list of tokens throughout generation.
|
||||
* Sets their log probs to `-inf` so that they are not generated.
|
||||
*/
|
||||
export class SuppressTokensLogitsProcessor extends LogitsProcessor {
|
||||
/**
|
||||
* Create a SuppressTokensLogitsProcessor.
|
||||
* @param {number[]} suppress_tokens The IDs of the tokens to suppress.
|
||||
*/
|
||||
constructor(suppress_tokens) {
|
||||
super();
|
||||
this.suppress_tokens = suppress_tokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* Suppress the specified tokens by setting their logits to -Infinity.
|
||||
* @param {bigint[][]} input_ids The input IDs.
|
||||
* @param {Tensor} logits The logits.
|
||||
* @returns {Tensor} The modified logits.
|
||||
*/
|
||||
_call(input_ids, logits) {
|
||||
for (let i = 0; i < input_ids.length; ++i) {
|
||||
const batch_logits_data = /** @type {Float32Array} */ (logits[i].data);
|
||||
for (const token_id of this.suppress_tokens) {
|
||||
batch_logits_data[token_id] = -Infinity;
|
||||
}
|
||||
}
|
||||
return logits;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A LogitsProcessor that suppresses a list of tokens as soon as the `generate` function starts
|
||||
* generating using `begin_index` tokens. This should ensure that the tokens defined by
|
||||
* `begin_suppress_tokens` at not sampled at the begining of the generation.
|
||||
*/
|
||||
export class SuppressTokensAtBeginLogitsProcessor extends LogitsProcessor {
|
||||
/**
|
||||
* Create a SuppressTokensAtBeginLogitsProcessor.
|
||||
* @param {number[]} begin_suppress_tokens The IDs of the tokens to suppress.
|
||||
* @param {number} begin_index The number of tokens to generate before suppressing tokens.
|
||||
*/
|
||||
constructor(begin_suppress_tokens, begin_index) {
|
||||
super();
|
||||
this.begin_suppress_tokens = begin_suppress_tokens;
|
||||
this.begin_index = begin_index;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the BOS token forcing to the logits.
|
||||
* @param {bigint[][]} input_ids The input IDs.
|
||||
* @param {Tensor} logits The logits.
|
||||
* @returns {Tensor} The logits with BOS token forcing.
|
||||
*/
|
||||
_call(input_ids, logits) {
|
||||
for (let i = 0; i < input_ids.length; ++i) {
|
||||
if (input_ids[i].length === this.begin_index) {
|
||||
const batch_logits_data = /** @type {Float32Array} */ (logits[i].data);
|
||||
for (const token_id of this.begin_suppress_tokens) {
|
||||
batch_logits_data[token_id] = -Infinity;
|
||||
}
|
||||
}
|
||||
}
|
||||
return logits;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A LogitsProcessor that handles adding timestamps to generated text.
|
||||
*/
|
||||
export class WhisperTimeStampLogitsProcessor extends LogitsProcessor {
|
||||
/**
|
||||
* Constructs a new WhisperTimeStampLogitsProcessor.
|
||||
* @param {import('../models/whisper/generation_whisper.js').WhisperGenerationConfig} generate_config The config object passed to the `generate()` method of a transformer model.
|
||||
* @param {number[]} init_tokens The initial tokens of the input sequence.
|
||||
*/
|
||||
constructor(generate_config, init_tokens) {
|
||||
super();
|
||||
this.eos_token_id = Array.isArray(generate_config.eos_token_id)
|
||||
? generate_config.eos_token_id[0]
|
||||
: generate_config.eos_token_id;
|
||||
|
||||
this.no_timestamps_token_id = generate_config.no_timestamps_token_id;
|
||||
this.timestamp_begin = this.no_timestamps_token_id + 1;
|
||||
|
||||
this.begin_index = init_tokens.length;
|
||||
if (init_tokens.at(-1) === this.no_timestamps_token_id) {
|
||||
this.begin_index -= 1;
|
||||
}
|
||||
this.max_initial_timestamp_index = generate_config.max_initial_timestamp_index;
|
||||
}
|
||||
|
||||
/**
|
||||
* Modify the logits to handle timestamp tokens.
|
||||
* @param {bigint[][]} input_ids The input sequence of tokens.
|
||||
* @param {Tensor} logits The logits output by the model.
|
||||
* @returns {Tensor} The modified logits.
|
||||
*/
|
||||
_call(input_ids, logits) {
|
||||
for (let i = 0; i < input_ids.length; ++i) {
|
||||
const batch_logits_data = /** @type {Float32Array} */ (logits[i].data);
|
||||
|
||||
// suppress <|notimestamps|> which is handled by without_timestamps
|
||||
batch_logits_data[this.no_timestamps_token_id] = -Infinity;
|
||||
|
||||
if (input_ids[i].length === this.begin_index) {
|
||||
batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity);
|
||||
continue;
|
||||
}
|
||||
|
||||
// timestamps have to appear in pairs, except directly before eos_token; mask logits accordingly
|
||||
const seq = input_ids[i].slice(this.begin_index);
|
||||
const last_was_timestamp = seq.length >= 1 && seq[seq.length - 1] >= this.timestamp_begin;
|
||||
const penultimate_was_timestamp = seq.length < 2 || seq[seq.length - 2] >= this.timestamp_begin;
|
||||
|
||||
if (last_was_timestamp) {
|
||||
if (penultimate_was_timestamp) {
|
||||
// has to be non-timestamp
|
||||
batch_logits_data.subarray(this.timestamp_begin).fill(-Infinity);
|
||||
} else {
|
||||
// cannot be normal text tokens
|
||||
batch_logits_data.subarray(0, this.eos_token_id).fill(-Infinity);
|
||||
}
|
||||
}
|
||||
|
||||
// apply the `max_initial_timestamp` option
|
||||
if (input_ids[i].length === this.begin_index && this.max_initial_timestamp_index !== null) {
|
||||
const last_allowed = this.timestamp_begin + this.max_initial_timestamp_index;
|
||||
batch_logits_data.subarray(last_allowed + 1).fill(-Infinity);
|
||||
}
|
||||
|
||||
// if sum of probability over timestamps is above any other token, sample timestamp
|
||||
const logprobs = log_softmax(batch_logits_data);
|
||||
const timestamp_logprob = Math.log(
|
||||
logprobs
|
||||
.subarray(this.timestamp_begin)
|
||||
.map(Math.exp)
|
||||
.reduce((a, b) => a + b),
|
||||
);
|
||||
const max_text_token_logprob = max(logprobs.subarray(0, this.timestamp_begin))[0];
|
||||
|
||||
if (timestamp_logprob > max_text_token_logprob) {
|
||||
batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity);
|
||||
}
|
||||
}
|
||||
|
||||
return logits;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A logits processor that disallows ngrams of a certain size to be repeated.
|
||||
*/
|
||||
export class NoRepeatNGramLogitsProcessor extends LogitsProcessor {
|
||||
/**
|
||||
* Create a NoRepeatNGramLogitsProcessor.
|
||||
* @param {number} no_repeat_ngram_size The no-repeat-ngram size. All ngrams of this size can only occur once.
|
||||
*/
|
||||
constructor(no_repeat_ngram_size) {
|
||||
super();
|
||||
this.no_repeat_ngram_size = no_repeat_ngram_size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate n-grams from a sequence of token ids.
|
||||
* @param {bigint[]} prevInputIds List of previous input ids
|
||||
* @returns {Map<string, number[]>} Map of generated n-grams
|
||||
*/
|
||||
getNgrams(prevInputIds) {
|
||||
const curLen = prevInputIds.length;
|
||||
|
||||
/**@type {number[][]} */
|
||||
const ngrams = [];
|
||||
for (let j = 0; j < curLen + 1 - this.no_repeat_ngram_size; ++j) {
|
||||
const ngram = [];
|
||||
for (let k = 0; k < this.no_repeat_ngram_size; ++k) {
|
||||
ngram.push(prevInputIds[j + k]);
|
||||
}
|
||||
ngrams.push(ngram.map(Number));
|
||||
}
|
||||
|
||||
/** @type {Map<string, number[]>} */
|
||||
const generatedNgram = new Map();
|
||||
for (const ngram of ngrams) {
|
||||
const prevNgram = ngram.slice(0, ngram.length - 1);
|
||||
const prevNgramKey = JSON.stringify(prevNgram);
|
||||
const prevNgramValue = generatedNgram.get(prevNgramKey) ?? [];
|
||||
prevNgramValue.push(ngram[ngram.length - 1]);
|
||||
generatedNgram.set(prevNgramKey, prevNgramValue);
|
||||
}
|
||||
return generatedNgram;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate n-grams from a sequence of token ids.
|
||||
* @param {Map<string, number[]>} bannedNgrams Map of banned n-grams
|
||||
* @param {bigint[]} prevInputIds List of previous input ids
|
||||
* @returns {number[]} Map of generated n-grams
|
||||
*/
|
||||
getGeneratedNgrams(bannedNgrams, prevInputIds) {
|
||||
const ngramIdx = prevInputIds.slice(prevInputIds.length + 1 - this.no_repeat_ngram_size, prevInputIds.length);
|
||||
const banned = bannedNgrams.get(JSON.stringify(ngramIdx.map(Number))) ?? [];
|
||||
return banned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate banned n-gram tokens
|
||||
* @param {bigint[]} prevInputIds List of previous input ids
|
||||
* @returns {number[]} Map of generated n-grams
|
||||
*/
|
||||
calcBannedNgramTokens(prevInputIds) {
|
||||
const bannedTokens = [];
|
||||
if (prevInputIds.length + 1 < this.no_repeat_ngram_size) {
|
||||
// return no banned tokens if we haven't generated no_repeat_ngram_size tokens yet
|
||||
return bannedTokens;
|
||||
} else {
|
||||
const generatedNgrams = this.getNgrams(prevInputIds);
|
||||
const bannedTokens = this.getGeneratedNgrams(generatedNgrams, prevInputIds);
|
||||
return bannedTokens;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the no-repeat-ngram processor to the logits.
|
||||
* @param {bigint[][]} input_ids The input IDs.
|
||||
* @param {Tensor} logits The logits.
|
||||
* @returns {Tensor} The logits with no-repeat-ngram processing.
|
||||
*/
|
||||
_call(input_ids, logits) {
|
||||
for (let i = 0; i < input_ids.length; ++i) {
|
||||
const batch_logits_data = /** @type {Float32Array} */ (logits[i].data);
|
||||
const bannedTokens = this.calcBannedNgramTokens(input_ids[i]);
|
||||
for (const token of bannedTokens) {
|
||||
batch_logits_data[token] = -Infinity;
|
||||
}
|
||||
}
|
||||
return logits;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A logits processor that prevents the repetition of previous tokens through a penalty.
|
||||
* This penalty is applied at most once per token. Note that, for decoder-only models like most LLMs,
|
||||
* the considered tokens include the prompt.
|
||||
*
|
||||
* In the original [paper](https://huggingface.co/papers/1909.05858), the authors suggest the use of a
|
||||
* penalty of around 1.2 to achieve a good balance between truthful generation and lack of repetition.
|
||||
* To penalize and reduce repetition, use `penalty` values above 1.0, where a higher value penalizes
|
||||
* more strongly. To reward and encourage repetition, use `penalty` values between 0.0 and 1.0, where
|
||||
* a lower value rewards more strongly.
|
||||
*/
|
||||
export class RepetitionPenaltyLogitsProcessor extends LogitsProcessor {
|
||||
/**
|
||||
* Create a RepetitionPenaltyLogitsProcessor.
|
||||
* @param {number} penalty The parameter for repetition penalty.
|
||||
* - 1.0 means no penalty. Above 1.0 penalizes previously generated tokens.
|
||||
* - Between 0.0 and 1.0 rewards previously generated tokens.
|
||||
*/
|
||||
constructor(penalty) {
|
||||
super();
|
||||
this.penalty = penalty;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the repetition penalty to the logits.
|
||||
* @param {bigint[][]} input_ids The input IDs.
|
||||
* @param {Tensor} logits The logits.
|
||||
* @returns {Tensor} The logits with repetition penalty processing.
|
||||
*/
|
||||
_call(input_ids, logits) {
|
||||
for (let i = 0; i < input_ids.length; ++i) {
|
||||
const batch_logits_data = /** @type {Float32Array} */ (logits[i].data);
|
||||
for (const input_id of new Set(input_ids[i])) {
|
||||
const token = Number(input_id);
|
||||
if (batch_logits_data[token] < 0) {
|
||||
batch_logits_data[token] *= this.penalty;
|
||||
} else {
|
||||
batch_logits_data[token] /= this.penalty;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return logits;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A logits processor that enforces a minimum number of tokens.
|
||||
*/
|
||||
export class MinLengthLogitsProcessor extends LogitsProcessor {
|
||||
/**
|
||||
* Create a MinLengthLogitsProcessor.
|
||||
* @param {number} min_length The minimum length below which the score of `eos_token_id` is set to negative infinity.
|
||||
* @param {number|number[]} eos_token_id The ID/IDs of the end-of-sequence token.
|
||||
*/
|
||||
constructor(min_length, eos_token_id) {
|
||||
super();
|
||||
this.min_length = min_length;
|
||||
this.eos_token_id = Array.isArray(eos_token_id) ? eos_token_id : [eos_token_id];
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply logit processor.
|
||||
* @param {bigint[][]} input_ids The input IDs.
|
||||
* @param {Tensor} logits The logits.
|
||||
* @returns {Tensor} The processed logits.
|
||||
*/
|
||||
_call(input_ids, logits) {
|
||||
for (let i = 0; i < input_ids.length; ++i) {
|
||||
if (input_ids[i].length < this.min_length) {
|
||||
const batch_logits_data = /** @type {Float32Array} */ (logits[i].data);
|
||||
|
||||
for (const eos_token of this.eos_token_id) {
|
||||
batch_logits_data[eos_token] = -Infinity;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return logits;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A logits processor that enforces a minimum number of new tokens.
|
||||
*/
|
||||
export class MinNewTokensLengthLogitsProcessor extends LogitsProcessor {
|
||||
/**
|
||||
* Create a MinNewTokensLengthLogitsProcessor.
|
||||
* @param {number} prompt_length_to_skip The input tokens length.
|
||||
* @param {number} min_new_tokens The minimum *new* tokens length below which the score of `eos_token_id` is set to negative infinity.
|
||||
* @param {number|number[]} eos_token_id The ID/IDs of the end-of-sequence token.
|
||||
*/
|
||||
constructor(prompt_length_to_skip, min_new_tokens, eos_token_id) {
|
||||
super();
|
||||
this.prompt_length_to_skip = prompt_length_to_skip;
|
||||
this.min_new_tokens = min_new_tokens;
|
||||
this.eos_token_id = Array.isArray(eos_token_id) ? eos_token_id : [eos_token_id];
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply logit processor.
|
||||
* @param {bigint[][]} input_ids The input IDs.
|
||||
* @param {Tensor} logits The logits.
|
||||
* @returns {Tensor} The processed logits.
|
||||
*/
|
||||
_call(input_ids, logits) {
|
||||
for (let i = 0; i < input_ids.length; ++i) {
|
||||
const new_tokens_length = input_ids[i].length - this.prompt_length_to_skip;
|
||||
if (new_tokens_length < this.min_new_tokens) {
|
||||
const batch_logits_data = /** @type {Float32Array} */ (logits[i].data);
|
||||
|
||||
for (const eos_token of this.eos_token_id) {
|
||||
batch_logits_data[eos_token] = -Infinity;
|
||||
}
|
||||
}
|
||||
}
|
||||
return logits;
|
||||
}
|
||||
}
|
||||
|
||||
export class NoBadWordsLogitsProcessor extends LogitsProcessor {
|
||||
/**
|
||||
* Create a `NoBadWordsLogitsProcessor`.
|
||||
* @param {number[][]} bad_words_ids List of list of token ids that are not allowed to be generated.
|
||||
* @param {number|number[]} eos_token_id The id of the *end-of-sequence* token. Optionally, use a list to set multiple *end-of-sequence* tokens.
|
||||
*/
|
||||
constructor(bad_words_ids, eos_token_id) {
|
||||
super();
|
||||
this.bad_words_ids = bad_words_ids;
|
||||
this.eos_token_id = Array.isArray(eos_token_id) ? eos_token_id : [eos_token_id];
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply logit processor.
|
||||
* @param {bigint[][]} input_ids The input IDs.
|
||||
* @param {Tensor} logits The logits.
|
||||
* @returns {Tensor} The processed logits.
|
||||
*/
|
||||
_call(input_ids, logits) {
|
||||
for (let i = 0; i < input_ids.length; ++i) {
|
||||
const batch_logits_data = /** @type {Float32Array} */ (logits[i].data);
|
||||
const ids = input_ids[i];
|
||||
for (const bad_word_ids of this.bad_words_ids) {
|
||||
// There aren't enough tokens to match the banned sequence
|
||||
if (ids.length < bad_word_ids.length - 1) continue;
|
||||
|
||||
// Whether to modify the logits of the last token in the bad word id sequence
|
||||
let mark = true;
|
||||
|
||||
// For each bad word in the list, if the current sequence of input ids ends with this sequence (excluding the last),
|
||||
// then we set the logits of the last bad word id to -Infinity.
|
||||
for (let j = 1; j <= bad_word_ids.length - 1; ++j) {
|
||||
// NOTE: We use != instead of !== to compare bigint and number
|
||||
// @ts-ignore
|
||||
if (bad_word_ids.at(-j - 1) != ids.at(-j)) {
|
||||
// We have found a mismatch
|
||||
mark = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (mark) {
|
||||
batch_logits_data[bad_word_ids.at(-1)] = -Infinity;
|
||||
}
|
||||
}
|
||||
}
|
||||
return logits;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* [`LogitsProcessor`] for classifier free guidance (CFG). The scores are split over the batch dimension,
|
||||
* where the first half correspond to the conditional logits (predicted from the input prompt) and the second half
|
||||
* correspond to the unconditional logits (predicted from an empty or 'null' prompt). The processor computes a
|
||||
* weighted average across the conditional and unconditional logits, parameterised by the `guidance_scale`.
|
||||
*
|
||||
* See [the paper](https://huggingface.co/papers/2306.05284) for more information.
|
||||
*/
|
||||
export class ClassifierFreeGuidanceLogitsProcessor extends LogitsProcessor {
|
||||
/**
|
||||
* Create a `ClassifierFreeGuidanceLogitsProcessor`.
|
||||
* @param {number} guidance_scale The guidance scale for classifier free guidance (CFG). CFG is enabled by setting `guidance_scale > 1`.
|
||||
* Higher guidance scale encourages the model to generate samples that are more closely linked to the input
|
||||
* prompt, usually at the expense of poorer quality.
|
||||
*/
|
||||
constructor(guidance_scale) {
|
||||
super();
|
||||
if (guidance_scale <= 1) {
|
||||
throw new Error(
|
||||
`Require guidance scale >1 to use the classifier free guidance processor, got guidance scale ${guidance_scale}.`,
|
||||
);
|
||||
}
|
||||
this.guidance_scale = guidance_scale;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply logit processor.
|
||||
* @param {bigint[][]} input_ids The input IDs.
|
||||
* @param {Tensor} logits The logits.
|
||||
* @returns {Tensor} The processed logits.
|
||||
*/
|
||||
_call(input_ids, logits) {
|
||||
if (logits.dims[0] !== 2 * input_ids.length) {
|
||||
throw new Error(
|
||||
`Logits should have twice the batch size of the input ids, the first half of batches corresponding to ` +
|
||||
`the conditional inputs, and the second half of batches corresponding to the unconditional inputs. Got ` +
|
||||
`batch size ${logits.dims[0]} for the logits and ${input_ids.length} for the input ids.`,
|
||||
);
|
||||
}
|
||||
|
||||
const unguided_bsz = input_ids.length;
|
||||
const cond_logits = logits.slice([0, unguided_bsz], null);
|
||||
const uncond_logits = logits.slice([unguided_bsz, logits.dims[0]], null);
|
||||
|
||||
// Merge into uncond_logits (to save memory). This is equivalent to the following:
|
||||
// scores = uncond_logits + (cond_logits - uncond_logits) * guidance_scale
|
||||
for (let i = 0; i < uncond_logits.data.length; ++i) {
|
||||
uncond_logits.data[i] += (cond_logits.data[i] - uncond_logits.data[i]) * this.guidance_scale;
|
||||
}
|
||||
|
||||
return uncond_logits;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* [`LogitsWarper`] for temperature (exponential scaling output probability distribution), which effectively means
|
||||
* that it can control the randomness of the predicted tokens. Often used together with [`TopPLogitsWarper`] and [`TopKLogitsWarper`].
|
||||
*/
|
||||
export class TemperatureLogitsWarper extends LogitsWarper {
|
||||
/**
|
||||
* Create a `TemperatureLogitsWarper`.
|
||||
* @param {number} temperature Strictly positive float value used to modulate the logits distribution.
|
||||
* A value smaller than `1` decreases randomness (and vice versa), with `0` being equivalent to shifting
|
||||
* all probability mass to the most likely token.
|
||||
*/
|
||||
constructor(temperature) {
|
||||
super();
|
||||
|
||||
if (typeof temperature !== 'number' || temperature <= 0) {
|
||||
let errorMessage = `\`temperature\` (=${temperature}) must be a strictly positive float, otherwise your next token scores will be invalid.`;
|
||||
|
||||
if (temperature === 0) {
|
||||
errorMessage += " If you're looking for greedy decoding strategies, set `do_sample=false`.";
|
||||
}
|
||||
}
|
||||
this.temperature = temperature;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply logit warper.
|
||||
* @param {bigint[][]} input_ids The input IDs.
|
||||
* @param {Tensor} logits The logits.
|
||||
* @returns {Tensor} The processed logits.
|
||||
*/
|
||||
_call(input_ids, logits) {
|
||||
const batch_logits_data = /** @type {Float32Array} */ (logits.data);
|
||||
for (let i = 0; i < batch_logits_data.length; ++i) {
|
||||
batch_logits_data[i] /= this.temperature;
|
||||
}
|
||||
return logits;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* [`LogitsWarper`] that performs top-p, i.e. restricting to top tokens summing to prob_cut_off <= prob_cut_off.
|
||||
* Often used together with [`TemperatureLogitsWarper`] and [`TopKLogitsWarper`].
|
||||
*/
|
||||
export class TopPLogitsWarper extends LogitsWarper {
|
||||
/**
|
||||
* Create a `TopPLogitsWarper`.
|
||||
* @param {number} top_p If set to < 1, only the smallest set of most probable tokens with
|
||||
* probabilities that add up to `top_p` or higher are kept for generation.
|
||||
* @param {Object} options Additional options for the top-p sampling.
|
||||
* @param {number} [options.filter_value=-Infinity] All filtered values will be set to this float value.
|
||||
* @param {number} [options.min_tokens_to_keep=1] Minimum number of tokens that cannot be filtered.
|
||||
*/
|
||||
constructor(top_p, { filter_value = -Infinity, min_tokens_to_keep = 1 } = {}) {
|
||||
super();
|
||||
if (top_p < 0 || top_p > 1.0) {
|
||||
throw new Error(`\`top_p\` must be a float > 0 and < 1, but is ${top_p}`);
|
||||
}
|
||||
if (!Number.isInteger(min_tokens_to_keep) || min_tokens_to_keep < 1) {
|
||||
throw new Error(`\`min_tokens_to_keep\` must be a positive integer, but is ${min_tokens_to_keep}`);
|
||||
}
|
||||
|
||||
this.top_p = top_p;
|
||||
this.filter_value = filter_value;
|
||||
this.min_tokens_to_keep = min_tokens_to_keep;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* [`LogitsWarper`] that performs top-k, i.e. restricting to the k highest probability elements.
|
||||
* Often used together with [`TemperatureLogitsWarper`] and [`TopPLogitsWarper`].
|
||||
*/
|
||||
export class TopKLogitsWarper extends LogitsWarper {
|
||||
/**
|
||||
* Create a `TopKLogitsWarper`.
|
||||
* @param {number} top_k If set to > 0, only the top `top_k` tokens are kept for generation.
|
||||
* @param {Object} options Additional options for the top-k sampling.
|
||||
* @param {number} [options.filter_value=-Infinity] All filtered values will be set to this float value.
|
||||
* @param {number} [options.min_tokens_to_keep=1] Minimum number of tokens that cannot be filtered.
|
||||
*/
|
||||
constructor(top_k, { filter_value = -Infinity, min_tokens_to_keep = 1 } = {}) {
|
||||
super();
|
||||
if (!Number.isInteger(top_k) || top_k < 0) {
|
||||
throw new Error(`\`top_k\` must be a positive integer, but is ${top_k}`);
|
||||
}
|
||||
|
||||
this.top_k = Math.max(top_k, min_tokens_to_keep);
|
||||
this.filter_value = filter_value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
/**
|
||||
* @module generation/logits_sampler
|
||||
*/
|
||||
|
||||
import { Callable } from '../utils/generic.js';
|
||||
import { Tensor, topk } from '../utils/tensor.js';
|
||||
|
||||
import { max, softmax } from '../utils/maths.js';
|
||||
import { _weightedIndex } from '../utils/random.js';
|
||||
import { GenerationConfig } from '../generation/configuration_utils.js';
|
||||
|
||||
/**
|
||||
* Sampler is a base class for all sampling methods used for text generation.
|
||||
*/
|
||||
export class LogitsSampler extends Callable {
|
||||
/**
|
||||
* Creates a new Sampler object with the specified generation config.
|
||||
* @param {GenerationConfig} generation_config The generation config.
|
||||
*/
|
||||
constructor(generation_config) {
|
||||
super();
|
||||
this.generation_config = generation_config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes the sampler, using the specified logits.
|
||||
* @param {Tensor} logits
|
||||
* @returns {Promise<[bigint, number][]>}
|
||||
*/
|
||||
async _call(logits) {
|
||||
// Sample from logits, of dims [batch, sequence_length, vocab_size].
|
||||
// If index is specified, sample from [batch, index, vocab_size].
|
||||
return this.sample(logits);
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract method for sampling the logits.
|
||||
* @param {Tensor} logits
|
||||
* @throws {Error} If not implemented in subclass.
|
||||
* @returns {Promise<[bigint, number][]>}
|
||||
*/
|
||||
async sample(logits) {
|
||||
throw Error('sample should be implemented in subclasses.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the specified logits as an array, with temperature applied.
|
||||
* @param {Tensor} logits
|
||||
* @param {number} index
|
||||
* @returns {Float32Array}
|
||||
*/
|
||||
getLogits(logits, index) {
|
||||
let vocabSize = logits.dims.at(-1);
|
||||
|
||||
let logs = /** @type {Float32Array} */ (logits.data);
|
||||
|
||||
if (index === -1) {
|
||||
logs = logs.slice(-vocabSize);
|
||||
} else {
|
||||
let startIndex = index * vocabSize;
|
||||
logs = logs.slice(startIndex, startIndex + vocabSize);
|
||||
}
|
||||
return logs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Selects an item randomly based on the specified probabilities.
|
||||
* @param {Float32Array} probabilities An array of probabilities to use for selection.
|
||||
* @returns {number} The index of the selected item.
|
||||
*/
|
||||
randomSelect(probabilities) {
|
||||
return _weightedIndex(probabilities);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a Sampler object based on the specified options.
|
||||
* @param {GenerationConfig} generation_config An object containing options for the sampler.
|
||||
* @returns {LogitsSampler} A Sampler object.
|
||||
*/
|
||||
static getSampler(generation_config) {
|
||||
// - *greedy decoding*: `num_beams=1` and `do_sample=False`
|
||||
// - *contrastive search*: `penalty_alpha>0` and `top_k>1`
|
||||
// - *multinomial sampling*: `num_beams=1` and `do_sample=True`
|
||||
// - *beam-search decoding*: `num_beams>1` and `do_sample=False`
|
||||
// - *beam-search multinomial sampling*: `num_beams>1` and `do_sample=True`
|
||||
// - *diverse beam-search decoding*: `num_beams>1` and `num_beam_groups>1`
|
||||
// - *constrained beam-search decoding*: `constraints!=None` or `force_words_ids!=None`
|
||||
|
||||
// NOTE: beam search is implemented directly into the generation function
|
||||
if (generation_config.do_sample) {
|
||||
return new MultinomialSampler(generation_config);
|
||||
} else if (generation_config.num_beams > 1) {
|
||||
return new BeamSearchSampler(generation_config);
|
||||
} else {
|
||||
if (generation_config.num_return_sequences > 1) {
|
||||
throw Error(
|
||||
`num_return_sequences has to be 1 when doing greedy search, but is ${generation_config.num_return_sequences}.`,
|
||||
);
|
||||
}
|
||||
return new GreedySampler(generation_config);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Class representing a Greedy Sampler.
|
||||
*/
|
||||
class GreedySampler extends LogitsSampler {
|
||||
/**
|
||||
* Sample the maximum probability of a given logits tensor.
|
||||
* @param {Tensor} logits
|
||||
* @returns {Promise<[bigint, number][]>} An array with a single tuple, containing the index of the maximum value and a meaningless score (since this is a greedy search).
|
||||
*/
|
||||
async sample(logits) {
|
||||
// NOTE: no need to do log_softmax here since we only take the maximum
|
||||
const argmax = max(logits.data)[1];
|
||||
|
||||
// Note: score is meaningless in this context, since we are performing
|
||||
// greedy search (p = 1 => log(p) = 0)
|
||||
return [[BigInt(argmax), 0]];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Class representing a MultinomialSampler.
|
||||
*/
|
||||
class MultinomialSampler extends LogitsSampler {
|
||||
/**
|
||||
* Sample from the logits.
|
||||
* @param {Tensor} logits
|
||||
* @returns {Promise<[bigint, number][]>}
|
||||
*/
|
||||
async sample(logits) {
|
||||
let k = logits.dims.at(-1); // defaults to vocab size
|
||||
if (this.generation_config.top_k > 0) {
|
||||
k = Math.min(this.generation_config.top_k, k);
|
||||
}
|
||||
|
||||
// Get top k tokens
|
||||
const [v, i] = await topk(logits, k);
|
||||
|
||||
// Compute softmax over logits
|
||||
const probabilities = softmax(/** @type {Float32Array} */ (v.data));
|
||||
|
||||
return Array.from({ length: this.generation_config.num_beams }, () => {
|
||||
const sampledIndex = this.randomSelect(probabilities);
|
||||
return [
|
||||
i.data[sampledIndex], // token id
|
||||
Math.log(probabilities[sampledIndex]), // score
|
||||
];
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Class representing a BeamSearchSampler.
|
||||
*/
|
||||
class BeamSearchSampler extends LogitsSampler {
|
||||
/**
|
||||
* Sample from the logits.
|
||||
* @param {Tensor} logits
|
||||
* @returns {Promise<[bigint, number][]>}
|
||||
*/
|
||||
async sample(logits) {
|
||||
let k = logits.dims.at(-1); // defaults to vocab size
|
||||
if (this.generation_config.top_k > 0) {
|
||||
k = Math.min(this.generation_config.top_k, k);
|
||||
}
|
||||
|
||||
// Get top k tokens
|
||||
const [v, i] = await topk(logits, k);
|
||||
|
||||
// Compute softmax over logits
|
||||
const probabilities = softmax(/** @type {Float32Array} */ (v.data));
|
||||
|
||||
return Array.from({ length: this.generation_config.num_beams }, (_, x) => {
|
||||
return [
|
||||
i.data[x], // token id
|
||||
Math.log(probabilities[x]), // score
|
||||
];
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
/**
|
||||
* @module generation/parameters
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} GenerationFunctionParametersBase
|
||||
* @property {import('../utils/tensor.js').Tensor} [inputs=null] (`Tensor` of varying shape depending on the modality, *optional*):
|
||||
* The sequence used as a prompt for the generation or as model inputs to the encoder. If `null` the
|
||||
* method initializes it with `bos_token_id` and a batch size of 1. For decoder-only models `inputs`
|
||||
* should be in the format of `input_ids`. For encoder-decoder models *inputs* can represent any of
|
||||
* `input_ids`, `input_values`, `input_features`, or `pixel_values`.
|
||||
* @property {import('./configuration_utils.js').GenerationConfig} [generation_config=null] (`GenerationConfig`, *optional*):
|
||||
* The generation configuration to be used as base parametrization for the generation call.
|
||||
* `**kwargs` passed to generate matching the attributes of `generation_config` will override them.
|
||||
* If `generation_config` is not provided, the default will be used, which has the following loading
|
||||
* priority:
|
||||
* - (1) from the `generation_config.json` model file, if it exists;
|
||||
* - (2) from the model configuration. Please note that unspecified parameters will inherit [`GenerationConfig`]'s
|
||||
* default values, whose documentation should be checked to parameterize generation.
|
||||
* @property {import('./logits_process.js').LogitsProcessorList} [logits_processor=null] (`LogitsProcessorList`, *optional*):
|
||||
* Custom logits processors that complement the default logits processors built from arguments and
|
||||
* generation config. If a logit processor is passed that is already created with the arguments or a
|
||||
* generation config an error is thrown. This feature is intended for advanced users.
|
||||
* @property {import('./stopping_criteria.js').StoppingCriteria|import('./stopping_criteria.js').StoppingCriteria[]|import('./stopping_criteria.js').StoppingCriteriaList} [stopping_criteria=null] (`StoppingCriteriaList`, *optional*):
|
||||
* Custom stopping criteria that complements the default stopping criteria built from arguments and a
|
||||
* generation config. If a stopping criteria is passed that is already created with the arguments or a
|
||||
* generation config an error is thrown. This feature is intended for advanced users.
|
||||
* @property {import('./streamers.js').BaseStreamer} [streamer=null] (`BaseStreamer`, *optional*):
|
||||
* Streamer object that will be used to stream the generated sequences. Generated tokens are passed
|
||||
* through `streamer.put(token_ids)` and the streamer is responsible for any further processing.
|
||||
* @property {number[]|import('../utils/tensor.js').Tensor} [decoder_input_ids=null] (`number[]` or `Tensor`, *optional*):
|
||||
* If the model is an encoder-decoder model, this argument is used to pass the `decoder_input_ids`.
|
||||
* @property {import('../cache_utils.js').DynamicCache | null} [past_key_values=null] (`DynamicCache`, *optional*):
|
||||
* A cache object that stores previously computed key/value states. When provided, the model will
|
||||
* use these cached states to avoid recomputing them, significantly speeding up sequential generation.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {GenerationFunctionParametersBase & Partial<import('./configuration_utils.js').GenerationConfig> & {[key: string]: unknown}} GenerationFunctionParameters
|
||||
*/
|
||||
|
||||
export {}; // Ensure this file is treated as a module
|
||||
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* @module generation/stopping_criteria
|
||||
*/
|
||||
|
||||
import { Callable } from '../utils/generic.js';
|
||||
|
||||
// NOTE:
|
||||
// Stopping Criteria returns a list of `batch_size` booleans, indicating whether each sequence in the batch should be stopped.
|
||||
|
||||
/**
|
||||
* Abstract base class for all stopping criteria that can be applied during generation.
|
||||
*/
|
||||
export class StoppingCriteria extends Callable {
|
||||
/**
|
||||
*
|
||||
* @param {number[][]} input_ids (`number[][]` of shape `(batch_size, sequence_length)`):
|
||||
* Indices of input sequence tokens in the vocabulary.
|
||||
* @param {number[][]} scores scores (`number[][]` of shape `(batch_size, config.vocab_size)`):
|
||||
* Prediction scores of a language modeling head. These can be scores for each vocabulary token before SoftMax
|
||||
* or scores for each vocabulary token after SoftMax.
|
||||
* @returns {boolean[]} A list of booleans indicating whether each sequence should be stopped.
|
||||
*/
|
||||
_call(input_ids, scores) {
|
||||
throw Error('StoppingCriteria needs to be subclassed');
|
||||
}
|
||||
}
|
||||
/**
|
||||
*/
|
||||
export class StoppingCriteriaList extends Callable {
|
||||
/**
|
||||
* Constructs a new instance of `StoppingCriteriaList`.
|
||||
*/
|
||||
constructor() {
|
||||
super();
|
||||
this.criteria = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a new stopping criterion to the list.
|
||||
*
|
||||
* @param {StoppingCriteria} item The stopping criterion to add.
|
||||
*/
|
||||
push(item) {
|
||||
this.criteria.push(item);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds multiple stopping criteria to the list.
|
||||
*
|
||||
* @param {StoppingCriteria|StoppingCriteriaList|StoppingCriteria[]} items The stopping criteria to add.
|
||||
*/
|
||||
extend(items) {
|
||||
if (items instanceof StoppingCriteriaList) {
|
||||
items = items.criteria;
|
||||
} else if (items instanceof StoppingCriteria) {
|
||||
items = [items];
|
||||
}
|
||||
this.criteria.push(...items);
|
||||
}
|
||||
|
||||
_call(input_ids, scores) {
|
||||
const is_done = new Array(input_ids.length).fill(false);
|
||||
for (const criterion of this.criteria) {
|
||||
const criterion_done = criterion(input_ids, scores);
|
||||
for (let i = 0; i < is_done.length; ++i) {
|
||||
is_done[i] ||= criterion_done[i];
|
||||
}
|
||||
}
|
||||
return is_done;
|
||||
}
|
||||
|
||||
[Symbol.iterator]() {
|
||||
return this.criteria.values();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This class can be used to stop generation whenever the full generated number of tokens exceeds `max_length`.
|
||||
* Keep in mind for decoder-only type of transformers, this will include the initial prompted tokens.
|
||||
*/
|
||||
export class MaxLengthCriteria extends StoppingCriteria {
|
||||
/**
|
||||
*
|
||||
* @param {number} max_length The maximum length that the output sequence can have in number of tokens.
|
||||
* @param {number} [max_position_embeddings=null] The maximum model length, as defined by the model's `config.max_position_embeddings` attribute.
|
||||
*/
|
||||
constructor(max_length, max_position_embeddings = null) {
|
||||
super();
|
||||
this.max_length = max_length;
|
||||
this.max_position_embeddings = max_position_embeddings;
|
||||
}
|
||||
|
||||
_call(input_ids) {
|
||||
return input_ids.map((ids) => ids.length >= this.max_length);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: add MaxTimeCriteria
|
||||
|
||||
/**
|
||||
* This class can be used to stop generation whenever the "end-of-sequence" token is generated.
|
||||
* By default, it uses the `model.generation_config.eos_token_id`.
|
||||
*/
|
||||
export class EosTokenCriteria extends StoppingCriteria {
|
||||
/**
|
||||
*
|
||||
* @param {number|number[]} eos_token_id The id of the *end-of-sequence* token.
|
||||
* Optionally, use a list to set multiple *end-of-sequence* tokens.
|
||||
*/
|
||||
constructor(eos_token_id) {
|
||||
super();
|
||||
if (!Array.isArray(eos_token_id)) {
|
||||
eos_token_id = [eos_token_id];
|
||||
}
|
||||
this.eos_token_id = eos_token_id;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {number[][]} input_ids
|
||||
* @param {number[][]} scores
|
||||
* @returns {boolean[]}
|
||||
*/
|
||||
_call(input_ids, scores) {
|
||||
return input_ids.map((ids) => {
|
||||
const last = ids.at(-1);
|
||||
// NOTE: We use == instead of === to allow for number/bigint comparison
|
||||
return this.eos_token_id.some((eos_id) => last == eos_id);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This class can be used to stop generation whenever the user interrupts the process.
|
||||
*/
|
||||
export class InterruptableStoppingCriteria extends StoppingCriteria {
|
||||
constructor() {
|
||||
super();
|
||||
this.interrupted = false;
|
||||
}
|
||||
|
||||
interrupt() {
|
||||
this.interrupted = true;
|
||||
}
|
||||
|
||||
reset() {
|
||||
this.interrupted = false;
|
||||
}
|
||||
|
||||
_call(input_ids, scores) {
|
||||
return new Array(input_ids.length).fill(this.interrupted);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
/**
|
||||
* @module generation/streamers
|
||||
*/
|
||||
|
||||
import { mergeArrays } from '../utils/core.js';
|
||||
import { apis } from '../env.js';
|
||||
|
||||
const is_chinese_char = (cp) =>
|
||||
(cp >= 0x4e00 && cp <= 0x9fff) ||
|
||||
(cp >= 0x3400 && cp <= 0x4dbf) ||
|
||||
(cp >= 0x20000 && cp <= 0x2a6df) ||
|
||||
(cp >= 0x2a700 && cp <= 0x2b73f) ||
|
||||
(cp >= 0x2b740 && cp <= 0x2b81f) ||
|
||||
(cp >= 0x2b820 && cp <= 0x2ceaf) ||
|
||||
(cp >= 0xf900 && cp <= 0xfaff) ||
|
||||
(cp >= 0x2f800 && cp <= 0x2fa1f);
|
||||
|
||||
export class BaseStreamer {
|
||||
/**
|
||||
* Function that is called by `.generate()` to push new tokens
|
||||
* @param {bigint[][]} value
|
||||
*/
|
||||
put(value) {
|
||||
throw Error('Not implemented');
|
||||
}
|
||||
|
||||
/**
|
||||
* Function that is called by `.generate()` to signal the end of generation
|
||||
*/
|
||||
end() {
|
||||
throw Error('Not implemented');
|
||||
}
|
||||
}
|
||||
|
||||
const stdout_write = apis.IS_PROCESS_AVAILABLE ? (x) => process.stdout.write(x) : (x) => console.log(x);
|
||||
|
||||
/**
|
||||
* Simple text streamer that prints the token(s) to stdout as soon as entire words are formed.
|
||||
*/
|
||||
export class TextStreamer extends BaseStreamer {
|
||||
/**
|
||||
*
|
||||
* @param {import('../tokenization_utils.js').PreTrainedTokenizer} tokenizer
|
||||
* @param {Object} options
|
||||
* @param {boolean} [options.skip_prompt=false] Whether to skip the prompt tokens
|
||||
* @param {boolean} [options.skip_special_tokens=true] Whether to skip special tokens when decoding
|
||||
* @param {function(string): void} [options.callback_function=null] Function to call when a piece of text is ready to display
|
||||
* @param {function(bigint[]): void} [options.token_callback_function=null] Function to call when a new token is generated
|
||||
* @param {Object} [options.decode_kwargs={}] Additional keyword arguments to pass to the tokenizer's decode method
|
||||
*/
|
||||
constructor(
|
||||
tokenizer,
|
||||
{
|
||||
skip_prompt = false,
|
||||
callback_function = null,
|
||||
token_callback_function = null,
|
||||
skip_special_tokens = true,
|
||||
decode_kwargs = {},
|
||||
...kwargs
|
||||
} = {},
|
||||
) {
|
||||
super();
|
||||
this.tokenizer = tokenizer;
|
||||
this.skip_prompt = skip_prompt;
|
||||
this.callback_function = callback_function ?? stdout_write;
|
||||
this.token_callback_function = token_callback_function;
|
||||
this.decode_kwargs = { skip_special_tokens, ...decode_kwargs, ...kwargs };
|
||||
|
||||
// variables used in the streaming process
|
||||
this.token_cache = [];
|
||||
this.print_len = 0;
|
||||
this.next_tokens_are_prompt = true;
|
||||
|
||||
// Track special token IDs for special handling during streaming.
|
||||
this.special_ids = new Set(this.tokenizer.all_special_ids.map(BigInt));
|
||||
}
|
||||
|
||||
/**
|
||||
* Receives tokens, decodes them, and prints them to stdout as soon as they form entire words.
|
||||
* @param {bigint[][]} value
|
||||
*/
|
||||
put(value) {
|
||||
if (value.length > 1) {
|
||||
throw Error('TextStreamer only supports batch size of 1');
|
||||
}
|
||||
|
||||
const is_prompt = this.next_tokens_are_prompt;
|
||||
if (is_prompt) {
|
||||
this.next_tokens_are_prompt = false;
|
||||
if (this.skip_prompt) return;
|
||||
}
|
||||
|
||||
const tokens = value[0];
|
||||
this.token_callback_function?.(tokens);
|
||||
|
||||
// Handle special tokens: flush any existing text, then print or skip them
|
||||
if (tokens.length === 1 && this.special_ids.has(tokens[0])) {
|
||||
if (this.decode_kwargs.skip_special_tokens) return;
|
||||
|
||||
// Flush any existing cached text first
|
||||
if (this.token_cache.length > 0) {
|
||||
const text = this.tokenizer.decode(this.token_cache, this.decode_kwargs);
|
||||
const printable_text = text.slice(this.print_len);
|
||||
this.on_finalized_text(printable_text, false);
|
||||
this.token_cache = [];
|
||||
this.print_len = 0;
|
||||
}
|
||||
// Print the special token immediately
|
||||
const special_text = this.tokenizer.decode(tokens, this.decode_kwargs);
|
||||
this.on_finalized_text(special_text, false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Add the new token to the cache and decodes the entire thing.
|
||||
this.token_cache = mergeArrays(this.token_cache, tokens);
|
||||
const text = this.tokenizer.decode(this.token_cache, this.decode_kwargs);
|
||||
|
||||
let printable_text;
|
||||
if (is_prompt || text.endsWith('\n')) {
|
||||
// After the symbol for a new line, we flush the cache.
|
||||
printable_text = text.slice(this.print_len);
|
||||
this.token_cache = [];
|
||||
this.print_len = 0;
|
||||
} else if (text.length > 0 && is_chinese_char(text.charCodeAt(text.length - 1))) {
|
||||
// If the last token is a CJK character, we print the characters.
|
||||
printable_text = text.slice(this.print_len);
|
||||
this.print_len += printable_text.length;
|
||||
} else {
|
||||
// Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words,
|
||||
// which may change with the subsequent token -- there are probably smarter ways to do this!)
|
||||
printable_text = text.slice(this.print_len, text.lastIndexOf(' ') + 1);
|
||||
this.print_len += printable_text.length;
|
||||
}
|
||||
|
||||
this.on_finalized_text(printable_text, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Flushes any remaining cache and prints a newline to stdout.
|
||||
*/
|
||||
end() {
|
||||
let printable_text;
|
||||
if (this.token_cache.length > 0) {
|
||||
const text = this.tokenizer.decode(this.token_cache, this.decode_kwargs);
|
||||
printable_text = text.slice(this.print_len);
|
||||
this.token_cache = [];
|
||||
this.print_len = 0;
|
||||
} else {
|
||||
printable_text = '';
|
||||
}
|
||||
this.next_tokens_are_prompt = true;
|
||||
this.on_finalized_text(printable_text, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints the new text to stdout. If the stream is ending, also prints a newline.
|
||||
* @param {string} text
|
||||
* @param {boolean} stream_end
|
||||
*/
|
||||
on_finalized_text(text, stream_end) {
|
||||
if (text.length > 0) {
|
||||
this.callback_function?.(text);
|
||||
}
|
||||
if (stream_end && this.callback_function === stdout_write && apis.IS_PROCESS_AVAILABLE) {
|
||||
this.callback_function?.('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility class to handle streaming of tokens generated by whisper speech-to-text models.
|
||||
* Callback functions are invoked when each of the following events occur:
|
||||
* - A new chunk starts (on_chunk_start)
|
||||
* - A new token is generated (callback_function)
|
||||
* - A chunk ends (on_chunk_end)
|
||||
* - The stream is finalized (on_finalize)
|
||||
*/
|
||||
export class WhisperTextStreamer extends TextStreamer {
|
||||
/**
|
||||
* @param {import('../models/whisper/tokenization_whisper.js').WhisperTokenizer} tokenizer
|
||||
* @param {Object} options
|
||||
* @param {boolean} [options.skip_prompt=false] Whether to skip the prompt tokens
|
||||
* @param {function(string): void} [options.callback_function=null] Function to call when a piece of text is ready to display
|
||||
* @param {function(bigint[]): void} [options.token_callback_function=null] Function to call when a new token is generated
|
||||
* @param {function(number): void} [options.on_chunk_start=null] Function to call when a new chunk starts
|
||||
* @param {function(number): void} [options.on_chunk_end=null] Function to call when a chunk ends
|
||||
* @param {function(): void} [options.on_finalize=null] Function to call when the stream is finalized
|
||||
* @param {number} [options.time_precision=0.02] Precision of the timestamps
|
||||
* @param {boolean} [options.skip_special_tokens=true] Whether to skip special tokens when decoding
|
||||
* @param {Object} [options.decode_kwargs={}] Additional keyword arguments to pass to the tokenizer's decode method
|
||||
*/
|
||||
constructor(
|
||||
tokenizer,
|
||||
{
|
||||
skip_prompt = false,
|
||||
callback_function = null,
|
||||
token_callback_function = null,
|
||||
on_chunk_start = null,
|
||||
on_chunk_end = null,
|
||||
on_finalize = null,
|
||||
time_precision = 0.02,
|
||||
skip_special_tokens = true,
|
||||
decode_kwargs = {},
|
||||
} = {},
|
||||
) {
|
||||
super(tokenizer, {
|
||||
skip_prompt,
|
||||
skip_special_tokens,
|
||||
callback_function,
|
||||
token_callback_function,
|
||||
decode_kwargs,
|
||||
});
|
||||
this.timestamp_begin = tokenizer.timestamp_begin;
|
||||
|
||||
this.on_chunk_start = on_chunk_start;
|
||||
this.on_chunk_end = on_chunk_end;
|
||||
this.on_finalize = on_finalize;
|
||||
|
||||
this.time_precision = time_precision;
|
||||
|
||||
this.waiting_for_timestamp = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {bigint[][]} value
|
||||
*/
|
||||
put(value) {
|
||||
if (value.length > 1) {
|
||||
throw Error('WhisperTextStreamer only supports batch size of 1');
|
||||
}
|
||||
const tokens = value[0];
|
||||
|
||||
// Check if the token is a timestamp
|
||||
if (tokens.length === 1) {
|
||||
const offset = Number(tokens[0]) - this.timestamp_begin;
|
||||
if (offset >= 0) {
|
||||
const time = offset * this.time_precision;
|
||||
if (this.waiting_for_timestamp) {
|
||||
this.on_chunk_end?.(time);
|
||||
} else {
|
||||
this.on_chunk_start?.(time);
|
||||
}
|
||||
this.waiting_for_timestamp = !this.waiting_for_timestamp; // Toggle
|
||||
|
||||
// NOTE: Timestamp tokens should not be printed. Although, since they
|
||||
// aren't classified as "special tokens", we need to handle them here.
|
||||
this.token_callback_function?.(tokens);
|
||||
return;
|
||||
}
|
||||
}
|
||||
return super.put(value);
|
||||
}
|
||||
|
||||
end() {
|
||||
super.end();
|
||||
this.on_finalize?.();
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
||||
import { PreTrainedModel } from '../modeling_utils.js';
|
||||
|
||||
export class AfmoePreTrainedModel extends PreTrainedModel {}
|
||||
export class AfmoeModel extends AfmoePreTrainedModel {}
|
||||
export class AfmoeForCausalLM extends AfmoePreTrainedModel {}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { PreTrainedModel } from '../modeling_utils.js';
|
||||
import { MaskedLMOutput, QuestionAnsweringModelOutput, SequenceClassifierOutput } from '../modeling_outputs.js';
|
||||
|
||||
export class AlbertPreTrainedModel extends PreTrainedModel {}
|
||||
export class AlbertModel extends AlbertPreTrainedModel {}
|
||||
export class AlbertForSequenceClassification extends AlbertPreTrainedModel {
|
||||
/**
|
||||
* Calls the model on new inputs.
|
||||
*
|
||||
* @param {Object} model_inputs The inputs to the model.
|
||||
* @returns {Promise<SequenceClassifierOutput>} returned object
|
||||
*/
|
||||
async _call(model_inputs) {
|
||||
return new SequenceClassifierOutput(await super._call(model_inputs));
|
||||
}
|
||||
}
|
||||
export class AlbertForQuestionAnswering extends AlbertPreTrainedModel {
|
||||
/**
|
||||
* Calls the model on new inputs.
|
||||
*
|
||||
* @param {Object} model_inputs The inputs to the model.
|
||||
* @returns {Promise<QuestionAnsweringModelOutput>} returned object
|
||||
*/
|
||||
async _call(model_inputs) {
|
||||
return new QuestionAnsweringModelOutput(await super._call(model_inputs));
|
||||
}
|
||||
}
|
||||
export class AlbertForMaskedLM extends AlbertPreTrainedModel {
|
||||
/**
|
||||
* Calls the model on new inputs.
|
||||
*
|
||||
* @param {Object} model_inputs The inputs to the model.
|
||||
* @returns {Promise<MaskedLMOutput>} returned object
|
||||
*/
|
||||
async _call(model_inputs) {
|
||||
return new MaskedLMOutput(await super._call(model_inputs));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import { PreTrainedTokenizer } from '../../tokenization_utils.js';
|
||||
|
||||
export class AlbertTokenizer extends PreTrainedTokenizer {
|
||||
return_token_type_ids = true;
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import { PreTrainedModel } from '../modeling_utils.js';
|
||||
|
||||
export class ApertusPreTrainedModel extends PreTrainedModel {}
|
||||
export class ApertusModel extends ApertusPreTrainedModel {}
|
||||
export class ApertusForCausalLM extends ApertusPreTrainedModel {}
|
||||
@@ -0,0 +1,5 @@
|
||||
import { PreTrainedModel } from '../modeling_utils.js';
|
||||
|
||||
export class ArceePreTrainedModel extends PreTrainedModel {}
|
||||
export class ArceeModel extends ArceePreTrainedModel {}
|
||||
export class ArceeForCausalLM extends ArceePreTrainedModel {}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js';
|
||||
import { Tensor } from '../../utils/tensor.js';
|
||||
import { mel_filter_bank, spectrogram, window_function } from '../../utils/audio.js';
|
||||
|
||||
export class ASTFeatureExtractor extends FeatureExtractor {
|
||||
constructor(config) {
|
||||
super(config);
|
||||
|
||||
const sampling_rate = this.config.sampling_rate;
|
||||
const mel_filters = mel_filter_bank(
|
||||
257, // num_frequency_bins
|
||||
this.config.num_mel_bins, // num_mel_filters
|
||||
20, // min_frequency
|
||||
Math.floor(sampling_rate / 2), // max_frequency
|
||||
sampling_rate, // sampling_rate
|
||||
null, // norm
|
||||
'kaldi', // mel_scale
|
||||
true, // triangularize_in_mel_space
|
||||
);
|
||||
this.mel_filters = mel_filters;
|
||||
|
||||
this.window = window_function(400, 'hann', {
|
||||
periodic: false,
|
||||
});
|
||||
|
||||
this.mean = this.config.mean;
|
||||
this.std = this.config.std;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the log-Mel spectrogram of the provided audio waveform.
|
||||
* @param {Float32Array|Float64Array} waveform The audio waveform to process.
|
||||
* @param {number} max_length The maximum number of frames to return.
|
||||
* @returns {Promise<Tensor>} An object containing the log-Mel spectrogram data as a Float32Array and its dimensions as an array of numbers.
|
||||
*/
|
||||
async _extract_fbank_features(waveform, max_length) {
|
||||
// NOTE: We don't pad/truncate since that is passed in as `max_num_frames`
|
||||
return spectrogram(
|
||||
waveform,
|
||||
this.window, // window
|
||||
400, // frame_length
|
||||
160, // hop_length
|
||||
{
|
||||
fft_length: 512,
|
||||
power: 2.0,
|
||||
center: false,
|
||||
preemphasis: 0.97,
|
||||
mel_filters: this.mel_filters,
|
||||
log_mel: 'log',
|
||||
mel_floor: 1.192092955078125e-7,
|
||||
remove_dc_offset: true,
|
||||
|
||||
// Custom
|
||||
max_num_frames: max_length,
|
||||
transpose: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Asynchronously extracts features from a given audio using the provided configuration.
|
||||
* @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array.
|
||||
* @returns {Promise<{ input_values: Tensor }>} A Promise resolving to an object containing the extracted input features as a Tensor.
|
||||
*/
|
||||
async _call(audio) {
|
||||
validate_audio_inputs(audio, 'ASTFeatureExtractor');
|
||||
|
||||
const features = await this._extract_fbank_features(audio, this.config.max_length);
|
||||
if (this.config.do_normalize) {
|
||||
// Normalize the input audio spectrogram to have mean=0, std=0.5
|
||||
const denom = this.std * 2;
|
||||
const features_data = features.data;
|
||||
for (let i = 0; i < features_data.length; ++i) {
|
||||
features_data[i] = (features_data[i] - this.mean) / denom;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
input_values: features.unsqueeze_(0),
|
||||
};
|
||||
}
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
import { PreTrainedModel } from '../modeling_utils.js';
|
||||
|
||||
export class ASTPreTrainedModel extends PreTrainedModel {}
|
||||
|
||||
/**
|
||||
* The bare AST Model transformer outputting raw hidden-states without any specific head on top.
|
||||
*/
|
||||
export class ASTModel extends ASTPreTrainedModel {}
|
||||
|
||||
/**
|
||||
* Audio Spectrogram Transformer model with an audio classification head on top
|
||||
* (a linear layer on top of the pooled output) e.g. for datasets like AudioSet, Speech Commands v2.
|
||||
*/
|
||||
export class ASTForAudioClassification extends ASTPreTrainedModel {}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { FEATURE_EXTRACTOR_NAME, GITHUB_ISSUE_URL } from '../../utils/constants.js';
|
||||
import { getModelJSON } from '../../utils/hub.js';
|
||||
import { FeatureExtractor } from '../../feature_extraction_utils.js';
|
||||
import * as AllFeatureExtractors from '../feature_extractors.js';
|
||||
|
||||
export class AutoFeatureExtractor {
|
||||
/** @type {typeof FeatureExtractor.from_pretrained} */
|
||||
static async from_pretrained(pretrained_model_name_or_path, options = {}) {
|
||||
const preprocessorConfig = await getModelJSON(
|
||||
pretrained_model_name_or_path,
|
||||
FEATURE_EXTRACTOR_NAME,
|
||||
true,
|
||||
options,
|
||||
);
|
||||
|
||||
// Determine feature extractor class
|
||||
const key = preprocessorConfig.feature_extractor_type;
|
||||
const feature_extractor_class = AllFeatureExtractors[key];
|
||||
|
||||
if (!feature_extractor_class) {
|
||||
throw new Error(`Unknown feature_extractor_type: '${key}'. Please report this at ${GITHUB_ISSUE_URL}.`);
|
||||
}
|
||||
|
||||
// Instantiate feature extractor
|
||||
return new feature_extractor_class(preprocessorConfig);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { getModelJSON } from '../../utils/hub.js';
|
||||
import { ImageProcessor } from '../../image_processors_utils.js';
|
||||
import * as AllImageProcessors from '../image_processors.js';
|
||||
import { GITHUB_ISSUE_URL, IMAGE_PROCESSOR_NAME } from '../../utils/constants.js';
|
||||
import { logger } from '../../utils/logger.js';
|
||||
|
||||
export class AutoImageProcessor {
|
||||
/** @type {typeof ImageProcessor.from_pretrained} */
|
||||
static async from_pretrained(pretrained_model_name_or_path, options = {}) {
|
||||
const preprocessorConfig = await getModelJSON(
|
||||
pretrained_model_name_or_path,
|
||||
IMAGE_PROCESSOR_NAME,
|
||||
true,
|
||||
options,
|
||||
);
|
||||
|
||||
// Determine image processor class
|
||||
const key = preprocessorConfig.image_processor_type ?? preprocessorConfig.feature_extractor_type;
|
||||
let image_processor_class = AllImageProcessors[key?.replace(/Fast$/, '')];
|
||||
|
||||
if (!image_processor_class) {
|
||||
if (key !== undefined) {
|
||||
// Only log a warning if the class is not found and the key is set.
|
||||
logger.warn(
|
||||
`Image processor type '${key}' not found, assuming base ImageProcessor. Please report this at ${GITHUB_ISSUE_URL}.`,
|
||||
);
|
||||
}
|
||||
image_processor_class = ImageProcessor;
|
||||
}
|
||||
|
||||
// Instantiate image processor
|
||||
return new image_processor_class(preprocessorConfig);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
/**
|
||||
* @file Definitions of all models available in Transformers.js.
|
||||
*
|
||||
* **Example:** Load and run an `AutoModel`.
|
||||
*
|
||||
* ```javascript
|
||||
* import { AutoModel, AutoTokenizer } from '@huggingface/transformers';
|
||||
*
|
||||
* const tokenizer = await AutoTokenizer.from_pretrained('Xenova/bert-base-uncased');
|
||||
* const model = await AutoModel.from_pretrained('Xenova/bert-base-uncased');
|
||||
*
|
||||
* const inputs = await tokenizer('I love transformers!');
|
||||
* const { logits } = await model(inputs);
|
||||
* // Tensor {
|
||||
* // data: Float32Array(183132) [-7.117443084716797, -7.107812881469727, -7.092104911804199, ...]
|
||||
* // dims: (3) [1, 6, 30522],
|
||||
* // type: "float32",
|
||||
* // size: 183132,
|
||||
* // }
|
||||
* ```
|
||||
*
|
||||
* We also provide other `AutoModel`s (listed below), which you can use in the same way as the Python library. For example:
|
||||
*
|
||||
* **Example:** Load and run an `AutoModelForSeq2SeqLM`.
|
||||
* ```javascript
|
||||
* import { AutoModelForSeq2SeqLM, AutoTokenizer } from '@huggingface/transformers';
|
||||
*
|
||||
* const tokenizer = await AutoTokenizer.from_pretrained('Xenova/t5-small');
|
||||
* const model = await AutoModelForSeq2SeqLM.from_pretrained('Xenova/t5-small');
|
||||
*
|
||||
* const { input_ids } = await tokenizer('translate English to German: I love transformers!');
|
||||
* const outputs = await model.generate(input_ids);
|
||||
* const decoded = tokenizer.decode(outputs[0], { skip_special_tokens: true });
|
||||
* // 'Ich liebe Transformatoren!'
|
||||
* ```
|
||||
*
|
||||
* @module models
|
||||
*/
|
||||
|
||||
import { AutoConfig } from '../../configs.js';
|
||||
import { PreTrainedModel } from '../modeling_utils.js';
|
||||
|
||||
import { CUSTOM_ARCHITECTURES, MODEL_CLASS_TYPE_MAPPING, MODEL_MAPPINGS } from '../registry.js';
|
||||
|
||||
import * as ALL_MODEL_FILES from '../models.js';
|
||||
import { logger } from '../../utils/logger.js';
|
||||
|
||||
/**
|
||||
* Base class of all AutoModels. Contains the `from_pretrained` function
|
||||
* which is used to instantiate pretrained models.
|
||||
*/
|
||||
class PretrainedMixin {
|
||||
/**
|
||||
* Mapping from model type to model class.
|
||||
* @type {Map<string, Object>[]}
|
||||
*/
|
||||
static MODEL_CLASS_MAPPINGS = null;
|
||||
|
||||
/**
|
||||
* Whether to attempt to instantiate the base class (`PretrainedModel`) if
|
||||
* the model type is not found in the mapping.
|
||||
*/
|
||||
static BASE_IF_FAIL = false;
|
||||
|
||||
/**
|
||||
* Check whether this AutoModel class supports a given model type.
|
||||
* @param {string} model_type The model type from config (e.g., 'bert', 'whisper').
|
||||
* @returns {boolean} Whether this class can handle the given model type.
|
||||
*/
|
||||
static supports(model_type) {
|
||||
if (!this.MODEL_CLASS_MAPPINGS) return false;
|
||||
for (const mapping of this.MODEL_CLASS_MAPPINGS) {
|
||||
if (mapping.has(model_type)) return true;
|
||||
}
|
||||
return this.BASE_IF_FAIL;
|
||||
}
|
||||
|
||||
/** @type {typeof PreTrainedModel.from_pretrained} */
|
||||
static async from_pretrained(
|
||||
pretrained_model_name_or_path,
|
||||
{
|
||||
progress_callback = null,
|
||||
config = null,
|
||||
cache_dir = null,
|
||||
local_files_only = false,
|
||||
revision = 'main',
|
||||
model_file_name = null,
|
||||
subfolder = 'onnx',
|
||||
device = null,
|
||||
dtype = null,
|
||||
use_external_data_format = null,
|
||||
session_options = {},
|
||||
} = {},
|
||||
) {
|
||||
const options = {
|
||||
progress_callback,
|
||||
config,
|
||||
cache_dir,
|
||||
local_files_only,
|
||||
revision,
|
||||
model_file_name,
|
||||
subfolder,
|
||||
device,
|
||||
dtype,
|
||||
use_external_data_format,
|
||||
session_options,
|
||||
};
|
||||
options.config = await AutoConfig.from_pretrained(pretrained_model_name_or_path, options);
|
||||
|
||||
if (!this.MODEL_CLASS_MAPPINGS) {
|
||||
throw new Error('`MODEL_CLASS_MAPPINGS` not implemented for this type of `AutoClass`: ' + this.name);
|
||||
}
|
||||
const { model_type } = options.config;
|
||||
for (const MODEL_CLASS_MAPPING of this.MODEL_CLASS_MAPPINGS) {
|
||||
let modelInfo = MODEL_CLASS_MAPPING.get(model_type);
|
||||
if (!modelInfo) {
|
||||
// As a fallback, we check if model_type is specified as the exact class
|
||||
for (const cls of MODEL_CLASS_MAPPING.values()) {
|
||||
if (cls[0] === model_type) {
|
||||
modelInfo = cls;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!modelInfo) continue; // Item not found in this mapping
|
||||
}
|
||||
return await ALL_MODEL_FILES[modelInfo].from_pretrained(pretrained_model_name_or_path, options);
|
||||
}
|
||||
|
||||
if (this.BASE_IF_FAIL) {
|
||||
if (!CUSTOM_ARCHITECTURES.has(model_type)) {
|
||||
logger.warn(`Unknown model class "${model_type}", attempting to construct from base class.`);
|
||||
}
|
||||
return await PreTrainedModel.from_pretrained(pretrained_model_name_or_path, options);
|
||||
} else {
|
||||
throw Error(`Unsupported model type: ${model_type}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModel.from_pretrained('Xenova/bert-base-uncased');
|
||||
*/
|
||||
export class AutoModel extends PretrainedMixin {
|
||||
/** @type {Map<string, Object>[]} */
|
||||
// @ts-ignore
|
||||
static MODEL_CLASS_MAPPINGS = MODEL_CLASS_TYPE_MAPPING.map((x) => x[0]);
|
||||
static BASE_IF_FAIL = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained sequence classification models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForSequenceClassification.from_pretrained('Xenova/distilbert-base-uncased-finetuned-sst-2-english');
|
||||
*/
|
||||
export class AutoModelForSequenceClassification extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained token classification models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForTokenClassification.from_pretrained('Xenova/distilbert-base-multilingual-cased-ner-hrl');
|
||||
*/
|
||||
export class AutoModelForTokenClassification extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained sequence-to-sequence models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForSeq2SeqLM.from_pretrained('Xenova/t5-small');
|
||||
*/
|
||||
export class AutoModelForSeq2SeqLM extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained sequence-to-sequence speech-to-text models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForSpeechSeq2Seq.from_pretrained('openai/whisper-tiny.en');
|
||||
*/
|
||||
export class AutoModelForSpeechSeq2Seq extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained sequence-to-sequence text-to-spectrogram models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForTextToSpectrogram.from_pretrained('microsoft/speecht5_tts');
|
||||
*/
|
||||
export class AutoModelForTextToSpectrogram extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained text-to-waveform models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForTextToSpectrogram.from_pretrained('facebook/mms-tts-eng');
|
||||
*/
|
||||
export class AutoModelForTextToWaveform extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained causal language models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForCausalLM.from_pretrained('Xenova/gpt2');
|
||||
*/
|
||||
export class AutoModelForCausalLM extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained masked language models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForMaskedLM.from_pretrained('Xenova/bert-base-uncased');
|
||||
*/
|
||||
export class AutoModelForMaskedLM extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_MASKED_LM_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained question answering models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForQuestionAnswering.from_pretrained('Xenova/distilbert-base-cased-distilled-squad');
|
||||
*/
|
||||
export class AutoModelForQuestionAnswering extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained vision-to-sequence models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForVision2Seq.from_pretrained('Xenova/vit-gpt2-image-captioning');
|
||||
*/
|
||||
export class AutoModelForVision2Seq extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained image classification models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForImageClassification.from_pretrained('Xenova/vit-base-patch16-224');
|
||||
*/
|
||||
export class AutoModelForImageClassification extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained image segmentation models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForImageSegmentation.from_pretrained('Xenova/detr-resnet-50-panoptic');
|
||||
*/
|
||||
export class AutoModelForImageSegmentation extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained image segmentation models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForSemanticSegmentation.from_pretrained('nvidia/segformer-b3-finetuned-cityscapes-1024-1024');
|
||||
*/
|
||||
export class AutoModelForSemanticSegmentation extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained universal image segmentation models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForUniversalSegmentation.from_pretrained('hf-internal-testing/tiny-random-MaskFormerForInstanceSegmentation');
|
||||
*/
|
||||
export class AutoModelForUniversalSegmentation extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained object detection models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForObjectDetection.from_pretrained('Xenova/detr-resnet-50');
|
||||
*/
|
||||
export class AutoModelForObjectDetection extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
export class AutoModelForZeroShotObjectDetection extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained mask generation models with the `from_pretrained` function.
|
||||
* The chosen model class is determined by the type specified in the model config.
|
||||
*
|
||||
* @example
|
||||
* const model = await AutoModelForMaskGeneration.from_pretrained('Xenova/sam-vit-base');
|
||||
*/
|
||||
export class AutoModelForMaskGeneration extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_MASK_GENERATION_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
export class AutoModelForCTC extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_CTC_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
export class AutoModelForAudioClassification extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
export class AutoModelForXVector extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
export class AutoModelForAudioFrameClassification extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
export class AutoModelForDocumentQuestionAnswering extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
export class AutoModelForImageMatting extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_IMAGE_MATTING_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
export class AutoModelForImageToImage extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_IMAGE_TO_IMAGE_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
export class AutoModelForDepthEstimation extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
export class AutoModelForNormalEstimation extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_NORMAL_ESTIMATION_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
export class AutoModelForPoseEstimation extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_POSE_ESTIMATION_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
export class AutoModelForImageFeatureExtraction extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_IMAGE_FEATURE_EXTRACTION_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
export class AutoModelForImageTextToText extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES];
|
||||
}
|
||||
|
||||
export class AutoModelForAudioTextToText extends PretrainedMixin {
|
||||
static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_AUDIO_TEXT_TO_TEXT_MAPPING_NAMES];
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { IMAGE_PROCESSOR_NAME } from '../../utils/constants.js';
|
||||
import { getModelJSON } from '../../utils/hub.js';
|
||||
import { Processor } from '../../processing_utils.js';
|
||||
|
||||
import * as AllProcessors from '../processors.js';
|
||||
import * as AllImageProcessors from '../image_processors.js';
|
||||
import * as AllFeatureExtractors from '../feature_extractors.js';
|
||||
|
||||
/**
|
||||
* @typedef {import('../../processing_utils.js').PretrainedProcessorOptions} PretrainedProcessorOptions
|
||||
*/
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained processors with the `from_pretrained` function.
|
||||
* The chosen processor class is determined by the type specified in the processor config.
|
||||
*
|
||||
* **Example:** Load a processor using `from_pretrained`.
|
||||
* ```javascript
|
||||
* let processor = await AutoProcessor.from_pretrained('openai/whisper-tiny.en');
|
||||
* ```
|
||||
*
|
||||
* **Example:** Run an image through a processor.
|
||||
* ```javascript
|
||||
* let processor = await AutoProcessor.from_pretrained('Xenova/clip-vit-base-patch16');
|
||||
* let image = await RawImage.read('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg');
|
||||
* let image_inputs = await processor(image);
|
||||
* // {
|
||||
* // "pixel_values": {
|
||||
* // "dims": [ 1, 3, 224, 224 ],
|
||||
* // "type": "float32",
|
||||
* // "data": Float32Array [ -1.558687686920166, -1.558687686920166, -1.5440893173217773, ... ],
|
||||
* // "size": 150528
|
||||
* // },
|
||||
* // "original_sizes": [
|
||||
* // [ 533, 800 ]
|
||||
* // ],
|
||||
* // "reshaped_input_sizes": [
|
||||
* // [ 224, 224 ]
|
||||
* // ]
|
||||
* // }
|
||||
* ```
|
||||
*/
|
||||
export class AutoProcessor {
|
||||
/** @type {typeof Processor.from_pretrained} */
|
||||
static async from_pretrained(pretrained_model_name_or_path, options = {}) {
|
||||
// TODO: first check for processor.json
|
||||
const preprocessorConfig = await getModelJSON(
|
||||
pretrained_model_name_or_path,
|
||||
IMAGE_PROCESSOR_NAME,
|
||||
true,
|
||||
options,
|
||||
);
|
||||
|
||||
const { image_processor_type, feature_extractor_type, processor_class } = preprocessorConfig;
|
||||
if (processor_class && AllProcessors[processor_class]) {
|
||||
return AllProcessors[processor_class].from_pretrained(pretrained_model_name_or_path, options);
|
||||
}
|
||||
|
||||
if (!image_processor_type && !feature_extractor_type) {
|
||||
throw new Error('No `image_processor_type` or `feature_extractor_type` found in the config.');
|
||||
}
|
||||
|
||||
const components = {};
|
||||
if (image_processor_type) {
|
||||
// Some image processors are saved with the "Fast" suffix, so we remove that if present.
|
||||
const image_processor_class = AllImageProcessors[image_processor_type.replace(/Fast$/, '')];
|
||||
if (!image_processor_class) {
|
||||
throw new Error(`Unknown image_processor_type: '${image_processor_type}'.`);
|
||||
}
|
||||
components.image_processor = new image_processor_class(preprocessorConfig);
|
||||
}
|
||||
|
||||
if (feature_extractor_type) {
|
||||
const image_processor_class = AllImageProcessors[feature_extractor_type];
|
||||
if (image_processor_class) {
|
||||
// Handle legacy case where image processors were specified as feature extractors
|
||||
components.image_processor = new image_processor_class(preprocessorConfig);
|
||||
} else {
|
||||
const feature_extractor_class = AllFeatureExtractors[feature_extractor_type];
|
||||
if (!feature_extractor_class) {
|
||||
throw new Error(`Unknown feature_extractor_type: '${feature_extractor_type}'.`);
|
||||
}
|
||||
components.feature_extractor = new feature_extractor_class(preprocessorConfig);
|
||||
}
|
||||
}
|
||||
|
||||
const config = {};
|
||||
return new Processor(config, components, null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import { PreTrainedTokenizer, loadTokenizer } from '../../tokenization_utils.js';
|
||||
import * as AllTokenizers from '../tokenizers.js';
|
||||
import { logger } from '../../utils/logger.js';
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained tokenizers with the `from_pretrained` function.
|
||||
* The chosen tokenizer class is determined by the type specified in the tokenizer config.
|
||||
*
|
||||
* **Example:** Create an `AutoTokenizer` and use it to tokenize a sentence.
|
||||
* This will automatically detect the tokenizer type based on the tokenizer class defined in `tokenizer_config.json`.
|
||||
*
|
||||
* ```javascript
|
||||
* import { AutoTokenizer } from '@huggingface/transformers';
|
||||
*
|
||||
* const tokenizer = await AutoTokenizer.from_pretrained('Xenova/bert-base-uncased');
|
||||
* const { input_ids } = await tokenizer('I love transformers!');
|
||||
* // Tensor {
|
||||
* // data: BigInt64Array(6) [101n, 1045n, 2293n, 19081n, 999n, 102n],
|
||||
* // dims: [1, 6],
|
||||
* // type: 'int64',
|
||||
* // size: 6,
|
||||
* // }
|
||||
* ```
|
||||
*/
|
||||
export class AutoTokenizer {
|
||||
/**
|
||||
* Instantiate one of the tokenizer classes of the library from a pretrained model.
|
||||
*
|
||||
* The tokenizer class to instantiate is selected based on the `tokenizer_class` property of the config object
|
||||
* (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible)
|
||||
*
|
||||
* @param {string} pretrained_model_name_or_path The name or path of the pretrained model. Can be either:
|
||||
* - A string, the *model id* of a pretrained tokenizer hosted inside a model repo on huggingface.co.
|
||||
* Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
|
||||
* user or organization name, like `dbmdz/bert-base-german-cased`.
|
||||
* - A path to a *directory* containing tokenizer files, e.g., `./my_model_directory/`.
|
||||
* @param {import('../../tokenization_utils.js').PretrainedTokenizerOptions} options Additional options for loading the tokenizer.
|
||||
*
|
||||
* @returns {Promise<PreTrainedTokenizer>} A new instance of the PreTrainedTokenizer class.
|
||||
*/
|
||||
static async from_pretrained(
|
||||
pretrained_model_name_or_path,
|
||||
{ progress_callback = null, config = null, cache_dir = null, local_files_only = false, revision = 'main' } = {},
|
||||
) {
|
||||
const [tokenizerJSON, tokenizerConfig] = await loadTokenizer(pretrained_model_name_or_path, {
|
||||
progress_callback,
|
||||
config,
|
||||
cache_dir,
|
||||
local_files_only,
|
||||
revision,
|
||||
});
|
||||
|
||||
// Some tokenizers are saved with the "Fast" suffix, so we remove that if present.
|
||||
const tokenizerName = tokenizerConfig.tokenizer_class?.replace(/Fast$/, '') ?? 'PreTrainedTokenizer';
|
||||
|
||||
let cls = AllTokenizers[tokenizerName];
|
||||
if (!cls) {
|
||||
logger.warn(`Unknown tokenizer class "${tokenizerName}", attempting to construct from base class.`);
|
||||
cls = PreTrainedTokenizer;
|
||||
}
|
||||
return new cls(tokenizerJSON, tokenizerConfig);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { PreTrainedModel } from '../modeling_utils.js';
|
||||
import { SequenceClassifierOutput } from '../modeling_outputs.js';
|
||||
|
||||
export class BartPretrainedModel extends PreTrainedModel {}
|
||||
|
||||
/**
|
||||
* The bare BART Model outputting raw hidden-states without any specific head on top.
|
||||
*/
|
||||
export class BartModel extends BartPretrainedModel {}
|
||||
|
||||
/**
|
||||
* The BART Model with a language modeling head. Can be used for summarization.
|
||||
*/
|
||||
export class BartForConditionalGeneration extends BartPretrainedModel {}
|
||||
|
||||
/**
|
||||
* Bart model with a sequence classification/head on top (a linear layer on top of the pooled output)
|
||||
*/
|
||||
export class BartForSequenceClassification extends BartPretrainedModel {
|
||||
/**
|
||||
* Calls the model on new inputs.
|
||||
*
|
||||
* @param {Object} model_inputs The inputs to the model.
|
||||
* @returns {Promise<SequenceClassifierOutput>} An object containing the model's output logits for sequence classification.
|
||||
*/
|
||||
async _call(model_inputs) {
|
||||
return new SequenceClassifierOutput(await super._call(model_inputs));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
import { PreTrainedTokenizer } from '../../tokenization_utils.js';
|
||||
|
||||
export class BartTokenizer extends PreTrainedTokenizer {}
|
||||
@@ -0,0 +1,3 @@
|
||||
import { ImageProcessor } from '../../image_processors_utils.js';
|
||||
|
||||
export class BeitFeatureExtractor extends ImageProcessor {}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { PreTrainedModel } from '../modeling_utils.js';
|
||||
import { SequenceClassifierOutput } from '../modeling_outputs.js';
|
||||
|
||||
export class BeitPreTrainedModel extends PreTrainedModel {}
|
||||
export class BeitModel extends BeitPreTrainedModel {}
|
||||
export class BeitForImageClassification extends BeitPreTrainedModel {
|
||||
/**
|
||||
* @param {any} model_inputs
|
||||
*/
|
||||
async _call(model_inputs) {
|
||||
return new SequenceClassifierOutput(await super._call(model_inputs));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import { PreTrainedModel } from '../modeling_utils.js';
|
||||
import {
|
||||
SequenceClassifierOutput,
|
||||
MaskedLMOutput,
|
||||
TokenClassifierOutput,
|
||||
QuestionAnsweringModelOutput,
|
||||
} from '../modeling_outputs.js';
|
||||
|
||||
export class BertPreTrainedModel extends PreTrainedModel {}
|
||||
export class BertModel extends BertPreTrainedModel {}
|
||||
|
||||
/**
|
||||
* BertForMaskedLM is a class representing a BERT model for masked language modeling.
|
||||
*/
|
||||
export class BertForMaskedLM extends BertPreTrainedModel {
|
||||
/**
|
||||
* Calls the model on new inputs.
|
||||
*
|
||||
* @param {Object} model_inputs The inputs to the model.
|
||||
* @returns {Promise<MaskedLMOutput>} An object containing the model's output logits for masked language modeling.
|
||||
*/
|
||||
async _call(model_inputs) {
|
||||
return new MaskedLMOutput(await super._call(model_inputs));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* BertForSequenceClassification is a class representing a BERT model for sequence classification.
|
||||
*/
|
||||
export class BertForSequenceClassification extends BertPreTrainedModel {
|
||||
/**
|
||||
* Calls the model on new inputs.
|
||||
*
|
||||
* @param {Object} model_inputs The inputs to the model.
|
||||
* @returns {Promise<SequenceClassifierOutput>} An object containing the model's output logits for sequence classification.
|
||||
*/
|
||||
async _call(model_inputs) {
|
||||
return new SequenceClassifierOutput(await super._call(model_inputs));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* BertForTokenClassification is a class representing a BERT model for token classification.
|
||||
*/
|
||||
export class BertForTokenClassification extends BertPreTrainedModel {
|
||||
/**
|
||||
* Calls the model on new inputs.
|
||||
*
|
||||
* @param {Object} model_inputs The inputs to the model.
|
||||
* @returns {Promise<TokenClassifierOutput>} An object containing the model's output logits for token classification.
|
||||
*/
|
||||
async _call(model_inputs) {
|
||||
return new TokenClassifierOutput(await super._call(model_inputs));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* BertForQuestionAnswering is a class representing a BERT model for question answering.
|
||||
*/
|
||||
export class BertForQuestionAnswering extends BertPreTrainedModel {
|
||||
/**
|
||||
* Calls the model on new inputs.
|
||||
*
|
||||
* @param {Object} model_inputs The inputs to the model.
|
||||
* @returns {Promise<QuestionAnsweringModelOutput>} An object containing the model's output logits for question answering.
|
||||
*/
|
||||
async _call(model_inputs) {
|
||||
return new QuestionAnsweringModelOutput(await super._call(model_inputs));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import { PreTrainedTokenizer } from '../../tokenization_utils.js';
|
||||
|
||||
export class BertTokenizer extends PreTrainedTokenizer {
|
||||
return_token_type_ids = true;
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
import { ImageProcessor } from '../../image_processors_utils.js';
|
||||
|
||||
export class BitImageProcessor extends ImageProcessor {}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { PreTrainedModel } from '../modeling_utils.js';
|
||||
|
||||
export class BlenderbotPreTrainedModel extends PreTrainedModel {}
|
||||
|
||||
/**
|
||||
* The bare Blenderbot Model outputting raw hidden-states without any specific head on top.
|
||||
*/
|
||||
export class BlenderbotModel extends BlenderbotPreTrainedModel {}
|
||||
|
||||
/**
|
||||
* The Blenderbot Model with a language modeling head. Can be used for summarization.
|
||||
*/
|
||||
export class BlenderbotForConditionalGeneration extends BlenderbotPreTrainedModel {}
|
||||
@@ -0,0 +1,3 @@
|
||||
import { PreTrainedTokenizer } from '../../tokenization_utils.js';
|
||||
|
||||
export class BlenderbotTokenizer extends PreTrainedTokenizer {}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { PreTrainedModel } from '../modeling_utils.js';
|
||||
|
||||
export class BlenderbotSmallPreTrainedModel extends PreTrainedModel {}
|
||||
|
||||
/**
|
||||
* The bare BlenderbotSmall Model outputting raw hidden-states without any specific head on top.
|
||||
*/
|
||||
export class BlenderbotSmallModel extends BlenderbotSmallPreTrainedModel {}
|
||||
|
||||
/**
|
||||
* The BlenderbotSmall Model with a language modeling head. Can be used for summarization.
|
||||
*/
|
||||
export class BlenderbotSmallForConditionalGeneration extends BlenderbotSmallPreTrainedModel {}
|
||||
@@ -0,0 +1,3 @@
|
||||
import { PreTrainedTokenizer } from '../../tokenization_utils.js';
|
||||
|
||||
export class BlenderbotSmallTokenizer extends PreTrainedTokenizer {}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { PreTrainedModel } from '../modeling_utils.js';
|
||||
|
||||
/**
|
||||
* The Bloom Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings).
|
||||
*/
|
||||
export class BloomPreTrainedModel extends PreTrainedModel {}
|
||||
|
||||
/**
|
||||
* The bare Bloom Model transformer outputting raw hidden-states without any specific head on top.
|
||||
*/
|
||||
export class BloomModel extends BloomPreTrainedModel {}
|
||||
|
||||
/**
|
||||
* The Bloom Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings).
|
||||
*/
|
||||
export class BloomForCausalLM extends BloomPreTrainedModel {}
|
||||
@@ -0,0 +1,3 @@
|
||||
import { PreTrainedTokenizer } from '../../tokenization_utils.js';
|
||||
|
||||
export class BloomTokenizer extends PreTrainedTokenizer {}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { PreTrainedModel } from '../modeling_utils.js';
|
||||
import {
|
||||
MaskedLMOutput,
|
||||
QuestionAnsweringModelOutput,
|
||||
SequenceClassifierOutput,
|
||||
TokenClassifierOutput,
|
||||
} from '../modeling_outputs.js';
|
||||
|
||||
export class CamembertPreTrainedModel extends PreTrainedModel {}
|
||||
|
||||
/**
|
||||
* The bare CamemBERT Model transformer outputting raw hidden-states without any specific head on top.
|
||||
*/
|
||||
export class CamembertModel extends CamembertPreTrainedModel {}
|
||||
|
||||
/**
|
||||
* CamemBERT Model with a `language modeling` head on top.
|
||||
*/
|
||||
export class CamembertForMaskedLM extends CamembertPreTrainedModel {
|
||||
/**
|
||||
* Calls the model on new inputs.
|
||||
*
|
||||
* @param {Object} model_inputs The inputs to the model.
|
||||
* @returns {Promise<MaskedLMOutput>} An object containing the model's output logits for masked language modeling.
|
||||
*/
|
||||
async _call(model_inputs) {
|
||||
return new MaskedLMOutput(await super._call(model_inputs));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* CamemBERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks.
|
||||
*/
|
||||
export class CamembertForSequenceClassification extends CamembertPreTrainedModel {
|
||||
/**
|
||||
* Calls the model on new inputs.
|
||||
*
|
||||
* @param {Object} model_inputs The inputs to the model.
|
||||
* @returns {Promise<SequenceClassifierOutput>} An object containing the model's output logits for sequence classification.
|
||||
*/
|
||||
async _call(model_inputs) {
|
||||
return new SequenceClassifierOutput(await super._call(model_inputs));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* CamemBERT Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks.
|
||||
*/
|
||||
export class CamembertForTokenClassification extends CamembertPreTrainedModel {
|
||||
/**
|
||||
* Calls the model on new inputs.
|
||||
*
|
||||
* @param {Object} model_inputs The inputs to the model.
|
||||
* @returns {Promise<TokenClassifierOutput>} An object containing the model's output logits for token classification.
|
||||
*/
|
||||
async _call(model_inputs) {
|
||||
return new TokenClassifierOutput(await super._call(model_inputs));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* CamemBERT Model with a span classification head on top for extractive question-answering tasks
|
||||
*/
|
||||
export class CamembertForQuestionAnswering extends CamembertPreTrainedModel {
|
||||
/**
|
||||
* Calls the model on new inputs.
|
||||
*
|
||||
* @param {Object} model_inputs The inputs to the model.
|
||||
* @returns {Promise<QuestionAnsweringModelOutput>} An object containing the model's output logits for question answering.
|
||||
*/
|
||||
async _call(model_inputs) {
|
||||
return new QuestionAnsweringModelOutput(await super._call(model_inputs));
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user