From 311a3666c4ea8b1faed9a612d354ffa1403c7e44 Mon Sep 17 00:00:00 2001 From: wehub-resource-sync Date: Mon, 13 Jul 2026 12:44:39 +0800 Subject: [PATCH] chore: import upstream snapshot with attribution --- .gitattributes | 2 + .github/ISSUE_TEMPLATE/1_bug-report.yml | 49 + .github/ISSUE_TEMPLATE/2_new_model.yml | 40 + .github/ISSUE_TEMPLATE/3_new_pipeline.yml | 39 + .github/ISSUE_TEMPLATE/4_feature-request.yml | 30 + .github/ISSUE_TEMPLATE/5_question.yml | 13 + .github/ISSUE_TEMPLATE/config.yml | 9 + .github/dependabot.yml | 11 + .github/workflows/documentation.yml | 19 + .github/workflows/pr-documentation.yml | 19 + .github/workflows/publish.yml | 28 + .github/workflows/tests.yml | 45 + .github/workflows/upload-pr-documentation.yml | 16 + .gitignore | 16 + .prettierignore | 17 + CONTRIBUTING.md | 367 ++ LICENSE | 202 + README.md | 456 ++ README.wehub.md | 7 + package.json | 63 + packages/transformers/docs/build/.gitignore | 3 + packages/transformers/docs/jsdoc-conf.json | 6 + .../transformers/docs/plugins/preprocess.js | 129 + .../transformers/docs/scripts/build_readme.py | 108 + .../transformers/docs/scripts/generate.js | 79 + .../docs/snippets/0_introduction.snippet | 16 + .../docs/snippets/1_quick-tour.snippet | 74 + .../docs/snippets/2_installation.snippet | 12 + .../docs/snippets/3_custom-usage.snippet | 24 + .../docs/snippets/4_supported-tasks.snippet | 71 + .../docs/snippets/5_supported-models.snippet | 216 + .../transformers/docs/source/_toctree.yml | 97 + .../transformers/docs/source/api/.gitignore | 3 + .../transformers/docs/source/custom_usage.md | 7 + .../transformers/docs/source/guides/dtypes.md | 156 + .../source/guides/node-audio-processing.md | 112 + .../docs/source/guides/private.md | 63 + .../transformers/docs/source/guides/webgpu.md | 88 + packages/transformers/docs/source/index.md | 46 + .../transformers/docs/source/installation.md | 7 + .../docs/source/integrations/vercel-ai-sdk.md | 286 ++ .../transformers/docs/source/pipelines.md | 252 + .../source/tutorials/browser-extension.md | 3 + .../docs/source/tutorials/electron.md | 3 + .../docs/source/tutorials/next-ai-sdk.md | 437 ++ .../docs/source/tutorials/next.md | 432 ++ .../docs/source/tutorials/node.md | 214 + .../docs/source/tutorials/react.md | 524 ++ .../docs/source/tutorials/vanilla-js.md | 302 ++ packages/transformers/jest.config.mjs | 192 + packages/transformers/package.json | 87 + packages/transformers/scripts/build.mjs | 25 + .../transformers/scripts/build/buildAll.mjs | 75 + .../scripts/build/buildAllWithWatch.mjs | 60 + .../transformers/scripts/build/constants.mjs | 38 + .../plugins/externalNodeBuiltinsPlugin.mjs | 14 + .../build/plugins/ignoreModulesPlugin.mjs | 49 + .../scripts/build/plugins/postBuildPlugin.mjs | 43 + .../build/plugins/stripNodePrefixPlugin.mjs | 15 + .../transformers/scripts/build/targets.mjs | 51 + packages/transformers/scripts/dev.mjs | 113 + packages/transformers/src/backends/onnx.js | 391 ++ .../src/backends/utils/cacheWasm.js | 101 + packages/transformers/src/cache_utils.js | 82 + packages/transformers/src/configs.js | 581 +++ packages/transformers/src/env.js | 293 ++ .../src/feature_extraction_utils.js | 53 + .../src/generation/configuration_utils.js | 386 ++ .../src/generation/logits_process.js | 751 +++ .../src/generation/logits_sampler.js | 183 + .../transformers/src/generation/parameters.js | 42 + .../src/generation/stopping_criteria.js | 153 + .../transformers/src/generation/streamers.js | 258 + .../src/image_processors_utils.js | 1094 +++++ .../src/models/afmoe/modeling_afmoe.js | 5 + .../src/models/albert/modeling_albert.js | 38 + .../src/models/albert/tokenization_albert.js | 5 + .../src/models/apertus/modeling_apertus.js | 5 + .../src/models/arcee/modeling_arcee.js | 5 + ...xtraction_audio_spectrogram_transformer.js | 82 + .../modeling_audio_spectrogram_transformer.js | 14 + .../models/auto/feature_extraction_auto.js | 27 + .../src/models/auto/image_processing_auto.js | 34 + .../src/models/auto/modeling_auto.js | 384 ++ .../src/models/auto/processing_auto.js | 90 + .../src/models/auto/tokenization_auto.js | 63 + .../src/models/bart/modeling_bart.js | 29 + .../src/models/bart/tokenization_bart.js | 3 + .../src/models/beit/image_processing_beit.js | 3 + .../src/models/beit/modeling_beit.js | 13 + .../src/models/bert/modeling_bert.js | 70 + .../src/models/bert/tokenization_bert.js | 5 + .../src/models/bit/image_processing_bit.js | 3 + .../models/blenderbot/modeling_blenderbot.js | 13 + .../blenderbot/tokenization_blenderbot.js | 3 + .../modeling_blenderbot_small.js | 13 + .../tokenization_blenderbot_small.js | 3 + .../src/models/bloom/modeling_bloom.js | 16 + .../src/models/bloom/tokenization_bloom.js | 3 + .../models/camembert/modeling_camembert.js | 74 + .../camembert/tokenization_camembert.js | 3 + .../feature_extraction_chatterbox.js | 22 + .../models/chatterbox/modeling_chatterbox.js | 187 + .../chatterbox/processing_chatterbox.js | 17 + .../image_processing_chinese_clip.js | 3 + .../chinese_clip/modeling_chinese_clip.js | 5 + .../models/chmv2/image_processing_chmv2.js | 3 + .../src/models/chmv2/modeling_chmv2.js | 4 + .../models/clap/feature_extraction_clap.js | 160 + .../src/models/clap/modeling_clap.js | 79 + .../src/models/clip/image_processing_clip.js | 4 + .../src/models/clip/modeling_clip.js | 150 + .../src/models/clip/tokenization_clip.js | 3 + .../src/models/clipseg/modeling_clipseg.js | 53 + .../code_llama/tokenization_code_llama.js | 3 + .../src/models/codegen/modeling_codegen.js | 13 + .../models/codegen/tokenization_codegen.js | 3 + .../src/models/cohere/modeling_cohere.js | 9 + .../src/models/cohere/tokenization_cohere.js | 3 + .../src/models/cohere2/modeling_cohere2.js | 5 + .../feature_extraction_cohere_asr.js | 117 + .../models/cohere_asr/modeling_cohere_asr.js | 11 + .../cohere_asr/processing_cohere_asr.js | 55 + .../cohere_asr/tokenization_cohere_asr.js | 3 + .../src/models/convbert/modeling_convbert.js | 76 + .../models/convbert/tokenization_convbert.js | 5 + .../convnext/image_processing_convnext.js | 44 + .../src/models/convnext/modeling_convnext.js | 21 + .../models/convnextv2/modeling_convnextv2.js | 21 + .../src/models/d_fine/modeling_d_fine.js | 13 + .../src/models/dac/feature_extraction_dac.js | 3 + .../src/models/dac/modeling_dac.js | 76 + .../src/models/deberta/modeling_deberta.js | 75 + .../models/deberta/tokenization_deberta.js | 5 + .../models/deberta_v2/modeling_deberta_v2.js | 75 + .../deberta_v2/tokenization_deberta_v2.js | 5 + .../modeling_decision_transformer.js | 9 + .../deepseek_v3/modeling_deepseek_v3.js | 5 + .../deepseek_v4/modeling_deepseek_v4.js | 5 + .../src/models/deit/image_processing_deit.js | 4 + .../src/models/deit/modeling_deit.js | 13 + .../depth_anything/modeling_depth_anything.js | 8 + .../models/depth_pro/modeling_depth_pro.js | 4 + .../src/models/detr/image_processing_detr.js | 51 + .../src/models/detr/modeling_detr.js | 54 + .../src/models/dinov2/modeling_dinov2.js | 21 + .../modeling_dinov2_with_registers.js | 21 + .../modeling_dinov3_convnext.js | 4 + .../dinov3_vit/image_processing_dinov3_vit.js | 3 + .../models/dinov3_vit/modeling_dinov3_vit.js | 4 + .../models/distilbert/modeling_distilbert.js | 70 + .../distilbert/tokenization_distilbert.js | 3 + .../models/donut/image_processing_donut.js | 29 + .../models/donut_swin/modeling_donut_swin.js | 79 + .../src/models/dpt/image_processing_dpt.js | 4 + .../src/models/dpt/modeling_dpt.js | 51 + .../image_processing_efficientnet.js | 12 + .../efficientnet/modeling_efficientnet.js | 21 + .../src/models/electra/modeling_electra.js | 77 + .../models/electra/tokenization_electra.js | 5 + .../encodec/feature_extraction_encodec.js | 33 + .../src/models/ernie4_5/modeling_ernie4_5.js | 7 + .../src/models/esm/modeling_esm.js | 55 + .../src/models/esm/tokenization_esm.js | 3 + .../src/models/eurobert/modeling_eurobert.js | 41 + .../src/models/exaone/modeling_exaone.js | 5 + .../src/models/falcon/modeling_falcon.js | 10 + .../src/models/falcon/tokenization_falcon.js | 3 + .../models/falcon_h1/modeling_falcon_h1.js | 5 + .../src/models/fastvit/modeling_fastvit.js | 13 + .../src/models/feature_extractors.js | 21 + .../models/florence2/modeling_florence2.js | 116 + .../models/florence2/processing_florence2.js | 134 + .../src/models/gemma/modeling_gemma.js | 13 + .../src/models/gemma/tokenization_gemma.js | 3 + .../src/models/gemma2/modeling_gemma2.js | 13 + .../models/gemma3/image_processing_gemma3.js | 3 + .../src/models/gemma3/modeling_gemma3.js | 16 + .../src/models/gemma3/processing_gemma3.js | 45 + .../gemma3n/feature_extraction_gemma3n.js | 89 + .../src/models/gemma3n/modeling_gemma3n.js | 122 + .../src/models/gemma3n/processing_gemma3n.js | 78 + .../gemma4/feature_extraction_gemma4.js | 74 + .../models/gemma4/image_processing_gemma4.js | 178 + .../src/models/gemma4/modeling_gemma4.js | 27 + .../src/models/gemma4/processing_gemma4.js | 108 + .../src/models/glm/modeling_glm.js | 5 + .../models/glm46v/image_processing_glm46v.js | 12 + .../src/models/glm46v/processing_glm46v.js | 5 + .../glm_moe_dsa/modeling_glm_moe_dsa.js | 5 + .../src/models/glm_ocr/modeling_glm_ocr.js | 78 + .../src/models/glpn/image_processing_glpn.js | 3 + .../src/models/glpn/modeling_glpn.js | 47 + .../src/models/gpt2/modeling_gpt2.js | 12 + .../src/models/gpt2/tokenization_gpt2.js | 3 + .../gpt_bigcode/modeling_gpt_bigcode.js | 6 + .../src/models/gpt_neo/modeling_gpt_neo.js | 6 + .../src/models/gpt_neox/modeling_gpt_neox.js | 6 + .../models/gpt_neox/tokenization_gpt_neox.js | 3 + .../src/models/gpt_oss/modeling_gpt_oss.js | 5 + .../src/models/gptj/modeling_gptj.js | 6 + .../src/models/granite/modeling_granite.js | 5 + .../feature_extraction_granite_speech.js | 58 + .../granite_speech/modeling_granite_speech.js | 5 + .../processing_granite_speech.js | 62 + .../modeling_granitemoehybrid.js | 5 + .../image_processing_grounding_dino.js | 25 + .../grounding_dino/modeling_grounding_dino.js | 4 + .../processing_grounding_dino.js | 100 + .../src/models/groupvit/modeling_groupvit.js | 4 + .../src/models/helium/modeling_helium.js | 5 + .../models/herbert/tokenization_herbert.js | 5 + .../src/models/hiera/modeling_hiera.js | 13 + .../src/models/hrm_text/modeling_hrm_text.js | 18 + .../src/models/hubert/modeling_hubert.js | 62 + .../modeling_hunyuan_v1_dense.js | 5 + .../idefics3/image_processing_idefics3.js | 224 + .../src/models/idefics3/modeling_idefics3.js | 15 + .../models/idefics3/processing_idefics3.js | 144 + .../src/models/ijepa/modeling_ijepa.js | 13 + .../src/models/image_processors.js | 52 + .../src/models/jais/modeling_jais.js | 13 + .../models/janus/image_processing_janus.js | 24 + .../src/models/janus/processing_janus.js | 120 + .../jina_clip/image_processing_jina_clip.js | 25 + .../models/jina_clip/modeling_jina_clip.js | 65 + .../models/jina_clip/processing_jina_clip.js | 22 + .../src/models/lfm2/modeling_lfm2.js | 5 + .../src/models/lfm2_moe/modeling_lfm2_moe.js | 5 + .../lfm2_vl/image_processing_lfm2_vl.js | 305 ++ .../src/models/lfm2_vl/modeling_lfm2_vl.js | 13 + .../src/models/lfm2_vl/processing_lfm2_vl.js | 77 + .../lighton_ocr/modeling_lighton_ocr.js | 3 + .../src/models/llama/modeling_llama.js | 13 + .../src/models/llama/tokenization_llama.js | 5 + .../src/models/llama4/modeling_llama4.js | 4 + .../src/models/llava/modeling_llava.js | 26 + .../src/models/llava/processing_llava.js | 41 + .../image_processing_llava_onevision.js | 3 + .../modeling_llava_onevision.js | 1 + .../src/models/longt5/modeling_longt5.js | 15 + .../src/models/m2m_100/modeling_m2m_100.js | 7 + .../models/m2m_100/tokenization_m2m_100.js | 34 + .../src/models/marian/modeling_marian.js | 7 + .../src/models/marian/tokenization_marian.js | 56 + .../image_processing_mask2former.js | 4 + .../maskformer/image_processing_maskformer.js | 17 + .../models/maskformer/modeling_maskformer.js | 5 + .../src/models/mbart/modeling_mbart.js | 31 + .../src/models/mbart/tokenization_mbart.js | 22 + .../models/mbart50/tokenization_mbart50.js | 3 + .../src/models/metric3d/modeling_metric3d.js | 4 + .../models/metric3dv2/modeling_metric3dv2.js | 4 + .../src/models/mgp_str/modeling_mgp_str.js | 30 + .../src/models/mgp_str/processing_mgp_str.js | 169 + .../models/mgp_str/tokenization_mgp_str.js | 3 + .../src/models/mimi/modeling_mimi.js | 76 + .../models/ministral/modeling_ministral.js | 5 + .../models/ministral3/modeling_ministral3.js | 5 + .../src/models/mistral/modeling_mistral.js | 10 + .../src/models/mistral3/modeling_mistral3.js | 3 + .../src/models/mistral4/modeling_mistral4.js | 5 + .../models/mobilebert/modeling_mobilebert.js | 50 + .../mobilebert/tokenization_mobilebert.js | 5 + .../models/mobilellm/modeling_mobilellm.js | 5 + .../image_processing_mobilenet_v1.js | 4 + .../mobilenet_v1/modeling_mobilenet_v1.js | 24 + .../image_processing_mobilenet_v2.js | 4 + .../mobilenet_v2/modeling_mobilenet_v2.js | 23 + .../image_processing_mobilenet_v3.js | 4 + .../mobilenet_v3/modeling_mobilenet_v3.js | 23 + .../image_processing_mobilenet_v4.js | 4 + .../mobilenet_v4/modeling_mobilenet_v4.js | 23 + .../mobilevit/image_processing_mobilevit.js | 4 + .../models/mobilevit/modeling_mobilevit.js | 14 + .../mobilevitv2/modeling_mobilevitv2.js | 14 + .../src/models/modeling_outputs.js | 149 + .../transformers/src/models/modeling_utils.js | 1726 +++++++ packages/transformers/src/models/models.js | 216 + .../models/modernbert/modeling_modernbert.js | 41 + .../modeling_modernbert_decoder.js | 5 + .../moonshine/feature_extraction_moonshine.js | 22 + .../models/moonshine/modeling_moonshine.js | 14 + .../models/moonshine/processing_moonshine.js | 20 + .../src/models/mpnet/modeling_mpnet.js | 74 + .../src/models/mpnet/tokenization_mpnet.js | 3 + .../src/models/mpt/modeling_mpt.js | 13 + .../src/models/mt5/modeling_mt5.js | 10 + .../multi_modality/modeling_multi_modality.js | 157 + .../src/models/musicgen/modeling_musicgen.js | 139 + .../src/models/nanochat/modeling_nanochat.js | 5 + .../models/nemotron_h/modeling_nemotron_h.js | 5 + .../src/models/neobert/modeling_neobert.js | 58 + .../src/models/nllb/tokenization_nllb.js | 35 + .../models/nomic_bert/modeling_nomic_bert.js | 4 + .../models/nougat/image_processing_nougat.js | 4 + .../src/models/nougat/tokenization_nougat.js | 3 + .../src/models/olmo/modeling_olmo.js | 5 + .../src/models/olmo2/modeling_olmo2.js | 5 + .../src/models/olmo3/modeling_olmo3.js | 5 + .../olmo_hybrid/modeling_olmo_hybrid.js | 5 + .../modeling_openai_privacy_filter.js | 17 + .../src/models/openelm/modeling_openelm.js | 6 + .../src/models/opt/modeling_opt.js | 13 + .../models/owlv2/image_processing_owlv2.js | 4 + .../src/models/owlv2/modeling_owlv2.js | 5 + .../models/owlvit/image_processing_owlvit.js | 9 + .../src/models/owlvit/modeling_owlvit.js | 5 + .../src/models/owlvit/processing_owlvit.js | 7 + .../models/paligemma/modeling_paligemma.js | 3 + .../models/paligemma/processing_paligemma.js | 70 + .../parakeet/feature_extraction_parakeet.js | 120 + .../src/models/parakeet/modeling_parakeet.js | 15 + .../patchtsmixer/modeling_patchtsmixer.js | 13 + .../src/models/patchtst/modeling_patchtst.js | 13 + .../src/models/phi/modeling_phi.js | 10 + .../src/models/phi3/modeling_phi3.js | 10 + .../models/phi3_v/image_processing_phi3_v.js | 165 + .../src/models/phi3_v/modeling_phi3_v.js | 75 + .../src/models/phi3_v/processing_phi3_v.js | 50 + .../pixtral/image_processing_pixtral.js | 35 + .../src/models/pixtral/processing_pixtral.js | 46 + .../transformers/src/models/processors.js | 36 + .../src/models/pvt/image_processing_pvt.js | 3 + .../src/models/pvt/modeling_pvt.js | 13 + .../pyannote/feature_extraction_pyannote.js | 79 + .../src/models/pyannote/modeling_pyannote.js | 77 + .../models/pyannote/processing_pyannote.js | 26 + .../src/models/qwen2/modeling_qwen2.js | 13 + .../src/models/qwen2/tokenization_qwen2.js | 3 + .../models/qwen2_5_vl/modeling_qwen2_5_vl.js | 9 + .../qwen2_5_vl/processing_qwen2_5_vl.js | 3 + .../models/qwen2_moe/modeling_qwen2_moe.js | 5 + .../qwen2_vl/image_processing_qwen2_vl.js | 63 + .../src/models/qwen2_vl/modeling_qwen2_vl.js | 354 ++ .../models/qwen2_vl/processing_qwen2_vl.js | 53 + .../src/models/qwen3/modeling_qwen3.js | 13 + .../src/models/qwen3_5/modeling_qwen3_5.js | 4 + .../qwen3_5_moe/modeling_qwen3_5_moe.js | 4 + .../models/qwen3_moe/modeling_qwen3_moe.js | 5 + .../models/qwen3_next/modeling_qwen3_next.js | 5 + .../src/models/qwen3_vl/modeling_qwen3_vl.js | 4 + .../models/qwen3_vl/processing_qwen3_vl.js | 3 + .../qwen3_vl_moe/modeling_qwen3_vl_moe.js | 4 + packages/transformers/src/models/registry.js | 701 +++ .../src/models/resnet/modeling_resnet.js | 24 + .../src/models/rf_detr/modeling_rf_detr.js | 15 + .../src/models/roberta/modeling_roberta.js | 70 + .../models/roberta/tokenization_roberta.js | 3 + .../src/models/roformer/modeling_roformer.js | 77 + .../models/roformer/tokenization_roformer.js | 5 + .../rt_detr/image_processing_rt_detr.js | 8 + .../src/models/rt_detr/modeling_rt_detr.js | 28 + .../models/rt_detr_v2/modeling_rt_detr_v2.js | 15 + .../src/models/sam/image_processing_sam.js | 230 + .../src/models/sam/modeling_sam.js | 145 + .../src/models/sam/processing_sam.js | 20 + .../src/models/sam2/image_processing_sam2.js | 1 + .../src/models/sam2/modeling_sam2.js | 95 + .../src/models/sam2/processing_sam2.js | 4 + .../src/models/sam3/image_processing_sam3.js | 1 + .../sapiens/image_processing_sapiens.js | 9 + .../src/models/sapiens/modeling_sapiens.js | 6 + .../feature_extraction_seamless_m4t.js | 159 + .../segformer/image_processing_segformer.js | 9 + .../models/segformer/modeling_segformer.js | 18 + packages/transformers/src/models/session.js | 285 ++ .../transformers/src/models/session_config.js | 170 + .../models/siglip/image_processing_siglip.js | 3 + .../src/models/siglip/modeling_siglip.js | 123 + .../src/models/siglip/tokenization_siglip.js | 3 + .../src/models/smollm3/modeling_smollm3.js | 5 + .../smolvlm/image_processing_smolvlm.js | 1 + .../src/models/smolvlm/modeling_smolvlm.js | 7 + .../src/models/smolvlm/processing_smolvlm.js | 1 + .../models/snac/feature_extraction_snac.js | 3 + .../src/models/snac/modeling_snac.js | 53 + .../models/solar_open/modeling_solar_open.js | 5 + .../speecht5/feature_extraction_speecht5.js | 3 + .../src/models/speecht5/modeling_speecht5.js | 164 + .../models/speecht5/processing_speecht5.js | 17 + .../models/speecht5/tokenization_speecht5.js | 3 + .../squeezebert/modeling_squeezebert.js | 38 + .../squeezebert/tokenization_squeezebert.js | 5 + .../src/models/stablelm/modeling_stablelm.js | 13 + .../models/starcoder2/modeling_starcoder2.js | 10 + .../modeling_style_text_to_speech_2.js | 4 + .../models/supertonic/modeling_supertonic.js | 86 + .../src/models/swin/modeling_swin.js | 14 + .../swin2sr/image_processing_swin2sr.js | 27 + .../src/models/swin2sr/modeling_swin2sr.js | 42 + .../transformers/src/models/t5/modeling_t5.js | 19 + .../src/models/t5/tokenization_t5.js | 3 + .../modeling_table_transformer.js | 24 + .../transformers/src/models/tokenizers.js | 48 + .../src/models/trocr/modeling_trocr.js | 8 + .../src/models/ultravox/modeling_ultravox.js | 19 + .../models/ultravox/processing_ultravox.js | 54 + .../models/unispeech/modeling_unispeech.js | 38 + .../unispeech_sat/modeling_unispeech_sat.js | 52 + .../models/vaultgemma/modeling_vaultgemma.js | 5 + .../modeling_vision_encoder_decoder.js | 17 + .../src/models/vit/image_processing_vit.js | 4 + .../src/models/vit/modeling_vit.js | 13 + .../src/models/vit_mae/modeling_vit_mae.js | 4 + .../src/models/vit_msn/modeling_vit_msn.js | 13 + .../vitmatte/image_processing_vitmatte.js | 51 + .../src/models/vitmatte/modeling_vitmatte.js | 64 + .../vitpose/image_processing_vitpose.js | 87 + .../src/models/vitpose/modeling_vitpose.js | 8 + .../src/models/vits/modeling_vits.js | 60 + .../src/models/vits/tokenization_vits.js | 22 + .../src/models/voxtral/modeling_voxtral.js | 3 + .../src/models/voxtral/processing_voxtral.js | 93 + .../feature_extraction_voxtral_realtime.js | 71 + .../modeling_voxtral_realtime.js | 248 + .../processing_voxtral_realtime.js | 113 + .../wav2vec2/feature_extraction_wav2vec2.js | 43 + .../src/models/wav2vec2/modeling_wav2vec2.js | 69 + .../models/wav2vec2/processing_wav2vec2.js | 17 + .../models/wav2vec2/tokenization_wav2vec2.js | 3 + .../wav2vec2_bert/modeling_wav2vec2_bert.js | 38 + .../processing_wav2vec2_with_lm.js | 17 + .../src/models/wavlm/modeling_wavlm.js | 171 + .../wespeaker/feature_extraction_wespeaker.js | 92 + .../modeling_wespeaker_resnet.js | 4 + .../src/models/whisper/common_whisper.js | 156 + .../whisper/feature_extraction_whisper.js | 80 + .../src/models/whisper/generation_whisper.js | 88 + .../src/models/whisper/modeling_whisper.js | 514 ++ .../src/models/whisper/processing_whisper.js | 20 + .../models/whisper/tokenization_whisper.js | 696 +++ .../src/models/xlm/modeling_xlm.js | 74 + .../src/models/xlm/tokenization_xlm.js | 13 + .../xlm_roberta/modeling_xlm_roberta.js | 70 + .../xlm_roberta/tokenization_xlm_roberta.js | 3 + .../models/yolos/image_processing_yolos.js | 9 + .../src/models/yolos/modeling_yolos.js | 28 + .../src/models/youtu/modeling_youtu.js | 5 + .../src/models/zaya/modeling_zaya.js | 5 + packages/transformers/src/ops/registry.js | 176 + packages/transformers/src/pipelines.js | 266 ++ packages/transformers/src/pipelines/_base.js | 144 + .../src/pipelines/audio-classification.js | 97 + .../pipelines/automatic-speech-recognition.js | 374 ++ .../src/pipelines/background-removal.js | 58 + .../src/pipelines/depth-estimation.js | 94 + .../pipelines/document-question-answering.js | 91 + .../src/pipelines/feature-extraction.js | 134 + .../transformers/src/pipelines/fill-mask.js | 130 + .../src/pipelines/image-classification.js | 108 + .../src/pipelines/image-feature-extraction.js | 80 + .../src/pipelines/image-segmentation.js | 195 + .../src/pipelines/image-to-image.js | 58 + .../src/pipelines/image-to-text.js | 74 + packages/transformers/src/pipelines/index.js | 313 ++ .../src/pipelines/object-detection.js | 92 + .../src/pipelines/question-answering.js | 141 + .../src/pipelines/summarization.js | 50 + .../src/pipelines/text-classification.js | 120 + .../src/pipelines/text-generation.js | 214 + .../src/pipelines/text-to-audio.js | 216 + .../src/pipelines/text2text-generation.js | 100 + .../src/pipelines/token-classification.js | 227 + .../transformers/src/pipelines/translation.js | 79 + .../zero-shot-audio-classification.js | 95 + .../src/pipelines/zero-shot-classification.js | 155 + .../zero-shot-image-classification.js | 99 + .../pipelines/zero-shot-object-detection.js | 173 + packages/transformers/src/processing_utils.js | 162 + .../transformers/src/tokenization_utils.js | 863 ++++ packages/transformers/src/transformers.js | 71 + packages/transformers/src/utils/audio.js | 884 ++++ packages/transformers/src/utils/cache.js | 90 + .../utils/cache/CrossOriginStorageCache.js | 252 + .../transformers/src/utils/cache/FileCache.js | 128 + .../src/utils/cache/cross-origin-storage.d.ts | 47 + packages/transformers/src/utils/constants.js | 8 + packages/transformers/src/utils/core.js | 304 ++ packages/transformers/src/utils/devices.js | 44 + packages/transformers/src/utils/dtypes.js | 146 + packages/transformers/src/utils/generic.js | 36 + packages/transformers/src/utils/hub.js | 580 +++ .../src/utils/hub/FileResponse.js | 121 + .../transformers/src/utils/hub/constants.js | 18 + packages/transformers/src/utils/hub/utils.js | 189 + packages/transformers/src/utils/image.js | 818 ++++ packages/transformers/src/utils/io.js | 49 + packages/transformers/src/utils/logger.js | 67 + packages/transformers/src/utils/lru_cache.js | 67 + packages/transformers/src/utils/maths.js | 1104 +++++ .../transformers/src/utils/memoize_promise.js | 45 + .../transformers/src/utils/model-loader.js | 111 + .../src/utils/model_registry/ModelRegistry.js | 382 ++ .../src/utils/model_registry/clear_cache.js | 128 + .../model_registry/get_available_dtypes.js | 68 + .../utils/model_registry/get_file_metadata.js | 156 + .../src/utils/model_registry/get_files.js | 42 + .../utils/model_registry/get_model_files.js | 115 + .../model_registry/get_pipeline_files.js | 62 + .../model_registry/get_processor_files.js | 20 + .../model_registry/get_tokenizer_files.js | 21 + .../src/utils/model_registry/is_cached.js | 169 + .../model_registry/resolve_model_type.js | 67 + packages/transformers/src/utils/random.js | 225 + packages/transformers/src/utils/tensor.js | 1673 +++++++ packages/transformers/src/utils/video.js | 126 + packages/transformers/tests/asset_cache.js | 74 + packages/transformers/tests/bundles.test.js | 37 + packages/transformers/tests/configs.test.js | 23 + packages/transformers/tests/exports.test.js | 41 + .../tests/feature_extractors.test.js | 5 + .../tests/image_processors.test.js | 5 + packages/transformers/tests/init.js | 125 + packages/transformers/tests/models.test.js | 81 + .../tests/models/afmoe/test_modeling_afmoe.js | 51 + .../models/albert/test_tokenization_albert.js | 183 + .../tests/models/arcee/test_modeling_arcee.js | 51 + ...xtraction_audio_spectrogram_transformer.js | 53 + .../models/beit/test_image_processing_beit.js | 31 + .../tests/models/bert/test_modeling_bert.js | 211 + .../models/bert/test_tokenization_bert.js | 1371 ++++++ .../models/bit/test_image_processing_bit.js | 31 + .../test_tokenization_blenderbot_small.js | 166 + .../tests/models/bloom/test_modeling_bloom.js | 50 + .../models/bloom/test_tokenization_bloom.js | 194 + .../chmv2/test_image_processing_chmv2.js | 48 + .../clap/test_feature_extraction_clap.js | 71 + .../models/clip/test_image_processing_clip.js | 33 + .../tests/models/clip/test_modeling_clip.js | 58 + .../models/clip/test_tokenization_clip.js | 166 + .../models/codegen/test_modeling_codegen.js | 51 + .../models/cohere/test_modeling_cohere.js | 90 + .../test_feature_extraction_cohere_asr.js | 93 + .../test_image_processing_convnext.js | 32 + .../tests/models/dac/test_modeling_dac.js | 88 + .../test_tokenization_deberta_v2.js | 304 ++ .../deepseek_v3/test_modeling_deepseek_v3.js | 51 + .../deepseek_v4/test_modeling_deepseek_v4.js | 75 + .../models/deit/test_image_processing_deit.js | 31 + .../models/detr/test_image_processing_detr.js | 34 + .../test_tokenization_distilbert.js | 315 ++ .../donut/test_image_processing_donut.js | 34 + .../models/dpt/test_image_processing_dpt.js | 77 + .../test_image_processing_efficientnet.js | 46 + .../tests/models/esm/test_tokenization_esm.js | 322 ++ .../models/eurobert/test_modeling_eurobert.js | 44 + .../models/falcon/test_tokenization_falcon.js | 244 + .../falcon_h1/test_modeling_falcon_h1.js | 51 + .../florence2/test_modeling_florence2.js | 83 + .../florence2/test_processor_florence2.js | 257 + .../tests/models/gemma/test_modeling_gemma.js | 51 + .../models/gemma/test_tokenization_gemma.js | 231 + .../models/gemma2/test_modeling_gemma2.js | 51 + .../models/gemma3/test_modeling_gemma3.js | 128 + .../gemma4/test_feature_extraction_gemma4.js | 91 + .../gemma4/test_image_processing_gemma4.js | 110 + .../models/gemma4/test_processor_gemma4.js | 180 + .../tests/models/glm/test_modeling_glm.js | 51 + .../models/glm46v/test_processor_glm46v.js | 43 + .../glm_moe_dsa/test_modeling_glm_moe_dsa.js | 51 + .../models/glm_ocr/test_modeling_glm_ocr.js | 64 + .../models/glpn/test_image_processing_glpn.js | 48 + .../tests/models/gpt2/test_modeling_gpt2.js | 51 + .../models/gpt2/test_tokenization_gpt2.js | 465 ++ .../gpt_bigcode/test_modeling_gpt_bigcode.js | 51 + .../models/gpt_neo/test_modeling_gpt_neo.js | 51 + .../models/gpt_neox/test_modeling_gpt_neox.js | 51 + .../tests/models/gptj/test_modeling_gptj.js | 51 + .../models/granite/test_modeling_granite.js | 50 + .../test_feature_extraction_granite_speech.js | 53 + .../test_modeling_granite_speech.js | 65 + .../test_processor_granite_speech.js | 59 + .../test_modeling_granitemoehybrid.js | 51 + .../test_modeling_grounding_dino.js | 44 + .../test_processor_grounding_dino.js | 57 + .../models/helium/test_modeling_helium.js | 51 + .../test_image_processing_idefics3.js | 107 + .../models/idefics3/test_modeling_idefics3.js | 142 + .../tests/models/jais/test_modeling_jais.js | 51 + .../models/janus/test_processor_janus.js | 47 + .../test_image_processing_jina_clip.js | 33 + .../jina_clip/test_processor_jina_clip.js | 44 + .../tests/models/lfm2/test_modeling_lfm2.js | 51 + .../lfm2_vl/test_image_processing_lfm2_vl.js | 120 + .../tests/models/llama/test_modeling_llama.js | 180 + .../models/llama/test_tokenization_llama.js | 1185 +++++ .../tests/models/llava/test_modeling_llava.js | 114 + .../m2m_100/test_tokenization_m2m_100.js | 179 + .../models/marian/test_modeling_marian.js | 51 + .../models/mgp_str/test_modeling_mgp_str.js | 84 + .../tests/models/mimi/test_modeling_mimi.js | 88 + .../models/mistral/test_modeling_mistral.js | 50 + .../models/mistral4/test_modeling_mistral4.js | 51 + .../test_image_processing_mobilevit.js | 90 + .../modernbert/test_modeling_modernbert.js | 180 + .../test_feature_extraction_moonshine.js | 30 + .../moonshine/test_modeling_moonshine.js | 50 + .../models/mpnet/test_tokenization_mpnet.js | 158 + .../tests/models/mpt/test_modeling_mpt.js | 51 + .../models/musicgen/test_modeling_musicgen.js | 61 + .../nemotron_h/test_modeling_nemotron_h.js | 51 + .../models/nllb/test_tokenization_nllb.js | 158 + .../nougat/test_image_processing_nougat.js | 33 + .../tests/models/olmo/test_modeling_olmo.js | 51 + .../tests/models/olmo2/test_modeling_olmo2.js | 51 + .../tests/models/opt/test_modeling_opt.js | 51 + .../owlvit/test_image_processing_owlvit.js | 31 + .../paligemma/test_modeling_paligemma.js | 52 + .../paligemma/test_processor_paligemma.js | 51 + .../test_modeling_patchtsmixer.js | 65 + .../models/patchtst/test_modeling_patchtst.js | 65 + .../phi3_v/test_image_processing_phi3_v.js | 93 + .../models/phi3_v/test_processor_phi3_v.js | 84 + .../models/pyannote/test_modeling_pyannote.js | 50 + .../models/qwen2/test_tokenization_qwen2.js | 158 + .../test_image_processing_qwen2_vl.js | 34 + .../models/qwen2_vl/test_modeling_qwen2_vl.js | 121 + .../qwen2_vl/test_processor_qwen2_vl.js | 44 + .../models/qwen3_5/test_modeling_qwen3_5.js | 190 + .../roberta/test_tokenization_roberta.js | 704 +++ .../models/sam/test_image_processing_sam.js | 95 + .../tests/models/sam/test_modeling_sam.js | 48 + .../test_feature_extraction_seamless_m4t.js | 65 + .../solar_open/test_modeling_solar_open.js | 51 + .../swin2sr/test_image_processing_swin2sr.js | 41 + .../tests/models/t5/test_modeling_t5.js | 96 + .../tests/models/t5/test_tokenization_t5.js | 293 ++ .../transformers/tests/models/test_strings.js | 125 + .../test_modeling_vision_encoder_decoder.js | 49 + .../models/vit/test_image_processing_vit.js | 31 + .../test_image_processing_vitmatte.js | 68 + .../vitpose/test_image_processing_vitpose.js | 50 + .../models/vits/test_tokenization_vits.js | 76 + ...est_feature_extraction_voxtral_realtime.js | 53 + .../test_modeling_voxtral_realtime.js | 164 + .../test_processor_voxtral_realtime.js | 91 + .../wav2vec2/test_tokenization_wav2vec2.js | 472 ++ ...est_feature_extraction_wespeaker_resnet.js | 56 + .../test_feature_extraction_whisper.js | 67 + .../models/whisper/test_modeling_whisper.js | 148 + .../whisper/test_tokenization_whisper.js | 868 ++++ .../test_tokenization_xlm_roberta.js | 332 ++ .../yolos/test_image_processing_yolos.js | 31 + .../tests/models/zaya/test_modeling_zaya.js | 51 + packages/transformers/tests/pipelines.test.js | 6 + .../test_pipelines_audio_classification.js | 80 + ..._pipelines_automatic_speech_recognition.js | 462 ++ .../test_pipelines_background_removal.js | 68 + .../test_pipelines_depth_estimation.js | 57 + ...t_pipelines_document_question_answering.js | 43 + .../test_pipelines_feature_extraction.js | 121 + .../pipelines/test_pipelines_fill_mask.js | 165 + .../test_pipelines_image_classification.js | 81 + ...test_pipelines_image_feature_extraction.js | 95 + .../test_pipelines_image_segmentation.js | 119 + .../test_pipelines_image_to_image.js | 56 + .../pipelines/test_pipelines_image_to_text.js | 51 + .../test_pipelines_object_detection.js | 131 + .../test_pipelines_question_answering.js | 49 + .../pipelines/test_pipelines_summarization.js | 40 + .../test_pipelines_text2text_generation.js | 40 + .../test_pipelines_text_classification.js | 107 + .../test_pipelines_text_generation.js | 317 ++ .../pipelines/test_pipelines_text_to_audio.js | 37 + .../test_pipelines_token_classification.js | 373 ++ .../pipelines/test_pipelines_translation.js | 42 + .../pipelines/test_pipelines_zero_shot.js | 100 + ...ipelines_zero_shot_audio_classification.js | 58 + ...ipelines_zero_shot_image_classification.js | 98 + ...st_pipelines_zero_shot_object_detection.js | 186 + .../transformers/tests/processors.test.js | 5 + .../tests/progress_callbacks.test.js | 310 ++ packages/transformers/tests/test_utils.js | 67 + .../transformers/tests/tokenizers.test.js | 728 +++ packages/transformers/tests/types.test.js | 353 ++ packages/transformers/tests/types/_base.ts | 12 + packages/transformers/tests/types/cache.ts | 37 + .../transformers/tests/types/pipelines.ts | 726 +++ .../transformers/tests/types/tokenizers.ts | 93 + .../transformers/tests/utils/audio.test.js | 363 ++ .../transformers/tests/utils/cache.test.js | 539 +++ .../tests/utils/custom_cache.test.js | 109 + .../tests/utils/generation.test.js | 591 +++ packages/transformers/tests/utils/hub.test.js | 52 + .../transformers/tests/utils/image.test.js | 89 + .../tests/utils/logits_process.test.js | 112 + .../tests/utils/lru_cache.test.js | 119 + .../transformers/tests/utils/maths.test.js | 195 + .../tests/utils/model_registry.test.js | 183 + .../transformers/tests/utils/random.test.js | 340 ++ .../transformers/tests/utils/tensor.test.js | 754 +++ .../tests/utils/tensor_ops.test.js | 323 ++ .../transformers/tests/utils/utils.test.js | 120 + packages/transformers/tsconfig.json | 24 + pnpm-lock.yaml | 4195 +++++++++++++++++ pnpm-workspace.yaml | 12 + scripts/dev.mjs | 102 + scripts/logger.mjs | 42 + scripts/prepareOutDir.mjs | 9 + scripts/rebuildPlugin.mjs | 42 + scripts/reportSize.mjs | 21 + tsconfig.json | 26 + 703 files changed, 71017 insertions(+) create mode 100644 .gitattributes create mode 100644 .github/ISSUE_TEMPLATE/1_bug-report.yml create mode 100644 .github/ISSUE_TEMPLATE/2_new_model.yml create mode 100644 .github/ISSUE_TEMPLATE/3_new_pipeline.yml create mode 100644 .github/ISSUE_TEMPLATE/4_feature-request.yml create mode 100644 .github/ISSUE_TEMPLATE/5_question.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/documentation.yml create mode 100644 .github/workflows/pr-documentation.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/tests.yml create mode 100644 .github/workflows/upload-pr-documentation.yml create mode 100644 .gitignore create mode 100644 .prettierignore create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 README.wehub.md create mode 100644 package.json create mode 100644 packages/transformers/docs/build/.gitignore create mode 100644 packages/transformers/docs/jsdoc-conf.json create mode 100644 packages/transformers/docs/plugins/preprocess.js create mode 100644 packages/transformers/docs/scripts/build_readme.py create mode 100644 packages/transformers/docs/scripts/generate.js create mode 100644 packages/transformers/docs/snippets/0_introduction.snippet create mode 100644 packages/transformers/docs/snippets/1_quick-tour.snippet create mode 100644 packages/transformers/docs/snippets/2_installation.snippet create mode 100644 packages/transformers/docs/snippets/3_custom-usage.snippet create mode 100644 packages/transformers/docs/snippets/4_supported-tasks.snippet create mode 100644 packages/transformers/docs/snippets/5_supported-models.snippet create mode 100644 packages/transformers/docs/source/_toctree.yml create mode 100644 packages/transformers/docs/source/api/.gitignore create mode 100644 packages/transformers/docs/source/custom_usage.md create mode 100644 packages/transformers/docs/source/guides/dtypes.md create mode 100644 packages/transformers/docs/source/guides/node-audio-processing.md create mode 100644 packages/transformers/docs/source/guides/private.md create mode 100644 packages/transformers/docs/source/guides/webgpu.md create mode 100644 packages/transformers/docs/source/index.md create mode 100644 packages/transformers/docs/source/installation.md create mode 100644 packages/transformers/docs/source/integrations/vercel-ai-sdk.md create mode 100644 packages/transformers/docs/source/pipelines.md create mode 100644 packages/transformers/docs/source/tutorials/browser-extension.md create mode 100644 packages/transformers/docs/source/tutorials/electron.md create mode 100644 packages/transformers/docs/source/tutorials/next-ai-sdk.md create mode 100644 packages/transformers/docs/source/tutorials/next.md create mode 100644 packages/transformers/docs/source/tutorials/node.md create mode 100644 packages/transformers/docs/source/tutorials/react.md create mode 100644 packages/transformers/docs/source/tutorials/vanilla-js.md create mode 100644 packages/transformers/jest.config.mjs create mode 100644 packages/transformers/package.json create mode 100644 packages/transformers/scripts/build.mjs create mode 100644 packages/transformers/scripts/build/buildAll.mjs create mode 100644 packages/transformers/scripts/build/buildAllWithWatch.mjs create mode 100644 packages/transformers/scripts/build/constants.mjs create mode 100644 packages/transformers/scripts/build/plugins/externalNodeBuiltinsPlugin.mjs create mode 100644 packages/transformers/scripts/build/plugins/ignoreModulesPlugin.mjs create mode 100644 packages/transformers/scripts/build/plugins/postBuildPlugin.mjs create mode 100644 packages/transformers/scripts/build/plugins/stripNodePrefixPlugin.mjs create mode 100644 packages/transformers/scripts/build/targets.mjs create mode 100644 packages/transformers/scripts/dev.mjs create mode 100644 packages/transformers/src/backends/onnx.js create mode 100644 packages/transformers/src/backends/utils/cacheWasm.js create mode 100644 packages/transformers/src/cache_utils.js create mode 100644 packages/transformers/src/configs.js create mode 100644 packages/transformers/src/env.js create mode 100644 packages/transformers/src/feature_extraction_utils.js create mode 100644 packages/transformers/src/generation/configuration_utils.js create mode 100644 packages/transformers/src/generation/logits_process.js create mode 100644 packages/transformers/src/generation/logits_sampler.js create mode 100644 packages/transformers/src/generation/parameters.js create mode 100644 packages/transformers/src/generation/stopping_criteria.js create mode 100644 packages/transformers/src/generation/streamers.js create mode 100644 packages/transformers/src/image_processors_utils.js create mode 100644 packages/transformers/src/models/afmoe/modeling_afmoe.js create mode 100644 packages/transformers/src/models/albert/modeling_albert.js create mode 100644 packages/transformers/src/models/albert/tokenization_albert.js create mode 100644 packages/transformers/src/models/apertus/modeling_apertus.js create mode 100644 packages/transformers/src/models/arcee/modeling_arcee.js create mode 100644 packages/transformers/src/models/audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.js create mode 100644 packages/transformers/src/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.js create mode 100644 packages/transformers/src/models/auto/feature_extraction_auto.js create mode 100644 packages/transformers/src/models/auto/image_processing_auto.js create mode 100644 packages/transformers/src/models/auto/modeling_auto.js create mode 100644 packages/transformers/src/models/auto/processing_auto.js create mode 100644 packages/transformers/src/models/auto/tokenization_auto.js create mode 100644 packages/transformers/src/models/bart/modeling_bart.js create mode 100644 packages/transformers/src/models/bart/tokenization_bart.js create mode 100644 packages/transformers/src/models/beit/image_processing_beit.js create mode 100644 packages/transformers/src/models/beit/modeling_beit.js create mode 100644 packages/transformers/src/models/bert/modeling_bert.js create mode 100644 packages/transformers/src/models/bert/tokenization_bert.js create mode 100644 packages/transformers/src/models/bit/image_processing_bit.js create mode 100644 packages/transformers/src/models/blenderbot/modeling_blenderbot.js create mode 100644 packages/transformers/src/models/blenderbot/tokenization_blenderbot.js create mode 100644 packages/transformers/src/models/blenderbot_small/modeling_blenderbot_small.js create mode 100644 packages/transformers/src/models/blenderbot_small/tokenization_blenderbot_small.js create mode 100644 packages/transformers/src/models/bloom/modeling_bloom.js create mode 100644 packages/transformers/src/models/bloom/tokenization_bloom.js create mode 100644 packages/transformers/src/models/camembert/modeling_camembert.js create mode 100644 packages/transformers/src/models/camembert/tokenization_camembert.js create mode 100644 packages/transformers/src/models/chatterbox/feature_extraction_chatterbox.js create mode 100644 packages/transformers/src/models/chatterbox/modeling_chatterbox.js create mode 100644 packages/transformers/src/models/chatterbox/processing_chatterbox.js create mode 100644 packages/transformers/src/models/chinese_clip/image_processing_chinese_clip.js create mode 100644 packages/transformers/src/models/chinese_clip/modeling_chinese_clip.js create mode 100644 packages/transformers/src/models/chmv2/image_processing_chmv2.js create mode 100644 packages/transformers/src/models/chmv2/modeling_chmv2.js create mode 100644 packages/transformers/src/models/clap/feature_extraction_clap.js create mode 100644 packages/transformers/src/models/clap/modeling_clap.js create mode 100644 packages/transformers/src/models/clip/image_processing_clip.js create mode 100644 packages/transformers/src/models/clip/modeling_clip.js create mode 100644 packages/transformers/src/models/clip/tokenization_clip.js create mode 100644 packages/transformers/src/models/clipseg/modeling_clipseg.js create mode 100644 packages/transformers/src/models/code_llama/tokenization_code_llama.js create mode 100644 packages/transformers/src/models/codegen/modeling_codegen.js create mode 100644 packages/transformers/src/models/codegen/tokenization_codegen.js create mode 100644 packages/transformers/src/models/cohere/modeling_cohere.js create mode 100644 packages/transformers/src/models/cohere/tokenization_cohere.js create mode 100644 packages/transformers/src/models/cohere2/modeling_cohere2.js create mode 100644 packages/transformers/src/models/cohere_asr/feature_extraction_cohere_asr.js create mode 100644 packages/transformers/src/models/cohere_asr/modeling_cohere_asr.js create mode 100644 packages/transformers/src/models/cohere_asr/processing_cohere_asr.js create mode 100644 packages/transformers/src/models/cohere_asr/tokenization_cohere_asr.js create mode 100644 packages/transformers/src/models/convbert/modeling_convbert.js create mode 100644 packages/transformers/src/models/convbert/tokenization_convbert.js create mode 100644 packages/transformers/src/models/convnext/image_processing_convnext.js create mode 100644 packages/transformers/src/models/convnext/modeling_convnext.js create mode 100644 packages/transformers/src/models/convnextv2/modeling_convnextv2.js create mode 100644 packages/transformers/src/models/d_fine/modeling_d_fine.js create mode 100644 packages/transformers/src/models/dac/feature_extraction_dac.js create mode 100644 packages/transformers/src/models/dac/modeling_dac.js create mode 100644 packages/transformers/src/models/deberta/modeling_deberta.js create mode 100644 packages/transformers/src/models/deberta/tokenization_deberta.js create mode 100644 packages/transformers/src/models/deberta_v2/modeling_deberta_v2.js create mode 100644 packages/transformers/src/models/deberta_v2/tokenization_deberta_v2.js create mode 100644 packages/transformers/src/models/decision_transformer/modeling_decision_transformer.js create mode 100644 packages/transformers/src/models/deepseek_v3/modeling_deepseek_v3.js create mode 100644 packages/transformers/src/models/deepseek_v4/modeling_deepseek_v4.js create mode 100644 packages/transformers/src/models/deit/image_processing_deit.js create mode 100644 packages/transformers/src/models/deit/modeling_deit.js create mode 100644 packages/transformers/src/models/depth_anything/modeling_depth_anything.js create mode 100644 packages/transformers/src/models/depth_pro/modeling_depth_pro.js create mode 100644 packages/transformers/src/models/detr/image_processing_detr.js create mode 100644 packages/transformers/src/models/detr/modeling_detr.js create mode 100644 packages/transformers/src/models/dinov2/modeling_dinov2.js create mode 100644 packages/transformers/src/models/dinov2_with_registers/modeling_dinov2_with_registers.js create mode 100644 packages/transformers/src/models/dinov3_convnext/modeling_dinov3_convnext.js create mode 100644 packages/transformers/src/models/dinov3_vit/image_processing_dinov3_vit.js create mode 100644 packages/transformers/src/models/dinov3_vit/modeling_dinov3_vit.js create mode 100644 packages/transformers/src/models/distilbert/modeling_distilbert.js create mode 100644 packages/transformers/src/models/distilbert/tokenization_distilbert.js create mode 100644 packages/transformers/src/models/donut/image_processing_donut.js create mode 100644 packages/transformers/src/models/donut_swin/modeling_donut_swin.js create mode 100644 packages/transformers/src/models/dpt/image_processing_dpt.js create mode 100644 packages/transformers/src/models/dpt/modeling_dpt.js create mode 100644 packages/transformers/src/models/efficientnet/image_processing_efficientnet.js create mode 100644 packages/transformers/src/models/efficientnet/modeling_efficientnet.js create mode 100644 packages/transformers/src/models/electra/modeling_electra.js create mode 100644 packages/transformers/src/models/electra/tokenization_electra.js create mode 100644 packages/transformers/src/models/encodec/feature_extraction_encodec.js create mode 100644 packages/transformers/src/models/ernie4_5/modeling_ernie4_5.js create mode 100644 packages/transformers/src/models/esm/modeling_esm.js create mode 100644 packages/transformers/src/models/esm/tokenization_esm.js create mode 100644 packages/transformers/src/models/eurobert/modeling_eurobert.js create mode 100644 packages/transformers/src/models/exaone/modeling_exaone.js create mode 100644 packages/transformers/src/models/falcon/modeling_falcon.js create mode 100644 packages/transformers/src/models/falcon/tokenization_falcon.js create mode 100644 packages/transformers/src/models/falcon_h1/modeling_falcon_h1.js create mode 100644 packages/transformers/src/models/fastvit/modeling_fastvit.js create mode 100644 packages/transformers/src/models/feature_extractors.js create mode 100644 packages/transformers/src/models/florence2/modeling_florence2.js create mode 100644 packages/transformers/src/models/florence2/processing_florence2.js create mode 100644 packages/transformers/src/models/gemma/modeling_gemma.js create mode 100644 packages/transformers/src/models/gemma/tokenization_gemma.js create mode 100644 packages/transformers/src/models/gemma2/modeling_gemma2.js create mode 100644 packages/transformers/src/models/gemma3/image_processing_gemma3.js create mode 100644 packages/transformers/src/models/gemma3/modeling_gemma3.js create mode 100644 packages/transformers/src/models/gemma3/processing_gemma3.js create mode 100644 packages/transformers/src/models/gemma3n/feature_extraction_gemma3n.js create mode 100644 packages/transformers/src/models/gemma3n/modeling_gemma3n.js create mode 100644 packages/transformers/src/models/gemma3n/processing_gemma3n.js create mode 100644 packages/transformers/src/models/gemma4/feature_extraction_gemma4.js create mode 100644 packages/transformers/src/models/gemma4/image_processing_gemma4.js create mode 100644 packages/transformers/src/models/gemma4/modeling_gemma4.js create mode 100644 packages/transformers/src/models/gemma4/processing_gemma4.js create mode 100644 packages/transformers/src/models/glm/modeling_glm.js create mode 100644 packages/transformers/src/models/glm46v/image_processing_glm46v.js create mode 100644 packages/transformers/src/models/glm46v/processing_glm46v.js create mode 100644 packages/transformers/src/models/glm_moe_dsa/modeling_glm_moe_dsa.js create mode 100644 packages/transformers/src/models/glm_ocr/modeling_glm_ocr.js create mode 100644 packages/transformers/src/models/glpn/image_processing_glpn.js create mode 100644 packages/transformers/src/models/glpn/modeling_glpn.js create mode 100644 packages/transformers/src/models/gpt2/modeling_gpt2.js create mode 100644 packages/transformers/src/models/gpt2/tokenization_gpt2.js create mode 100644 packages/transformers/src/models/gpt_bigcode/modeling_gpt_bigcode.js create mode 100644 packages/transformers/src/models/gpt_neo/modeling_gpt_neo.js create mode 100644 packages/transformers/src/models/gpt_neox/modeling_gpt_neox.js create mode 100644 packages/transformers/src/models/gpt_neox/tokenization_gpt_neox.js create mode 100644 packages/transformers/src/models/gpt_oss/modeling_gpt_oss.js create mode 100644 packages/transformers/src/models/gptj/modeling_gptj.js create mode 100644 packages/transformers/src/models/granite/modeling_granite.js create mode 100644 packages/transformers/src/models/granite_speech/feature_extraction_granite_speech.js create mode 100644 packages/transformers/src/models/granite_speech/modeling_granite_speech.js create mode 100644 packages/transformers/src/models/granite_speech/processing_granite_speech.js create mode 100644 packages/transformers/src/models/granitemoehybrid/modeling_granitemoehybrid.js create mode 100644 packages/transformers/src/models/grounding_dino/image_processing_grounding_dino.js create mode 100644 packages/transformers/src/models/grounding_dino/modeling_grounding_dino.js create mode 100644 packages/transformers/src/models/grounding_dino/processing_grounding_dino.js create mode 100644 packages/transformers/src/models/groupvit/modeling_groupvit.js create mode 100644 packages/transformers/src/models/helium/modeling_helium.js create mode 100644 packages/transformers/src/models/herbert/tokenization_herbert.js create mode 100644 packages/transformers/src/models/hiera/modeling_hiera.js create mode 100644 packages/transformers/src/models/hrm_text/modeling_hrm_text.js create mode 100644 packages/transformers/src/models/hubert/modeling_hubert.js create mode 100644 packages/transformers/src/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.js create mode 100644 packages/transformers/src/models/idefics3/image_processing_idefics3.js create mode 100644 packages/transformers/src/models/idefics3/modeling_idefics3.js create mode 100644 packages/transformers/src/models/idefics3/processing_idefics3.js create mode 100644 packages/transformers/src/models/ijepa/modeling_ijepa.js create mode 100644 packages/transformers/src/models/image_processors.js create mode 100644 packages/transformers/src/models/jais/modeling_jais.js create mode 100644 packages/transformers/src/models/janus/image_processing_janus.js create mode 100644 packages/transformers/src/models/janus/processing_janus.js create mode 100644 packages/transformers/src/models/jina_clip/image_processing_jina_clip.js create mode 100644 packages/transformers/src/models/jina_clip/modeling_jina_clip.js create mode 100644 packages/transformers/src/models/jina_clip/processing_jina_clip.js create mode 100644 packages/transformers/src/models/lfm2/modeling_lfm2.js create mode 100644 packages/transformers/src/models/lfm2_moe/modeling_lfm2_moe.js create mode 100644 packages/transformers/src/models/lfm2_vl/image_processing_lfm2_vl.js create mode 100644 packages/transformers/src/models/lfm2_vl/modeling_lfm2_vl.js create mode 100644 packages/transformers/src/models/lfm2_vl/processing_lfm2_vl.js create mode 100644 packages/transformers/src/models/lighton_ocr/modeling_lighton_ocr.js create mode 100644 packages/transformers/src/models/llama/modeling_llama.js create mode 100644 packages/transformers/src/models/llama/tokenization_llama.js create mode 100644 packages/transformers/src/models/llama4/modeling_llama4.js create mode 100644 packages/transformers/src/models/llava/modeling_llava.js create mode 100644 packages/transformers/src/models/llava/processing_llava.js create mode 100644 packages/transformers/src/models/llava_onevision/image_processing_llava_onevision.js create mode 100644 packages/transformers/src/models/llava_onevision/modeling_llava_onevision.js create mode 100644 packages/transformers/src/models/longt5/modeling_longt5.js create mode 100644 packages/transformers/src/models/m2m_100/modeling_m2m_100.js create mode 100644 packages/transformers/src/models/m2m_100/tokenization_m2m_100.js create mode 100644 packages/transformers/src/models/marian/modeling_marian.js create mode 100644 packages/transformers/src/models/marian/tokenization_marian.js create mode 100644 packages/transformers/src/models/mask2former/image_processing_mask2former.js create mode 100644 packages/transformers/src/models/maskformer/image_processing_maskformer.js create mode 100644 packages/transformers/src/models/maskformer/modeling_maskformer.js create mode 100644 packages/transformers/src/models/mbart/modeling_mbart.js create mode 100644 packages/transformers/src/models/mbart/tokenization_mbart.js create mode 100644 packages/transformers/src/models/mbart50/tokenization_mbart50.js create mode 100644 packages/transformers/src/models/metric3d/modeling_metric3d.js create mode 100644 packages/transformers/src/models/metric3dv2/modeling_metric3dv2.js create mode 100644 packages/transformers/src/models/mgp_str/modeling_mgp_str.js create mode 100644 packages/transformers/src/models/mgp_str/processing_mgp_str.js create mode 100644 packages/transformers/src/models/mgp_str/tokenization_mgp_str.js create mode 100644 packages/transformers/src/models/mimi/modeling_mimi.js create mode 100644 packages/transformers/src/models/ministral/modeling_ministral.js create mode 100644 packages/transformers/src/models/ministral3/modeling_ministral3.js create mode 100644 packages/transformers/src/models/mistral/modeling_mistral.js create mode 100644 packages/transformers/src/models/mistral3/modeling_mistral3.js create mode 100644 packages/transformers/src/models/mistral4/modeling_mistral4.js create mode 100644 packages/transformers/src/models/mobilebert/modeling_mobilebert.js create mode 100644 packages/transformers/src/models/mobilebert/tokenization_mobilebert.js create mode 100644 packages/transformers/src/models/mobilellm/modeling_mobilellm.js create mode 100644 packages/transformers/src/models/mobilenet_v1/image_processing_mobilenet_v1.js create mode 100644 packages/transformers/src/models/mobilenet_v1/modeling_mobilenet_v1.js create mode 100644 packages/transformers/src/models/mobilenet_v2/image_processing_mobilenet_v2.js create mode 100644 packages/transformers/src/models/mobilenet_v2/modeling_mobilenet_v2.js create mode 100644 packages/transformers/src/models/mobilenet_v3/image_processing_mobilenet_v3.js create mode 100644 packages/transformers/src/models/mobilenet_v3/modeling_mobilenet_v3.js create mode 100644 packages/transformers/src/models/mobilenet_v4/image_processing_mobilenet_v4.js create mode 100644 packages/transformers/src/models/mobilenet_v4/modeling_mobilenet_v4.js create mode 100644 packages/transformers/src/models/mobilevit/image_processing_mobilevit.js create mode 100644 packages/transformers/src/models/mobilevit/modeling_mobilevit.js create mode 100644 packages/transformers/src/models/mobilevitv2/modeling_mobilevitv2.js create mode 100644 packages/transformers/src/models/modeling_outputs.js create mode 100644 packages/transformers/src/models/modeling_utils.js create mode 100644 packages/transformers/src/models/models.js create mode 100644 packages/transformers/src/models/modernbert/modeling_modernbert.js create mode 100644 packages/transformers/src/models/modernbert_decoder/modeling_modernbert_decoder.js create mode 100644 packages/transformers/src/models/moonshine/feature_extraction_moonshine.js create mode 100644 packages/transformers/src/models/moonshine/modeling_moonshine.js create mode 100644 packages/transformers/src/models/moonshine/processing_moonshine.js create mode 100644 packages/transformers/src/models/mpnet/modeling_mpnet.js create mode 100644 packages/transformers/src/models/mpnet/tokenization_mpnet.js create mode 100644 packages/transformers/src/models/mpt/modeling_mpt.js create mode 100644 packages/transformers/src/models/mt5/modeling_mt5.js create mode 100644 packages/transformers/src/models/multi_modality/modeling_multi_modality.js create mode 100644 packages/transformers/src/models/musicgen/modeling_musicgen.js create mode 100644 packages/transformers/src/models/nanochat/modeling_nanochat.js create mode 100644 packages/transformers/src/models/nemotron_h/modeling_nemotron_h.js create mode 100644 packages/transformers/src/models/neobert/modeling_neobert.js create mode 100644 packages/transformers/src/models/nllb/tokenization_nllb.js create mode 100644 packages/transformers/src/models/nomic_bert/modeling_nomic_bert.js create mode 100644 packages/transformers/src/models/nougat/image_processing_nougat.js create mode 100644 packages/transformers/src/models/nougat/tokenization_nougat.js create mode 100644 packages/transformers/src/models/olmo/modeling_olmo.js create mode 100644 packages/transformers/src/models/olmo2/modeling_olmo2.js create mode 100644 packages/transformers/src/models/olmo3/modeling_olmo3.js create mode 100644 packages/transformers/src/models/olmo_hybrid/modeling_olmo_hybrid.js create mode 100644 packages/transformers/src/models/openai_privacy_filter/modeling_openai_privacy_filter.js create mode 100644 packages/transformers/src/models/openelm/modeling_openelm.js create mode 100644 packages/transformers/src/models/opt/modeling_opt.js create mode 100644 packages/transformers/src/models/owlv2/image_processing_owlv2.js create mode 100644 packages/transformers/src/models/owlv2/modeling_owlv2.js create mode 100644 packages/transformers/src/models/owlvit/image_processing_owlvit.js create mode 100644 packages/transformers/src/models/owlvit/modeling_owlvit.js create mode 100644 packages/transformers/src/models/owlvit/processing_owlvit.js create mode 100644 packages/transformers/src/models/paligemma/modeling_paligemma.js create mode 100644 packages/transformers/src/models/paligemma/processing_paligemma.js create mode 100644 packages/transformers/src/models/parakeet/feature_extraction_parakeet.js create mode 100644 packages/transformers/src/models/parakeet/modeling_parakeet.js create mode 100644 packages/transformers/src/models/patchtsmixer/modeling_patchtsmixer.js create mode 100644 packages/transformers/src/models/patchtst/modeling_patchtst.js create mode 100644 packages/transformers/src/models/phi/modeling_phi.js create mode 100644 packages/transformers/src/models/phi3/modeling_phi3.js create mode 100644 packages/transformers/src/models/phi3_v/image_processing_phi3_v.js create mode 100644 packages/transformers/src/models/phi3_v/modeling_phi3_v.js create mode 100644 packages/transformers/src/models/phi3_v/processing_phi3_v.js create mode 100644 packages/transformers/src/models/pixtral/image_processing_pixtral.js create mode 100644 packages/transformers/src/models/pixtral/processing_pixtral.js create mode 100644 packages/transformers/src/models/processors.js create mode 100644 packages/transformers/src/models/pvt/image_processing_pvt.js create mode 100644 packages/transformers/src/models/pvt/modeling_pvt.js create mode 100644 packages/transformers/src/models/pyannote/feature_extraction_pyannote.js create mode 100644 packages/transformers/src/models/pyannote/modeling_pyannote.js create mode 100644 packages/transformers/src/models/pyannote/processing_pyannote.js create mode 100644 packages/transformers/src/models/qwen2/modeling_qwen2.js create mode 100644 packages/transformers/src/models/qwen2/tokenization_qwen2.js create mode 100644 packages/transformers/src/models/qwen2_5_vl/modeling_qwen2_5_vl.js create mode 100644 packages/transformers/src/models/qwen2_5_vl/processing_qwen2_5_vl.js create mode 100644 packages/transformers/src/models/qwen2_moe/modeling_qwen2_moe.js create mode 100644 packages/transformers/src/models/qwen2_vl/image_processing_qwen2_vl.js create mode 100644 packages/transformers/src/models/qwen2_vl/modeling_qwen2_vl.js create mode 100644 packages/transformers/src/models/qwen2_vl/processing_qwen2_vl.js create mode 100644 packages/transformers/src/models/qwen3/modeling_qwen3.js create mode 100644 packages/transformers/src/models/qwen3_5/modeling_qwen3_5.js create mode 100644 packages/transformers/src/models/qwen3_5_moe/modeling_qwen3_5_moe.js create mode 100644 packages/transformers/src/models/qwen3_moe/modeling_qwen3_moe.js create mode 100644 packages/transformers/src/models/qwen3_next/modeling_qwen3_next.js create mode 100644 packages/transformers/src/models/qwen3_vl/modeling_qwen3_vl.js create mode 100644 packages/transformers/src/models/qwen3_vl/processing_qwen3_vl.js create mode 100644 packages/transformers/src/models/qwen3_vl_moe/modeling_qwen3_vl_moe.js create mode 100644 packages/transformers/src/models/registry.js create mode 100644 packages/transformers/src/models/resnet/modeling_resnet.js create mode 100644 packages/transformers/src/models/rf_detr/modeling_rf_detr.js create mode 100644 packages/transformers/src/models/roberta/modeling_roberta.js create mode 100644 packages/transformers/src/models/roberta/tokenization_roberta.js create mode 100644 packages/transformers/src/models/roformer/modeling_roformer.js create mode 100644 packages/transformers/src/models/roformer/tokenization_roformer.js create mode 100644 packages/transformers/src/models/rt_detr/image_processing_rt_detr.js create mode 100644 packages/transformers/src/models/rt_detr/modeling_rt_detr.js create mode 100644 packages/transformers/src/models/rt_detr_v2/modeling_rt_detr_v2.js create mode 100644 packages/transformers/src/models/sam/image_processing_sam.js create mode 100644 packages/transformers/src/models/sam/modeling_sam.js create mode 100644 packages/transformers/src/models/sam/processing_sam.js create mode 100644 packages/transformers/src/models/sam2/image_processing_sam2.js create mode 100644 packages/transformers/src/models/sam2/modeling_sam2.js create mode 100644 packages/transformers/src/models/sam2/processing_sam2.js create mode 100644 packages/transformers/src/models/sam3/image_processing_sam3.js create mode 100644 packages/transformers/src/models/sapiens/image_processing_sapiens.js create mode 100644 packages/transformers/src/models/sapiens/modeling_sapiens.js create mode 100644 packages/transformers/src/models/seamless_m4t/feature_extraction_seamless_m4t.js create mode 100644 packages/transformers/src/models/segformer/image_processing_segformer.js create mode 100644 packages/transformers/src/models/segformer/modeling_segformer.js create mode 100644 packages/transformers/src/models/session.js create mode 100644 packages/transformers/src/models/session_config.js create mode 100644 packages/transformers/src/models/siglip/image_processing_siglip.js create mode 100644 packages/transformers/src/models/siglip/modeling_siglip.js create mode 100644 packages/transformers/src/models/siglip/tokenization_siglip.js create mode 100644 packages/transformers/src/models/smollm3/modeling_smollm3.js create mode 100644 packages/transformers/src/models/smolvlm/image_processing_smolvlm.js create mode 100644 packages/transformers/src/models/smolvlm/modeling_smolvlm.js create mode 100644 packages/transformers/src/models/smolvlm/processing_smolvlm.js create mode 100644 packages/transformers/src/models/snac/feature_extraction_snac.js create mode 100644 packages/transformers/src/models/snac/modeling_snac.js create mode 100644 packages/transformers/src/models/solar_open/modeling_solar_open.js create mode 100644 packages/transformers/src/models/speecht5/feature_extraction_speecht5.js create mode 100644 packages/transformers/src/models/speecht5/modeling_speecht5.js create mode 100644 packages/transformers/src/models/speecht5/processing_speecht5.js create mode 100644 packages/transformers/src/models/speecht5/tokenization_speecht5.js create mode 100644 packages/transformers/src/models/squeezebert/modeling_squeezebert.js create mode 100644 packages/transformers/src/models/squeezebert/tokenization_squeezebert.js create mode 100644 packages/transformers/src/models/stablelm/modeling_stablelm.js create mode 100644 packages/transformers/src/models/starcoder2/modeling_starcoder2.js create mode 100644 packages/transformers/src/models/style_text_to_speech_2/modeling_style_text_to_speech_2.js create mode 100644 packages/transformers/src/models/supertonic/modeling_supertonic.js create mode 100644 packages/transformers/src/models/swin/modeling_swin.js create mode 100644 packages/transformers/src/models/swin2sr/image_processing_swin2sr.js create mode 100644 packages/transformers/src/models/swin2sr/modeling_swin2sr.js create mode 100644 packages/transformers/src/models/t5/modeling_t5.js create mode 100644 packages/transformers/src/models/t5/tokenization_t5.js create mode 100644 packages/transformers/src/models/table_transformer/modeling_table_transformer.js create mode 100644 packages/transformers/src/models/tokenizers.js create mode 100644 packages/transformers/src/models/trocr/modeling_trocr.js create mode 100644 packages/transformers/src/models/ultravox/modeling_ultravox.js create mode 100644 packages/transformers/src/models/ultravox/processing_ultravox.js create mode 100644 packages/transformers/src/models/unispeech/modeling_unispeech.js create mode 100644 packages/transformers/src/models/unispeech_sat/modeling_unispeech_sat.js create mode 100644 packages/transformers/src/models/vaultgemma/modeling_vaultgemma.js create mode 100644 packages/transformers/src/models/vision_encoder_decoder/modeling_vision_encoder_decoder.js create mode 100644 packages/transformers/src/models/vit/image_processing_vit.js create mode 100644 packages/transformers/src/models/vit/modeling_vit.js create mode 100644 packages/transformers/src/models/vit_mae/modeling_vit_mae.js create mode 100644 packages/transformers/src/models/vit_msn/modeling_vit_msn.js create mode 100644 packages/transformers/src/models/vitmatte/image_processing_vitmatte.js create mode 100644 packages/transformers/src/models/vitmatte/modeling_vitmatte.js create mode 100644 packages/transformers/src/models/vitpose/image_processing_vitpose.js create mode 100644 packages/transformers/src/models/vitpose/modeling_vitpose.js create mode 100644 packages/transformers/src/models/vits/modeling_vits.js create mode 100644 packages/transformers/src/models/vits/tokenization_vits.js create mode 100644 packages/transformers/src/models/voxtral/modeling_voxtral.js create mode 100644 packages/transformers/src/models/voxtral/processing_voxtral.js create mode 100644 packages/transformers/src/models/voxtral_realtime/feature_extraction_voxtral_realtime.js create mode 100644 packages/transformers/src/models/voxtral_realtime/modeling_voxtral_realtime.js create mode 100644 packages/transformers/src/models/voxtral_realtime/processing_voxtral_realtime.js create mode 100644 packages/transformers/src/models/wav2vec2/feature_extraction_wav2vec2.js create mode 100644 packages/transformers/src/models/wav2vec2/modeling_wav2vec2.js create mode 100644 packages/transformers/src/models/wav2vec2/processing_wav2vec2.js create mode 100644 packages/transformers/src/models/wav2vec2/tokenization_wav2vec2.js create mode 100644 packages/transformers/src/models/wav2vec2_bert/modeling_wav2vec2_bert.js create mode 100644 packages/transformers/src/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.js create mode 100644 packages/transformers/src/models/wavlm/modeling_wavlm.js create mode 100644 packages/transformers/src/models/wespeaker/feature_extraction_wespeaker.js create mode 100644 packages/transformers/src/models/wespeaker_resnet/modeling_wespeaker_resnet.js create mode 100644 packages/transformers/src/models/whisper/common_whisper.js create mode 100644 packages/transformers/src/models/whisper/feature_extraction_whisper.js create mode 100644 packages/transformers/src/models/whisper/generation_whisper.js create mode 100644 packages/transformers/src/models/whisper/modeling_whisper.js create mode 100644 packages/transformers/src/models/whisper/processing_whisper.js create mode 100644 packages/transformers/src/models/whisper/tokenization_whisper.js create mode 100644 packages/transformers/src/models/xlm/modeling_xlm.js create mode 100644 packages/transformers/src/models/xlm/tokenization_xlm.js create mode 100644 packages/transformers/src/models/xlm_roberta/modeling_xlm_roberta.js create mode 100644 packages/transformers/src/models/xlm_roberta/tokenization_xlm_roberta.js create mode 100644 packages/transformers/src/models/yolos/image_processing_yolos.js create mode 100644 packages/transformers/src/models/yolos/modeling_yolos.js create mode 100644 packages/transformers/src/models/youtu/modeling_youtu.js create mode 100644 packages/transformers/src/models/zaya/modeling_zaya.js create mode 100644 packages/transformers/src/ops/registry.js create mode 100644 packages/transformers/src/pipelines.js create mode 100644 packages/transformers/src/pipelines/_base.js create mode 100644 packages/transformers/src/pipelines/audio-classification.js create mode 100644 packages/transformers/src/pipelines/automatic-speech-recognition.js create mode 100644 packages/transformers/src/pipelines/background-removal.js create mode 100644 packages/transformers/src/pipelines/depth-estimation.js create mode 100644 packages/transformers/src/pipelines/document-question-answering.js create mode 100644 packages/transformers/src/pipelines/feature-extraction.js create mode 100644 packages/transformers/src/pipelines/fill-mask.js create mode 100644 packages/transformers/src/pipelines/image-classification.js create mode 100644 packages/transformers/src/pipelines/image-feature-extraction.js create mode 100644 packages/transformers/src/pipelines/image-segmentation.js create mode 100644 packages/transformers/src/pipelines/image-to-image.js create mode 100644 packages/transformers/src/pipelines/image-to-text.js create mode 100644 packages/transformers/src/pipelines/index.js create mode 100644 packages/transformers/src/pipelines/object-detection.js create mode 100644 packages/transformers/src/pipelines/question-answering.js create mode 100644 packages/transformers/src/pipelines/summarization.js create mode 100644 packages/transformers/src/pipelines/text-classification.js create mode 100644 packages/transformers/src/pipelines/text-generation.js create mode 100644 packages/transformers/src/pipelines/text-to-audio.js create mode 100644 packages/transformers/src/pipelines/text2text-generation.js create mode 100644 packages/transformers/src/pipelines/token-classification.js create mode 100644 packages/transformers/src/pipelines/translation.js create mode 100644 packages/transformers/src/pipelines/zero-shot-audio-classification.js create mode 100644 packages/transformers/src/pipelines/zero-shot-classification.js create mode 100644 packages/transformers/src/pipelines/zero-shot-image-classification.js create mode 100644 packages/transformers/src/pipelines/zero-shot-object-detection.js create mode 100644 packages/transformers/src/processing_utils.js create mode 100644 packages/transformers/src/tokenization_utils.js create mode 100644 packages/transformers/src/transformers.js create mode 100644 packages/transformers/src/utils/audio.js create mode 100644 packages/transformers/src/utils/cache.js create mode 100644 packages/transformers/src/utils/cache/CrossOriginStorageCache.js create mode 100644 packages/transformers/src/utils/cache/FileCache.js create mode 100644 packages/transformers/src/utils/cache/cross-origin-storage.d.ts create mode 100644 packages/transformers/src/utils/constants.js create mode 100644 packages/transformers/src/utils/core.js create mode 100644 packages/transformers/src/utils/devices.js create mode 100644 packages/transformers/src/utils/dtypes.js create mode 100644 packages/transformers/src/utils/generic.js create mode 100755 packages/transformers/src/utils/hub.js create mode 100644 packages/transformers/src/utils/hub/FileResponse.js create mode 100644 packages/transformers/src/utils/hub/constants.js create mode 100644 packages/transformers/src/utils/hub/utils.js create mode 100644 packages/transformers/src/utils/image.js create mode 100644 packages/transformers/src/utils/io.js create mode 100644 packages/transformers/src/utils/logger.js create mode 100644 packages/transformers/src/utils/lru_cache.js create mode 100644 packages/transformers/src/utils/maths.js create mode 100644 packages/transformers/src/utils/memoize_promise.js create mode 100644 packages/transformers/src/utils/model-loader.js create mode 100644 packages/transformers/src/utils/model_registry/ModelRegistry.js create mode 100644 packages/transformers/src/utils/model_registry/clear_cache.js create mode 100644 packages/transformers/src/utils/model_registry/get_available_dtypes.js create mode 100644 packages/transformers/src/utils/model_registry/get_file_metadata.js create mode 100644 packages/transformers/src/utils/model_registry/get_files.js create mode 100644 packages/transformers/src/utils/model_registry/get_model_files.js create mode 100644 packages/transformers/src/utils/model_registry/get_pipeline_files.js create mode 100644 packages/transformers/src/utils/model_registry/get_processor_files.js create mode 100644 packages/transformers/src/utils/model_registry/get_tokenizer_files.js create mode 100644 packages/transformers/src/utils/model_registry/is_cached.js create mode 100644 packages/transformers/src/utils/model_registry/resolve_model_type.js create mode 100644 packages/transformers/src/utils/random.js create mode 100644 packages/transformers/src/utils/tensor.js create mode 100644 packages/transformers/src/utils/video.js create mode 100644 packages/transformers/tests/asset_cache.js create mode 100644 packages/transformers/tests/bundles.test.js create mode 100644 packages/transformers/tests/configs.test.js create mode 100644 packages/transformers/tests/exports.test.js create mode 100644 packages/transformers/tests/feature_extractors.test.js create mode 100644 packages/transformers/tests/image_processors.test.js create mode 100644 packages/transformers/tests/init.js create mode 100644 packages/transformers/tests/models.test.js create mode 100644 packages/transformers/tests/models/afmoe/test_modeling_afmoe.js create mode 100644 packages/transformers/tests/models/albert/test_tokenization_albert.js create mode 100644 packages/transformers/tests/models/arcee/test_modeling_arcee.js create mode 100644 packages/transformers/tests/models/audio_spectrogram_transformer/test_feature_extraction_audio_spectrogram_transformer.js create mode 100644 packages/transformers/tests/models/beit/test_image_processing_beit.js create mode 100644 packages/transformers/tests/models/bert/test_modeling_bert.js create mode 100644 packages/transformers/tests/models/bert/test_tokenization_bert.js create mode 100644 packages/transformers/tests/models/bit/test_image_processing_bit.js create mode 100644 packages/transformers/tests/models/blenderbot_small/test_tokenization_blenderbot_small.js create mode 100644 packages/transformers/tests/models/bloom/test_modeling_bloom.js create mode 100644 packages/transformers/tests/models/bloom/test_tokenization_bloom.js create mode 100644 packages/transformers/tests/models/chmv2/test_image_processing_chmv2.js create mode 100644 packages/transformers/tests/models/clap/test_feature_extraction_clap.js create mode 100644 packages/transformers/tests/models/clip/test_image_processing_clip.js create mode 100644 packages/transformers/tests/models/clip/test_modeling_clip.js create mode 100644 packages/transformers/tests/models/clip/test_tokenization_clip.js create mode 100644 packages/transformers/tests/models/codegen/test_modeling_codegen.js create mode 100644 packages/transformers/tests/models/cohere/test_modeling_cohere.js create mode 100644 packages/transformers/tests/models/cohere_asr/test_feature_extraction_cohere_asr.js create mode 100644 packages/transformers/tests/models/convnext/test_image_processing_convnext.js create mode 100644 packages/transformers/tests/models/dac/test_modeling_dac.js create mode 100644 packages/transformers/tests/models/deberta_v2/test_tokenization_deberta_v2.js create mode 100644 packages/transformers/tests/models/deepseek_v3/test_modeling_deepseek_v3.js create mode 100644 packages/transformers/tests/models/deepseek_v4/test_modeling_deepseek_v4.js create mode 100644 packages/transformers/tests/models/deit/test_image_processing_deit.js create mode 100644 packages/transformers/tests/models/detr/test_image_processing_detr.js create mode 100644 packages/transformers/tests/models/distilbert/test_tokenization_distilbert.js create mode 100644 packages/transformers/tests/models/donut/test_image_processing_donut.js create mode 100644 packages/transformers/tests/models/dpt/test_image_processing_dpt.js create mode 100644 packages/transformers/tests/models/efficientnet/test_image_processing_efficientnet.js create mode 100644 packages/transformers/tests/models/esm/test_tokenization_esm.js create mode 100644 packages/transformers/tests/models/eurobert/test_modeling_eurobert.js create mode 100644 packages/transformers/tests/models/falcon/test_tokenization_falcon.js create mode 100644 packages/transformers/tests/models/falcon_h1/test_modeling_falcon_h1.js create mode 100644 packages/transformers/tests/models/florence2/test_modeling_florence2.js create mode 100644 packages/transformers/tests/models/florence2/test_processor_florence2.js create mode 100644 packages/transformers/tests/models/gemma/test_modeling_gemma.js create mode 100644 packages/transformers/tests/models/gemma/test_tokenization_gemma.js create mode 100644 packages/transformers/tests/models/gemma2/test_modeling_gemma2.js create mode 100644 packages/transformers/tests/models/gemma3/test_modeling_gemma3.js create mode 100644 packages/transformers/tests/models/gemma4/test_feature_extraction_gemma4.js create mode 100644 packages/transformers/tests/models/gemma4/test_image_processing_gemma4.js create mode 100644 packages/transformers/tests/models/gemma4/test_processor_gemma4.js create mode 100644 packages/transformers/tests/models/glm/test_modeling_glm.js create mode 100644 packages/transformers/tests/models/glm46v/test_processor_glm46v.js create mode 100644 packages/transformers/tests/models/glm_moe_dsa/test_modeling_glm_moe_dsa.js create mode 100644 packages/transformers/tests/models/glm_ocr/test_modeling_glm_ocr.js create mode 100644 packages/transformers/tests/models/glpn/test_image_processing_glpn.js create mode 100644 packages/transformers/tests/models/gpt2/test_modeling_gpt2.js create mode 100644 packages/transformers/tests/models/gpt2/test_tokenization_gpt2.js create mode 100644 packages/transformers/tests/models/gpt_bigcode/test_modeling_gpt_bigcode.js create mode 100644 packages/transformers/tests/models/gpt_neo/test_modeling_gpt_neo.js create mode 100644 packages/transformers/tests/models/gpt_neox/test_modeling_gpt_neox.js create mode 100644 packages/transformers/tests/models/gptj/test_modeling_gptj.js create mode 100644 packages/transformers/tests/models/granite/test_modeling_granite.js create mode 100644 packages/transformers/tests/models/granite_speech/test_feature_extraction_granite_speech.js create mode 100644 packages/transformers/tests/models/granite_speech/test_modeling_granite_speech.js create mode 100644 packages/transformers/tests/models/granite_speech/test_processor_granite_speech.js create mode 100644 packages/transformers/tests/models/granitemoehybrid/test_modeling_granitemoehybrid.js create mode 100644 packages/transformers/tests/models/grounding_dino/test_modeling_grounding_dino.js create mode 100644 packages/transformers/tests/models/grounding_dino/test_processor_grounding_dino.js create mode 100644 packages/transformers/tests/models/helium/test_modeling_helium.js create mode 100644 packages/transformers/tests/models/idefics3/test_image_processing_idefics3.js create mode 100644 packages/transformers/tests/models/idefics3/test_modeling_idefics3.js create mode 100644 packages/transformers/tests/models/jais/test_modeling_jais.js create mode 100644 packages/transformers/tests/models/janus/test_processor_janus.js create mode 100644 packages/transformers/tests/models/jina_clip/test_image_processing_jina_clip.js create mode 100644 packages/transformers/tests/models/jina_clip/test_processor_jina_clip.js create mode 100644 packages/transformers/tests/models/lfm2/test_modeling_lfm2.js create mode 100644 packages/transformers/tests/models/lfm2_vl/test_image_processing_lfm2_vl.js create mode 100644 packages/transformers/tests/models/llama/test_modeling_llama.js create mode 100644 packages/transformers/tests/models/llama/test_tokenization_llama.js create mode 100644 packages/transformers/tests/models/llava/test_modeling_llava.js create mode 100644 packages/transformers/tests/models/m2m_100/test_tokenization_m2m_100.js create mode 100644 packages/transformers/tests/models/marian/test_modeling_marian.js create mode 100644 packages/transformers/tests/models/mgp_str/test_modeling_mgp_str.js create mode 100644 packages/transformers/tests/models/mimi/test_modeling_mimi.js create mode 100644 packages/transformers/tests/models/mistral/test_modeling_mistral.js create mode 100644 packages/transformers/tests/models/mistral4/test_modeling_mistral4.js create mode 100644 packages/transformers/tests/models/mobilevit/test_image_processing_mobilevit.js create mode 100644 packages/transformers/tests/models/modernbert/test_modeling_modernbert.js create mode 100644 packages/transformers/tests/models/moonshine/test_feature_extraction_moonshine.js create mode 100644 packages/transformers/tests/models/moonshine/test_modeling_moonshine.js create mode 100644 packages/transformers/tests/models/mpnet/test_tokenization_mpnet.js create mode 100644 packages/transformers/tests/models/mpt/test_modeling_mpt.js create mode 100644 packages/transformers/tests/models/musicgen/test_modeling_musicgen.js create mode 100644 packages/transformers/tests/models/nemotron_h/test_modeling_nemotron_h.js create mode 100644 packages/transformers/tests/models/nllb/test_tokenization_nllb.js create mode 100644 packages/transformers/tests/models/nougat/test_image_processing_nougat.js create mode 100644 packages/transformers/tests/models/olmo/test_modeling_olmo.js create mode 100644 packages/transformers/tests/models/olmo2/test_modeling_olmo2.js create mode 100644 packages/transformers/tests/models/opt/test_modeling_opt.js create mode 100644 packages/transformers/tests/models/owlvit/test_image_processing_owlvit.js create mode 100644 packages/transformers/tests/models/paligemma/test_modeling_paligemma.js create mode 100644 packages/transformers/tests/models/paligemma/test_processor_paligemma.js create mode 100644 packages/transformers/tests/models/patchtsmixer/test_modeling_patchtsmixer.js create mode 100644 packages/transformers/tests/models/patchtst/test_modeling_patchtst.js create mode 100644 packages/transformers/tests/models/phi3_v/test_image_processing_phi3_v.js create mode 100644 packages/transformers/tests/models/phi3_v/test_processor_phi3_v.js create mode 100644 packages/transformers/tests/models/pyannote/test_modeling_pyannote.js create mode 100644 packages/transformers/tests/models/qwen2/test_tokenization_qwen2.js create mode 100644 packages/transformers/tests/models/qwen2_vl/test_image_processing_qwen2_vl.js create mode 100644 packages/transformers/tests/models/qwen2_vl/test_modeling_qwen2_vl.js create mode 100644 packages/transformers/tests/models/qwen2_vl/test_processor_qwen2_vl.js create mode 100644 packages/transformers/tests/models/qwen3_5/test_modeling_qwen3_5.js create mode 100644 packages/transformers/tests/models/roberta/test_tokenization_roberta.js create mode 100644 packages/transformers/tests/models/sam/test_image_processing_sam.js create mode 100644 packages/transformers/tests/models/sam/test_modeling_sam.js create mode 100644 packages/transformers/tests/models/seamless_m4t/test_feature_extraction_seamless_m4t.js create mode 100644 packages/transformers/tests/models/solar_open/test_modeling_solar_open.js create mode 100644 packages/transformers/tests/models/swin2sr/test_image_processing_swin2sr.js create mode 100644 packages/transformers/tests/models/t5/test_modeling_t5.js create mode 100644 packages/transformers/tests/models/t5/test_tokenization_t5.js create mode 100644 packages/transformers/tests/models/test_strings.js create mode 100644 packages/transformers/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.js create mode 100644 packages/transformers/tests/models/vit/test_image_processing_vit.js create mode 100644 packages/transformers/tests/models/vitmatte/test_image_processing_vitmatte.js create mode 100644 packages/transformers/tests/models/vitpose/test_image_processing_vitpose.js create mode 100644 packages/transformers/tests/models/vits/test_tokenization_vits.js create mode 100644 packages/transformers/tests/models/voxtral_realtime/test_feature_extraction_voxtral_realtime.js create mode 100644 packages/transformers/tests/models/voxtral_realtime/test_modeling_voxtral_realtime.js create mode 100644 packages/transformers/tests/models/voxtral_realtime/test_processor_voxtral_realtime.js create mode 100644 packages/transformers/tests/models/wav2vec2/test_tokenization_wav2vec2.js create mode 100644 packages/transformers/tests/models/wespeaker_resnet/test_feature_extraction_wespeaker_resnet.js create mode 100644 packages/transformers/tests/models/whisper/test_feature_extraction_whisper.js create mode 100644 packages/transformers/tests/models/whisper/test_modeling_whisper.js create mode 100644 packages/transformers/tests/models/whisper/test_tokenization_whisper.js create mode 100644 packages/transformers/tests/models/xlm_roberta/test_tokenization_xlm_roberta.js create mode 100644 packages/transformers/tests/models/yolos/test_image_processing_yolos.js create mode 100644 packages/transformers/tests/models/zaya/test_modeling_zaya.js create mode 100644 packages/transformers/tests/pipelines.test.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_audio_classification.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_automatic_speech_recognition.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_background_removal.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_depth_estimation.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_document_question_answering.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_feature_extraction.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_fill_mask.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_image_classification.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_image_feature_extraction.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_image_segmentation.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_image_to_image.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_image_to_text.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_object_detection.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_question_answering.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_summarization.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_text2text_generation.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_text_classification.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_text_generation.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_text_to_audio.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_token_classification.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_translation.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_zero_shot.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_zero_shot_audio_classification.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_zero_shot_image_classification.js create mode 100644 packages/transformers/tests/pipelines/test_pipelines_zero_shot_object_detection.js create mode 100644 packages/transformers/tests/processors.test.js create mode 100644 packages/transformers/tests/progress_callbacks.test.js create mode 100644 packages/transformers/tests/test_utils.js create mode 100644 packages/transformers/tests/tokenizers.test.js create mode 100644 packages/transformers/tests/types.test.js create mode 100644 packages/transformers/tests/types/_base.ts create mode 100644 packages/transformers/tests/types/cache.ts create mode 100644 packages/transformers/tests/types/pipelines.ts create mode 100644 packages/transformers/tests/types/tokenizers.ts create mode 100644 packages/transformers/tests/utils/audio.test.js create mode 100644 packages/transformers/tests/utils/cache.test.js create mode 100644 packages/transformers/tests/utils/custom_cache.test.js create mode 100644 packages/transformers/tests/utils/generation.test.js create mode 100644 packages/transformers/tests/utils/hub.test.js create mode 100644 packages/transformers/tests/utils/image.test.js create mode 100644 packages/transformers/tests/utils/logits_process.test.js create mode 100644 packages/transformers/tests/utils/lru_cache.test.js create mode 100644 packages/transformers/tests/utils/maths.test.js create mode 100644 packages/transformers/tests/utils/model_registry.test.js create mode 100644 packages/transformers/tests/utils/random.test.js create mode 100644 packages/transformers/tests/utils/tensor.test.js create mode 100644 packages/transformers/tests/utils/tensor_ops.test.js create mode 100644 packages/transformers/tests/utils/utils.test.js create mode 100644 packages/transformers/tsconfig.json create mode 100644 pnpm-lock.yaml create mode 100644 pnpm-workspace.yaml create mode 100644 scripts/dev.mjs create mode 100644 scripts/logger.mjs create mode 100644 scripts/prepareOutDir.mjs create mode 100644 scripts/rebuildPlugin.mjs create mode 100644 scripts/reportSize.mjs create mode 100644 tsconfig.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..ed6e196 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +index.html -linguist-detectable +assets/** linguist-vendored diff --git a/.github/ISSUE_TEMPLATE/1_bug-report.yml b/.github/ISSUE_TEMPLATE/1_bug-report.yml new file mode 100644 index 0000000..f30a68c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/1_bug-report.yml @@ -0,0 +1,49 @@ +name: "🐛 Bug Report" +description: Submit a bug report to help us improve transformers.js +labels: ["bug"] +body: + - type: textarea + id: system-info + attributes: + label: System Info + description: Please share your system info with us. If you are using other JS libraries/frameworks (e.g., React or Next.js), please include their versions too. + placeholder: transformers.js version, browser (if applicable), operating system, Node.js version, bundlers, ... + validations: + required: true + + - type: checkboxes + id: environment-or-platform + attributes: + label: Environment/Platform + description: "The environment I am running in, or platform I am developing for:" + options: + - label: "Website/web-app" + - label: "Browser extension" + - label: "Server-side (e.g., Node.js, Deno, Bun)" + - label: "Desktop app (e.g., Electron)" + - label: "Other (e.g., VSCode extension)" + + - type: textarea + id: description + validations: + required: true + attributes: + label: Description + description: A clear and concise description of the bug, as well as what you expected to happen. + + - type: textarea + id: reproduction + validations: + required: true + attributes: + label: Reproduction + description: | + Please provide a code sample that reproduces the problem you ran into. If you have code snippets, error messages, or stack traces, please provide them here as well. + Important! Use [code tags](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#syntax-highlighting) to correctly format your code. + + placeholder: | + Steps to reproduce the behavior: + + 1. + 2. + 3. diff --git a/.github/ISSUE_TEMPLATE/2_new_model.yml b/.github/ISSUE_TEMPLATE/2_new_model.yml new file mode 100644 index 0000000..74b1844 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/2_new_model.yml @@ -0,0 +1,40 @@ +name: "🌟 New model addition" +description: Submit a proposal/request to implement a new model +labels: ["new model"] + +body: + - type: textarea + id: description-request + validations: + required: true + attributes: + label: Model description + description: | + Include important information about the model. + + - type: checkboxes + id: information-tasks + attributes: + label: Prerequisites + description: | + Please note that Transformers.js relies on the model first being supported in [🤗 Transformers](https://github.com/huggingface/transformers) and [🤗 Optimum](https://github.com/huggingface/optimum). If the model you are requesting is not yet supported by either of them, feel free to open up a model request there too. + options: + - label: "The model is supported in Transformers (i.e., listed [here](https://huggingface.co/docs/transformers/index#supported-models-and-frameworks))" + - label: "The model can be exported to ONNX with Optimum (i.e., listed [here](https://huggingface.co/docs/optimum/main/en/exporters/onnx/overview))" + + - type: textarea + id: additional-info + attributes: + label: Additional information + description: | + Please provide additional information about the model here. + If the model is already supported in Transformers, you can provide example Python code to help ensure the JavaScript implementation (and output) matches the original version. + + - type: textarea + id: contribution + validations: + required: true + attributes: + label: Your contribution + description: | + Is there any way that you could help, e.g. by submitting a PR? diff --git a/.github/ISSUE_TEMPLATE/3_new_pipeline.yml b/.github/ISSUE_TEMPLATE/3_new_pipeline.yml new file mode 100644 index 0000000..3428e92 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/3_new_pipeline.yml @@ -0,0 +1,39 @@ +name: "🔧 New pipeline addition" +description: Submit a proposal/request to implement a new pipeline +labels: ["new pipeline"] + +body: + - type: textarea + id: description-request + validations: + required: true + attributes: + label: Pipeline description + description: | + Include important information about the pipeline. + + - type: checkboxes + id: information-tasks + attributes: + label: Prerequisites + description: | + Please note that Transformers.js relies on the pipeline first being supported in [🤗 Transformers](https://github.com/huggingface/transformers). If the pipeline you are requesting is not yet supported by Transformers, feel free to open up a feature request for it there too. + options: + - label: "The pipeline is supported in Transformers (i.e., listed [here](https://huggingface.co/docs/transformers/main_classes/pipelines))" + - label: "The task is listed [here](https://huggingface.co/tasks)" + + - type: textarea + id: additional-info + attributes: + label: Additional information + description: | + Please provide additional information about the pipeline here. If the pipeline is already supported in Transformers, you can provide example Python code to help ensure the JavaScript implementation (and output) matches the original version. + + - type: textarea + id: contribution + validations: + required: true + attributes: + label: Your contribution + description: | + Is there any way that you could help, e.g. by submitting a PR? diff --git a/.github/ISSUE_TEMPLATE/4_feature-request.yml b/.github/ISSUE_TEMPLATE/4_feature-request.yml new file mode 100644 index 0000000..f7da458 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/4_feature-request.yml @@ -0,0 +1,30 @@ +name: "🚀 Feature request" +description: Submit a proposal/request for a new transformers.js feature +labels: ["enhancement"] +body: + - type: textarea + id: feature-request + validations: + required: true + attributes: + label: Feature request + description: | + A clear and concise description of the feature proposal. If the feature is already part of the [Transformers](https://github.com/huggingface/transformers) library, please provide relevant links or example usage. + + - type: textarea + id: motivation + validations: + required: true + attributes: + label: Motivation + description: | + Please outline the motivation for the proposal. Why is it important that we add this feature? What is your intended use case? + + - type: textarea + id: contribution + validations: + required: true + attributes: + label: Your contribution + description: | + Is there any way that you could help, e.g. by submitting a PR? diff --git a/.github/ISSUE_TEMPLATE/5_question.yml b/.github/ISSUE_TEMPLATE/5_question.yml new file mode 100644 index 0000000..34bc317 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/5_question.yml @@ -0,0 +1,13 @@ +name: "🙋 Question" +description: Ask a question about the library +labels: ["question"] + +body: + - type: textarea + id: question + validations: + required: true + attributes: + label: Question + description: | + Please enter your question here... diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..906f1ac --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,9 @@ +blank_issues_enabled: true +version: 2.1 +contact_links: + - name: Models on the Hugging Face Hub + url: https://huggingface.co/models?library=transformers.js + about: Open a Pull request / Discussion related to a specific model checkpoint directly on the Hugging Face Hub + - name: Documentation + url: https://huggingface.co/docs/transformers.js + about: View the transformers.js documentation diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..15f7bdd --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + cooldown: + default-days: 7 + groups: + actions: + patterns: ["*"] diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 0000000..fc376b0 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,19 @@ +name: Build documentation + +on: + workflow_dispatch: + push: + branches: + - main + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@90b4ee2c10b81b5c1a6367c4e6fc9e2fb510a7e3 # main + with: + commit_sha: ${{ github.sha }} + package: transformers.js + path_to_docs: transformers.js/packages/transformers/docs/source + pre_command: cd transformers.js && corepack enable && ONNXRUNTIME_NODE_INSTALL=skip pnpm install --frozen-lockfile && pnpm --filter @huggingface/transformers docs-api + additional_args: --not_python_module + secrets: + hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} diff --git a/.github/workflows/pr-documentation.yml b/.github/workflows/pr-documentation.yml new file mode 100644 index 0000000..af5dd38 --- /dev/null +++ b/.github/workflows/pr-documentation.yml @@ -0,0 +1,19 @@ +name: Build PR Documentation + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@90b4ee2c10b81b5c1a6367c4e6fc9e2fb510a7e3 # main + with: + commit_sha: ${{ github.sha }} + pr_number: ${{ github.event.number }} + package: transformers.js + path_to_docs: transformers.js/packages/transformers/docs/source + pre_command: cd transformers.js && corepack enable && ONNXRUNTIME_NODE_INSTALL=skip pnpm install --frozen-lockfile && pnpm --filter @huggingface/transformers docs-api + additional_args: --not_python_module diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..e4eb041 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,28 @@ +name: Publish Package to npmjs + +on: + release: + types: [published] + +env: + ONNXRUNTIME_NODE_INSTALL: skip + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Install pnpm + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + # Setup .npmrc file to publish to npm + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: "24.10.0" + registry-url: "https://registry.npmjs.org" + cache: "pnpm" + - run: pnpm install --frozen-lockfile + - run: pnpm build + - name: Publish to npm + run: pnpm --filter @huggingface/transformers publish --access public --no-git-checks + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..73e3304 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,45 @@ +name: Unit tests + +on: + push: + branches: + - main + pull_request: + branches: + - main + types: + - opened + - reopened + - synchronize + - ready_for_review + +env: + TESTING_REMOTELY: true + ONNXRUNTIME_NODE_INSTALL: skip + +jobs: + build: + if: github.event.pull_request.draft == false + runs-on: + group: aws-general-8-plus + strategy: + matrix: + node-version: [18, 20, 22] + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Install pnpm + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: ${{ matrix.node-version }} + cache: "pnpm" + - run: pnpm install --frozen-lockfile + - run: pnpm build + + # Setup the testing environment + - run: git lfs install && GIT_CLONE_PROTECTION_ACTIVE=false git clone https://huggingface.co/hf-internal-testing/tiny-random-T5ForConditionalGeneration ./packages/transformers/models/hf-internal-testing/tiny-random-T5ForConditionalGeneration + + # Actually run tests + - run: pnpm test diff --git a/.github/workflows/upload-pr-documentation.yml b/.github/workflows/upload-pr-documentation.yml new file mode 100644 index 0000000..688bbf8 --- /dev/null +++ b/.github/workflows/upload-pr-documentation.yml @@ -0,0 +1,16 @@ +name: Upload PR Documentation + +on: + workflow_run: + workflows: ["Build PR Documentation"] + types: + - completed + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@9ad2de8582b56c017cb530c1165116d40433f1c6 # main + with: + package_name: transformers.js + secrets: + hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} + comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..21c721c --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +__pycache__ +.vscode +.cache +.DS_STORE +.idea +node_modules +deno.lock +package-lock.json + +# Do not track build artifacts/generated files +packages/*/dist +packages/*/types +*.tsbuildinfo + +# Do not track coverage reports +packages/*/coverage diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..d2ea801 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,17 @@ +# Ignore artifacts and non-source files +.github +*.py +*.md + +# Build outputs & caches +**/dist +**/types +**/node_modules +**/coverage +**/.cache +**/docs/build +*.tsbuildinfo + +# Lock files +pnpm-lock.yaml + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c98dc60 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,367 @@ + + +# Contribute to 🤗 Transformers.js + +Everyone is welcome to contribute, and we value everybody's contribution. Code +contributions are not the only way to help the community. Answering questions, helping +others, and improving the documentation are also immensely valuable. + +It also helps us if you spread the word! Reference the library in blog posts +about the awesome projects it made possible, shout out on Twitter every time it has +helped you, or simply ⭐️ the repository to say thank you. + +**This guide was heavily inspired by the awesome [scikit-learn guide to contributing](https://github.com/scikit-learn/scikit-learn/blob/main/CONTRIBUTING.md) and our friends at [transformers](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md).** + +## Ways to contribute + +There are several ways you can contribute to 🤗 Transformers.js: + +* Fix outstanding issues with the existing code. +* Submit issues related to bugs or desired new features. +* Implement new models. +* Contribute to the examples or to the documentation. + +## Fixing outstanding issues + +If you notice an issue with the existing code and have a fix in mind, feel free to [start contributing](#create-a-pull-request) and open a Pull Request! + +## Submitting a bug-related issue or feature request + +Do your best to follow these guidelines when submitting a bug-related issue or a feature +request. It will make it easier for us to come back to you quickly and with good +feedback. + +### Did you find a bug? + +The 🤗 Transformers.js library is robust and reliable thanks to users who report the problems they encounter. + +Before you report an issue, we would really appreciate it if you could **make sure the bug was not +already reported** (use the search bar on GitHub under Issues). Your issue should also be related to bugs in the library itself, and not your code. + +To create a new issue, please [use one of the templates](https://github.com/huggingface/transformers.js/issues/new/choose) we prepared for you. Most likely the [Bug Report](https://github.com/huggingface/transformers.js/issues/new?template=1_bug-report.yml). + +### Do you want a new feature? + +If there is a new feature you'd like to see in 🤗 Transformers.js, please open an issue and describe: + +1. What is the *motivation* behind this feature? Is it related to a problem or frustration with the library? Is it a feature related to something you need for a project? Is it something you worked on and think it could benefit the community? Whatever it is, we'd love to hear about it! +2. Describe your requested feature in as much detail as possible. The more you can tell us about it, the better we'll be able to help you. +3. Provide a *code snippet* that demonstrates the feature's usage. +4. If the feature is related to a paper, please include a link. + +If your issue is well written we're already 80% of the way there by the time you create it. + +We have added [a template](https://github.com/huggingface/transformers.js/issues/new?template=4_feature-request.yml) to help you get started with your issue. + +## Do you want to implement a new model? + +New models are constantly released and if you want to request support for a new model, please use the [template for new model requests](https://github.com/huggingface/transformers.js/issues/new?template=2_new_model.yml). + +If you are willing to contribute the model yourself, let us know so we can help you add it to 🤗 Transformers.js! The process of adding support for a new model architecture has three main phases: **exporting the model to ONNX**, then **wiring it into the library**, and finally **adding tests**. + +### 1. Export the Model to ONNX + +Transformers.js runs models using ONNX Runtime. Before adding a model to the library, you need an ONNX export of it. + +- For LLMs, we recommend exporting with [microsoft/onnxruntime-genai](https://github.com/microsoft/onnxruntime-genai) +- For other models, we recommend exporting with [huggingface/optimum-onnx](https://github.com/huggingface/optimum-onnx) + +Once exported, upload the ONNX files to the Hugging Face Hub alongside the model's original config, tokenizer, and other preprocessing files so they can be loaded with `from_pretrained`. + +--- + +### 2. Add the Model to the Library + +Every model in Transformers.js is built from the same pieces: + +- **A model class**: extends `PreTrainedModel`, which handles all ONNX inference, generation, and KV-cache management +- **Task head classes**: thin wrappers that wrap the output in the right output object (e.g. `MaskedLMOutput`) +- **A tokenizer and/or processor**: only needed if the model requires a custom one; most models reuse an existing class + +All model files live under `packages/transformers/src/models//`. Look at an existing model of the same type to understand what's needed; most are just a few lines. + +#### Model class + +Every model file exports a base class and one or more task heads. For the vast majority of models, these are empty subclasses. All the logic lives in `PreTrainedModel`. + +**Decoder-only LLM:** + +```js +import { PreTrainedModel } from '../modeling_utils.js'; + +export class MyModelPreTrainedModel extends PreTrainedModel {} +export class MyModelModel extends MyModelPreTrainedModel {} +export class MyModelForCausalLM extends MyModelPreTrainedModel {} +``` + +**Encoder-only model:** + +```js +import { PreTrainedModel } from '../modeling_utils.js'; +import { MaskedLMOutput, SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class MyModelPreTrainedModel extends PreTrainedModel {} +export class MyModelModel extends MyModelPreTrainedModel {} + +export class MyModelForMaskedLM extends MyModelPreTrainedModel { + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +export class MyModelForSequenceClassification extends MyModelPreTrainedModel { + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} +``` + +Only add the task heads the model actually supports. The available output classes (`MaskedLMOutput`, `TokenClassifierOutput`, `Seq2SeqLMOutput`, etc.) are all in `modeling_outputs.js`. + +#### Tokenizer and processor + +Most models reuse an existing tokenizer (e.g. all Llama-family models use `LlamaTokenizer`). Only create a new one if the model genuinely needs custom tokenization or preprocessing logic. + +| What | File | Barrel to update | +| --- | --- | --- | +| Custom tokenizer | `src/models//tokenization_.js` | `src/models/tokenizers.js` | +| Custom image processor | `src/models//image_processing_.js` | `src/models/image_processors.js` | +| Custom multimodal processor | `src/models//processing_.js` | `src/models/processors.js` | +| Custom audio/feature extractor | `src/models//feature_extraction_.js` | `src/models/feature_extractors.js` | + +The class name must match the `tokenizer_class` or `processor_class` field in the model's `tokenizer_config.json` / `preprocessor_config.json` on the Hub. + +#### Wiring it up + +Once the model file is written, three more files need updating: + +1. **`src/models/models.js`**: add `export * from './/modeling_.js'` +2. **`src/models/registry.js`**: map the `model_type` string (from `config.json`) to the class names, and set the correct loading category (`EncoderOnly`, `DecoderOnly`, `Seq2Seq`, etc.) +3. **`src/configs.js`**: for generative models, add a `case` in `getNormalizedConfig()` to map the model's config field names to the normalized names the KV-cache runtime expects + +Look at a similar existing model in each file to see exactly what to add. + +--- + +### 3. Write Tests + +Create `packages/transformers/tests/models//test_modeling_.js`. The test runner auto-discovers files by this naming convention. No registration needed. + +Use a small, fast model. The convention is to use a `tiny-random-*` model from `hf-internal-testing/` on the Hub. If one doesn't exist for your architecture, generate one with the `transformers` Python library: + +```python +from transformers import AutoConfig, AutoModelForCausalLM +config = AutoConfig.for_model("my_model", num_hidden_layers=2, hidden_size=64, ...) +model = AutoModelForCausalLM.from_config(config) +model.push_to_hub("hf-internal-testing/tiny-random-MyModelForCausalLM") +``` + +**Test file structure:** + +```js +import { MyModelForCausalLM, MyModelTokenizer } from "../../../src/transformers.js"; +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("MyModelForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-MyModelForCausalLM"; + let model, tokenizer; + + beforeAll(async () => { + model = await MyModelForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await MyModelTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it("batch_size=1", async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ ...inputs, max_length: 10 }); + expect(outputs.tolist()).toEqual([[/* expected token ids */]]); + }, MAX_TEST_EXECUTION_TIME); + + it("batch_size>1", async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ ...inputs, max_length: 10 }); + expect(outputs.tolist()).toEqual([[...], [...]]); + }, MAX_TEST_EXECUTION_TIME); + + afterAll(async () => { await model?.dispose(); }, MAX_MODEL_DISPOSE_TIME); + }); +}; +``` + +Run your tests with: + +```bash +# All tests +pnpm test + +# Only your model's tests +pnpm --filter @huggingface/transformers test -t "MyModelForCausalLM" +``` + + +## Create a Pull Request + +Before writing any code, we strongly advise you to search through the existing PRs or +issues to make sure nobody is already working on the same thing. If you are +unsure, it is always a good idea to open an issue to get some feedback. + +You will need basic `git` proficiency to contribute to 🤗 Transformers.js. +While `git` is not the easiest tool to use, it has the greatest +manual. Type `git --help` in a shell and enjoy! If you prefer books, [Pro +Git](https://git-scm.com/book/en/v2) is a very good reference. + +### Prerequisites + +You'll need the following tools installed to contribute to 🤗 Transformers.js: + +- **[Node.js v18](https://nodejs.org/)** or above +- **[pnpm](https://pnpm.io/)** - Fast, disk space efficient package manager + +To install pnpm: +```bash +npm install -g pnpm +``` + +Follow the steps below to start contributing: + +1. Fork the [repository](https://github.com/huggingface/transformers.js) by + clicking on the **[Fork](https://github.com/huggingface/transformers.js/fork)** button on the repository's page. This creates a copy of the code + under your GitHub user account. + +2. Clone your fork to your local disk, and add the base repository as a remote: + + ```bash + git clone git@github.com:/transformers.js.git + cd transformers.js + ``` + +3. Create a new branch to hold your development changes: + + ```bash + git checkout -b a-descriptive-name-for-my-changes + ``` + +> 🚨 **Do not** work on the `main` branch! + +4. Set up a development environment by running the following command: + ```bash + pnpm install + ``` +5. Develop the features in your branch. +6. Now you can go to your fork of the repository on GitHub and click on **Pull Request** to open a pull request. Make sure you tick off all the boxes on our [checklist](#pull-request-checklist) below. When you're ready, you can send your changes to the project maintainers for review. +7. It's ok if maintainers request changes, it happens to our core contributors + too! So everyone can see the changes in the pull request, work in your local + branch and push the changes to your fork. They will automatically appear in + the pull request. + +### Pull request checklist +☐ The pull request title should summarize your contribution. +☐ If your pull request addresses an issue, please mention the issue number in the pull +request description to make sure they are linked (and people viewing the issue know you +are working on it). +☐ To indicate a work in progress please prefix the title with `[WIP]`. These are +useful to avoid duplicated work, and to differentiate it from PRs ready to be merged. +☐ Make sure existing tests pass (`pnpm test`). +☐ Make sure the build completes successfully (`pnpm build`). +☐ Make sure your code is [formatted properly with Prettier](#code-formatting) (`pnpm format:check`). +☐ If adding a new feature, also add tests for it. +☐ If your changes affect user-facing functionality, update the relevant documentation. + +### Tests +We are using [Jest](https://jestjs.io/) to execute unit-tests. All tests can be found in `packages/transformers/tests` and have to end with `.test.js` + +Execute all tests +```bash +pnpm test +``` + +Execute tests for a specific package +```bash +pnpm --filter @huggingface/transformers test +``` + +Execute a specific test file +```bash +cd packages/transformers +pnpm test -- ./tests/models.test.js +``` + +### Style guide + +#### Code formatting +We use [Prettier](https://prettier.io/) to maintain consistent code formatting across the project. Please ensure your code is formatted before submitting a pull request. + +**Format all files:** +```bash +pnpm format +``` + +**Check formatting without making changes:** +```bash +pnpm format:check +``` + +**IDE Integration (recommended)** + +We recommend setting up Prettier in your IDE to format on save: + +**Visual Studio Code:** +1. Install the [Prettier extension](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode) +2. Open Settings (Ctrl+, or Cmd+,) +3. Search for "format on save" +4. Enable "Editor: Format On Save" +5. Set Prettier as your default formatter: search for "default formatter" and select "Prettier - Code formatter" + +**IntelliJ IDEA / WebStorm:** +1. Go to `Settings` → `Languages & Frameworks` → `JavaScript` → `Prettier` +2. Set the Prettier package path (usually `node_modules/prettier`) +3. Check "On save" under "Run for files" +4. Add file patterns: `{**/*,*}.{js,ts,jsx,tsx,json,css,scss,md}` +5. Click "Apply" and "OK" + +## Project Structure + +This project uses **pnpm workspaces** to manage multiple packages in a monorepo. Currently, there is one workspace: + +- `packages/transformers` - The main Transformers.js library + +This structure allows for better organization and makes it easier to add framework-specific integrations in the future. + +## How to make changes to transformers.js + +### Development workflow + +The recommended way to develop and test changes is to use the watch mode build and install from the local package: + +1. Start the build in watch mode: + ```bash + pnpm dev + ``` + This will automatically rebuild the library whenever you make changes to the source code. + +2. Create a separate test project and install transformers.js from your local development directory: + ```bash + mkdir my-test-project + cd my-test-project + npm init -y + npm install file:/path/to/transformers.js/packages/transformers + ``` + Replace `/path/to/transformers.js` with the actual path to your cloned repository. + +3. Make your changes to the transformers.js source code in the main repository. The watch mode will automatically rebuild the library. + +4. Test your changes in your test project. The changes will be automatically reflected since the package is linked via the `file:` protocol. + +This workflow allows for rapid iteration and testing during development. \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..93772b0 --- /dev/null +++ b/README.md @@ -0,0 +1,456 @@ + + +

+
+ + + + transformers.js javascript library logo + +
+

+ +

+ NPM + NPM Downloads + jsDelivr Hits + License + Documentation +

+ + +

+

State-of-the-art Machine Learning for the Web

+

+ +Run 🤗 Transformers directly in your browser, with no need for a server! + +Transformers.js is designed to be functionally equivalent to Hugging Face's [transformers](https://github.com/huggingface/transformers) python library, meaning you can run the same pretrained models using a very similar API. These models support common tasks in different modalities, such as: + - 📝 **Natural Language Processing**: text classification, named entity recognition, question answering, language modeling, summarization, translation, multiple choice, and text generation. + - 🖼️ **Computer Vision**: image classification, object detection, segmentation, and depth estimation. + - 🗣️ **Audio**: automatic speech recognition, audio classification, and text-to-speech. + - 🐙 **Multimodal**: embeddings, zero-shot audio classification, zero-shot image classification, and zero-shot object detection. + +Transformers.js uses [ONNX Runtime](https://onnxruntime.ai/) to run models in the browser. The best part about it, is that you can easily [convert](#convert-your-models-to-onnx) your pretrained PyTorch, TensorFlow, or JAX models to ONNX using [🤗 Optimum](https://github.com/huggingface/optimum#onnx--onnx-runtime). + +For more information, check out the full [documentation](https://huggingface.co/docs/transformers.js). + + +## Installation + + +To install via [NPM](https://www.npmjs.com/package/@huggingface/transformers), run: +```bash +npm i @huggingface/transformers +``` + +Alternatively, you can use it in vanilla JS, without any bundler, by using a CDN or static hosting. For example, using [ES Modules](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Modules), you can import the library with: +```html + +``` + + +## Quick tour + + +It's super simple to translate from existing code! Just like the python library, we support the `pipeline` API. Pipelines group together a pretrained model with preprocessing of inputs and postprocessing of outputs, making it the easiest way to run models with the library. + + + + + + + + + + +
Python (original)Javascript (ours)
+ +```python +from transformers import pipeline + +# Allocate a pipeline for sentiment-analysis +pipe = pipeline('sentiment-analysis') + +out = pipe('I love transformers!') +# [{'label': 'POSITIVE', 'score': 0.999806941}] +``` + + + +```javascript +import { pipeline } from '@huggingface/transformers'; + +// Allocate a pipeline for sentiment-analysis +const pipe = await pipeline('sentiment-analysis'); + +const out = await pipe('I love transformers!'); +// [{'label': 'POSITIVE', 'score': 0.999817686}] +``` + +
+ + +You can also use a different model by specifying the model id or path as the second argument to the `pipeline` function. For example: +```javascript +// Use a different model for sentiment-analysis +const pipe = await pipeline('sentiment-analysis', 'Xenova/bert-base-multilingual-uncased-sentiment'); +``` + +By default, when running in the browser, the model will be run on your CPU (via WASM). If you would like +to run the model on your GPU (via WebGPU), you can do this by setting `device: 'webgpu'`, for example: +```javascript +// Run the model on WebGPU +const pipe = await pipeline('sentiment-analysis', 'Xenova/distilbert-base-uncased-finetuned-sst-2-english', { + device: 'webgpu', +}); +``` + +For more information, check out the [WebGPU guide](https://huggingface.co/docs/transformers.js/guides/webgpu). + +> [!WARNING] +> The WebGPU API is still experimental in many browsers, so if you run into any issues, +> please file a [bug report](https://github.com/huggingface/transformers.js/issues/new?title=%5BWebGPU%5D%20Error%20running%20MODEL_ID_GOES_HERE&assignees=&labels=bug,webgpu&projects=&template=1_bug-report.yml). + +In resource-constrained environments, such as web browsers, it is advisable to use a quantized version of +the model to lower bandwidth and optimize performance. This can be achieved by adjusting the `dtype` option, +which allows you to select the appropriate data type for your model. While the available options may vary +depending on the specific model, typical choices include `"fp32"` (default for WebGPU), `"fp16"`, `"q8"` +(default for WASM), and `"q4"`. For more information, check out the [quantization guide](https://huggingface.co/docs/transformers.js/guides/dtypes). +```javascript +// Run the model at 4-bit quantization +const pipe = await pipeline('sentiment-analysis', 'Xenova/distilbert-base-uncased-finetuned-sst-2-english', { + dtype: 'q4', +}); +``` + +Ready to dive in? Explore our wide variety of demo applications and templates [here](https://github.com/huggingface/transformers.js-examples). You can also launch your own project instantly using the official Transformers.js [template](https://huggingface.co/new-space?template=static-templates%2Ftransformers.js) on Hugging Face! + + + +## Custom usage + + + +By default, Transformers.js uses [hosted pretrained models](https://huggingface.co/models?library=transformers.js) and [precompiled WASM binaries](https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0/dist/), which should work out-of-the-box. You can customize this as follows: + +### Settings + +```javascript +import { env } from '@huggingface/transformers'; + +// Specify a custom location for models (defaults to '/models/'). +env.localModelPath = '/path/to/models/'; + +// Disable the loading of remote models from the Hugging Face Hub: +env.allowRemoteModels = false; + +// Set location of .wasm files. Defaults to use a CDN. +env.backends.onnx.wasm.wasmPaths = '/path/to/files/'; +``` + +For a full list of available settings, check out the [API Reference](https://huggingface.co/docs/transformers.js/api/env). + +### Convert your models to ONNX + +We recommend using [Optimum](https://github.com/huggingface/optimum-onnx) to convert your PyTorch models to ONNX in a single command. For the full list of supported architectures, check out the [Optimum documentation](https://huggingface.co/docs/optimum-onnx/onnx/overview). + + +## Supported tasks/models + +Here is the list of all tasks and architectures currently supported by Transformers.js. If you don't see your task/model listed here or it is not yet supported, feel free to open up a feature request [here](https://github.com/huggingface/transformers.js/issues/new/choose). + +To find compatible models on the Hub, select the "transformers.js" library tag in the filter menu (or visit [this link](https://huggingface.co/models?library=transformers.js)). You can refine your search by selecting the task you're interested in (e.g., [text-classification](https://huggingface.co/models?pipeline_tag=text-classification&library=transformers.js)). + + +### Tasks + +#### Natural Language Processing + +| Task | ID | Description | Supported? | +|--------------------------|----|-------------|------------| +| [Fill-Mask](https://huggingface.co/tasks/fill-mask) | `fill-mask` | Masking some of the words in a sentence and predicting which words should replace those masks. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.FillMaskPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=fill-mask&library=transformers.js) | +| [Question Answering](https://huggingface.co/tasks/question-answering) | `question-answering` | Retrieve the answer to a question from a given text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.QuestionAnsweringPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=question-answering&library=transformers.js) | +| [Sentence Similarity](https://huggingface.co/tasks/sentence-similarity) | `sentence-similarity` | Determining how similar two texts are. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.FeatureExtractionPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=sentence-similarity&library=transformers.js) | +| [Summarization](https://huggingface.co/tasks/summarization) | `summarization` | Producing a shorter version of a document while preserving its important information. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.SummarizationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=summarization&library=transformers.js) | +| [Table Question Answering](https://huggingface.co/tasks/table-question-answering) | `table-question-answering` | Answering a question about information from a given table. | ❌ | +| [Text Classification](https://huggingface.co/tasks/text-classification) | `text-classification` or `sentiment-analysis` | Assigning a label or class to a given text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TextClassificationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=text-classification&library=transformers.js) | +| [Text Generation](https://huggingface.co/tasks/text-generation#completion-generation-models) | `text-generation` | Producing new text by predicting the next word in a sequence. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TextGenerationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=text-generation&library=transformers.js) | +| [Text-to-text Generation](https://huggingface.co/tasks/text-generation#text-to-text-generation-models) | `text2text-generation` | Converting one text sequence into another text sequence. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.Text2TextGenerationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=text2text-generation&library=transformers.js) | +| [Token Classification](https://huggingface.co/tasks/token-classification) | `token-classification` or `ner` | Assigning a label to each token in a text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TokenClassificationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=token-classification&library=transformers.js) | +| [Translation](https://huggingface.co/tasks/translation) | `translation` | Converting text from one language to another. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TranslationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=translation&library=transformers.js) | +| [Zero-Shot Classification](https://huggingface.co/tasks/zero-shot-classification) | `zero-shot-classification` | Classifying text into classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotClassificationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=zero-shot-classification&library=transformers.js) | +| [Feature Extraction](https://huggingface.co/tasks/feature-extraction) | `feature-extraction` | Transforming raw data into numerical features that can be processed while preserving the information in the original dataset. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.FeatureExtractionPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=feature-extraction&library=transformers.js) | + +#### Vision + +| Task | ID | Description | Supported? | +|--------------------------|----|-------------|------------| +| [Background Removal](https://huggingface.co/tasks/image-segmentation#background-removal) | `background-removal` | Isolating the main subject of an image by removing or making the background transparent. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.BackgroundRemovalPipeline)
[(models)](https://huggingface.co/models?other=background-removal&library=transformers.js) | +| [Depth Estimation](https://huggingface.co/tasks/depth-estimation) | `depth-estimation` | Predicting the depth of objects present in an image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.DepthEstimationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=depth-estimation&library=transformers.js) | +| [Image Classification](https://huggingface.co/tasks/image-classification) | `image-classification` | Assigning a label or class to an entire image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageClassificationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=image-classification&library=transformers.js) | +| [Image Segmentation](https://huggingface.co/tasks/image-segmentation) | `image-segmentation` | Divides an image into segments where each pixel is mapped to an object. This task has multiple variants such as instance segmentation, panoptic segmentation and semantic segmentation. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageSegmentationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=image-segmentation&library=transformers.js) | +| [Image-to-Image](https://huggingface.co/tasks/image-to-image) | `image-to-image` | Transforming a source image to match the characteristics of a target image or a target image domain. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageToImagePipeline)
[(models)](https://huggingface.co/models?pipeline_tag=image-to-image&library=transformers.js) | +| [Mask Generation](https://huggingface.co/tasks/mask-generation) | `mask-generation` | Generate masks for the objects in an image. | ❌ | +| [Object Detection](https://huggingface.co/tasks/object-detection) | `object-detection` | Identify objects of certain defined classes within an image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ObjectDetectionPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=object-detection&library=transformers.js) | +| [Video Classification](https://huggingface.co/tasks/video-classification) | n/a | Assigning a label or class to an entire video. | ❌ | +| [Unconditional Image Generation](https://huggingface.co/tasks/unconditional-image-generation) | n/a | Generating images with no condition in any context (like a prompt text or another image). | ❌ | +| [Image Feature Extraction](https://huggingface.co/tasks/image-feature-extraction) | `image-feature-extraction` | Transforming raw data into numerical features that can be processed while preserving the information in the original image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageFeatureExtractionPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=image-feature-extraction&library=transformers.js) | + +#### Audio + +| Task | ID | Description | Supported? | +|--------------------------|----|-------------|------------| +| [Audio Classification](https://huggingface.co/tasks/audio-classification) | `audio-classification` | Assigning a label or class to a given audio. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.AudioClassificationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=audio-classification&library=transformers.js) | +| [Audio-to-Audio](https://huggingface.co/tasks/audio-to-audio) | n/a | Generating audio from an input audio source. | ❌ | +| [Automatic Speech Recognition](https://huggingface.co/tasks/automatic-speech-recognition) | `automatic-speech-recognition` | Transcribing a given audio into text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.AutomaticSpeechRecognitionPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=automatic-speech-recognition&library=transformers.js) | +| [Text-to-Speech](https://huggingface.co/tasks/text-to-speech) | `text-to-speech` or `text-to-audio` | Generating natural-sounding speech given text input. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TextToAudioPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=text-to-audio&library=transformers.js) | + + +#### Tabular + +| Task | ID | Description | Supported? | +|--------------------------|----|-------------|------------| +| [Tabular Classification](https://huggingface.co/tasks/tabular-classification) | n/a | Classifying a target category (a group) based on set of attributes. | ❌ | +| [Tabular Regression](https://huggingface.co/tasks/tabular-regression) | n/a | Predicting a numerical value given a set of attributes. | ❌ | + + +#### Multimodal + +| Task | ID | Description | Supported? | +|--------------------------|----|-------------|------------| +| [Document Question Answering](https://huggingface.co/tasks/document-question-answering) | `document-question-answering` | Answering questions on document images. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.DocumentQuestionAnsweringPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=document-question-answering&library=transformers.js) | +| [Image-to-Text](https://huggingface.co/tasks/image-to-text) | `image-to-text` | Output text from a given image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageToTextPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=image-to-text&library=transformers.js) | +| [Text-to-Image](https://huggingface.co/tasks/text-to-image) | `text-to-image` | Generates images from input text. | ❌ | +| [Visual Question Answering](https://huggingface.co/tasks/visual-question-answering) | `visual-question-answering` | Answering open-ended questions based on an image. | ❌ | +| [Zero-Shot Audio Classification](https://huggingface.co/learn/audio-course/chapter4/classification_models#zero-shot-audio-classification) | `zero-shot-audio-classification` | Classifying audios into classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotAudioClassificationPipeline)
[(models)](https://huggingface.co/models?other=zero-shot-audio-classification&library=transformers.js) | +| [Zero-Shot Image Classification](https://huggingface.co/tasks/zero-shot-image-classification) | `zero-shot-image-classification` | Classifying images into classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotImageClassificationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=zero-shot-image-classification&library=transformers.js) | +| [Zero-Shot Object Detection](https://huggingface.co/tasks/zero-shot-object-detection) | `zero-shot-object-detection` | Identify objects of classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotObjectDetectionPipeline)
[(models)](https://huggingface.co/models?other=zero-shot-object-detection&library=transformers.js) | + + +#### Reinforcement Learning + +| Task | ID | Description | Supported? | +|--------------------------|----|-------------|------------| +| [Reinforcement Learning](https://huggingface.co/tasks/reinforcement-learning) | n/a | Learning from actions by interacting with an environment through trial and error and receiving rewards (negative or positive) as feedback. | ✅ | + + + +### Models + +1. **[AFMoE](https://huggingface.co/docs/transformers/model_doc/afmoe)** (from Arcee AI, Prime Intellect, and DatologyAI) released with the paper [Arcee Trinity Large Technical Report](https://huggingface.co/papers/2602.17004), by Varun Singh, Lucas Krauss, Sami Jaghouar, Matej Sirovatka, Charles Goddard, Fares Obied, Jack Min Ong, Jannik Straube, Fern, Aria Harley, Conner Stewart, Colin Kealty, Maziyar Panahi, Simon Kirsten, Anushka Deshpande, Anneketh Vij, Arthur Bresnu, Pranav Veldurthi, Raghav Ravishankar, Hardik Bishnoi, Mark McQuade, Johannes Hagemann, Lucas Atkins. +1. **[ALBERT](https://huggingface.co/docs/transformers/model_doc/albert)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://huggingface.co/papers/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. +1. **[Arcee](https://huggingface.co/docs/transformers/model_doc/arcee)** (from Arcee AI) released with the blog post [Announcing Arcee Foundation Models](https://www.arcee.ai/blog/announcing-the-arcee-foundation-model-family) by Fernando Fernandes, Varun Singh, Charles Goddard, Lucas Atkins, Mark McQuade, Maziyar Panahi, Conner Stewart, Colin Kealty, Raghav Ravishankar, Lucas Krauss, Anneketh Vij, Pranav Veldurthi, Abhishek Thakur, Julien Simon, Scott Zembsch, Benjamin Langer, Aleksiej Cecocho, Maitri Patel. +1. **[Audio Spectrogram Transformer](https://huggingface.co/docs/transformers/model_doc/audio-spectrogram-transformer)** (from MIT) released with the paper [AST: Audio Spectrogram Transformer](https://huggingface.co/papers/2104.01778) by Yuan Gong, Yu-An Chung, James Glass. +1. **[BART](https://huggingface.co/docs/transformers/model_doc/bart)** (from Facebook) released with the paper [BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension](https://huggingface.co/papers/1910.13461) by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer. +1. **[BEiT](https://huggingface.co/docs/transformers/model_doc/beit)** (from Microsoft) released with the paper [BEiT: BERT Pre-Training of Image Transformers](https://huggingface.co/papers/2106.08254) by Hangbo Bao, Li Dong, Furu Wei. +1. **[BERT](https://huggingface.co/docs/transformers/model_doc/bert)** (from Google) released with the paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://huggingface.co/papers/1810.04805) by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. +1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://huggingface.co/papers/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. +1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://huggingface.co/papers/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. +1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). +1. **[CamemBERT](https://huggingface.co/docs/transformers/model_doc/camembert)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://huggingface.co/papers/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. +1. **[CHMv2](https://huggingface.co/docs/transformers/main/model_doc/chmv2)** (from Meta) released with the paper [CHMv2: Improvements in Global Canopy Height Mapping using DINOv3](https://huggingface.co/papers/2603.06382) by John Brandt, Seungeun Yi, Jamie Tolan, Xinyuan Li, Peter Potapov, Jessica Ertel, Justine Spore, Huy V. Vo, Michaël Ramamonjisoa, Patrick Labatut, Piotr Bojanowski, Camille Couprie. +1. **Chatterbox** (from Resemble AI) released with the repository [Chatterbox TTS](https://github.com/resemble-ai/chatterbox) by the Resemble AI team. +1. **[Chinese-CLIP](https://huggingface.co/docs/transformers/model_doc/chinese_clip)** (from OFA-Sys) released with the paper [Chinese CLIP: Contrastive Vision-Language Pretraining in Chinese](https://huggingface.co/papers/2211.01335) by An Yang, Junshu Pan, Junyang Lin, Rui Men, Yichang Zhang, Jingren Zhou, Chang Zhou. +1. **[CLAP](https://huggingface.co/docs/transformers/model_doc/clap)** (from LAION-AI) released with the paper [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation](https://huggingface.co/papers/2211.06687) by Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov. +1. **[CLIP](https://huggingface.co/docs/transformers/model_doc/clip)** (from OpenAI) released with the paper [Learning Transferable Visual Models From Natural Language Supervision](https://huggingface.co/papers/2103.00020) by Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, Gretchen Krueger, Ilya Sutskever. +1. **[CLIPSeg](https://huggingface.co/docs/transformers/model_doc/clipseg)** (from University of Göttingen) released with the paper [Image Segmentation Using Text and Image Prompts](https://huggingface.co/papers/2112.10003) by Timo Lüddecke and Alexander Ecker. +1. **[CodeGen](https://huggingface.co/docs/transformers/model_doc/codegen)** (from Salesforce) released with the paper [A Conversational Paradigm for Program Synthesis](https://huggingface.co/papers/2203.13474) by Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, Caiming Xiong. +1. **[CodeLlama](https://huggingface.co/docs/transformers/model_doc/llama_code)** (from MetaAI) released with the paper [Code Llama: Open Foundation Models for Code](https://ai.meta.com/research/publications/code-llama-open-foundation-models-for-code/) by Baptiste Rozière, Jonas Gehring, Fabian Gloeckle, Sten Sootla, Itai Gat, Xiaoqing Ellen Tan, Yossi Adi, Jingyu Liu, Tal Remez, Jérémy Rapin, Artyom Kozhevnikov, Ivan Evtimov, Joanna Bitton, Manish Bhatt, Cristian Canton Ferrer, Aaron Grattafiori, Wenhan Xiong, Alexandre Défossez, Jade Copet, Faisal Azhar, Hugo Touvron, Louis Martin, Nicolas Usunier, Thomas Scialom, Gabriel Synnaeve. +1. **[Cohere](https://huggingface.co/docs/transformers/main/model_doc/cohere)** (from Cohere) released with the blog post [Command-R: Retrieval Augmented Generation at Production Scale](https://cohere.com/blog/command-r) by Cohere. +1. **[Cohere2](https://huggingface.co/docs/transformers/main/model_doc/cohere2)** (from Cohere) released with the blog post [Introducing Command R7B: Fast and efficient generative AI](https://cohere.com/blog/command-r7b) by Cohere. +1. **[Cohere ASR](https://huggingface.co/docs/transformers/main/model_doc/cohere_asr)** (from Cohere) released with the blog post [Introducing Cohere Transcribe: a new state-of-the-art in open-source speech recognition](https://cohere.com/blog/transcribe) by Cohere. +1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://huggingface.co/papers/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. +1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://huggingface.co/papers/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://huggingface.co/papers/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. +1. **[D-FINE](https://huggingface.co/docs/transformers/model_doc/d_fine)** (from University of Science and Technology of China) released with the paper [D-FINE: Redefine Regression Task in DETRs as Fine-grained Distribution Refinement](https://huggingface.co/papers/2410.13842) by Yansong Peng, Hebei Li, Peixi Wu, Yueyi Zhang, Xiaoyan Sun, Feng Wu. +1. **[DAC](https://huggingface.co/docs/transformers/model_doc/dac)** (from Descript) released with the paper [Descript Audio Codec: High-Fidelity Audio Compression with Improved RVQGAN](https://huggingface.co/papers/2306.06546) by Rithesh Kumar, Prem Seetharaman, Alejandro Luebs, Ishaan Kumar, Kundan Kumar. +1. **[DeBERTa](https://huggingface.co/docs/transformers/model_doc/deberta)** (from Microsoft) released with the paper [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://huggingface.co/papers/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. +1. **[DeBERTa-v2](https://huggingface.co/docs/transformers/model_doc/deberta-v2)** (from Microsoft) released with the paper [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://huggingface.co/papers/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. +1. **[Decision Transformer](https://huggingface.co/docs/transformers/model_doc/decision_transformer)** (from Berkeley/Facebook/Google) released with the paper [Decision Transformer: Reinforcement Learning via Sequence Modeling](https://huggingface.co/papers/2106.01345) by Lili Chen, Kevin Lu, Aravind Rajeswaran, Kimin Lee, Aditya Grover, Michael Laskin, Pieter Abbeel, Aravind Srinivas, Igor Mordatch. +1. **[DeepSeek-V3](https://huggingface.co/docs/transformers/model_doc/deepseek_v3)** (from DeepSeek-AI) released with the paper [DeepSeek-V3 Technical Report](https://huggingface.co/papers/2412.19437) by the DeepSeek-AI team. +1. **[DeepSeek-V4](https://huggingface.co/docs/transformers/model_doc/deepseek_v4)** (from DeepSeek-AI) released with the paper [DeepSeek-V4: Towards Highly Efficient Million-Token Context Intelligence](https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf) by the DeepSeek-AI team. +1. **[DeiT](https://huggingface.co/docs/transformers/model_doc/deit)** (from Facebook) released with the paper [Training data-efficient image transformers & distillation through attention](https://huggingface.co/papers/2012.12877) by Hugo Touvron, Matthieu Cord, Matthijs Douze, Francisco Massa, Alexandre Sablayrolles, Hervé Jégou. +1. **[Depth Anything](https://huggingface.co/docs/transformers/main/model_doc/depth_anything)** (from University of Hong Kong and TikTok) released with the paper [Depth Anything: Unleashing the Power of Large-Scale Unlabeled Data](https://huggingface.co/papers/2401.10891) by Lihe Yang, Bingyi Kang, Zilong Huang, Xiaogang Xu, Jiashi Feng, Hengshuang Zhao. +1. **Depth Pro** (from Apple) released with the paper [Depth Pro: Sharp Monocular Metric Depth in Less Than a Second](https://huggingface.co/papers/2410.02073) by Aleksei Bochkovskii, Amaël Delaunoy, Hugo Germain, Marcel Santos, Yichao Zhou, Stephan R. Richter, Vladlen Koltun. +1. **[DETR](https://huggingface.co/docs/transformers/model_doc/detr)** (from Facebook) released with the paper [End-to-End Object Detection with Transformers](https://huggingface.co/papers/2005.12872) by Nicolas Carion, Francisco Massa, Gabriel Synnaeve, Nicolas Usunier, Alexander Kirillov, Sergey Zagoruyko. +1. **[DINOv2](https://huggingface.co/docs/transformers/model_doc/dinov2)** (from Meta AI) released with the paper [DINOv2: Learning Robust Visual Features without Supervision](https://huggingface.co/papers/2304.07193) by Maxime Oquab, Timothée Darcet, Théo Moutakanni, Huy Vo, Marc Szafraniec, Vasil Khalidov, Pierre Fernandez, Daniel Haziza, Francisco Massa, Alaaeldin El-Nouby, Mahmoud Assran, Nicolas Ballas, Wojciech Galuba, Russell Howes, Po-Yao Huang, Shang-Wen Li, Ishan Misra, Michael Rabbat, Vasu Sharma, Gabriel Synnaeve, Hu Xu, Hervé Jegou, Julien Mairal, Patrick Labatut, Armand Joulin, Piotr Bojanowski. +1. **[DINOv2 with Registers](https://huggingface.co/docs/transformers/model_doc/dinov2_with_registers)** (from Meta AI) released with the paper [DINOv2 with Registers](https://huggingface.co/papers/2309.16588) by Timothée Darcet, Maxime Oquab, Julien Mairal, Piotr Bojanowski. +1. **[DINOv3](https://huggingface.co/docs/transformers/model_doc/dinov3)** (from Meta AI) released with the paper [DINOv3](https://huggingface.co/papers/2508.10104) by Oriane Siméoni, Huy V. Vo, Maximilian Seitzer, Federico Baldassarre, Maxime Oquab, Cijo Jose, Vasil Khalidov, Marc Szafraniec, Seungeun Yi, Michaël Ramamonjisoa, Francisco Massa, Daniel Haziza, Luca Wehrstedt, Jianyuan Wang, Timothée Darcet, Théo Moutakanni, Leonel Sentana, Claire Roberts, Andrea Vedaldi, Jamie Tolan, John Brandt, Camille Couprie, Julien Mairal, Hervé Jégou, Patrick Labatut, Piotr Bojanowski. +1. **[DistilBERT](https://huggingface.co/docs/transformers/model_doc/distilbert)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://huggingface.co/papers/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/research_projects/distillation), RoBERTa into [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/research_projects/distillation), Multilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/research_projects/distillation) and a German version of DistilBERT. +1. **[DiT](https://huggingface.co/docs/transformers/model_doc/dit)** (from Microsoft Research) released with the paper [DiT: Self-supervised Pre-training for Document Image Transformer](https://huggingface.co/papers/2203.02378) by Junlong Li, Yiheng Xu, Tengchao Lv, Lei Cui, Cha Zhang, Furu Wei. +1. **[Donut](https://huggingface.co/docs/transformers/model_doc/donut)** (from NAVER), released together with the paper [OCR-free Document Understanding Transformer](https://huggingface.co/papers/2111.15664) by Geewook Kim, Teakgyu Hong, Moonbin Yim, Jeongyeon Nam, Jinyoung Park, Jinyeong Yim, Wonseok Hwang, Sangdoo Yun, Dongyoon Han, Seunghyun Park. +1. **[DPT](https://huggingface.co/docs/transformers/master/model_doc/dpt)** (from Intel Labs) released with the paper [Vision Transformers for Dense Prediction](https://huggingface.co/papers/2103.13413) by René Ranftl, Alexey Bochkovskiy, Vladlen Koltun. +1. **[EdgeTAM](https://huggingface.co/docs/transformers/model_doc/edgetam)** (from Facebook) released with the paper [EdgeTAM: On-Device Track Anything Model](https://huggingface.co/papers/2501.07256) by Chong Zhou, Chenchen Zhu, Yunyang Xiong, Saksham Suri, Fanyi Xiao, Lemeng Wu, Raghuraman Krishnamoorthi, Bo Dai, Chen Change Loy, Vikas Chandra, Bilge Soran. +1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://huggingface.co/papers/1905.11946) by Mingxing Tan, Quoc V. Le. +1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://huggingface.co/papers/2003.10555) by Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning. +1. **ERNIE-4.5** (from Baidu ERNIE Team) released with the blog post [Announcing the Open Source Release of the ERNIE 4.5 Model Family](https://ernie.baidu.com/blog/posts/ernie4.5/) by the Baidu ERNIE Team. +1. **[ESM](https://huggingface.co/docs/transformers/model_doc/esm)** (from Meta AI) are transformer protein language models. **ESM-1b** was released with the paper [Biological structure and function emerge from scaling unsupervised learning to 250 million protein sequences](https://www.pnas.org/content/118/15/e2016239118) by Alexander Rives, Joshua Meier, Tom Sercu, Siddharth Goyal, Zeming Lin, Jason Liu, Demi Guo, Myle Ott, C. Lawrence Zitnick, Jerry Ma, and Rob Fergus. **ESM-1v** was released with the paper [Language models enable zero-shot prediction of the effects of mutations on protein function](https://doi.org/10.1101/2021.07.09.450648) by Joshua Meier, Roshan Rao, Robert Verkuil, Jason Liu, Tom Sercu and Alexander Rives. **ESM-2 and ESMFold** were released with the paper [Language models of protein sequences at the scale of evolution enable accurate structure prediction](https://doi.org/10.1101/2022.07.20.500902) by Zeming Lin, Halil Akin, Roshan Rao, Brian Hie, Zhongkai Zhu, Wenting Lu, Allan dos Santos Costa, Maryam Fazel-Zarandi, Tom Sercu, Sal Candido, Alexander Rives. +1. **[EuroBERT](https://huggingface.co/docs/transformers/model_doc/eurobert)** (from the EuroBERT team) released with the paper [EuroBERT: Scaling Multilingual Encoders for European Languages](https://huggingface.co/papers/2503.05500) by Nicolas Boizard, Hippolyte Gisserot-Boukhlef, Duarte M. Alves, André Martins, Ayoub Hammal, Caio Corro, Céline Hudelot, Emmanuel Malherbe, Etienne Malaboeuf, Fanny Jourdan, Gabriel Hautreux, João Alves, Kevin El-Haddad, Manuel Faysse, Maxime Peyrard, Nuno M. Guerreiro, Patrick Fernandes, Ricardo Rei, Pierre Colombo. +1. **EXAONE** (from LG AI Research) released with the papers [EXAONE 3.0 7.8B Instruction Tuned Language Model](https://huggingface.co/papers/2408.03541) and [EXAONE 3.5: Series of Large Language Models for Real-world Use Cases](https://huggingface.co/papers/2412.04862) by the LG AI Research team. +1. **[Falcon](https://huggingface.co/docs/transformers/model_doc/falcon)** (from Technology Innovation Institute) by Almazrouei, Ebtesam and Alobeidli, Hamza and Alshamsi, Abdulaziz and Cappelli, Alessandro and Cojocaru, Ruxandra and Debbah, Merouane and Goffinet, Etienne and Heslow, Daniel and Launay, Julien and Malartic, Quentin and Noune, Badreddine and Pannier, Baptiste and Penedo, Guilherme. +1. **[FalconH1](https://huggingface.co/docs/transformers/model_doc/falcon_h1)** (from Technology Innovation Institute) released with the blog post [Falcon-H1-Tiny: A series of extremely small, yet powerful language models redefining capabilities at small scale](https://huggingface.co/spaces/tiiuae/tiny-h1-blogpost) by the Falcon-LLM team. +1. **FastViT** (from Apple) released with the paper [FastViT: A Fast Hybrid Vision Transformer using Structural Reparameterization](https://huggingface.co/papers/2303.14189) by Pavan Kumar Anasosalu Vasu, James Gabriel, Jeff Zhu, Oncel Tuzel and Anurag Ranjan. +1. **[FLAN-T5](https://huggingface.co/docs/transformers/model_doc/flan-t5)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-t5-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei +1. **Florence2** (from Microsoft) released with the paper [Florence-2: Advancing a Unified Representation for a Variety of Vision Tasks](https://huggingface.co/papers/2311.06242) by Bin Xiao, Haiping Wu, Weijian Xu, Xiyang Dai, Houdong Hu, Yumao Lu, Michael Zeng, Ce Liu, Lu Yuan. +1. **[Gemma](https://huggingface.co/docs/transformers/main/model_doc/gemma)** (from Google) released with the blog post [Gemma: Open Models Based on Gemini Technology and Research](https://blog.google/technology/developers/gemma-open-models/) by the Gemma Google team. +1. **[Gemma2](https://huggingface.co/docs/transformers/main/model_doc/gemma2)** (from Google) released with the blog post [Gemma2: Open Models Based on Gemini Technology and Research](https://blog.google/technology/developers/google-gemma-2/) by the Gemma Google team. +1. **[Gemma3](https://huggingface.co/docs/transformers/main/model_doc/gemma3)** (from Google) released with the blog post [Introducing Gemma 3: The most capable model you can run on a single GPU or TPU](https://blog.google/technology/developers/gemma-3/) by the Gemma Google team. +1. **[Gemma3n](https://huggingface.co/docs/transformers/main/model_doc/gemma3n)** (from Google) released with the blog post [Announcing Gemma 3n preview: powerful, efficient, mobile-first AI](https://developers.googleblog.com/en/introducing-gemma-3n/) by the Gemma Google team. +1. **[Gemma4](https://huggingface.co/docs/transformers/main/model_doc/gemma4)** (from Google) released with the blog post [Gemma 4](https://blog.google/innovation-and-ai/technology/developers-tools/gemma-4/) by the Gemma Google team. +1. **[GLM](https://huggingface.co/docs/transformers/main/model_doc/glm)** (from the GLM Team, THUDM & ZhipuAI) released with the paper [ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools](https://huggingface.co/papers/2406.12793v2) by Team GLM: Aohan Zeng, Bin Xu, Bowen Wang, Chenhui Zhang, Da Yin, Dan Zhang, Diego Rojas, Guanyu Feng, Hanlin Zhao, Hanyu Lai, Hao Yu, Hongning Wang, Jiadai Sun, Jiajie Zhang, Jiale Cheng, Jiayi Gui, Jie Tang, Jing Zhang, Jingyu Sun, Juanzi Li, Lei Zhao, Lindong Wu, Lucen Zhong, Mingdao Liu, Minlie Huang, Peng Zhang, Qinkai Zheng, Rui Lu, Shuaiqi Duan, Shudan Zhang, Shulin Cao, Shuxun Yang, Weng Lam Tam, Wenyi Zhao, Xiao Liu, Xiao Xia, Xiaohan Zhang, Xiaotao Gu, Xin Lv, Xinghan Liu, Xinyi Liu, Xinyue Yang, Xixuan Song, Xunkai Zhang, Yifan An, Yifan Xu, Yilin Niu, Yuantao Yang, Yueyan Li, Yushi Bai, Yuxiao Dong, Zehan Qi, Zhaoyu Wang, Zhen Yang, Zhengxiao Du, Zhenyu Hou, Zihan Wang. +1. **[GLM-MoE-DSA](https://huggingface.co/docs/transformers/main/model_doc/glm_moe_dsa)** (from the GLM Team, ZhipuAI & Tsinghua University) released with the paper [GLM-5: from Vibe Coding to Agentic Engineering](https://huggingface.co/papers/2602.15763) by Team GLM. +1. **[GLM-OCR](https://huggingface.co/docs/transformers/main/model_doc/glm_ocr)** (from the GLM Team, ZhipuAI & Tsinghua University) released with the paper [GLM-OCR Technical Report](https://huggingface.co/papers/2603.10910) by Team GLM: Shuaiqi Duan, Yadong Xue, Weihan Wang, Zhe Su, Huan Liu, Sheng Yang, Guobing Gan, Guo Wang, Zihan Wang, Shengdong Yan, Dexin Jin, Yuxuan Zhang, Guohong Wen, Yanfeng Wang, Yutao Zhang, Xiaohan Zhang, Wenyi Hong, Yukuo Cen, Da Yin, Bin Chen, Wenmeng Yu, Xiaotao Gu, Jie Tang. +1. **[GLPN](https://huggingface.co/docs/transformers/model_doc/glpn)** (from KAIST) released with the paper [Global-Local Path Networks for Monocular Depth Estimation with Vertical CutDepth](https://huggingface.co/papers/2201.07436) by Doyeon Kim, Woonghyun Ga, Pyungwhan Ahn, Donggyu Joo, Sehwan Chun, Junmo Kim. +1. **[GPT Neo](https://huggingface.co/docs/transformers/model_doc/gpt_neo)** (from EleutherAI) released in the repository [EleutherAI/gpt-neo](https://github.com/EleutherAI/gpt-neo) by Sid Black, Stella Biderman, Leo Gao, Phil Wang and Connor Leahy. +1. **[GPT NeoX](https://huggingface.co/docs/transformers/model_doc/gpt_neox)** (from EleutherAI) released with the paper [GPT-NeoX-20B: An Open-Source Autoregressive Language Model](https://huggingface.co/papers/2204.06745) by Sid Black, Stella Biderman, Eric Hallahan, Quentin Anthony, Leo Gao, Laurence Golding, Horace He, Connor Leahy, Kyle McDonell, Jason Phang, Michael Pieler, USVSN Sai Prashanth, Shivanshu Purohit, Laria Reynolds, Jonathan Tow, Ben Wang, Samuel Weinbach +1. **[GPT OSS](https://huggingface.co/docs/transformers/model_doc/gpt_oss)** (from OpenAI) released with the blog [Introducing gpt-oss](https://openai.com/index/introducing-gpt-oss/) by Sandhini Agarwal, Lama Ahmad, Jason Ai, Sam Altman, Andy Applebaum, Edwin Arbus, Rahul K. Arora, Yu Bai, Bowen Baker, Haiming Bao, Boaz Barak, Ally Bennett, Tyler Bertao, Nivedita Brett, Eugene Brevdo, Greg Brockman, Sebastien Bubeck, Che Chang, Kai Chen, Mark Chen, Enoch Cheung, Aidan Clark, Dan Cook, Marat Dukhan, Casey Dvorak, Kevin Fives, Vlad Fomenko, Timur Garipov, Kristian Georgiev, Mia Glaese, Tarun Gogineni, Adam Goucher, Lukas Gross, Katia Gil Guzman, John Hallman, Jackie Hehir, Johannes Heidecke, Alec Helyar, Haitang Hu, Romain Huet, Jacob Huh, Saachi Jain, Zach Johnson, Chris Koch, Irina Kofman, Dominik Kundel, Jason Kwon, Volodymyr Kyrylov, Elaine Ya Le, Guillaume Leclerc, James Park Lennon, Scott Lessans, Mario Lezcano-Casado, Yuanzhi Li, Zhuohan Li, Ji Lin, Jordan Liss, Lily (Xiaoxuan) Liu, Jiancheng Liu, Kevin Lu, Chris Lu, Zoran Martinovic, Lindsay McCallum, Josh McGrath, Scott McKinney, Aidan McLaughlin, Song Mei, Steve Mostovoy, Tong Mu, Gideon Myles, Alexander Neitz, Alex Nichol, Jakub Pachocki, Alex Paino, Dana Palmie, Ashley Pantuliano, Giambattista Parascandolo, Jongsoo Park, Leher Pathak, Carolina Paz, Ludovic Peran, Dmitry Pimenov, Michelle Pokrass, Elizabeth Proehl, Huida Qiu, Gaby Raila, Filippo Raso, Hongyu Ren, Kimmy Richardson, David Robinson, Bob Rotsted, Hadi Salman, Suvansh Sanjeev, Max Schwarzer, D. Sculley, Harshit Sikchi, Kendal Simon, Karan Singhal, Yang Song, Dane Stuckey, Zhiqing Sun, Philippe Tillet, Sam Toizer, Foivos Tsimpourlas, Nikhil Vyas, Eric Wallace, Xin Wang, Miles Wang, Olivia Watkins, Kevin Weil, Amy Wendling, Kevin Whinnery, Cedric Whitney, Hannah Wong, Lin Yang, Yu Yang, Michihiro Yasunaga, Kristen Ying, Wojciech Zaremba, Wenting Zhan, Cyril Zhang, Brian Zhang, Eddie Zhang, Shengjia Zhao. +1. **[GPT-2](https://huggingface.co/docs/transformers/model_doc/gpt2)** (from OpenAI) released with the paper [Language Models are Unsupervised Multitask Learners](https://blog.openai.com/better-language-models/) by Alec Radford*, Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever**. +1. **[GPT-J](https://huggingface.co/docs/transformers/model_doc/gptj)** (from EleutherAI) released in the repository [kingoflolz/mesh-transformer-jax](https://github.com/kingoflolz/mesh-transformer-jax/) by Ben Wang and Aran Komatsuzaki. +1. **[GPTBigCode](https://huggingface.co/docs/transformers/model_doc/gpt_bigcode)** (from BigCode) released with the paper [SantaCoder: don't reach for the stars!](https://huggingface.co/papers/2301.03988) by Loubna Ben Allal, Raymond Li, Denis Kocetkov, Chenghao Mou, Christopher Akiki, Carlos Munoz Ferrandis, Niklas Muennighoff, Mayank Mishra, Alex Gu, Manan Dey, Logesh Kumar Umapathi, Carolyn Jane Anderson, Yangtian Zi, Joel Lamy Poirier, Hailey Schoelkopf, Sergey Troshin, Dmitry Abulkhanov, Manuel Romero, Michael Lappert, Francesco De Toni, Bernardo García del Río, Qian Liu, Shamik Bose, Urvashi Bhattacharyya, Terry Yue Zhuo, Ian Yu, Paulo Villegas, Marco Zocca, Sourab Mangrulkar, David Lansky, Huu Nguyen, Danish Contractor, Luis Villa, Jia Li, Dzmitry Bahdanau, Yacine Jernite, Sean Hughes, Daniel Fried, Arjun Guha, Harm de Vries, Leandro von Werra. +1. **[Granite](https://huggingface.co/docs/transformers/main/model_doc/granite)** (from IBM) released with the paper [Power Scheduler: A Batch Size and Token Number Agnostic Learning Rate Scheduler](https://huggingface.co/papers/2408.13359) by Yikang Shen, Matthew Stallone, Mayank Mishra, Gaoyuan Zhang, Shawn Tan, Aditya Prasad, Adriana Meza Soria, David D. Cox, Rameswar Panda. +1. **[GraniteMoeHybrid](https://huggingface.co/docs/transformers/main/model_doc/granitemoehybrid)** (from IBM) released with the blog post [IBM Granite 4.0: hyper-efficient, high performance hybrid models for enterprise](https://www.ibm.com/new/announcements/ibm-granite-4-0-hyper-efficient-high-performance-hybrid-models) by the IBM Granite team. +1. **[Granite Speech](https://huggingface.co/docs/transformers/main/model_doc/granite_speech)** (from IBM) released with the paper [Granite-speech: open-source speech-aware LLMs with strong English ASR capabilities](https://huggingface.co/papers/2505.08699) by the IBM Granite team. +1. **[Grounding DINO](https://huggingface.co/docs/transformers/model_doc/grounding-dino)** (from IDEA-Research) released with the paper [Grounding DINO: Marrying DINO with Grounded Pre-Training for Open-Set Object Detection](https://huggingface.co/papers/2303.05499) by Shilong Liu, Zhaoyang Zeng, Tianhe Ren, Feng Li, Hao Zhang, Jie Yang, Qing Jiang, Chunyuan Li, Jianwei Yang, Hang Su, Jun Zhu, Lei Zhang. +1. **[GroupViT](https://huggingface.co/docs/transformers/model_doc/groupvit)** (from UCSD, NVIDIA) released with the paper [GroupViT: Semantic Segmentation Emerges from Text Supervision](https://huggingface.co/papers/2202.11094) by Jiarui Xu, Shalini De Mello, Sifei Liu, Wonmin Byeon, Thomas Breuel, Jan Kautz, Xiaolong Wang. +1. **[Helium](https://huggingface.co/docs/transformers/main/model_doc/helium)** (from the Kyutai Team) released with the blog post [Announcing Helium-1 Preview](https://kyutai.org/2025/01/13/helium.html) by the Kyutai Team. +1. **[HerBERT](https://huggingface.co/docs/transformers/model_doc/herbert)** (from Allegro.pl, AGH University of Science and Technology) released with the paper [KLEJ: Comprehensive Benchmark for Polish Language Understanding](https://www.aclweb.org/anthology/2020.acl-main.111.pdf) by Piotr Rybak, Robert Mroczkowski, Janusz Tracz, Ireneusz Gawlik. +1. **[HRM-Text](https://huggingface.co/docs/transformers/model_doc/hrm_text)** (from Sapient Intelligence) released with the paper [Hierarchical Reasoning Model](https://huggingface.co/papers/2506.21734) by Guan Wang, Jin Li, Yuhao Sun, Xing Chen, Changling Liu, Yue Wu, Meng Lu, Sen Song, Yasin Abbasi Yadkori. +1. **[Hiera](https://huggingface.co/docs/transformers/model_doc/hiera)** (from Meta) released with the paper [Hiera: A Hierarchical Vision Transformer without the Bells-and-Whistles](https://huggingface.co/papers/2306.00989) by Chaitanya Ryali, Yuan-Ting Hu, Daniel Bolya, Chen Wei, Haoqi Fan, Po-Yao Huang, Vaibhav Aggarwal, Arkabandhu Chowdhury, Omid Poursaeed, Judy Hoffman, Jitendra Malik, Yanghao Li, Christoph Feichtenhofer. +1. **[Hubert](https://huggingface.co/docs/transformers/model_doc/hubert)** (from Facebook) released with the paper [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units](https://huggingface.co/papers/2106.07447) by Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia, Ruslan Salakhutdinov, Abdelrahman Mohamed. +1. **[HunYuanDenseV1](https://huggingface.co/docs/transformers/model_doc/hunyuan_v1_dense)** (from Tencent) released with the paper [Hunyuan-Large: An Open-Source MoE Model with 52 Billion Activated Parameters by Tencent](https://huggingface.co/papers/2411.02265) by the Tencent team. +1. **[I-JEPA](https://huggingface.co/docs/transformers/model_doc/ijepa)** (from Meta) released with the paper [Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture](https://huggingface.co/papers/2301.08243) by Mahmoud Assran, Quentin Duval, Ishan Misra, Piotr Bojanowski, Pascal Vincent, Michael Rabbat, Yann LeCun, Nicolas Ballas. +1. **[Idefics3](https://huggingface.co/docs/transformers/model_doc/idefics3)** (from Hugging Face) released with the paper [Building and better understanding vision-language models: insights and future directions](https://huggingface.co/papers/2408.12637) by Hugo Laurençon, Andrés Marafioti, Victor Sanh, Léo Tronchon. +1. **JAIS** (from Core42) released with the paper [Jais and Jais-chat: Arabic-Centric Foundation and Instruction-Tuned Open Generative Large Language Models](https://huggingface.co/papers/2308.16149) by Neha Sengupta, Sunil Kumar Sahu, Bokang Jia, Satheesh Katipomu, Haonan Li, Fajri Koto, William Marshall, Gurpreet Gosal, Cynthia Liu, Zhiming Chen, Osama Mohammed Afzal, Samta Kamboj, Onkar Pandit, Rahul Pal, Lalit Pradhan, Zain Muhammad Mujahid, Massa Baali, Xudong Han, Sondos Mahmoud Bsharat, Alham Fikri Aji, Zhiqiang Shen, Zhengzhong Liu, Natalia Vassilieva, Joel Hestness, Andy Hock, Andrew Feldman, Jonathan Lee, Andrew Jackson, Hector Xuguang Ren, Preslav Nakov, Timothy Baldwin, Eric Xing. +1. **Janus** (from DeepSeek) released with the paper [Janus: Decoupling Visual Encoding for Unified Multimodal Understanding and Generation](https://huggingface.co/papers/2410.13848) Chengyue Wu, Xiaokang Chen, Zhiyu Wu, Yiyang Ma, Xingchao Liu, Zizheng Pan, Wen Liu, Zhenda Xie, Xingkai Yu, Chong Ruan, Ping Luo. +1. **JinaCLIP** (from Jina AI) released with the paper [Jina CLIP: Your CLIP Model Is Also Your Text Retriever](https://huggingface.co/papers/2405.20204) by Andreas Koukounas, Georgios Mastrapas, Michael Günther, Bo Wang, Scott Martens, Isabelle Mohr, Saba Sturua, Mohammad Kalim Akram, Joan Fontanals Martínez, Saahil Ognawala, Susana Guzman, Maximilian Werk, Nan Wang, Han Xiao. +1. **LiteWhisper** (from University of Washington, Kotoba Technologies) released with the paper [LiteASR: Efficient Automatic Speech Recognition with Low-Rank Approximation](https://huggingface.co/papers/2502.20583) by Keisuke Kamahori, Jungo Kasai, Noriyuki Kojima, Baris Kasikci. +1. **[LongT5](https://huggingface.co/docs/transformers/model_doc/longt5)** (from Google AI) released with the paper [LongT5: Efficient Text-To-Text Transformer for Long Sequences](https://huggingface.co/papers/2112.07916) by Mandy Guo, Joshua Ainslie, David Uthus, Santiago Ontanon, Jianmo Ni, Yun-Hsuan Sung, Yinfei Yang. +1. **[LFM2](https://huggingface.co/docs/transformers/model_doc/lfm2)** (from Liquid AI) released with the blog post [Introducing LFM2: The Fastest On-Device Foundation Models on the Market](https://www.liquid.ai/blog/liquid-foundation-models-v2-our-second-series-of-generative-ai-models) by the Liquid AI Team. +1. **[Lfm2Moe](https://huggingface.co/docs/transformers/model_doc/lfm2_moe)** (from Liquid AI) released with the blog post [LFM2-8B-A1B: An Efficient On-device Mixture-of-Experts](https://www.liquid.ai/blog/lfm2-8b-a1b-an-efficient-on-device-mixture-of-experts) by the Liquid AI Team. +1. **[LFM2-VL](https://huggingface.co/docs/transformers/model_doc/lfm2_vl)** (from Liquid AI) released with the blog post [LFM2-VL: Efficient Vision-Language Models](https://www.liquid.ai/blog/lfm2-vl-efficient-vision-language-models) by the Liquid AI Team. +1. **[LightOnOcr](https://huggingface.co/docs/transformers/model_doc/lighton_ocr)** (from LightOn) released with the blog post [LightOnOCR-1B: The Case for End-to-End and Efficient Domain-Specific Vision-Language Models for OCR](https://huggingface.co/blog/lightonai/lightonocr) by the LightOn Team. +1. **[LLaMA](https://huggingface.co/docs/transformers/model_doc/llama)** (from The FAIR team of Meta AI) released with the paper [LLaMA: Open and Efficient Foundation Language Models](https://huggingface.co/papers/2302.13971) by Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, Guillaume Lample. +1. **[Llama2](https://huggingface.co/docs/transformers/model_doc/llama2)** (from The FAIR team of Meta AI) released with the paper [Llama2: Open Foundation and Fine-Tuned Chat Models](https://huggingface.co/papers/2307.09288) by Hugo Touvron, Louis Martin, Kevin Stone, Peter Albert, Amjad Almahairi, Yasmine Babaei, Nikolay Bashlykov, Soumya Batra, Prajjwal Bhargava, Shruti Bhosale, Dan Bikel, Lukas Blecher, Cristian Canton Ferrer, Moya Chen, Guillem Cucurull, David Esiobu, Jude Fernandes, Jeremy Fu, Wenyin Fu, Brian Fuller, Cynthia Gao, Vedanuj Goswami, Naman Goyal, Anthony Hartshorn, Saghar Hosseini, Rui Hou, Hakan Inan, Marcin Kardas, Viktor Kerkez Madian Khabsa, Isabel Kloumann, Artem Korenev, Punit Singh Koura, Marie-Anne Lachaux, Thibaut Lavril, Jenya Lee, Diana Liskovich, Yinghai Lu, Yuning Mao, Xavier Martinet, Todor Mihaylov, Pushka rMishra, Igor Molybog, Yixin Nie, Andrew Poulton, Jeremy Reizenstein, Rashi Rungta, Kalyan Saladi, Alan Schelten, Ruan Silva, Eric Michael Smith, Ranjan Subramanian, Xiaoqing EllenTan, Binh Tang, Ross Taylor, Adina Williams, Jian Xiang Kuan, Puxin Xu, Zheng Yan, Iliyan Zarov, Yuchen Zhang, Angela Fan, Melanie Kambadur, Sharan Narang, Aurelien Rodriguez, Robert Stojnic, Sergey Edunov, Thomas Scialom. +1. **[Llama3](https://huggingface.co/docs/transformers/model_doc/llama3)** (from The FAIR team of Meta AI) released with the paper [The Llama 3 Herd of Models](https://huggingface.co/papers/2407.21783) by the Llama Team at Meta. +1. **[Llama4](https://huggingface.co/docs/transformers/model_doc/llama4)** (from The FAIR team of Meta AI) released with the blog post [The Llama 4 herd: The beginning of a new era of natively multimodal AI innovation](https://ai.meta.com/blog/llama-4-multimodal-intelligence/) by the Llama Team at Meta. +1. **[LLaVa](https://huggingface.co/docs/transformers/model_doc/llava)** (from Microsoft Research & University of Wisconsin-Madison) released with the paper [Visual Instruction Tuning](https://huggingface.co/papers/2304.08485) by Haotian Liu, Chunyuan Li, Yuheng Li and Yong Jae Lee. +1. **[LLaVA-OneVision](https://huggingface.co/docs/transformers/model_doc/llava_onevision)** (from ByteDance & NTU & CUHK & HKUST) released with the paper [LLaVA-OneVision: Easy Visual Task Transfer](https://huggingface.co/papers/2408.03326) by Bo Li, Yuanhan Zhang, Dong Guo, Renrui Zhang, Feng Li, Hao Zhang, Kaichen Zhang, Yanwei Li, Ziwei Liu, Chunyuan Li +1. **[M2M100](https://huggingface.co/docs/transformers/model_doc/m2m_100)** (from Facebook) released with the paper [Beyond English-Centric Multilingual Machine Translation](https://huggingface.co/papers/2010.11125) by Angela Fan, Shruti Bhosale, Holger Schwenk, Zhiyi Ma, Ahmed El-Kishky, Siddharth Goyal, Mandeep Baines, Onur Celebi, Guillaume Wenzek, Vishrav Chaudhary, Naman Goyal, Tom Birch, Vitaliy Liptchinsky, Sergey Edunov, Edouard Grave, Michael Auli, Armand Joulin. +1. **[MarianMT](https://huggingface.co/docs/transformers/model_doc/marian)** Machine translation models trained using [OPUS](http://opus.nlpl.eu/) data by Jörg Tiedemann. The [Marian Framework](https://marian-nmt.github.io/) is being developed by the Microsoft Translator Team. +1. **[MaskFormer](https://huggingface.co/docs/transformers/model_doc/maskformer)** (from Meta and UIUC) released with the paper [Per-Pixel Classification is Not All You Need for Semantic Segmentation](https://huggingface.co/papers/2107.06278) by Bowen Cheng, Alexander G. Schwing, Alexander Kirillov. +1. **[mBART](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Denoising Pre-training for Neural Machine Translation](https://huggingface.co/papers/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer. +1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://huggingface.co/papers/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. +1. **Metric3D** released with the paper [Metric3D: Towards Zero-shot Metric 3D Prediction from A Single Image](https://huggingface.co/papers/2307.10984) by Wei Yin, Chi Zhang, Hao Chen, Zhipeng Cai, Gang Yu, Kaixuan Wang, Xiaozhi Chen, Chunhua Shen. +1. **Metric3Dv2** released with the paper [Metric3Dv2: A Versatile Monocular Geometric Foundation Model for Zero-shot Metric Depth and Surface Normal Estimation](https://huggingface.co/papers/2404.15506) by Mu Hu, Wei Yin, Chi Zhang, Zhipeng Cai, Xiaoxiao Long, Kaixuan Wang, Hao Chen, Gang Yu, Chunhua Shen, Shaojie Shen. +1. **[MusicGen](https://huggingface.co/docs/transformers/model_doc/musicgen)** (from Meta) released with the paper [Simple and Controllable Music Generation](https://huggingface.co/papers/2306.05284) by Jade Copet, Felix Kreuk, Itai Gat, Tal Remez, David Kant, Gabriel Synnaeve, Yossi Adi and Alexandre Défossez. +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://huggingface.co/papers/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. +1. **[Mimi](https://huggingface.co/docs/transformers/model_doc/mimi)** (from Kyutai) released with the paper [Moshi: a speech-text foundation model for real-time dialogue](https://huggingface.co/papers/2410.00037) by Alexandre Défossez, Laurent Mazaré, Manu Orsini, Amélie Royer, Patrick Pérez, Hervé Jégou, Edouard Grave and Neil Zeghidour. +1. **[Ministral](https://huggingface.co/docs/transformers/model_doc/ministral)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team. +1. **[Ministral3](https://huggingface.co/docs/transformers/model_doc/ministral3)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team. +1. **[Mistral](https://huggingface.co/docs/transformers/model_doc/mistral)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team: Albert Jiang, Alexandre Sablayrolles, Arthur Mensch, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Florian Bressand, Gianna Lengyel, Guillaume Lample, Lélio Renard Lavaud, Lucile Saulnier, Marie-Anne Lachaux, Pierre Stock, Teven Le Scao, Thibaut Lavril, Thomas Wang, Timothée Lacroix, William El Sayed. +1. **[Mistral3](https://huggingface.co/docs/transformers/model_doc/mistral3)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team. +1. **[Mistral4](https://huggingface.co/docs/transformers/model_doc/mistral4)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team. +1. **[MMS](https://huggingface.co/docs/transformers/model_doc/mms)** (from Facebook) released with the paper [Scaling Speech Technology to 1,000+ Languages](https://huggingface.co/papers/2305.13516) by Vineel Pratap, Andros Tjandra, Bowen Shi, Paden Tomasello, Arun Babu, Sayani Kundu, Ali Elkahky, Zhaoheng Ni, Apoorv Vyas, Maryam Fazel-Zarandi, Alexei Baevski, Yossi Adi, Xiaohui Zhang, Wei-Ning Hsu, Alexis Conneau, Michael Auli. +1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (from CMU/Google Brain) released with the paper [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://huggingface.co/papers/2004.02984) by Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou. +1. **MobileCLIP** (from Apple) released with the paper [MobileCLIP: Fast Image-Text Models through Multi-Modal Reinforced Training](https://huggingface.co/papers/2311.17049) by Pavan Kumar Anasosalu Vasu, Hadi Pouransari, Fartash Faghri, Raviteja Vemulapalli, Oncel Tuzel. +1. **MobileLLM** (from Meta) released with the paper [MobileLLM: Optimizing Sub-billion Parameter Language Models for On-Device Use Cases](https://huggingface.co/papers/2402.14905) by Zechun Liu, Changsheng Zhao, Forrest Iandola, Chen Lai, Yuandong Tian, Igor Fedorov, Yunyang Xiong, Ernie Chang, Yangyang Shi, Raghuraman Krishnamoorthi, Liangzhen Lai, Vikas Chandra. +1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://huggingface.co/papers/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam. +1. **[MobileNetV2](https://huggingface.co/docs/transformers/model_doc/mobilenet_v2)** (from Google Inc.) released with the paper [MobileNetV2: Inverted Residuals and Linear Bottlenecks](https://huggingface.co/papers/1801.04381) by Mark Sandler, Andrew Howard, Menglong Zhu, Andrey Zhmoginov, Liang-Chieh Chen. +1. **MobileNetV3** (from Google Inc.) released with the paper [Searching for MobileNetV3](https://huggingface.co/papers/1905.02244) by Andrew Howard, Mark Sandler, Grace Chu, Liang-Chieh Chen, Bo Chen, Mingxing Tan, Weijun Wang, Yukun Zhu, Ruoming Pang, Vijay Vasudevan, Quoc V. Le, Hartwig Adam. +1. **MobileNetV4** (from Google Inc.) released with the paper [MobileNetV4 - Universal Models for the Mobile Ecosystem](https://huggingface.co/papers/2404.10518) by Danfeng Qin, Chas Leichner, Manolis Delakis, Marco Fornoni, Shixin Luo, Fan Yang, Weijun Wang, Colby Banbury, Chengxi Ye, Berkin Akin, Vaibhav Aggarwal, Tenghui Zhu, Daniele Moro, Andrew Howard. +1. **[MobileViT](https://huggingface.co/docs/transformers/model_doc/mobilevit)** (from Apple) released with the paper [MobileViT: Light-weight, General-purpose, and Mobile-friendly Vision Transformer](https://huggingface.co/papers/2110.02178) by Sachin Mehta and Mohammad Rastegari. +1. **[MobileViTV2](https://huggingface.co/docs/transformers/model_doc/mobilevitv2)** (from Apple) released with the paper [Separable Self-attention for Mobile Vision Transformers](https://huggingface.co/papers/2206.02680) by Sachin Mehta and Mohammad Rastegari. +1. **[ModernBERT](https://huggingface.co/docs/transformers/model_doc/modernbert)** (from Answer.AI and LightOn) released with the paper [Smarter, Better, Faster, Longer: A Modern Bidirectional Encoder for Fast, Memory Efficient, and Long Context Finetuning and Inference](https://huggingface.co/papers/2412.13663) by Benjamin Warner, Antoine Chaffin, Benjamin Clavié, Orion Weller, Oskar Hallström, Said Taghadouini, Alexis Gallagher, Raja Biswas, Faisal Ladhak, Tom Aarsen, Nathan Cooper, Griffin Adams, Jeremy Howard, Iacopo Poli. +1. **[ModernBERT Decoder](https://huggingface.co/docs/transformers/model_doc/modernbert-decoder)** (from Johns Hopkins University and LightOn) released with the paper [Seq vs Seq: An Open Suite of Paired Encoders and Decoders](https://huggingface.co/papers/2507.11412) by Orion Weller, Kathryn Ricci, Marc Marone, Antoine Chaffin, Dawn Lawrie, Benjamin Van Durme. +1. **Moondream1** released in the repository [moondream](https://github.com/vikhyat/moondream) by vikhyat. +1. **[Moonshine](https://huggingface.co/docs/transformers/model_doc/moonshine)** (from Useful Sensors) released with the paper [Moonshine: Speech Recognition for Live Transcription and Voice Commands](https://huggingface.co/papers/2410.15608) by Nat Jeffries, Evan King, Manjunath Kudlur, Guy Nicholson, James Wang, Pete Warden. +1. **[MPNet](https://huggingface.co/docs/transformers/model_doc/mpnet)** (from Microsoft Research) released with the paper [MPNet: Masked and Permuted Pre-training for Language Understanding](https://huggingface.co/papers/2004.09297) by Kaitao Song, Xu Tan, Tao Qin, Jianfeng Lu, Tie-Yan Liu. +1. **[MPT](https://huggingface.co/docs/transformers/model_doc/mpt)** (from MosaicML) released with the repository [llm-foundry](https://github.com/mosaicml/llm-foundry/) by the MosaicML NLP Team. +1. **[MT5](https://huggingface.co/docs/transformers/model_doc/mt5)** (from Google AI) released with the paper [mT5: A massively multilingual pre-trained text-to-text transformer](https://huggingface.co/papers/2010.11934) by Linting Xue, Noah Constant, Adam Roberts, Mihir Kale, Rami Al-Rfou, Aditya Siddhant, Aditya Barua, Colin Raffel. +1. **[NanoChat](https://huggingface.co/docs/transformers/model_doc/nanochat)** released with the repository [nanochat: The best ChatGPT that $100 can buy](https://github.com/karpathy/nanochat) by Andrej Karpathy. +1. **[Nemotron](https://huggingface.co/docs/transformers/model_doc/nemotron)** (from NVIDIA) released with the blog post [NVIDIA AI Foundation Models: Build Custom Enterprise Chatbots and Co-Pilots with Production-Ready LLMs](https://developer.nvidia.com/blog/nvidia-ai-foundation-models-build-custom-enterprise-chatbots-and-co-pilots-with-production-ready-llms/) by the Nemotron team. +1. **[Nemotron-H](https://huggingface.co/docs/transformers/model_doc/nemotron_h)** (from NVIDIA) released with the papers [Nemotron-H: A Family of Accurate and Efficient Hybrid Mamba-Transformer Models](https://huggingface.co/papers/2504.03624), [NVIDIA Nemotron 3: Efficient and Open Intelligence](https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-White-Paper.pdf), [Nemotron 3 Nano: Open, Efficient Mixture-of-Experts Hybrid Mamba-Transformer Model for Agentic Reasoning](https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Nano-Technical-Report.pdf), and [Nemotron 3 Super: Open, Efficient Mixture-of-Experts Hybrid Mamba-Transformer Model for Agentic Reasoning](https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Super-Technical-Report.pdf) by the Nemotron team. +1. **NeoBERT** (from Chandar Research Lab) released with the paper [NeoBERT: A Next-Generation BERT](https://huggingface.co/papers/2502.19587) by Lola Le Breton, Quentin Fournier, Mariam El Mezouar, John X. Morris, Sarath Chandar. +1. **[NLLB](https://huggingface.co/docs/transformers/model_doc/nllb)** (from Meta) released with the paper [No Language Left Behind: Scaling Human-Centered Machine Translation](https://huggingface.co/papers/2207.04672) by the NLLB team. +1. **[Nougat](https://huggingface.co/docs/transformers/model_doc/nougat)** (from Meta AI) released with the paper [Nougat: Neural Optical Understanding for Academic Documents](https://huggingface.co/papers/2308.13418) by Lukas Blecher, Guillem Cucurull, Thomas Scialom, Robert Stojnic. +1. **[OLMo](https://huggingface.co/docs/transformers/master/model_doc/olmo)** (from Ai2) released with the paper [OLMo: Accelerating the Science of Language Models](https://huggingface.co/papers/2402.00838) by Dirk Groeneveld, Iz Beltagy, Pete Walsh, Akshita Bhagia, Rodney Kinney, Oyvind Tafjord, Ananya Harsh Jha, Hamish Ivison, Ian Magnusson, Yizhong Wang, Shane Arora, David Atkinson, Russell Authur, Khyathi Raghavi Chandu, Arman Cohan, Jennifer Dumas, Yanai Elazar, Yuling Gu, Jack Hessel, Tushar Khot, William Merrill, Jacob Morrison, Niklas Muennighoff, Aakanksha Naik, Crystal Nam, Matthew E. Peters, Valentina Pyatkin, Abhilasha Ravichander, Dustin Schwenk, Saurabh Shah, Will Smith, Emma Strubell, Nishant Subramani, Mitchell Wortsman, Pradeep Dasigi, Nathan Lambert, Kyle Richardson, Luke Zettlemoyer, Jesse Dodge, Kyle Lo, Luca Soldaini, Noah A. Smith, Hannaneh Hajishirzi. +1. **[OLMo2](https://huggingface.co/docs/transformers/master/model_doc/olmo2)** (from Ai2) released with the blog [OLMo 2: The best fully open language model to date](https://allenai.org/blog/olmo2) by the Ai2 OLMo team. +1. **[OLMo3](https://huggingface.co/docs/transformers/master/model_doc/olmo3)** (from Ai2) released with the blog [Olmo 3: Charting a path through the model flow to lead open-source AI](https://allenai.org/blog/olmo3) by the Ai2 OLMo team. +1. **[Olmo Hybrid](https://huggingface.co/docs/transformers/master/model_doc/olmo_hybrid)** (from Ai2) released with the blog [Introducing Olmo Hybrid: Combining transformers and linear RNNs for superior scaling](https://allenai.org/blog/olmohybrid) by the Ai2 OLMo team. +1. **OpenELM** (from Apple) released with the paper [OpenELM: An Efficient Language Model Family with Open-source Training and Inference Framework](https://huggingface.co/papers/2404.14619) by Sachin Mehta, Mohammad Hossein Sekhavat, Qingqing Cao, Maxwell Horton, Yanzi Jin, Chenfan Sun, Iman Mirzadeh, Mahyar Najibi, Dmitry Belenko, Peter Zatloukal, Mohammad Rastegari. +1. **[OPT](https://huggingface.co/docs/transformers/master/model_doc/opt)** (from Meta AI) released with the paper [OPT: Open Pre-trained Transformer Language Models](https://huggingface.co/papers/2205.01068) by Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen et al. +1. **[OWL-ViT](https://huggingface.co/docs/transformers/model_doc/owlvit)** (from Google AI) released with the paper [Simple Open-Vocabulary Object Detection with Vision Transformers](https://huggingface.co/papers/2205.06230) by Matthias Minderer, Alexey Gritsenko, Austin Stone, Maxim Neumann, Dirk Weissenborn, Alexey Dosovitskiy, Aravindh Mahendran, Anurag Arnab, Mostafa Dehghani, Zhuoran Shen, Xiao Wang, Xiaohua Zhai, Thomas Kipf, and Neil Houlsby. +1. **[OWLv2](https://huggingface.co/docs/transformers/model_doc/owlv2)** (from Google AI) released with the paper [Scaling Open-Vocabulary Object Detection](https://huggingface.co/papers/2306.09683) by Matthias Minderer, Alexey Gritsenko, Neil Houlsby. +1. **[PaliGemma](https://huggingface.co/docs/transformers/main/model_doc/paligemma)** (from Google) released with the papers [PaliGemma: A versatile 3B VLM for transfer](https://huggingface.co/papers/2407.07726) and [PaliGemma 2: A Family of Versatile VLMs for Transfer](https://huggingface.co/papers/2412.03555) by the PaliGemma Google team. +1. **[Parakeet](https://huggingface.co/docs/transformers/main/model_doc/parakeet)** (from NVIDIA) released with the blog post [Introducing the Parakeet ASR family](https://developer.nvidia.com/blog/pushing-the-boundaries-of-speech-recognition-with-nemo-parakeet-asr-models/) by the NVIDIA NeMo team. +1. **[PatchTSMixer](https://huggingface.co/docs/transformers/main/model_doc/patchtsmixer)** (from IBM) released with the paper [TSMixer: Lightweight MLP-Mixer Model for Multivariate Time Series Forecasting](https://huggingface.co/papers/2306.09364) by Vijay Ekambaram, Arindam Jati, Nam Nguyen, Phanwadee Sinthong, Jayant Kalagnanam. +1. **[PatchTST](https://huggingface.co/docs/transformers/main/model_doc/patchtst)** (from Princeton University, IBM) released with the paper [A Time Series is Worth 64 Words: Long-term Forecasting with Transformers](https://huggingface.co/papers/2211.14730) by Yuqi Nie, Nam H. Nguyen, Phanwadee Sinthong, Jayant Kalagnanam. +1. **[Phi](https://huggingface.co/docs/transformers/main/model_doc/phi)** (from Microsoft) released with the papers - [Textbooks Are All You Need](https://huggingface.co/papers/2306.11644) by Suriya Gunasekar, Yi Zhang, Jyoti Aneja, Caio César Teodoro Mendes, Allie Del Giorno, Sivakanth Gopi, Mojan Javaheripi, Piero Kauffmann, Gustavo de Rosa, Olli Saarikivi, Adil Salim, Shital Shah, Harkirat Singh Behl, Xin Wang, Sébastien Bubeck, Ronen Eldan, Adam Tauman Kalai, Yin Tat Lee and Yuanzhi Li, [Textbooks Are All You Need II: phi-1.5 technical report](https://huggingface.co/papers/2309.05463) by Yuanzhi Li, Sébastien Bubeck, Ronen Eldan, Allie Del Giorno, Suriya Gunasekar and Yin Tat Lee. +1. **[Phi3](https://huggingface.co/docs/transformers/main/model_doc/phi3)** (from Microsoft) released with the paper [Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone](https://huggingface.co/papers/2404.14219v2) by Marah Abdin, Sam Ade Jacobs, Ammar Ahmad Awan, Jyoti Aneja, Ahmed Awadallah, Hany Awadalla, Nguyen Bach, Amit Bahree, Arash Bakhtiari, Harkirat Behl, Alon Benhaim, Misha Bilenko, Johan Bjorck, Sébastien Bubeck, Martin Cai, Caio César Teodoro Mendes, Weizhu Chen, Vishrav Chaudhary, Parul Chopra, Allie Del Giorno, Gustavo de Rosa, Matthew Dixon, Ronen Eldan, Dan Iter, Amit Garg, Abhishek Goswami, Suriya Gunasekar, Emman Haider, Junheng Hao, Russell J. Hewett, Jamie Huynh, Mojan Javaheripi, Xin Jin, Piero Kauffmann, Nikos Karampatziakis, Dongwoo Kim, Mahoud Khademi, Lev Kurilenko, James R. Lee, Yin Tat Lee, Yuanzhi Li, Chen Liang, Weishung Liu, Eric Lin, Zeqi Lin, Piyush Madan, Arindam Mitra, Hardik Modi, Anh Nguyen, Brandon Norick, Barun Patra, Daniel Perez-Becker, Thomas Portet, Reid Pryzant, Heyang Qin, Marko Radmilac, Corby Rosset, Sambudha Roy, Olatunji Ruwase, Olli Saarikivi, Amin Saied, Adil Salim, Michael Santacroce, Shital Shah, Ning Shang, Hiteshi Sharma, Xia Song, Masahiro Tanaka, Xin Wang, Rachel Ward, Guanhua Wang, Philipp Witte, Michael Wyatt, Can Xu, Jiahang Xu, Sonali Yadav, Fan Yang, Ziyi Yang, Donghan Yu, Chengruidong Zhang, Cyril Zhang, Jianwen Zhang, Li Lyna Zhang, Yi Zhang, Yue Zhang, Yunan Zhang, Xiren Zhou. +1. **Phi3V** (from Microsoft) released with the paper [Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone](https://huggingface.co/papers/2404.14219v4) by Marah Abdin, Jyoti Aneja, Hany Awadalla, Ahmed Awadallah, Ammar Ahmad Awan, Nguyen Bach, Amit Bahree, Arash Bakhtiari, Jianmin Bao, Harkirat Behl, Alon Benhaim, Misha Bilenko, Johan Bjorck, Sébastien Bubeck, Martin Cai, Qin Cai, Vishrav Chaudhary, Dong Chen, Dongdong Chen, Weizhu Chen, Yen-Chun Chen, Yi-Ling Chen, Hao Cheng, Parul Chopra, Xiyang Dai, Matthew Dixon, Ronen Eldan, Victor Fragoso, Jianfeng Gao, Mei Gao, Min Gao, Amit Garg, Allie Del Giorno, Abhishek Goswami, Suriya Gunasekar, Emman Haider, Junheng Hao, Russell J. Hewett, Wenxiang Hu, Jamie Huynh, Dan Iter, Sam Ade Jacobs, Mojan Javaheripi, Xin Jin, Nikos Karampatziakis, Piero Kauffmann, Mahoud Khademi, Dongwoo Kim, Young Jin Kim, Lev Kurilenko, James R. Lee, Yin Tat Lee, Yuanzhi Li, Yunsheng Li, Chen Liang, Lars Liden, Xihui Lin, Zeqi Lin, Ce Liu, Liyuan Liu, Mengchen Liu, Weishung Liu, Xiaodong Liu, Chong Luo, Piyush Madan, Ali Mahmoudzadeh, David Majercak, Matt Mazzola, Caio César Teodoro Mendes, Arindam Mitra, Hardik Modi, Anh Nguyen, Brandon Norick, Barun Patra, Daniel Perez-Becker, Thomas Portet, Reid Pryzant, Heyang Qin, Marko Radmilac, Liliang Ren, Gustavo de Rosa, Corby Rosset, Sambudha Roy, Olatunji Ruwase, Olli Saarikivi, Amin Saied, Adil Salim, Michael Santacroce, Shital Shah, Ning Shang, Hiteshi Sharma, Yelong Shen, Swadheen Shukla, Xia Song, Masahiro Tanaka, Andrea Tupini, Praneetha Vaddamanu, Chunyu Wang, Guanhua Wang, Lijuan Wang , Shuohang Wang, Xin Wang, Yu Wang, Rachel Ward, Wen Wen, Philipp Witte, Haiping Wu, Xiaoxia Wu, Michael Wyatt, Bin Xiao, Can Xu, Jiahang Xu, Weijian Xu, Jilong Xue, Sonali Yadav, Fan Yang, Jianwei Yang, Yifan Yang, Ziyi Yang, Donghan Yu, Lu Yuan, Chenruidong Zhang, Cyril Zhang, Jianwen Zhang, Li Lyna Zhang, Yi Zhang, Yue Zhang, Yunan Zhang, Xiren Zhou. +1. **[PVT](https://huggingface.co/docs/transformers/main/model_doc/pvt)** (from Nanjing University, The University of Hong Kong etc.) released with the paper [Pyramid Vision Transformer: A Versatile Backbone for Dense Prediction without Convolutions](https://huggingface.co/papers/2102.12122) by Wenhai Wang, Enze Xie, Xiang Li, Deng-Ping Fan, Kaitao Song, Ding Liang, Tong Lu, Ping Luo, Ling Shao. +1. **PyAnnote** released in the repository [pyannote/pyannote-audio](https://github.com/pyannote/pyannote-audio) by Hervé Bredin. +1. **[Qwen2](https://huggingface.co/docs/transformers/model_doc/qwen2)** (from the Qwen team, Alibaba Group) released with the papers [Qwen Technical Report](https://huggingface.co/papers/2309.16609) and [Qwen2 Technical Report](https://huggingface.co/papers/2407.10671) by Jinze Bai, Shuai Bai, Yunfei Chu, Zeyu Cui, Kai Dang, Xiaodong Deng, Yang Fan, Wenbin Ge, Yu Han, Fei Huang, Binyuan Hui, Luo Ji, Mei Li, Junyang Lin, Runji Lin, Dayiheng Liu, Gao Liu, Chengqiang Lu, Keming Lu, Jianxin Ma, Rui Men, Xingzhang Ren, Xuancheng Ren, Chuanqi Tan, Sinan Tan, Jianhong Tu, Peng Wang, Shijie Wang, Wei Wang, Shengguang Wu, Benfeng Xu, Jin Xu, An Yang, Hao Yang, Jian Yang, Shusheng Yang, Yang Yao, Bowen Yu, Hongyi Yuan, Zheng Yuan, Jianwei Zhang, Xingxuan Zhang, Yichang Zhang, Zhenru Zhang, Chang Zhou, Jingren Zhou, Xiaohuan Zhou and Tianhang Zhu. +1. **[Qwen2 MoE](https://huggingface.co/docs/transformers/model_doc/qwen2_moe)** (from the Qwen team, Alibaba Group) released with the paper [Qwen2 Technical Report](https://huggingface.co/papers/2407.10671) by Jinze Bai, Shuai Bai, Yunfei Chu, Zeyu Cui, Kai Dang, Xiaodong Deng, Yang Fan, Wenbin Ge, Yu Han, Fei Huang, Binyuan Hui, Luo Ji, Mei Li, Junyang Lin, Runji Lin, Dayiheng Liu, Gao Liu, Chengqiang Lu, Keming Lu, Jianxin Ma, Rui Men, Xingzhang Ren, Xuancheng Ren, Chuanqi Tan, Sinan Tan, Jianhong Tu, Peng Wang, Shijie Wang, Wei Wang, Shengguang Wu, Benfeng Xu, Jin Xu, An Yang, Hao Yang, Jian Yang, Shusheng Yang, Yang Yao, Bowen Yu, Hongyi Yuan, Zheng Yuan, Jianwei Zhang, Xingxuan Zhang, Yichang Zhang, Zhenru Zhang, Chang Zhou, Jingren Zhou, Xiaohuan Zhou and Tianhang Zhu. +1. **[Qwen2-VL](https://huggingface.co/docs/transformers/model_doc/qwen2_vl)** (from the Qwen team, Alibaba Group) released with the papers [Qwen-VL: A Versatile Vision-Language Model for Understanding, Localization, Text Reading, and Beyond](https://huggingface.co/papers/2308.12966) and [Qwen2-VL: Enhancing Vision-Language Model's Perception of the World at Any Resolution](https://huggingface.co/papers/2409.12191) by Peng Wang, Shuai Bai, Sinan Tan, Shijie Wang, Zhihao Fan, Jinze Bai, Keqin Chen, Xuejing Liu, Jialin Wang, Wenbin Ge, Yang Fan, Kai Dang, Mengfei Du, Xuancheng Ren, Rui Men, Dayiheng Liu, Chang Zhou, Jingren Zhou, Junyang Lin. +1. **[Qwen2.5-VL](https://huggingface.co/docs/transformers/model_doc/qwen2_5_vl)** (from the Qwen team, Alibaba Group) released with the paper [Qwen2.5-VL Technical Report](https://huggingface.co/papers/2502.13923) by Shuai Bai, Keqin Chen, Xuejing Liu, Jialin Wang, Wenbin Ge, Sibo Song, Kai Dang, Peng Wang, Shijie Wang, Jun Tang, Humen Zhong, Yuanzhi Zhu, Mingkun Yang, Zhaohai Li, Jianqiang Wan, Pengfei Wang, Wei Ding, Zheren Fu, Yiheng Xu, Jiabo Ye, Xi Zhang, Tianbao Xie, Zesen Cheng, Hang Zhang, Zhibo Yang, Haiyang Xu, Junyang Lin. +1. **[Qwen3](https://huggingface.co/docs/transformers/en/model_doc/qwen3)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3: Think Deeper, Act Faster](https://qwen.ai/blog?id=qwen3) by the Qwen team. +1. **[Qwen3 MoE](https://huggingface.co/docs/transformers/en/model_doc/qwen3_moe)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3: Think Deeper, Act Faster](https://qwen.ai/blog?id=qwen3) by the Qwen team. +1. **[Qwen3 Next](https://huggingface.co/docs/transformers/en/model_doc/qwen3_next)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3-Next: Towards Ultimate Training & Inference Efficiency](https://qwen.ai/blog?id=qwen3-next) by the Qwen team. +1. **[Qwen3-VL](https://huggingface.co/docs/transformers/en/model_doc/qwen3_vl)** (from the Qwen team, Alibaba Group) released with the paper [Qwen3-VL Technical Report](https://huggingface.co/papers/2511.21631) by Shuai Bai, Yuxuan Cai, Ruizhe Chen, Keqin Chen, Xionghui Chen, Zesen Cheng, Lianghao Deng, Wei Ding, Chang Gao, Chunjiang Ge, Wenbin Ge, Zhifang Guo, Qidong Huang, Jie Huang, Fei Huang, Binyuan Hui, Shutong Jiang, Zhaohai Li, Mingsheng Li, Mei Li, Kaixin Li, Zicheng Lin, Junyang Lin, Xuejing Liu, Jiawei Liu, Chenglong Liu, Yang Liu, Dayiheng Liu, Shixuan Liu, Dunjie Lu, Ruilin Luo, Chenxu Lv, Rui Men, Lingchen Meng, Xuancheng Ren, Xingzhang Ren, Sibo Song, Yuchong Sun, Jun Tang, Jianhong Tu, Jianqiang Wan, Peng Wang, Pengfei Wang, Qiuyue Wang, Yuxuan Wang, Tianbao Xie, Yiheng Xu, Haiyang Xu, Jin Xu, Zhibo Yang, Mingkun Yang, Jianxin Yang, An Yang, Bowen Yu, Fei Zhang, Hang Zhang, Xi Zhang, Bo Zheng, Humen Zhong, Jingren Zhou, Fan Zhou, Jing Zhou, Yuanzhi Zhu, Ke Zhu. +1. **[Qwen3-VL MoE](https://huggingface.co/docs/transformers/en/model_doc/qwen3_vl_moe)** (from the Qwen team, Alibaba Group) released with the paper [Qwen3-VL Technical Report](https://huggingface.co/papers/2511.21631) by Shuai Bai, Yuxuan Cai, Ruizhe Chen, Keqin Chen, Xionghui Chen, Zesen Cheng, Lianghao Deng, Wei Ding, Chang Gao, Chunjiang Ge, Wenbin Ge, Zhifang Guo, Qidong Huang, Jie Huang, Fei Huang, Binyuan Hui, Shutong Jiang, Zhaohai Li, Mingsheng Li, Mei Li, Kaixin Li, Zicheng Lin, Junyang Lin, Xuejing Liu, Jiawei Liu, Chenglong Liu, Yang Liu, Dayiheng Liu, Shixuan Liu, Dunjie Lu, Ruilin Luo, Chenxu Lv, Rui Men, Lingchen Meng, Xuancheng Ren, Xingzhang Ren, Sibo Song, Yuchong Sun, Jun Tang, Jianhong Tu, Jianqiang Wan, Peng Wang, Pengfei Wang, Qiuyue Wang, Yuxuan Wang, Tianbao Xie, Yiheng Xu, Haiyang Xu, Jin Xu, Zhibo Yang, Mingkun Yang, Jianxin Yang, An Yang, Bowen Yu, Fei Zhang, Hang Zhang, Xi Zhang, Bo Zheng, Humen Zhong, Jingren Zhou, Fan Zhou, Jing Zhou, Yuanzhi Zhu, Ke Zhu. +1. **[Qwen3.5](https://huggingface.co/docs/transformers/en/model_doc/qwen3_5)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3.5: Towards Native Multimodal Agents](https://qwen.ai/blog?id=qwen3.5) by the Qwen team. +1. **[Qwen3.5 Moe](https://huggingface.co/docs/transformers/en/model_doc/qwen3_5_moe)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3.5: Towards Native Multimodal Agents](https://qwen.ai/blog?id=qwen3.5) by the Qwen team. +1. **[ResNet](https://huggingface.co/docs/transformers/model_doc/resnet)** (from Microsoft Research) released with the paper [Deep Residual Learning for Image Recognition](https://huggingface.co/papers/1512.03385) by Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. +1. **[RF-DETR](https://huggingface.co/docs/transformers/model_doc/rf_detr)** (from Roboflow) released with the blog post [RF-DETR: A SOTA Real-Time Object Detection Model](https://blog.roboflow.com/rf-detr/) by Peter Robicheaux, James Gallagher, Joseph Nelson, Isaac Robinson. +1. **[RoBERTa](https://huggingface.co/docs/transformers/model_doc/roberta)** (from Facebook), released together with the paper [RoBERTa: A Robustly Optimized BERT Pretraining Approach](https://huggingface.co/papers/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. +1. **[RoFormer](https://huggingface.co/docs/transformers/model_doc/roformer)** (from ZhuiyiTechnology), released together with the paper [RoFormer: Enhanced Transformer with Rotary Position Embedding](https://huggingface.co/papers/2104.09864) by Jianlin Su and Yu Lu and Shengfeng Pan and Bo Wen and Yunfeng Liu. +1. **[RT-DETR](https://huggingface.co/docs/transformers/model_doc/rt_detr)** (from Baidu), released together with the paper [DETRs Beat YOLOs on Real-time Object Detection](https://huggingface.co/papers/2304.08069) by Yian Zhao, Wenyu Lv, Shangliang Xu, Jinman Wei, Guanzhong Wang, Qingqing Dang, Yi Liu, Jie Chen. +1. **[RT-DETRv2](https://huggingface.co/docs/transformers/model_doc/rt_detr_v2)** (from Baidu), released together with the paper [RT-DETRv2: Improved Baseline with Bag-of-Freebies for Real-Time Detection Transformer](https://huggingface.co/papers/2407.17140) by Wenyu Lv, Yian Zhao, Qinyao Chang, Kui Huang, Guanzhong Wang, Yi Liu. +1. **Sapiens** (from Meta AI) released with the paper [Sapiens: Foundation for Human Vision Models](https://huggingface.co/papers/2408.12569) by Rawal Khirodkar, Timur Bagautdinov, Julieta Martinez, Su Zhaoen, Austin James, Peter Selednik, Stuart Anderson, Shunsuke Saito. +1. **[SegFormer](https://huggingface.co/docs/transformers/model_doc/segformer)** (from NVIDIA) released with the paper [SegFormer: Simple and Efficient Design for Semantic Segmentation with Transformers](https://huggingface.co/papers/2105.15203) by Enze Xie, Wenhai Wang, Zhiding Yu, Anima Anandkumar, Jose M. Alvarez, Ping Luo. +1. **[Segment Anything](https://huggingface.co/docs/transformers/model_doc/sam)** (from Meta AI) released with the paper [Segment Anything](https://huggingface.co/papers/2304.02643v1.pdf) by Alexander Kirillov, Eric Mintun, Nikhila Ravi, Hanzi Mao, Chloe Rolland, Laura Gustafson, Tete Xiao, Spencer Whitehead, Alex Berg, Wan-Yen Lo, Piotr Dollar, Ross Girshick. +1. **[Segment Anything 2](https://huggingface.co/docs/transformers/model_doc/sam2)** (from Meta AI) released with the paper [SAM 2: Segment Anything in Images and Videos](https://huggingface.co/papers/2408.00714) by Nikhila Ravi, Valentin Gabeur, Yuan-Ting Hu, Ronghang Hu, Chaitanya Ryali, Tengyu Ma, Haitham Khedr, Roman Rädle, Chloe Rolland, Laura Gustafson, Eric Mintun, Junting Pan, Kalyan Vasudev Alwala, Nicolas Carion, Chao-Yuan Wu, Ross Girshick, Piotr Dollár, Christoph Feichtenhofer. +1. **[Segment Anything 3](https://huggingface.co/docs/transformers/model_doc/sam3)** (from Meta Superintelligence Labs) released with the paper [SAM 3: Segment Anything with Concepts](https://ai.meta.com/research/publications/sam-3-segment-anything-with-concepts/) by SAM 3D Team, Xingyu Chen, Fu-Jen Chu, Pierre Gleize, Kevin J Liang, Alexander Sax, Hao Tang, Weiyao Wang, Michelle Guo, Thibaut Hardin, Xiang Li, Aohan Lin, Jiawei Liu, Ziqi Ma, Anushka Sagar, Bowen Song, Xiaodong Wang, Jianing Yang, Bowen Zhang, Piotr Dollar, Georgia Gkioxari, Matt Feiszli, Jitendra Malik, Nicolas Carion, Laura Gustafson, Yuan-Ting Hu, Shoubhik Debnath, Ronghang Hu, Didac Suris Coll-Vinent, Chaitanya Ryali, Kalyan Vasudev Alwala, Haitham Khedr, Andrew Huang, Jie Lei, Tengyu Ma, Baishan Guo, Arpit Kalla, Markus Marks, Joseph Greer, Meng Wang, Peize Sun, Roman Rädle, Triantafyllos Afouras, Effrosyni Mavroudi, Katherine Xu, Tsung-Han Wu, Yu Zhou, Liliane Momeni, Rishi Hazra, Shuangrui Ding, Sagar Vaze, Francois Porcher, Feng Li, Siyuan Li, Aishwarya Kamath, Ho Kei Cheng, Piotr Dollar, Nikhila Ravi, Kate Saenko, Pengchuan Zhang, Christoph Feichtenhofer. +1. **[SigLIP](https://huggingface.co/docs/transformers/main/model_doc/siglip)** (from Google AI) released with the paper [Sigmoid Loss for Language Image Pre-Training](https://huggingface.co/papers/2303.15343) by Xiaohua Zhai, Basil Mustafa, Alexander Kolesnikov, Lucas Beyer. +1. **[SmolLM3](https://huggingface.co/docs/transformers/main/model_doc/smollm3) (from Hugging Face) released with the blog post [SmolLM3: smol, multilingual, long-context reasoner](https://huggingface.co/blog/smollm3) by the Hugging Face TB Research team. +1. **[SmolVLM](https://huggingface.co/docs/transformers/main/model_doc/smolvlm) (from Hugging Face) released with the blog posts [SmolVLM - small yet mighty Vision Language Model](https://huggingface.co/blog/smolvlm) and [SmolVLM Grows Smaller – Introducing the 250M & 500M Models!](https://huggingface.co/blog/smolervlm) by the Hugging Face TB Research team. +1. **[SolarOpen](https://huggingface.co/docs/transformers/main/model_doc/solar_open)** (from Upstage) released with the paper [Solar Open Technical Report](https://huggingface.co/papers/2601.07022) by Sungrae Park, Sanghoon Kim, Jungho Cho, Gyoungjin Gim, Dawoon Jung, Mikyoung Cha, Eunhae Choo, Taekgyu Hong, Minbyul Jeong, SeHwan Joo, Minsoo Khang, Eunwon Kim, Minjeong Kim, Sujeong Kim, Yunsu Kim, Hyeonju Lee, Seunghyun Lee, Sukyung Lee, Siyoung Park, Gyungin Shin, Inseo Song, Wonho Song, Seonghoon Yang, Seungyoun Yi, Sanghoon Yoon, Jeonghyun Ko, Seyoung Song, Keunwoo Choi, Hwalsuk Lee, Sunghun Kim, Du-Seong Chang, Kyunghyun Cho, Junsuk Choe, Hwaran Lee, Jae-Gil Lee, KyungTae Lim, Alice Oh. +1. **SNAC** (from Papla Media, ETH Zurich) released with the paper [SNAC: Multi-Scale Neural Audio Codec](https://huggingface.co/papers/2410.14411) by Hubert Siuzdak, Florian Grötschla, Luca A. Lanzendörfer. +1. **[SpeechT5](https://huggingface.co/docs/transformers/model_doc/speecht5)** (from Microsoft Research) released with the paper [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://huggingface.co/papers/2110.07205) by Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei. +1. **[SqueezeBERT](https://huggingface.co/docs/transformers/model_doc/squeezebert)** (from Berkeley) released with the paper [SqueezeBERT: What can computer vision teach NLP about efficient neural networks?](https://huggingface.co/papers/2006.11316) by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W. Keutzer. +1. **[StableLm](https://huggingface.co/docs/transformers/model_doc/stablelm)** (from Stability AI) released with the paper [StableLM 3B 4E1T (Technical Report)](https://stability.wandb.io/stability-llm/stable-lm/reports/StableLM-3B-4E1T--VmlldzoyMjU4?accessToken=u3zujipenkx5g7rtcj9qojjgxpconyjktjkli2po09nffrffdhhchq045vp0wyfo) by Jonathan Tow, Marco Bellagente, Dakota Mahan, Carlos Riquelme Ruiz, Duy Phung, Maksym Zhuravinskyi, Nathan Cooper, Nikhil Pinnaparaju, Reshinth Adithyan, and James Baicoianu. +1. **[Starcoder2](https://huggingface.co/docs/transformers/main/model_doc/starcoder2)** (from BigCode team) released with the paper [StarCoder 2 and The Stack v2: The Next Generation](https://huggingface.co/papers/2402.19173) by Anton Lozhkov, Raymond Li, Loubna Ben Allal, Federico Cassano, Joel Lamy-Poirier, Nouamane Tazi, Ao Tang, Dmytro Pykhtar, Jiawei Liu, Yuxiang Wei, Tianyang Liu, Max Tian, Denis Kocetkov, Arthur Zucker, Younes Belkada, Zijian Wang, Qian Liu, Dmitry Abulkhanov, Indraneil Paul, Zhuang Li, Wen-Ding Li, Megan Risdal, Jia Li, Jian Zhu, Terry Yue Zhuo, Evgenii Zheltonozhskii, Nii Osae Osae Dade, Wenhao Yu, Lucas Krauß, Naman Jain, Yixuan Su, Xuanli He, Manan Dey, Edoardo Abati, Yekun Chai, Niklas Muennighoff, Xiangru Tang, Muhtasham Oblokulov, Christopher Akiki, Marc Marone, Chenghao Mou, Mayank Mishra, Alex Gu, Binyuan Hui, Tri Dao, Armel Zebaze, Olivier Dehaene, Nicolas Patry, Canwen Xu, Julian McAuley, Han Hu, Torsten Scholak, Sebastien Paquet, Jennifer Robinson, Carolyn Jane Anderson, Nicolas Chapados, Mostofa Patwary, Nima Tajbakhsh, Yacine Jernite, Carlos Muñoz Ferrandis, Lingming Zhang, Sean Hughes, Thomas Wolf, Arjun Guha, Leandro von Werra, and Harm de Vries. +1. **StyleTTS 2** (from Columbia University) released with the paper [StyleTTS 2: Towards Human-Level Text-to-Speech through Style Diffusion and Adversarial Training with Large Speech Language Models](https://huggingface.co/papers/2306.07691) by Yinghao Aaron Li, Cong Han, Vinay S. Raghavan, Gavin Mischler, Nima Mesgarani. +1. **Supertonic** (from Supertone) released with the paper [SupertonicTTS: Towards Highly Efficient and Streamlined Text-to-Speech System](https://huggingface.co/papers/2503.23108) by Hyeongju Kim, Jinhyeok Yang, Yechan Yu, Seunghun Ji, Jacob Morton, Frederik Bous, Joon Byun, Juheon Lee. +1. **[Swin Transformer](https://huggingface.co/docs/transformers/model_doc/swin)** (from Microsoft) released with the paper [Swin Transformer: Hierarchical Vision Transformer using Shifted Windows](https://huggingface.co/papers/2103.14030) by Ze Liu, Yutong Lin, Yue Cao, Han Hu, Yixuan Wei, Zheng Zhang, Stephen Lin, Baining Guo. +1. **[Swin2SR](https://huggingface.co/docs/transformers/model_doc/swin2sr)** (from University of Würzburg) released with the paper [Swin2SR: SwinV2 Transformer for Compressed Image Super-Resolution and Restoration](https://huggingface.co/papers/2209.11345) by Marcos V. Conde, Ui-Jin Choi, Maxime Burchi, Radu Timofte. +1. **[T5](https://huggingface.co/docs/transformers/model_doc/t5)** (from Google AI) released with the paper [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://huggingface.co/papers/1910.10683) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu. +1. **[T5v1.1](https://huggingface.co/docs/transformers/model_doc/t5v1.1)** (from Google AI) released in the repository [google-research/text-to-text-transfer-transformer](https://github.com/google-research/text-to-text-transfer-transformer/blob/main/released_checkpoints.md#t511) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu. +1. **[Table Transformer](https://huggingface.co/docs/transformers/model_doc/table-transformer)** (from Microsoft Research) released with the paper [PubTables-1M: Towards Comprehensive Table Extraction From Unstructured Documents](https://huggingface.co/papers/2110.00061) by Brandon Smock, Rohith Pesala, Robin Abraham. +1. **[TrOCR](https://huggingface.co/docs/transformers/model_doc/trocr)** (from Microsoft), released together with the paper [TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models](https://huggingface.co/papers/2109.10282) by Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei. +1. **Ultravox** (from Fixie.ai) released with the repository [fixie-ai/ultravox](https://github.com/fixie-ai/ultravox) by the Fixie.ai team. +1. **[UniSpeech](https://huggingface.co/docs/transformers/model_doc/unispeech)** (from Microsoft Research) released with the paper [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled Data](https://huggingface.co/papers/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei, Michael Zeng, Xuedong Huang. +1. **[UniSpeechSat](https://huggingface.co/docs/transformers/model_doc/unispeech-sat)** (from Microsoft Research) released with the paper [UNISPEECH-SAT: UNIVERSAL SPEECH REPRESENTATION LEARNING WITH SPEAKER AWARE PRE-TRAINING](https://huggingface.co/papers/2110.05752) by Sanyuan Chen, Yu Wu, Chengyi Wang, Zhengyang Chen, Zhuo Chen, Shujie Liu, Jian Wu, Yao Qian, Furu Wei, Jinyu Li, Xiangzhan Yu. +1. **[VaultGemma](https://huggingface.co/docs/transformers/main/model_doc/vaultgemma)** (from Google) released with the technical report [VaultGemma: A Differentially Private Gemma Model](https://services.google.com/fh/files/blogs/vaultgemma_tech_report.pdf) by the VaultGemma Google team. +1. **[Vision Transformer (ViT)](https://huggingface.co/docs/transformers/model_doc/vit)** (from Google AI) released with the paper [An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale](https://huggingface.co/papers/2010.11929) by Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, Georg Heigold, Sylvain Gelly, Jakob Uszkoreit, Neil Houlsby. +1. **[ViTMAE](https://huggingface.co/docs/transformers/model_doc/vit_mae)** (from Meta AI) released with the paper [Masked Autoencoders Are Scalable Vision Learners](https://huggingface.co/papers/2111.06377) by Kaiming He, Xinlei Chen, Saining Xie, Yanghao Li, Piotr Dollár, Ross Girshick. +1. **[ViTMatte](https://huggingface.co/docs/transformers/model_doc/vitmatte)** (from HUST-VL) released with the paper [ViTMatte: Boosting Image Matting with Pretrained Plain Vision Transformers](https://huggingface.co/papers/2305.15272) by Jingfeng Yao, Xinggang Wang, Shusheng Yang, Baoyuan Wang. +1. **[ViTMSN](https://huggingface.co/docs/transformers/model_doc/vit_msn)** (from Meta AI) released with the paper [Masked Siamese Networks for Label-Efficient Learning](https://huggingface.co/papers/2204.07141) by Mahmoud Assran, Mathilde Caron, Ishan Misra, Piotr Bojanowski, Florian Bordes, Pascal Vincent, Armand Joulin, Michael Rabbat, Nicolas Ballas. +1. **[ViTPose](https://huggingface.co/docs/transformers/model_doc/vitpose)** (from The University of Sydney) released with the paper [ViTPose: Simple Vision Transformer Baselines for Human Pose Estimation](https://huggingface.co/papers/2204.12484) by Yufei Xu, Jing Zhang, Qiming Zhang, Dacheng Tao. +1. **[VITS](https://huggingface.co/docs/transformers/model_doc/vits)** (from Kakao Enterprise) released with the paper [Conditional Variational Autoencoder with Adversarial Learning for End-to-End Text-to-Speech](https://huggingface.co/papers/2106.06103) by Jaehyeon Kim, Jungil Kong, Juhee Son. +1. **[Voxtral](https://huggingface.co/docs/transformers/model_doc/voxtral)** (from Mistral AI) released with the paper [Voxtral](https://huggingface.co/papers/2507.13264) by Alexander H. Liu, Andy Ehrenberg, Andy Lo, Clément Denoix, Corentin Barreau, Guillaume Lample, Jean-Malo Delignon, Khyathi Raghavi Chandu, Patrick von Platen, Pavankumar Reddy Muddireddy, Sanchit Gandhi, Soham Ghosh, Srijan Mishra, Thomas Foubert, Abhinav Rastogi, Adam Yang, Albert Q. Jiang, Alexandre Sablayrolles, Amélie Héliou, Amélie Martin, Anmol Agarwal, Antoine Roux, Arthur Darcet, Arthur Mensch, Baptiste Bout, Baptiste Rozière, Baudouin De Monicault, Chris Bamford, Christian Wallenwein, Christophe Renaudin, Clémence Lanfranchi, Darius Dabert, Devendra Singh Chaplot, Devon Mizelle, Diego de las Casas, Elliot Chane-Sane, Emilien Fugier, Emma Bou Hanna, Gabrielle Berrada, Gauthier Delerce, Gauthier Guinet, Georgii Novikov, Guillaume Martin, Himanshu Jaju, Jan Ludziejewski, Jason Rute, Jean-Hadrien Chabran, Jessica Chudnovsky, Joachim Studnia, Joep Barmentlo, Jonas Amar, Josselin Somerville Roberts, Julien Denize, Karan Saxena, Karmesh Yadav, Kartik Khandelwal, Kush Jain, Lélio Renard Lavaud, Léonard Blier, Lingxiao Zhao, Louis Martin, Lucile Saulnier, Luyu Gao, Marie Pellat, Mathilde Guillaumin, Mathis Felardos, Matthieu Dinot, Maxime Darrin, Maximilian Augustin, Mickaël Seznec, Neha Gupta, Nikhil Raghuraman, Olivier Duchenne, Patricia Wang, Patryk Saffer, Paul Jacob, Paul Wambergue, Paula Kurylowicz, Philomène Chagniot, Pierre Stock, Pravesh Agrawal, Rémi Delacourt, Romain Sauvestre, Roman Soletskyi, Sagar Vaze, Sandeep Subramanian, Saurabh Garg, Shashwat Dalal, Siddharth Gandhi, Sumukh Aithal, Szymon Antoniak, Teven Le Scao, Thibault Schueller, Thibaut Lavril, Thomas Robert, Thomas Wang, Timothée Lacroix, Tom Bewley, Valeriia Nemychnikova, Victor Paltz , Virgile Richard, Wen-Ding Li, William Marshall, Xuanyu Zhang, Yihan Wan, Yunhao Tang. +1. **[Voxtral Realtime](https://huggingface.co/docs/transformers/model_doc/voxtral_realtime)** (from Mistral AI) released with the paper [Voxtral Realtime](https://huggingface.co/papers/2602.11298) by Alexander H. Liu, Andy Ehrenberg, Andy Lo, Chen-Yo Sun, Guillaume Lample, Jean-Malo Delignon, Khyathi Raghavi Chandu, Patrick von Platen, Pavankumar Reddy Muddireddy, Rohin Arora, Sanchit Gandhi, Sandeep Subramanian, Soham Ghosh, Srijan Mishra, Abhinav Rastogi, Adrien Sadé, Alan Jeffares, Albert Jiang, Alexandre Cahill, Alexandre Gavaudan, Alexandre Sablayrolles, Amélie Héliou, Amos You, Andrew Bai, Angele Lenglemetz, Anmol Agarwal, Anton Eliseev, Antonia Calvi, Arjun Majumdar, Avi Sooriyarachchi, Baptiste Bout, Baptiste Rozière, Baudouin De Monicault, Benjamin Tibi, Charlotte Cronjäger, Clémence Lanfranchi, Connor Chen, Corentin Barreau, Corentin Sautier, Cyprien Courtot, Darius Dabert, Diego de las Casas, Elizaveta Demyanenko, Elliot Chane-Sane, Enguerrand Paquin, Etienne Goffinet, Fabien Niel, Faruk Ahmed, Federico Baldassarre, Gabrielle Berrada, Gaëtan Ecrepont, Gauthier Guinet, Genevieve Hayes, Georgii Novikov, Giada Pistilli, Guillaume Kunsch, Guillaume Martin, Guillaume Raille, Gunjan Dhanuka, Gunshi Gupta, Han Zhou, Harshil Shah, Hope McGovern, Hugo Thimonier, Indraneel Mukherjee, Irene Zhang, Jaeyoung Kim, Jan Ludziejewski, Jason Rute, Joachim Studnia, John Harvill, Jonas Amar, Joséphine Delas, Josselin Somerville Roberts, Julien Tauran, Karmesh Yadav, Kartik Khandelwal, Kilian Tep, Kush Jain, Laurence Aitchison, Laurent Fainsin, Léonard Blier, Lingxiao Zhao, Louis Martin, Lucile Saulnier, Luyu Gao, Maarten Buyl, Manan Sharma, Margaret Jennings, Marie Pellat, Mark Prins, Martin Alexandre, Mathieu Poirée, Mathilde Guillaumin, Matthieu Dinot, Matthieu Futeral, Maxime Darrin, Maximilian Augustin, Mert Unsal, Mia Chiquier , Minh-Quang Pham, Nathan Grinsztajn, Neha Gupta, Olivier Bousquet, Olivier Duchenne, Patricia Wang, Paul Jacob, Paul Wambergue, Paula Kurylowicz, Philippe Pinel, Philomène Chagniot, Pierre Stock, Piotr Miłoś, Prateek Gupta, Pravesh Agrawal, Quentin Torroba, Ram Ramrakhya, Rishi Shah, Romain Sauvestre, Roman Soletskyi, Rosalie Millner, Rupert Menneer, Sagar Vaze, Samuel Barry, Samuel Humeau, Sean Cha, Shashwat Verma, Siddhant Waghjale, Siddharth Gandhi, Simon Lepage, Sumukh Aithal, Szymon Antoniak, Teven Le Scao, Théo Cachet, Theo Simon Sorg, Thibaut Lavril, Thomas Chabal, Thomas Foubert, Thomas Robert, Thomas Wang, Tim Lawson, Tom Bewley, Tom Edwards, Tyler Wang, Umar Jamil, Umberto Tomasini, Valeriia Nemychnikova, Van Phung, Vedant Nanda, Victor Jouault, Vincent Maladière, Virgile Richard, Vladislav Bataev, Wassim Bouaziz, Wen-Ding Li, William Havard, William Marshall, Xinghui Li, Xingran Guo, Xinyu Yang, Yannic Neuhaus, Yassine El Ouahidi, Yassir Bendou, Yihan Wang, Yimu Pan, Zaccharie Ramzi, Zhenlin Xu. +1. **[Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2)** (from Facebook AI) released with the paper [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations](https://huggingface.co/papers/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael Auli. +1. **[Wav2Vec2-BERT](https://huggingface.co/docs/transformers/main/model_doc/wav2vec2-bert)** (from Meta AI) released with the paper [Seamless: Multilingual Expressive and Streaming Speech Translation](https://ai.meta.com/research/publications/seamless-multilingual-expressive-and-streaming-speech-translation/) by the Seamless Communication team. +1. **[WavLM](https://huggingface.co/docs/transformers/model_doc/wavlm)** (from Microsoft Research) released with the paper [WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://huggingface.co/papers/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei. +1. **[Whisper](https://huggingface.co/docs/transformers/model_doc/whisper)** (from OpenAI) released with the paper [Robust Speech Recognition via Large-Scale Weak Supervision](https://cdn.openai.com/papers/whisper.pdf) by Alec Radford, Jong Wook Kim, Tao Xu, Greg Brockman, Christine McLeavey, Ilya Sutskever. +1. **[XLM](https://huggingface.co/docs/transformers/model_doc/xlm)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://huggingface.co/papers/1901.07291) by Guillaume Lample and Alexis Conneau. +1. **[XLM-RoBERTa](https://huggingface.co/docs/transformers/model_doc/xlm-roberta)** (from Facebook AI), released together with the paper [Unsupervised Cross-lingual Representation Learning at Scale](https://huggingface.co/papers/1911.02116) by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov. +1. **[YOLOS](https://huggingface.co/docs/transformers/model_doc/yolos)** (from Huazhong University of Science & Technology) released with the paper [You Only Look at One Sequence: Rethinking Transformer in Vision through Object Detection](https://huggingface.co/papers/2106.00666) by Yuxin Fang, Bencheng Liao, Xinggang Wang, Jiemin Fang, Jiyang Qi, Rui Wu, Jianwei Niu, Wenyu Liu. +1. **[Youtu-LLM](https://huggingface.co/docs/transformers/model_doc/youtu)** (from the Tencent Youtu Team) released with the paper [Youtu-LLM: Unlocking the Native Agentic Potential for Lightweight Large Language Models](https://huggingface.co/papers/2512.24618) by Junru Lu, Jiarui Qin, Lingfeng Qiao, Yinghui Li, Xinyi Dai, Bo Ke, Jianfeng He, Ruizhi Qiao, Di Yin, Xing Sun, Yunsheng Wu, Yinsong Liu, Shuangyin Liu, Mingkong Tang, Haodong Lin, Jiayi Kuang, Fanxu Meng, Xiaojuan Tang, Yunjia Xi, Junjie Huang, Haotong Yang, Zhenyi Shen, Yangning Li, Qianwen Zhang, Yifei Yu, Siyu An, Junnan Dong, Qiufeng Wang, Jie Wang, Keyu Chen, Wei Wen, Taian Guo, Zhifeng Shen, Daohai Yu, Jiahao Li, Ke Li, Zongyi Li, Xiaoyu Tan. +1. **Zaya** (from Zyphra) released with the paper [ZAYA1-8B Technical Report](https://huggingface.co/papers/2605.05365) by Robert Washbourne, Rishi Iyer, Tomas Figliolia, Henry Zheng, Ryan Lorig-Roach, Sungyeon Yang, Pritish Yuvraj, Quentin Anthony, Yury Tokpanov, Xiao Yang, Ganesh Nanduru, Stephen Ebert, Praneeth Medepalli, Skyler Szot, Srivatsan Rajagopal, Alex Ong, Bhavana Mehta, Beren Millidge. + diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..66903b6 --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目：`huggingface/transformers.js` +- 原始仓库：https://github.com/huggingface/transformers.js +- 导入方式：上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源，不代表 WeHub 是原项目作者 diff --git a/package.json b/package.json new file mode 100644 index 0000000..af64a6f --- /dev/null +++ b/package.json @@ -0,0 +1,63 @@ +{ + "name": "@huggingface/transformers-monorepo", + "version": "0.0.1", + "private": true, + "description": "Monorepo for Hugging Face Transformers.js and framework integrations", + "type": "module", + "packageManager": "pnpm@10.28.1", + "scripts": { + "build": "pnpm -r build", + "test": "pnpm -r test", + "format": "prettier --write .", + "format:check": "prettier --check .", + "dev": "node scripts/dev.mjs" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/huggingface/transformers.js.git" + }, + "author": "Hugging Face", + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/huggingface/transformers.js/issues" + }, + "homepage": "https://github.com/huggingface/transformers.js#readme", + "devDependencies": { + "prettier": "3.8.1", + "typescript": "5.9.3" + }, + "prettier": { + "overrides": [ + { + "files": [ + "./packages/*/src/**/*.{js,mjs,cjs,ts}" + ], + "options": { + "singleQuote": true, + "tabWidth": 4, + "printWidth": 120 + } + }, + { + "files": [ + "./packages/*/tests/**/*.{js,mjs,cjs,ts}" + ], + "options": { + "tabWidth": 2, + "printWidth": 10000000 + } + }, + { + "files": [ + "./scripts/**/*.{js,mjs,cjs,ts}", + "./packages/*/scripts/**/*.{js,mjs,cjs,ts}", + "./packages/*/docs/**/*.{js,mjs,cjs,ts}" + ], + "options": { + "tabWidth": 2, + "printWidth": 160 + } + } + ] + } +} diff --git a/packages/transformers/docs/build/.gitignore b/packages/transformers/docs/build/.gitignore new file mode 100644 index 0000000..10b0e9e --- /dev/null +++ b/packages/transformers/docs/build/.gitignore @@ -0,0 +1,3 @@ +# This folder contains the build files for the documentation. Do not commit these files. +* +!.gitignore diff --git a/packages/transformers/docs/jsdoc-conf.json b/packages/transformers/docs/jsdoc-conf.json new file mode 100644 index 0000000..0b0d58d --- /dev/null +++ b/packages/transformers/docs/jsdoc-conf.json @@ -0,0 +1,6 @@ +{ + "source": { + "excludePattern": "" + }, + "plugins": ["./plugins/preprocess.js"] +} diff --git a/packages/transformers/docs/plugins/preprocess.js b/packages/transformers/docs/plugins/preprocess.js new file mode 100644 index 0000000..a277601 --- /dev/null +++ b/packages/transformers/docs/plugins/preprocess.js @@ -0,0 +1,129 @@ +/** + * JSDoc plugin to transform TypeScript-style type expressions into JSDoc-compatible syntax. + */ + +function extractBalancedBraces(text, start) { + if (text[start] !== "{") return null; + let depth = 1, + i = start + 1; + while (i < text.length && depth > 0) { + if (text[i] === "{") depth++; + else if (text[i] === "}") depth--; + i++; + } + return depth === 0 ? { content: text.slice(start + 1, i - 1), endIndex: i } : null; +} + +function stripLeadingGenericParams(expr) { + if (expr[0] !== "<") return expr; + let depth = 1, + i = 1; + while (i < expr.length && depth > 0) { + if (expr[i] === "<") depth++; + else if (expr[i] === ">") depth--; + i++; + } + return depth === 0 ? expr.slice(i) : expr; +} + +function hasTopLevelConditional(expr) { + let angle = 0, + paren = 0, + brace = 0, + bracket = 0; + + for (let i = 0; i < expr.length; ++i) { + const ch = expr[i]; + if (ch === "<") angle++; + else if (ch === ">") angle = Math.max(0, angle - 1); + else if (ch === "(") paren++; + else if (ch === ")") paren = Math.max(0, paren - 1); + else if (ch === "{") brace++; + else if (ch === "}") brace = Math.max(0, brace - 1); + else if (ch === "[") bracket++; + else if (ch === "]") bracket = Math.max(0, bracket - 1); + else if (ch === "?" && angle === 0 && paren === 0 && brace === 0 && bracket === 0) { + return true; + } + } + return false; +} + +function transformType(expr) { + let result = expr, + prev = ""; + while (result !== prev) { + prev = result; + result = stripLeadingGenericParams(result); // (...) -> (...) + if (/^Promise p?.slice(1) || "any") // import() -> Type + .replace(/\w+\[['"][^\]]+['"]\]\s+extends\s+[^}]+/g, "any") // X['p'] extends ... -> any + .replace(/(\w+)(?:<[^>]+>)?\[[^\]]+\]/g, "$1") // Type[x] or Type[x] -> Type + .replace(/keyof\s+typeof\s+\w+/g, "string") // keyof typeof X -> string + .replace(/typeof\s+\w+/g, "Object") // typeof X -> Object + .replace(/\binfer\s+\w+/g, "any") // infer K -> any + .replace(/\breadonly\s+/g, "") // readonly T -> T + .replace(/\(\s*\w[\w<>, ]*\s+extends\s+\w+\s*\?[^)]*\)/g, "any") // (X extends Y ? A : B) -> any + .replace(/(? any (unwrap parens around simple types, not after words like "function") + .replace(/(\w+)\?\s*:/g, "$1:") // x?: T -> x: T + .replace(/;\s*([}\n])/g, " $1") + .replace(/;\s+/g, ", ") // semicolons -> commas + .replace(/\{\s*\[\w+\s+in\s+[^\]]+\][^}]*\}/g, "Object") // mapped types -> Object + .replace(/new\s*\([^)]*\)\s*=>\s*\{[^}]*\}/g, "Function") // new () => {...} -> Function + .replace(/new\s*\([^)]*\)\s*=>\s*\w+/g, "Function") // new () => T -> Function + .replace(/\([^()]*\)\s*=>\s*\w[\w<>|[\], ]*/g, "Function") // () => T -> Function + .replace(/\{[^{}]*\}\[\]/g, "Array") // {x:T}[] -> Array + .replace(/(\w+)<[^>]+>\[\]/g, "Array.<$1>") // T[] -> Array. + .replace(/\([^()]+\)\[\]/g, "Array") // (A|B)[] -> Array + .replace(/(\w+)\[\]/g, "Array") // T[] -> Array (simple) + .replace(/\[\w+\]/g, "Array") // [T] single-element tuple -> Array + .replace(/\[[^\[\]]*,[^\[\]]*\]/g, "Array") // tuples with commas -> Array + .replace(/\bnew\s+([A-Z]\w*)\b/g, "$1") // new Type -> Type + .replace(/,?\s*\[\s*\w+\s*:\s*\w+\s*\]\s*:\s*\w+/g, "") // [key: string]: any -> (removed) + .replace(/\(\s*(\w+)\s*&\s*\{\s*\}\s*\)/g, "$1") // (string & {}) -> string + .replace(/\s*&\s*\{\s*\}/g, ""); // string & {} -> string + if (!result.includes("=>")) result = result.replace(/\s*&\s*/g, "|"); // A & B -> A|B + } + return result; +} + +function transformComment(comment) { + const tagRegex = + /@(?:type|param|returns?|typedef|property|prop|callback|template|augments|extends|class|constructor|member|var|const|enum|throws?)[^\S\n]*\{/g; + let result = "", + lastIndex = 0, + match; + while ((match = tagRegex.exec(comment)) !== null) { + const braceStart = match.index + match[0].length - 1; + result += comment.slice(lastIndex, braceStart); + const extracted = extractBalancedBraces(comment, braceStart); + if (extracted) { + result += "{" + transformType(extracted.content) + "}"; + lastIndex = tagRegex.lastIndex = extracted.endIndex; + } else { + result += "{"; + lastIndex = braceStart + 1; + } + } + return result + comment.slice(lastIndex); +} + +export const handlers = { + beforeParse: (e) => { + e.source = e.source + .replace(/\/\*\*[\s\S]*?\*\//g, transformComment) // transform JSDoc comments + .replace(/\b(\d+)n\b/g, "BigInt($1)"); // BigInt literals + }, +}; diff --git a/packages/transformers/docs/scripts/build_readme.py b/packages/transformers/docs/scripts/build_readme.py new file mode 100644 index 0000000..033e79b --- /dev/null +++ b/packages/transformers/docs/scripts/build_readme.py @@ -0,0 +1,108 @@ + +import re +README_TEMPLATE = """ + +

+
+ + + + transformers.js javascript library logo + +
+

+ +

+ NPM + NPM Downloads + jsDelivr Hits + License + Documentation +

+ +{intro} + +## Installation + +{installation} + +## Quick tour + +{quick_tour} + +## Custom usage + +{custom_usage} + +## Supported tasks/models + +Here is the list of all tasks and architectures currently supported by Transformers.js. If you don't see your task/model listed here or it is not yet supported, feel free to open up a feature request [here](https://github.com/huggingface/transformers.js/issues/new/choose). + +To find compatible models on the Hub, select the "transformers.js" library tag in the filter menu (or visit [this link](https://huggingface.co/models?library=transformers.js)). You can refine your search by selecting the task you're interested in (e.g., [text-classification](https://huggingface.co/models?pipeline_tag=text-classification&library=transformers.js)). + +{tasks} + +{models} +""" + + +FILES_TO_INCLUDE = dict( + intro='./docs/snippets/0_introduction.snippet', + quick_tour='./docs/snippets/1_quick-tour.snippet', + installation='./docs/snippets/2_installation.snippet', + custom_usage='./docs/snippets/3_custom-usage.snippet', + tasks='./docs/snippets/4_supported-tasks.snippet', + models='./docs/snippets/5_supported-models.snippet', +) + +DOCS_BASE_URL = 'https://huggingface.co/docs/transformers.js' + +# Map of custom links to replace, typically used for links to other sections of the README. +CUSTOM_LINK_MAP = { + '/custom_usage#convert-your-models-to-onnx': '#convert-your-models-to-onnx', + './api/env': DOCS_BASE_URL + '/api/env', + './guides/webgpu': DOCS_BASE_URL + '/guides/webgpu', + './guides/dtypes': DOCS_BASE_URL + '/guides/dtypes', +} + + +def main(): + + file_data = {} + for key, file_path in FILES_TO_INCLUDE.items(): + with open(file_path, encoding='utf-8') as f: + file_data[key] = f.read() + + # Fix links: + # NOTE: This regex does not match all markdown links, but works for the ones we need to replace. + LINK_RE = r'(?<=\])\((.+?)\)' + + def replace_fn(match): + link = match.group(1) + + if link in CUSTOM_LINK_MAP: + link = CUSTOM_LINK_MAP[link] + + elif link.startswith('/'): + # Link to docs + link = DOCS_BASE_URL + link + + elif link.startswith('./'): + # Relative link to file + pass + + elif link.startswith('http'): + # Link to external site + pass + + return f'({link})' + + result = README_TEMPLATE.format(**file_data) + result = re.sub(LINK_RE, replace_fn, result, count=0, flags=re.MULTILINE) + + with open('README.md', 'w', encoding='utf-8') as f: + f.write(result) + + +if __name__ == '__main__': + main() diff --git a/packages/transformers/docs/scripts/generate.js b/packages/transformers/docs/scripts/generate.js new file mode 100644 index 0000000..f651032 --- /dev/null +++ b/packages/transformers/docs/scripts/generate.js @@ -0,0 +1,79 @@ +// Based on [this tutorial](https://github.com/jsdoc2md/jsdoc-to-markdown/wiki/How-to-create-one-output-file-per-class). + +import fs from "node:fs"; +import path from "node:path"; +import url from "node:url"; + +import jsdoc2md from "jsdoc-to-markdown"; + +const docs = path.dirname(path.dirname(url.fileURLToPath(import.meta.url))); +const root = path.dirname(docs); + +// jsdoc config file +const conf = path.join(docs, "jsdoc-conf.json"); + +// input and output paths +const inputFile = path.join(root, "/src/**/*.js"); +const outputDir = path.join(root, "/docs/source/api/"); + +// get template data +const templateData = await jsdoc2md.getTemplateData({ + files: inputFile, + configure: conf, + "no-cache": true, +}); + +// reduce templateData to an array of module names +const moduleNames = templateData.reduce((moduleNames, identifier) => { + if (identifier.kind === "module") { + moduleNames.push(identifier.name); + } + return moduleNames; +}, []); + +// Clear all existing .md files from output directory (recursively) +if (fs.existsSync(outputDir)) { + const existingFiles = fs.readdirSync(outputDir, { recursive: true }); + for (const file of existingFiles) { + if (file.endsWith(".md")) { + fs.unlinkSync(path.join(outputDir, file)); + } + } +} + +// create a documentation file for each module +for (const moduleName of moduleNames) { + const template = `{{#module name="${moduleName}"}}{{>docs}}{{/module}}`; + console.log(`rendering ${moduleName}, template: ${template}`); + let output = await jsdoc2md.render({ + data: templateData, + template: template, + "heading-depth": 1, + "no-gfm": true, + "name-format": "backticks", + "no-cache": true, + separators: true, + configure: conf, + }); + + // Post-processing + output = output.replace(/(^#+\s.+)/gm, "$1\n"); // Add new line after each header + + // Remove tags from headers + output = output.replace(/^#+\s.+$/gm, (match) => match.replace(/<\/?code>/g, "")); + + // Replace all generated marker names with ids (for linking), and add group class + output = output.replace(/<\/a>/g, ''); + + // Unescape some of the characters which jsdoc2md escapes: + // TODO: May need to extend this list + output = output.replace(/\\([|_&*])/gm, "$1"); + + output = output.replaceAll("new exports.", "new "); + + const outputPath = path.resolve(outputDir, `${moduleName}.md`); + + console.log(`Writing to ${outputPath}`); + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.writeFileSync(outputPath, output); +} diff --git a/packages/transformers/docs/snippets/0_introduction.snippet b/packages/transformers/docs/snippets/0_introduction.snippet new file mode 100644 index 0000000..34d71bc --- /dev/null +++ b/packages/transformers/docs/snippets/0_introduction.snippet @@ -0,0 +1,16 @@ + +

+

State-of-the-art Machine Learning for the Web

+

+ +Run 🤗 Transformers directly in your browser, with no need for a server! + +Transformers.js is designed to be functionally equivalent to Hugging Face's [transformers](https://github.com/huggingface/transformers) python library, meaning you can run the same pretrained models using a very similar API. These models support common tasks in different modalities, such as: + - 📝 **Natural Language Processing**: text classification, named entity recognition, question answering, language modeling, summarization, translation, multiple choice, and text generation. + - 🖼️ **Computer Vision**: image classification, object detection, segmentation, and depth estimation. + - 🗣️ **Audio**: automatic speech recognition, audio classification, and text-to-speech. + - 🐙 **Multimodal**: embeddings, zero-shot audio classification, zero-shot image classification, and zero-shot object detection. + +Transformers.js uses [ONNX Runtime](https://onnxruntime.ai/) to run models in the browser. The best part about it, is that you can easily [convert](#convert-your-models-to-onnx) your pretrained PyTorch, TensorFlow, or JAX models to ONNX using [🤗 Optimum](https://github.com/huggingface/optimum#onnx--onnx-runtime). + +For more information, check out the full [documentation](https://huggingface.co/docs/transformers.js). diff --git a/packages/transformers/docs/snippets/1_quick-tour.snippet b/packages/transformers/docs/snippets/1_quick-tour.snippet new file mode 100644 index 0000000..fbfc292 --- /dev/null +++ b/packages/transformers/docs/snippets/1_quick-tour.snippet @@ -0,0 +1,74 @@ + +It's super simple to translate from existing code! Just like the python library, we support the `pipeline` API. Pipelines group together a pretrained model with preprocessing of inputs and postprocessing of outputs, making it the easiest way to run models with the library. + + + + + + + + + + +
Python (original)Javascript (ours)
+ +```python +from transformers import pipeline + +# Allocate a pipeline for sentiment-analysis +pipe = pipeline('sentiment-analysis') + +out = pipe('I love transformers!') +# [{'label': 'POSITIVE', 'score': 0.999806941}] +``` + + + +```javascript +import { pipeline } from '@huggingface/transformers'; + +// Allocate a pipeline for sentiment-analysis +const pipe = await pipeline('sentiment-analysis'); + +const out = await pipe('I love transformers!'); +// [{'label': 'POSITIVE', 'score': 0.999817686}] +``` + +
+ + +You can also use a different model by specifying the model id or path as the second argument to the `pipeline` function. For example: +```javascript +// Use a different model for sentiment-analysis +const pipe = await pipeline('sentiment-analysis', 'Xenova/bert-base-multilingual-uncased-sentiment'); +``` + +By default, when running in the browser, the model will be run on your CPU (via WASM). If you would like +to run the model on your GPU (via WebGPU), you can do this by setting `device: 'webgpu'`, for example: +```javascript +// Run the model on WebGPU +const pipe = await pipeline('sentiment-analysis', 'Xenova/distilbert-base-uncased-finetuned-sst-2-english', { + device: 'webgpu', +}); +``` + +For more information, check out the [WebGPU guide](./guides/webgpu). + +> [!WARNING] +> The WebGPU API is still experimental in many browsers, so if you run into any issues, +> please file a [bug report](https://github.com/huggingface/transformers.js/issues/new?title=%5BWebGPU%5D%20Error%20running%20MODEL_ID_GOES_HERE&assignees=&labels=bug,webgpu&projects=&template=1_bug-report.yml). + +In resource-constrained environments, such as web browsers, it is advisable to use a quantized version of +the model to lower bandwidth and optimize performance. This can be achieved by adjusting the `dtype` option, +which allows you to select the appropriate data type for your model. While the available options may vary +depending on the specific model, typical choices include `"fp32"` (default for WebGPU), `"fp16"`, `"q8"` +(default for WASM), and `"q4"`. For more information, check out the [quantization guide](./guides/dtypes). +```javascript +// Run the model at 4-bit quantization +const pipe = await pipeline('sentiment-analysis', 'Xenova/distilbert-base-uncased-finetuned-sst-2-english', { + dtype: 'q4', +}); +``` + +Ready to dive in? Explore our wide variety of demo applications and templates [here](https://github.com/huggingface/transformers.js-examples). You can also launch your own project instantly using the official Transformers.js [template](https://huggingface.co/new-space?template=static-templates%2Ftransformers.js) on Hugging Face! + diff --git a/packages/transformers/docs/snippets/2_installation.snippet b/packages/transformers/docs/snippets/2_installation.snippet new file mode 100644 index 0000000..5c2cba6 --- /dev/null +++ b/packages/transformers/docs/snippets/2_installation.snippet @@ -0,0 +1,12 @@ + +To install via [NPM](https://www.npmjs.com/package/@huggingface/transformers), run: +```bash +npm i @huggingface/transformers +``` + +Alternatively, you can use it in vanilla JS, without any bundler, by using a CDN or static hosting. For example, using [ES Modules](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Modules), you can import the library with: +```html + +``` diff --git a/packages/transformers/docs/snippets/3_custom-usage.snippet b/packages/transformers/docs/snippets/3_custom-usage.snippet new file mode 100644 index 0000000..ca48358 --- /dev/null +++ b/packages/transformers/docs/snippets/3_custom-usage.snippet @@ -0,0 +1,24 @@ + + +By default, Transformers.js uses [hosted pretrained models](https://huggingface.co/models?library=transformers.js) and [precompiled WASM binaries](https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0/dist/), which should work out-of-the-box. You can customize this as follows: + +### Settings + +```javascript +import { env } from '@huggingface/transformers'; + +// Specify a custom location for models (defaults to '/models/'). +env.localModelPath = '/path/to/models/'; + +// Disable the loading of remote models from the Hugging Face Hub: +env.allowRemoteModels = false; + +// Set location of .wasm files. Defaults to use a CDN. +env.backends.onnx.wasm.wasmPaths = '/path/to/files/'; +``` + +For a full list of available settings, check out the [API Reference](./api/env). + +### Convert your models to ONNX + +We recommend using [Optimum](https://github.com/huggingface/optimum-onnx) to convert your PyTorch models to ONNX in a single command. For the full list of supported architectures, check out the [Optimum documentation](https://huggingface.co/docs/optimum-onnx/onnx/overview). diff --git a/packages/transformers/docs/snippets/4_supported-tasks.snippet b/packages/transformers/docs/snippets/4_supported-tasks.snippet new file mode 100644 index 0000000..3e1a3ea --- /dev/null +++ b/packages/transformers/docs/snippets/4_supported-tasks.snippet @@ -0,0 +1,71 @@ + +### Tasks + +#### Natural Language Processing + +| Task | ID | Description | Supported? | +|--------------------------|----|-------------|------------| +| [Fill-Mask](https://huggingface.co/tasks/fill-mask) | `fill-mask` | Masking some of the words in a sentence and predicting which words should replace those masks. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.FillMaskPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=fill-mask&library=transformers.js) | +| [Question Answering](https://huggingface.co/tasks/question-answering) | `question-answering` | Retrieve the answer to a question from a given text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.QuestionAnsweringPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=question-answering&library=transformers.js) | +| [Sentence Similarity](https://huggingface.co/tasks/sentence-similarity) | `sentence-similarity` | Determining how similar two texts are. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.FeatureExtractionPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=sentence-similarity&library=transformers.js) | +| [Summarization](https://huggingface.co/tasks/summarization) | `summarization` | Producing a shorter version of a document while preserving its important information. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.SummarizationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=summarization&library=transformers.js) | +| [Table Question Answering](https://huggingface.co/tasks/table-question-answering) | `table-question-answering` | Answering a question about information from a given table. | ❌ | +| [Text Classification](https://huggingface.co/tasks/text-classification) | `text-classification` or `sentiment-analysis` | Assigning a label or class to a given text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TextClassificationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=text-classification&library=transformers.js) | +| [Text Generation](https://huggingface.co/tasks/text-generation#completion-generation-models) | `text-generation` | Producing new text by predicting the next word in a sequence. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TextGenerationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=text-generation&library=transformers.js) | +| [Text-to-text Generation](https://huggingface.co/tasks/text-generation#text-to-text-generation-models) | `text2text-generation` | Converting one text sequence into another text sequence. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.Text2TextGenerationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=text2text-generation&library=transformers.js) | +| [Token Classification](https://huggingface.co/tasks/token-classification) | `token-classification` or `ner` | Assigning a label to each token in a text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TokenClassificationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=token-classification&library=transformers.js) | +| [Translation](https://huggingface.co/tasks/translation) | `translation` | Converting text from one language to another. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TranslationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=translation&library=transformers.js) | +| [Zero-Shot Classification](https://huggingface.co/tasks/zero-shot-classification) | `zero-shot-classification` | Classifying text into classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotClassificationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=zero-shot-classification&library=transformers.js) | +| [Feature Extraction](https://huggingface.co/tasks/feature-extraction) | `feature-extraction` | Transforming raw data into numerical features that can be processed while preserving the information in the original dataset. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.FeatureExtractionPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=feature-extraction&library=transformers.js) | + +#### Vision + +| Task | ID | Description | Supported? | +|--------------------------|----|-------------|------------| +| [Background Removal](https://huggingface.co/tasks/image-segmentation#background-removal) | `background-removal` | Isolating the main subject of an image by removing or making the background transparent. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.BackgroundRemovalPipeline)
[(models)](https://huggingface.co/models?other=background-removal&library=transformers.js) | +| [Depth Estimation](https://huggingface.co/tasks/depth-estimation) | `depth-estimation` | Predicting the depth of objects present in an image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.DepthEstimationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=depth-estimation&library=transformers.js) | +| [Image Classification](https://huggingface.co/tasks/image-classification) | `image-classification` | Assigning a label or class to an entire image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageClassificationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=image-classification&library=transformers.js) | +| [Image Segmentation](https://huggingface.co/tasks/image-segmentation) | `image-segmentation` | Divides an image into segments where each pixel is mapped to an object. This task has multiple variants such as instance segmentation, panoptic segmentation and semantic segmentation. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageSegmentationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=image-segmentation&library=transformers.js) | +| [Image-to-Image](https://huggingface.co/tasks/image-to-image) | `image-to-image` | Transforming a source image to match the characteristics of a target image or a target image domain. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageToImagePipeline)
[(models)](https://huggingface.co/models?pipeline_tag=image-to-image&library=transformers.js) | +| [Mask Generation](https://huggingface.co/tasks/mask-generation) | `mask-generation` | Generate masks for the objects in an image. | ❌ | +| [Object Detection](https://huggingface.co/tasks/object-detection) | `object-detection` | Identify objects of certain defined classes within an image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ObjectDetectionPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=object-detection&library=transformers.js) | +| [Video Classification](https://huggingface.co/tasks/video-classification) | n/a | Assigning a label or class to an entire video. | ❌ | +| [Unconditional Image Generation](https://huggingface.co/tasks/unconditional-image-generation) | n/a | Generating images with no condition in any context (like a prompt text or another image). | ❌ | +| [Image Feature Extraction](https://huggingface.co/tasks/image-feature-extraction) | `image-feature-extraction` | Transforming raw data into numerical features that can be processed while preserving the information in the original image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageFeatureExtractionPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=image-feature-extraction&library=transformers.js) | + +#### Audio + +| Task | ID | Description | Supported? | +|--------------------------|----|-------------|------------| +| [Audio Classification](https://huggingface.co/tasks/audio-classification) | `audio-classification` | Assigning a label or class to a given audio. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.AudioClassificationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=audio-classification&library=transformers.js) | +| [Audio-to-Audio](https://huggingface.co/tasks/audio-to-audio) | n/a | Generating audio from an input audio source. | ❌ | +| [Automatic Speech Recognition](https://huggingface.co/tasks/automatic-speech-recognition) | `automatic-speech-recognition` | Transcribing a given audio into text. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.AutomaticSpeechRecognitionPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=automatic-speech-recognition&library=transformers.js) | +| [Text-to-Speech](https://huggingface.co/tasks/text-to-speech) | `text-to-speech` or `text-to-audio` | Generating natural-sounding speech given text input. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.TextToAudioPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=text-to-audio&library=transformers.js) | + + +#### Tabular + +| Task | ID | Description | Supported? | +|--------------------------|----|-------------|------------| +| [Tabular Classification](https://huggingface.co/tasks/tabular-classification) | n/a | Classifying a target category (a group) based on set of attributes. | ❌ | +| [Tabular Regression](https://huggingface.co/tasks/tabular-regression) | n/a | Predicting a numerical value given a set of attributes. | ❌ | + + +#### Multimodal + +| Task | ID | Description | Supported? | +|--------------------------|----|-------------|------------| +| [Document Question Answering](https://huggingface.co/tasks/document-question-answering) | `document-question-answering` | Answering questions on document images. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.DocumentQuestionAnsweringPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=document-question-answering&library=transformers.js) | +| [Image-to-Text](https://huggingface.co/tasks/image-to-text) | `image-to-text` | Output text from a given image. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ImageToTextPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=image-to-text&library=transformers.js) | +| [Text-to-Image](https://huggingface.co/tasks/text-to-image) | `text-to-image` | Generates images from input text. | ❌ | +| [Visual Question Answering](https://huggingface.co/tasks/visual-question-answering) | `visual-question-answering` | Answering open-ended questions based on an image. | ❌ | +| [Zero-Shot Audio Classification](https://huggingface.co/learn/audio-course/chapter4/classification_models#zero-shot-audio-classification) | `zero-shot-audio-classification` | Classifying audios into classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotAudioClassificationPipeline)
[(models)](https://huggingface.co/models?other=zero-shot-audio-classification&library=transformers.js) | +| [Zero-Shot Image Classification](https://huggingface.co/tasks/zero-shot-image-classification) | `zero-shot-image-classification` | Classifying images into classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotImageClassificationPipeline)
[(models)](https://huggingface.co/models?pipeline_tag=zero-shot-image-classification&library=transformers.js) | +| [Zero-Shot Object Detection](https://huggingface.co/tasks/zero-shot-object-detection) | `zero-shot-object-detection` | Identify objects of classes that are unseen during training. | ✅ [(docs)](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.ZeroShotObjectDetectionPipeline)
[(models)](https://huggingface.co/models?other=zero-shot-object-detection&library=transformers.js) | + + +#### Reinforcement Learning + +| Task | ID | Description | Supported? | +|--------------------------|----|-------------|------------| +| [Reinforcement Learning](https://huggingface.co/tasks/reinforcement-learning) | n/a | Learning from actions by interacting with an environment through trial and error and receiving rewards (negative or positive) as feedback. | ✅ | diff --git a/packages/transformers/docs/snippets/5_supported-models.snippet b/packages/transformers/docs/snippets/5_supported-models.snippet new file mode 100644 index 0000000..59bf5dc --- /dev/null +++ b/packages/transformers/docs/snippets/5_supported-models.snippet @@ -0,0 +1,216 @@ + +### Models + +1. **[AFMoE](https://huggingface.co/docs/transformers/model_doc/afmoe)** (from Arcee AI, Prime Intellect, and DatologyAI) released with the paper [Arcee Trinity Large Technical Report](https://huggingface.co/papers/2602.17004), by Varun Singh, Lucas Krauss, Sami Jaghouar, Matej Sirovatka, Charles Goddard, Fares Obied, Jack Min Ong, Jannik Straube, Fern, Aria Harley, Conner Stewart, Colin Kealty, Maziyar Panahi, Simon Kirsten, Anushka Deshpande, Anneketh Vij, Arthur Bresnu, Pranav Veldurthi, Raghav Ravishankar, Hardik Bishnoi, Mark McQuade, Johannes Hagemann, Lucas Atkins. +1. **[ALBERT](https://huggingface.co/docs/transformers/model_doc/albert)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://huggingface.co/papers/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. +1. **[Arcee](https://huggingface.co/docs/transformers/model_doc/arcee)** (from Arcee AI) released with the blog post [Announcing Arcee Foundation Models](https://www.arcee.ai/blog/announcing-the-arcee-foundation-model-family) by Fernando Fernandes, Varun Singh, Charles Goddard, Lucas Atkins, Mark McQuade, Maziyar Panahi, Conner Stewart, Colin Kealty, Raghav Ravishankar, Lucas Krauss, Anneketh Vij, Pranav Veldurthi, Abhishek Thakur, Julien Simon, Scott Zembsch, Benjamin Langer, Aleksiej Cecocho, Maitri Patel. +1. **[Audio Spectrogram Transformer](https://huggingface.co/docs/transformers/model_doc/audio-spectrogram-transformer)** (from MIT) released with the paper [AST: Audio Spectrogram Transformer](https://huggingface.co/papers/2104.01778) by Yuan Gong, Yu-An Chung, James Glass. +1. **[BART](https://huggingface.co/docs/transformers/model_doc/bart)** (from Facebook) released with the paper [BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension](https://huggingface.co/papers/1910.13461) by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer. +1. **[BEiT](https://huggingface.co/docs/transformers/model_doc/beit)** (from Microsoft) released with the paper [BEiT: BERT Pre-Training of Image Transformers](https://huggingface.co/papers/2106.08254) by Hangbo Bao, Li Dong, Furu Wei. +1. **[BERT](https://huggingface.co/docs/transformers/model_doc/bert)** (from Google) released with the paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://huggingface.co/papers/1810.04805) by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. +1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://huggingface.co/papers/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. +1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://huggingface.co/papers/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. +1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). +1. **[CamemBERT](https://huggingface.co/docs/transformers/model_doc/camembert)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://huggingface.co/papers/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. +1. **[CHMv2](https://huggingface.co/docs/transformers/main/model_doc/chmv2)** (from Meta) released with the paper [CHMv2: Improvements in Global Canopy Height Mapping using DINOv3](https://huggingface.co/papers/2603.06382) by John Brandt, Seungeun Yi, Jamie Tolan, Xinyuan Li, Peter Potapov, Jessica Ertel, Justine Spore, Huy V. Vo, Michaël Ramamonjisoa, Patrick Labatut, Piotr Bojanowski, Camille Couprie. +1. **Chatterbox** (from Resemble AI) released with the repository [Chatterbox TTS](https://github.com/resemble-ai/chatterbox) by the Resemble AI team. +1. **[Chinese-CLIP](https://huggingface.co/docs/transformers/model_doc/chinese_clip)** (from OFA-Sys) released with the paper [Chinese CLIP: Contrastive Vision-Language Pretraining in Chinese](https://huggingface.co/papers/2211.01335) by An Yang, Junshu Pan, Junyang Lin, Rui Men, Yichang Zhang, Jingren Zhou, Chang Zhou. +1. **[CLAP](https://huggingface.co/docs/transformers/model_doc/clap)** (from LAION-AI) released with the paper [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation](https://huggingface.co/papers/2211.06687) by Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov. +1. **[CLIP](https://huggingface.co/docs/transformers/model_doc/clip)** (from OpenAI) released with the paper [Learning Transferable Visual Models From Natural Language Supervision](https://huggingface.co/papers/2103.00020) by Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, Gretchen Krueger, Ilya Sutskever. +1. **[CLIPSeg](https://huggingface.co/docs/transformers/model_doc/clipseg)** (from University of Göttingen) released with the paper [Image Segmentation Using Text and Image Prompts](https://huggingface.co/papers/2112.10003) by Timo Lüddecke and Alexander Ecker. +1. **[CodeGen](https://huggingface.co/docs/transformers/model_doc/codegen)** (from Salesforce) released with the paper [A Conversational Paradigm for Program Synthesis](https://huggingface.co/papers/2203.13474) by Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, Caiming Xiong. +1. **[CodeLlama](https://huggingface.co/docs/transformers/model_doc/llama_code)** (from MetaAI) released with the paper [Code Llama: Open Foundation Models for Code](https://ai.meta.com/research/publications/code-llama-open-foundation-models-for-code/) by Baptiste Rozière, Jonas Gehring, Fabian Gloeckle, Sten Sootla, Itai Gat, Xiaoqing Ellen Tan, Yossi Adi, Jingyu Liu, Tal Remez, Jérémy Rapin, Artyom Kozhevnikov, Ivan Evtimov, Joanna Bitton, Manish Bhatt, Cristian Canton Ferrer, Aaron Grattafiori, Wenhan Xiong, Alexandre Défossez, Jade Copet, Faisal Azhar, Hugo Touvron, Louis Martin, Nicolas Usunier, Thomas Scialom, Gabriel Synnaeve. +1. **[Cohere](https://huggingface.co/docs/transformers/main/model_doc/cohere)** (from Cohere) released with the blog post [Command-R: Retrieval Augmented Generation at Production Scale](https://cohere.com/blog/command-r) by Cohere. +1. **[Cohere2](https://huggingface.co/docs/transformers/main/model_doc/cohere2)** (from Cohere) released with the blog post [Introducing Command R7B: Fast and efficient generative AI](https://cohere.com/blog/command-r7b) by Cohere. +1. **[Cohere ASR](https://huggingface.co/docs/transformers/main/model_doc/cohere_asr)** (from Cohere) released with the blog post [Introducing Cohere Transcribe: a new state-of-the-art in open-source speech recognition](https://cohere.com/blog/transcribe) by Cohere. +1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://huggingface.co/papers/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. +1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://huggingface.co/papers/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://huggingface.co/papers/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. +1. **[D-FINE](https://huggingface.co/docs/transformers/model_doc/d_fine)** (from University of Science and Technology of China) released with the paper [D-FINE: Redefine Regression Task in DETRs as Fine-grained Distribution Refinement](https://huggingface.co/papers/2410.13842) by Yansong Peng, Hebei Li, Peixi Wu, Yueyi Zhang, Xiaoyan Sun, Feng Wu. +1. **[DAC](https://huggingface.co/docs/transformers/model_doc/dac)** (from Descript) released with the paper [Descript Audio Codec: High-Fidelity Audio Compression with Improved RVQGAN](https://huggingface.co/papers/2306.06546) by Rithesh Kumar, Prem Seetharaman, Alejandro Luebs, Ishaan Kumar, Kundan Kumar. +1. **[DeBERTa](https://huggingface.co/docs/transformers/model_doc/deberta)** (from Microsoft) released with the paper [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://huggingface.co/papers/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. +1. **[DeBERTa-v2](https://huggingface.co/docs/transformers/model_doc/deberta-v2)** (from Microsoft) released with the paper [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://huggingface.co/papers/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. +1. **[Decision Transformer](https://huggingface.co/docs/transformers/model_doc/decision_transformer)** (from Berkeley/Facebook/Google) released with the paper [Decision Transformer: Reinforcement Learning via Sequence Modeling](https://huggingface.co/papers/2106.01345) by Lili Chen, Kevin Lu, Aravind Rajeswaran, Kimin Lee, Aditya Grover, Michael Laskin, Pieter Abbeel, Aravind Srinivas, Igor Mordatch. +1. **[DeepSeek-V3](https://huggingface.co/docs/transformers/model_doc/deepseek_v3)** (from DeepSeek-AI) released with the paper [DeepSeek-V3 Technical Report](https://huggingface.co/papers/2412.19437) by the DeepSeek-AI team. +1. **[DeepSeek-V4](https://huggingface.co/docs/transformers/model_doc/deepseek_v4)** (from DeepSeek-AI) released with the paper [DeepSeek-V4: Towards Highly Efficient Million-Token Context Intelligence](https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf) by the DeepSeek-AI team. +1. **[DeiT](https://huggingface.co/docs/transformers/model_doc/deit)** (from Facebook) released with the paper [Training data-efficient image transformers & distillation through attention](https://huggingface.co/papers/2012.12877) by Hugo Touvron, Matthieu Cord, Matthijs Douze, Francisco Massa, Alexandre Sablayrolles, Hervé Jégou. +1. **[Depth Anything](https://huggingface.co/docs/transformers/main/model_doc/depth_anything)** (from University of Hong Kong and TikTok) released with the paper [Depth Anything: Unleashing the Power of Large-Scale Unlabeled Data](https://huggingface.co/papers/2401.10891) by Lihe Yang, Bingyi Kang, Zilong Huang, Xiaogang Xu, Jiashi Feng, Hengshuang Zhao. +1. **Depth Pro** (from Apple) released with the paper [Depth Pro: Sharp Monocular Metric Depth in Less Than a Second](https://huggingface.co/papers/2410.02073) by Aleksei Bochkovskii, Amaël Delaunoy, Hugo Germain, Marcel Santos, Yichao Zhou, Stephan R. Richter, Vladlen Koltun. +1. **[DETR](https://huggingface.co/docs/transformers/model_doc/detr)** (from Facebook) released with the paper [End-to-End Object Detection with Transformers](https://huggingface.co/papers/2005.12872) by Nicolas Carion, Francisco Massa, Gabriel Synnaeve, Nicolas Usunier, Alexander Kirillov, Sergey Zagoruyko. +1. **[DINOv2](https://huggingface.co/docs/transformers/model_doc/dinov2)** (from Meta AI) released with the paper [DINOv2: Learning Robust Visual Features without Supervision](https://huggingface.co/papers/2304.07193) by Maxime Oquab, Timothée Darcet, Théo Moutakanni, Huy Vo, Marc Szafraniec, Vasil Khalidov, Pierre Fernandez, Daniel Haziza, Francisco Massa, Alaaeldin El-Nouby, Mahmoud Assran, Nicolas Ballas, Wojciech Galuba, Russell Howes, Po-Yao Huang, Shang-Wen Li, Ishan Misra, Michael Rabbat, Vasu Sharma, Gabriel Synnaeve, Hu Xu, Hervé Jegou, Julien Mairal, Patrick Labatut, Armand Joulin, Piotr Bojanowski. +1. **[DINOv2 with Registers](https://huggingface.co/docs/transformers/model_doc/dinov2_with_registers)** (from Meta AI) released with the paper [DINOv2 with Registers](https://huggingface.co/papers/2309.16588) by Timothée Darcet, Maxime Oquab, Julien Mairal, Piotr Bojanowski. +1. **[DINOv3](https://huggingface.co/docs/transformers/model_doc/dinov3)** (from Meta AI) released with the paper [DINOv3](https://huggingface.co/papers/2508.10104) by Oriane Siméoni, Huy V. Vo, Maximilian Seitzer, Federico Baldassarre, Maxime Oquab, Cijo Jose, Vasil Khalidov, Marc Szafraniec, Seungeun Yi, Michaël Ramamonjisoa, Francisco Massa, Daniel Haziza, Luca Wehrstedt, Jianyuan Wang, Timothée Darcet, Théo Moutakanni, Leonel Sentana, Claire Roberts, Andrea Vedaldi, Jamie Tolan, John Brandt, Camille Couprie, Julien Mairal, Hervé Jégou, Patrick Labatut, Piotr Bojanowski. +1. **[DistilBERT](https://huggingface.co/docs/transformers/model_doc/distilbert)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://huggingface.co/papers/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/research_projects/distillation), RoBERTa into [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/research_projects/distillation), Multilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/research_projects/distillation) and a German version of DistilBERT. +1. **[DiT](https://huggingface.co/docs/transformers/model_doc/dit)** (from Microsoft Research) released with the paper [DiT: Self-supervised Pre-training for Document Image Transformer](https://huggingface.co/papers/2203.02378) by Junlong Li, Yiheng Xu, Tengchao Lv, Lei Cui, Cha Zhang, Furu Wei. +1. **[Donut](https://huggingface.co/docs/transformers/model_doc/donut)** (from NAVER), released together with the paper [OCR-free Document Understanding Transformer](https://huggingface.co/papers/2111.15664) by Geewook Kim, Teakgyu Hong, Moonbin Yim, Jeongyeon Nam, Jinyoung Park, Jinyeong Yim, Wonseok Hwang, Sangdoo Yun, Dongyoon Han, Seunghyun Park. +1. **[DPT](https://huggingface.co/docs/transformers/master/model_doc/dpt)** (from Intel Labs) released with the paper [Vision Transformers for Dense Prediction](https://huggingface.co/papers/2103.13413) by René Ranftl, Alexey Bochkovskiy, Vladlen Koltun. +1. **[EdgeTAM](https://huggingface.co/docs/transformers/model_doc/edgetam)** (from Facebook) released with the paper [EdgeTAM: On-Device Track Anything Model](https://huggingface.co/papers/2501.07256) by Chong Zhou, Chenchen Zhu, Yunyang Xiong, Saksham Suri, Fanyi Xiao, Lemeng Wu, Raghuraman Krishnamoorthi, Bo Dai, Chen Change Loy, Vikas Chandra, Bilge Soran. +1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://huggingface.co/papers/1905.11946) by Mingxing Tan, Quoc V. Le. +1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://huggingface.co/papers/2003.10555) by Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning. +1. **ERNIE-4.5** (from Baidu ERNIE Team) released with the blog post [Announcing the Open Source Release of the ERNIE 4.5 Model Family](https://ernie.baidu.com/blog/posts/ernie4.5/) by the Baidu ERNIE Team. +1. **[ESM](https://huggingface.co/docs/transformers/model_doc/esm)** (from Meta AI) are transformer protein language models. **ESM-1b** was released with the paper [Biological structure and function emerge from scaling unsupervised learning to 250 million protein sequences](https://www.pnas.org/content/118/15/e2016239118) by Alexander Rives, Joshua Meier, Tom Sercu, Siddharth Goyal, Zeming Lin, Jason Liu, Demi Guo, Myle Ott, C. Lawrence Zitnick, Jerry Ma, and Rob Fergus. **ESM-1v** was released with the paper [Language models enable zero-shot prediction of the effects of mutations on protein function](https://doi.org/10.1101/2021.07.09.450648) by Joshua Meier, Roshan Rao, Robert Verkuil, Jason Liu, Tom Sercu and Alexander Rives. **ESM-2 and ESMFold** were released with the paper [Language models of protein sequences at the scale of evolution enable accurate structure prediction](https://doi.org/10.1101/2022.07.20.500902) by Zeming Lin, Halil Akin, Roshan Rao, Brian Hie, Zhongkai Zhu, Wenting Lu, Allan dos Santos Costa, Maryam Fazel-Zarandi, Tom Sercu, Sal Candido, Alexander Rives. +1. **[EuroBERT](https://huggingface.co/docs/transformers/model_doc/eurobert)** (from the EuroBERT team) released with the paper [EuroBERT: Scaling Multilingual Encoders for European Languages](https://huggingface.co/papers/2503.05500) by Nicolas Boizard, Hippolyte Gisserot-Boukhlef, Duarte M. Alves, André Martins, Ayoub Hammal, Caio Corro, Céline Hudelot, Emmanuel Malherbe, Etienne Malaboeuf, Fanny Jourdan, Gabriel Hautreux, João Alves, Kevin El-Haddad, Manuel Faysse, Maxime Peyrard, Nuno M. Guerreiro, Patrick Fernandes, Ricardo Rei, Pierre Colombo. +1. **EXAONE** (from LG AI Research) released with the papers [EXAONE 3.0 7.8B Instruction Tuned Language Model](https://huggingface.co/papers/2408.03541) and [EXAONE 3.5: Series of Large Language Models for Real-world Use Cases](https://huggingface.co/papers/2412.04862) by the LG AI Research team. +1. **[Falcon](https://huggingface.co/docs/transformers/model_doc/falcon)** (from Technology Innovation Institute) by Almazrouei, Ebtesam and Alobeidli, Hamza and Alshamsi, Abdulaziz and Cappelli, Alessandro and Cojocaru, Ruxandra and Debbah, Merouane and Goffinet, Etienne and Heslow, Daniel and Launay, Julien and Malartic, Quentin and Noune, Badreddine and Pannier, Baptiste and Penedo, Guilherme. +1. **[FalconH1](https://huggingface.co/docs/transformers/model_doc/falcon_h1)** (from Technology Innovation Institute) released with the blog post [Falcon-H1-Tiny: A series of extremely small, yet powerful language models redefining capabilities at small scale](https://huggingface.co/spaces/tiiuae/tiny-h1-blogpost) by the Falcon-LLM team. +1. **FastViT** (from Apple) released with the paper [FastViT: A Fast Hybrid Vision Transformer using Structural Reparameterization](https://huggingface.co/papers/2303.14189) by Pavan Kumar Anasosalu Vasu, James Gabriel, Jeff Zhu, Oncel Tuzel and Anurag Ranjan. +1. **[FLAN-T5](https://huggingface.co/docs/transformers/model_doc/flan-t5)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-t5-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei +1. **Florence2** (from Microsoft) released with the paper [Florence-2: Advancing a Unified Representation for a Variety of Vision Tasks](https://huggingface.co/papers/2311.06242) by Bin Xiao, Haiping Wu, Weijian Xu, Xiyang Dai, Houdong Hu, Yumao Lu, Michael Zeng, Ce Liu, Lu Yuan. +1. **[Gemma](https://huggingface.co/docs/transformers/main/model_doc/gemma)** (from Google) released with the blog post [Gemma: Open Models Based on Gemini Technology and Research](https://blog.google/technology/developers/gemma-open-models/) by the Gemma Google team. +1. **[Gemma2](https://huggingface.co/docs/transformers/main/model_doc/gemma2)** (from Google) released with the blog post [Gemma2: Open Models Based on Gemini Technology and Research](https://blog.google/technology/developers/google-gemma-2/) by the Gemma Google team. +1. **[Gemma3](https://huggingface.co/docs/transformers/main/model_doc/gemma3)** (from Google) released with the blog post [Introducing Gemma 3: The most capable model you can run on a single GPU or TPU](https://blog.google/technology/developers/gemma-3/) by the Gemma Google team. +1. **[Gemma3n](https://huggingface.co/docs/transformers/main/model_doc/gemma3n)** (from Google) released with the blog post [Announcing Gemma 3n preview: powerful, efficient, mobile-first AI](https://developers.googleblog.com/en/introducing-gemma-3n/) by the Gemma Google team. +1. **[Gemma4](https://huggingface.co/docs/transformers/main/model_doc/gemma4)** (from Google) released with the blog post [Gemma 4](https://blog.google/innovation-and-ai/technology/developers-tools/gemma-4/) by the Gemma Google team. +1. **[GLM](https://huggingface.co/docs/transformers/main/model_doc/glm)** (from the GLM Team, THUDM & ZhipuAI) released with the paper [ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools](https://huggingface.co/papers/2406.12793v2) by Team GLM: Aohan Zeng, Bin Xu, Bowen Wang, Chenhui Zhang, Da Yin, Dan Zhang, Diego Rojas, Guanyu Feng, Hanlin Zhao, Hanyu Lai, Hao Yu, Hongning Wang, Jiadai Sun, Jiajie Zhang, Jiale Cheng, Jiayi Gui, Jie Tang, Jing Zhang, Jingyu Sun, Juanzi Li, Lei Zhao, Lindong Wu, Lucen Zhong, Mingdao Liu, Minlie Huang, Peng Zhang, Qinkai Zheng, Rui Lu, Shuaiqi Duan, Shudan Zhang, Shulin Cao, Shuxun Yang, Weng Lam Tam, Wenyi Zhao, Xiao Liu, Xiao Xia, Xiaohan Zhang, Xiaotao Gu, Xin Lv, Xinghan Liu, Xinyi Liu, Xinyue Yang, Xixuan Song, Xunkai Zhang, Yifan An, Yifan Xu, Yilin Niu, Yuantao Yang, Yueyan Li, Yushi Bai, Yuxiao Dong, Zehan Qi, Zhaoyu Wang, Zhen Yang, Zhengxiao Du, Zhenyu Hou, Zihan Wang. +1. **[GLM-MoE-DSA](https://huggingface.co/docs/transformers/main/model_doc/glm_moe_dsa)** (from the GLM Team, ZhipuAI & Tsinghua University) released with the paper [GLM-5: from Vibe Coding to Agentic Engineering](https://huggingface.co/papers/2602.15763) by Team GLM. +1. **[GLM-OCR](https://huggingface.co/docs/transformers/main/model_doc/glm_ocr)** (from the GLM Team, ZhipuAI & Tsinghua University) released with the paper [GLM-OCR Technical Report](https://huggingface.co/papers/2603.10910) by Team GLM: Shuaiqi Duan, Yadong Xue, Weihan Wang, Zhe Su, Huan Liu, Sheng Yang, Guobing Gan, Guo Wang, Zihan Wang, Shengdong Yan, Dexin Jin, Yuxuan Zhang, Guohong Wen, Yanfeng Wang, Yutao Zhang, Xiaohan Zhang, Wenyi Hong, Yukuo Cen, Da Yin, Bin Chen, Wenmeng Yu, Xiaotao Gu, Jie Tang. +1. **[GLPN](https://huggingface.co/docs/transformers/model_doc/glpn)** (from KAIST) released with the paper [Global-Local Path Networks for Monocular Depth Estimation with Vertical CutDepth](https://huggingface.co/papers/2201.07436) by Doyeon Kim, Woonghyun Ga, Pyungwhan Ahn, Donggyu Joo, Sehwan Chun, Junmo Kim. +1. **[GPT Neo](https://huggingface.co/docs/transformers/model_doc/gpt_neo)** (from EleutherAI) released in the repository [EleutherAI/gpt-neo](https://github.com/EleutherAI/gpt-neo) by Sid Black, Stella Biderman, Leo Gao, Phil Wang and Connor Leahy. +1. **[GPT NeoX](https://huggingface.co/docs/transformers/model_doc/gpt_neox)** (from EleutherAI) released with the paper [GPT-NeoX-20B: An Open-Source Autoregressive Language Model](https://huggingface.co/papers/2204.06745) by Sid Black, Stella Biderman, Eric Hallahan, Quentin Anthony, Leo Gao, Laurence Golding, Horace He, Connor Leahy, Kyle McDonell, Jason Phang, Michael Pieler, USVSN Sai Prashanth, Shivanshu Purohit, Laria Reynolds, Jonathan Tow, Ben Wang, Samuel Weinbach +1. **[GPT OSS](https://huggingface.co/docs/transformers/model_doc/gpt_oss)** (from OpenAI) released with the blog [Introducing gpt-oss](https://openai.com/index/introducing-gpt-oss/) by Sandhini Agarwal, Lama Ahmad, Jason Ai, Sam Altman, Andy Applebaum, Edwin Arbus, Rahul K. Arora, Yu Bai, Bowen Baker, Haiming Bao, Boaz Barak, Ally Bennett, Tyler Bertao, Nivedita Brett, Eugene Brevdo, Greg Brockman, Sebastien Bubeck, Che Chang, Kai Chen, Mark Chen, Enoch Cheung, Aidan Clark, Dan Cook, Marat Dukhan, Casey Dvorak, Kevin Fives, Vlad Fomenko, Timur Garipov, Kristian Georgiev, Mia Glaese, Tarun Gogineni, Adam Goucher, Lukas Gross, Katia Gil Guzman, John Hallman, Jackie Hehir, Johannes Heidecke, Alec Helyar, Haitang Hu, Romain Huet, Jacob Huh, Saachi Jain, Zach Johnson, Chris Koch, Irina Kofman, Dominik Kundel, Jason Kwon, Volodymyr Kyrylov, Elaine Ya Le, Guillaume Leclerc, James Park Lennon, Scott Lessans, Mario Lezcano-Casado, Yuanzhi Li, Zhuohan Li, Ji Lin, Jordan Liss, Lily (Xiaoxuan) Liu, Jiancheng Liu, Kevin Lu, Chris Lu, Zoran Martinovic, Lindsay McCallum, Josh McGrath, Scott McKinney, Aidan McLaughlin, Song Mei, Steve Mostovoy, Tong Mu, Gideon Myles, Alexander Neitz, Alex Nichol, Jakub Pachocki, Alex Paino, Dana Palmie, Ashley Pantuliano, Giambattista Parascandolo, Jongsoo Park, Leher Pathak, Carolina Paz, Ludovic Peran, Dmitry Pimenov, Michelle Pokrass, Elizabeth Proehl, Huida Qiu, Gaby Raila, Filippo Raso, Hongyu Ren, Kimmy Richardson, David Robinson, Bob Rotsted, Hadi Salman, Suvansh Sanjeev, Max Schwarzer, D. Sculley, Harshit Sikchi, Kendal Simon, Karan Singhal, Yang Song, Dane Stuckey, Zhiqing Sun, Philippe Tillet, Sam Toizer, Foivos Tsimpourlas, Nikhil Vyas, Eric Wallace, Xin Wang, Miles Wang, Olivia Watkins, Kevin Weil, Amy Wendling, Kevin Whinnery, Cedric Whitney, Hannah Wong, Lin Yang, Yu Yang, Michihiro Yasunaga, Kristen Ying, Wojciech Zaremba, Wenting Zhan, Cyril Zhang, Brian Zhang, Eddie Zhang, Shengjia Zhao. +1. **[GPT-2](https://huggingface.co/docs/transformers/model_doc/gpt2)** (from OpenAI) released with the paper [Language Models are Unsupervised Multitask Learners](https://blog.openai.com/better-language-models/) by Alec Radford*, Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever**. +1. **[GPT-J](https://huggingface.co/docs/transformers/model_doc/gptj)** (from EleutherAI) released in the repository [kingoflolz/mesh-transformer-jax](https://github.com/kingoflolz/mesh-transformer-jax/) by Ben Wang and Aran Komatsuzaki. +1. **[GPTBigCode](https://huggingface.co/docs/transformers/model_doc/gpt_bigcode)** (from BigCode) released with the paper [SantaCoder: don't reach for the stars!](https://huggingface.co/papers/2301.03988) by Loubna Ben Allal, Raymond Li, Denis Kocetkov, Chenghao Mou, Christopher Akiki, Carlos Munoz Ferrandis, Niklas Muennighoff, Mayank Mishra, Alex Gu, Manan Dey, Logesh Kumar Umapathi, Carolyn Jane Anderson, Yangtian Zi, Joel Lamy Poirier, Hailey Schoelkopf, Sergey Troshin, Dmitry Abulkhanov, Manuel Romero, Michael Lappert, Francesco De Toni, Bernardo García del Río, Qian Liu, Shamik Bose, Urvashi Bhattacharyya, Terry Yue Zhuo, Ian Yu, Paulo Villegas, Marco Zocca, Sourab Mangrulkar, David Lansky, Huu Nguyen, Danish Contractor, Luis Villa, Jia Li, Dzmitry Bahdanau, Yacine Jernite, Sean Hughes, Daniel Fried, Arjun Guha, Harm de Vries, Leandro von Werra. +1. **[Granite](https://huggingface.co/docs/transformers/main/model_doc/granite)** (from IBM) released with the paper [Power Scheduler: A Batch Size and Token Number Agnostic Learning Rate Scheduler](https://huggingface.co/papers/2408.13359) by Yikang Shen, Matthew Stallone, Mayank Mishra, Gaoyuan Zhang, Shawn Tan, Aditya Prasad, Adriana Meza Soria, David D. Cox, Rameswar Panda. +1. **[GraniteMoeHybrid](https://huggingface.co/docs/transformers/main/model_doc/granitemoehybrid)** (from IBM) released with the blog post [IBM Granite 4.0: hyper-efficient, high performance hybrid models for enterprise](https://www.ibm.com/new/announcements/ibm-granite-4-0-hyper-efficient-high-performance-hybrid-models) by the IBM Granite team. +1. **[Granite Speech](https://huggingface.co/docs/transformers/main/model_doc/granite_speech)** (from IBM) released with the paper [Granite-speech: open-source speech-aware LLMs with strong English ASR capabilities](https://huggingface.co/papers/2505.08699) by the IBM Granite team. +1. **[Grounding DINO](https://huggingface.co/docs/transformers/model_doc/grounding-dino)** (from IDEA-Research) released with the paper [Grounding DINO: Marrying DINO with Grounded Pre-Training for Open-Set Object Detection](https://huggingface.co/papers/2303.05499) by Shilong Liu, Zhaoyang Zeng, Tianhe Ren, Feng Li, Hao Zhang, Jie Yang, Qing Jiang, Chunyuan Li, Jianwei Yang, Hang Su, Jun Zhu, Lei Zhang. +1. **[GroupViT](https://huggingface.co/docs/transformers/model_doc/groupvit)** (from UCSD, NVIDIA) released with the paper [GroupViT: Semantic Segmentation Emerges from Text Supervision](https://huggingface.co/papers/2202.11094) by Jiarui Xu, Shalini De Mello, Sifei Liu, Wonmin Byeon, Thomas Breuel, Jan Kautz, Xiaolong Wang. +1. **[Helium](https://huggingface.co/docs/transformers/main/model_doc/helium)** (from the Kyutai Team) released with the blog post [Announcing Helium-1 Preview](https://kyutai.org/2025/01/13/helium.html) by the Kyutai Team. +1. **[HerBERT](https://huggingface.co/docs/transformers/model_doc/herbert)** (from Allegro.pl, AGH University of Science and Technology) released with the paper [KLEJ: Comprehensive Benchmark for Polish Language Understanding](https://www.aclweb.org/anthology/2020.acl-main.111.pdf) by Piotr Rybak, Robert Mroczkowski, Janusz Tracz, Ireneusz Gawlik. +1. **[HRM-Text](https://huggingface.co/docs/transformers/model_doc/hrm_text)** (from Sapient Intelligence) released with the paper [Hierarchical Reasoning Model](https://huggingface.co/papers/2506.21734) by Guan Wang, Jin Li, Yuhao Sun, Xing Chen, Changling Liu, Yue Wu, Meng Lu, Sen Song, Yasin Abbasi Yadkori. +1. **[Hiera](https://huggingface.co/docs/transformers/model_doc/hiera)** (from Meta) released with the paper [Hiera: A Hierarchical Vision Transformer without the Bells-and-Whistles](https://huggingface.co/papers/2306.00989) by Chaitanya Ryali, Yuan-Ting Hu, Daniel Bolya, Chen Wei, Haoqi Fan, Po-Yao Huang, Vaibhav Aggarwal, Arkabandhu Chowdhury, Omid Poursaeed, Judy Hoffman, Jitendra Malik, Yanghao Li, Christoph Feichtenhofer. +1. **[Hubert](https://huggingface.co/docs/transformers/model_doc/hubert)** (from Facebook) released with the paper [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units](https://huggingface.co/papers/2106.07447) by Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia, Ruslan Salakhutdinov, Abdelrahman Mohamed. +1. **[HunYuanDenseV1](https://huggingface.co/docs/transformers/model_doc/hunyuan_v1_dense)** (from Tencent) released with the paper [Hunyuan-Large: An Open-Source MoE Model with 52 Billion Activated Parameters by Tencent](https://huggingface.co/papers/2411.02265) by the Tencent team. +1. **[I-JEPA](https://huggingface.co/docs/transformers/model_doc/ijepa)** (from Meta) released with the paper [Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture](https://huggingface.co/papers/2301.08243) by Mahmoud Assran, Quentin Duval, Ishan Misra, Piotr Bojanowski, Pascal Vincent, Michael Rabbat, Yann LeCun, Nicolas Ballas. +1. **[Idefics3](https://huggingface.co/docs/transformers/model_doc/idefics3)** (from Hugging Face) released with the paper [Building and better understanding vision-language models: insights and future directions](https://huggingface.co/papers/2408.12637) by Hugo Laurençon, Andrés Marafioti, Victor Sanh, Léo Tronchon. +1. **JAIS** (from Core42) released with the paper [Jais and Jais-chat: Arabic-Centric Foundation and Instruction-Tuned Open Generative Large Language Models](https://huggingface.co/papers/2308.16149) by Neha Sengupta, Sunil Kumar Sahu, Bokang Jia, Satheesh Katipomu, Haonan Li, Fajri Koto, William Marshall, Gurpreet Gosal, Cynthia Liu, Zhiming Chen, Osama Mohammed Afzal, Samta Kamboj, Onkar Pandit, Rahul Pal, Lalit Pradhan, Zain Muhammad Mujahid, Massa Baali, Xudong Han, Sondos Mahmoud Bsharat, Alham Fikri Aji, Zhiqiang Shen, Zhengzhong Liu, Natalia Vassilieva, Joel Hestness, Andy Hock, Andrew Feldman, Jonathan Lee, Andrew Jackson, Hector Xuguang Ren, Preslav Nakov, Timothy Baldwin, Eric Xing. +1. **Janus** (from DeepSeek) released with the paper [Janus: Decoupling Visual Encoding for Unified Multimodal Understanding and Generation](https://huggingface.co/papers/2410.13848) Chengyue Wu, Xiaokang Chen, Zhiyu Wu, Yiyang Ma, Xingchao Liu, Zizheng Pan, Wen Liu, Zhenda Xie, Xingkai Yu, Chong Ruan, Ping Luo. +1. **JinaCLIP** (from Jina AI) released with the paper [Jina CLIP: Your CLIP Model Is Also Your Text Retriever](https://huggingface.co/papers/2405.20204) by Andreas Koukounas, Georgios Mastrapas, Michael Günther, Bo Wang, Scott Martens, Isabelle Mohr, Saba Sturua, Mohammad Kalim Akram, Joan Fontanals Martínez, Saahil Ognawala, Susana Guzman, Maximilian Werk, Nan Wang, Han Xiao. +1. **LiteWhisper** (from University of Washington, Kotoba Technologies) released with the paper [LiteASR: Efficient Automatic Speech Recognition with Low-Rank Approximation](https://huggingface.co/papers/2502.20583) by Keisuke Kamahori, Jungo Kasai, Noriyuki Kojima, Baris Kasikci. +1. **[LongT5](https://huggingface.co/docs/transformers/model_doc/longt5)** (from Google AI) released with the paper [LongT5: Efficient Text-To-Text Transformer for Long Sequences](https://huggingface.co/papers/2112.07916) by Mandy Guo, Joshua Ainslie, David Uthus, Santiago Ontanon, Jianmo Ni, Yun-Hsuan Sung, Yinfei Yang. +1. **[LFM2](https://huggingface.co/docs/transformers/model_doc/lfm2)** (from Liquid AI) released with the blog post [Introducing LFM2: The Fastest On-Device Foundation Models on the Market](https://www.liquid.ai/blog/liquid-foundation-models-v2-our-second-series-of-generative-ai-models) by the Liquid AI Team. +1. **[Lfm2Moe](https://huggingface.co/docs/transformers/model_doc/lfm2_moe)** (from Liquid AI) released with the blog post [LFM2-8B-A1B: An Efficient On-device Mixture-of-Experts](https://www.liquid.ai/blog/lfm2-8b-a1b-an-efficient-on-device-mixture-of-experts) by the Liquid AI Team. +1. **[LFM2-VL](https://huggingface.co/docs/transformers/model_doc/lfm2_vl)** (from Liquid AI) released with the blog post [LFM2-VL: Efficient Vision-Language Models](https://www.liquid.ai/blog/lfm2-vl-efficient-vision-language-models) by the Liquid AI Team. +1. **[LightOnOcr](https://huggingface.co/docs/transformers/model_doc/lighton_ocr)** (from LightOn) released with the blog post [LightOnOCR-1B: The Case for End-to-End and Efficient Domain-Specific Vision-Language Models for OCR](https://huggingface.co/blog/lightonai/lightonocr) by the LightOn Team. +1. **[LLaMA](https://huggingface.co/docs/transformers/model_doc/llama)** (from The FAIR team of Meta AI) released with the paper [LLaMA: Open and Efficient Foundation Language Models](https://huggingface.co/papers/2302.13971) by Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, Guillaume Lample. +1. **[Llama2](https://huggingface.co/docs/transformers/model_doc/llama2)** (from The FAIR team of Meta AI) released with the paper [Llama2: Open Foundation and Fine-Tuned Chat Models](https://huggingface.co/papers/2307.09288) by Hugo Touvron, Louis Martin, Kevin Stone, Peter Albert, Amjad Almahairi, Yasmine Babaei, Nikolay Bashlykov, Soumya Batra, Prajjwal Bhargava, Shruti Bhosale, Dan Bikel, Lukas Blecher, Cristian Canton Ferrer, Moya Chen, Guillem Cucurull, David Esiobu, Jude Fernandes, Jeremy Fu, Wenyin Fu, Brian Fuller, Cynthia Gao, Vedanuj Goswami, Naman Goyal, Anthony Hartshorn, Saghar Hosseini, Rui Hou, Hakan Inan, Marcin Kardas, Viktor Kerkez Madian Khabsa, Isabel Kloumann, Artem Korenev, Punit Singh Koura, Marie-Anne Lachaux, Thibaut Lavril, Jenya Lee, Diana Liskovich, Yinghai Lu, Yuning Mao, Xavier Martinet, Todor Mihaylov, Pushka rMishra, Igor Molybog, Yixin Nie, Andrew Poulton, Jeremy Reizenstein, Rashi Rungta, Kalyan Saladi, Alan Schelten, Ruan Silva, Eric Michael Smith, Ranjan Subramanian, Xiaoqing EllenTan, Binh Tang, Ross Taylor, Adina Williams, Jian Xiang Kuan, Puxin Xu, Zheng Yan, Iliyan Zarov, Yuchen Zhang, Angela Fan, Melanie Kambadur, Sharan Narang, Aurelien Rodriguez, Robert Stojnic, Sergey Edunov, Thomas Scialom. +1. **[Llama3](https://huggingface.co/docs/transformers/model_doc/llama3)** (from The FAIR team of Meta AI) released with the paper [The Llama 3 Herd of Models](https://huggingface.co/papers/2407.21783) by the Llama Team at Meta. +1. **[Llama4](https://huggingface.co/docs/transformers/model_doc/llama4)** (from The FAIR team of Meta AI) released with the blog post [The Llama 4 herd: The beginning of a new era of natively multimodal AI innovation](https://ai.meta.com/blog/llama-4-multimodal-intelligence/) by the Llama Team at Meta. +1. **[LLaVa](https://huggingface.co/docs/transformers/model_doc/llava)** (from Microsoft Research & University of Wisconsin-Madison) released with the paper [Visual Instruction Tuning](https://huggingface.co/papers/2304.08485) by Haotian Liu, Chunyuan Li, Yuheng Li and Yong Jae Lee. +1. **[LLaVA-OneVision](https://huggingface.co/docs/transformers/model_doc/llava_onevision)** (from ByteDance & NTU & CUHK & HKUST) released with the paper [LLaVA-OneVision: Easy Visual Task Transfer](https://huggingface.co/papers/2408.03326) by Bo Li, Yuanhan Zhang, Dong Guo, Renrui Zhang, Feng Li, Hao Zhang, Kaichen Zhang, Yanwei Li, Ziwei Liu, Chunyuan Li +1. **[M2M100](https://huggingface.co/docs/transformers/model_doc/m2m_100)** (from Facebook) released with the paper [Beyond English-Centric Multilingual Machine Translation](https://huggingface.co/papers/2010.11125) by Angela Fan, Shruti Bhosale, Holger Schwenk, Zhiyi Ma, Ahmed El-Kishky, Siddharth Goyal, Mandeep Baines, Onur Celebi, Guillaume Wenzek, Vishrav Chaudhary, Naman Goyal, Tom Birch, Vitaliy Liptchinsky, Sergey Edunov, Edouard Grave, Michael Auli, Armand Joulin. +1. **[MarianMT](https://huggingface.co/docs/transformers/model_doc/marian)** Machine translation models trained using [OPUS](http://opus.nlpl.eu/) data by Jörg Tiedemann. The [Marian Framework](https://marian-nmt.github.io/) is being developed by the Microsoft Translator Team. +1. **[MaskFormer](https://huggingface.co/docs/transformers/model_doc/maskformer)** (from Meta and UIUC) released with the paper [Per-Pixel Classification is Not All You Need for Semantic Segmentation](https://huggingface.co/papers/2107.06278) by Bowen Cheng, Alexander G. Schwing, Alexander Kirillov. +1. **[mBART](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Denoising Pre-training for Neural Machine Translation](https://huggingface.co/papers/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer. +1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://huggingface.co/papers/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. +1. **Metric3D** released with the paper [Metric3D: Towards Zero-shot Metric 3D Prediction from A Single Image](https://huggingface.co/papers/2307.10984) by Wei Yin, Chi Zhang, Hao Chen, Zhipeng Cai, Gang Yu, Kaixuan Wang, Xiaozhi Chen, Chunhua Shen. +1. **Metric3Dv2** released with the paper [Metric3Dv2: A Versatile Monocular Geometric Foundation Model for Zero-shot Metric Depth and Surface Normal Estimation](https://huggingface.co/papers/2404.15506) by Mu Hu, Wei Yin, Chi Zhang, Zhipeng Cai, Xiaoxiao Long, Kaixuan Wang, Hao Chen, Gang Yu, Chunhua Shen, Shaojie Shen. +1. **[MusicGen](https://huggingface.co/docs/transformers/model_doc/musicgen)** (from Meta) released with the paper [Simple and Controllable Music Generation](https://huggingface.co/papers/2306.05284) by Jade Copet, Felix Kreuk, Itai Gat, Tal Remez, David Kant, Gabriel Synnaeve, Yossi Adi and Alexandre Défossez. +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://huggingface.co/papers/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. +1. **[Mimi](https://huggingface.co/docs/transformers/model_doc/mimi)** (from Kyutai) released with the paper [Moshi: a speech-text foundation model for real-time dialogue](https://huggingface.co/papers/2410.00037) by Alexandre Défossez, Laurent Mazaré, Manu Orsini, Amélie Royer, Patrick Pérez, Hervé Jégou, Edouard Grave and Neil Zeghidour. +1. **[Ministral](https://huggingface.co/docs/transformers/model_doc/ministral)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team. +1. **[Ministral3](https://huggingface.co/docs/transformers/model_doc/ministral3)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team. +1. **[Mistral](https://huggingface.co/docs/transformers/model_doc/mistral)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team: Albert Jiang, Alexandre Sablayrolles, Arthur Mensch, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Florian Bressand, Gianna Lengyel, Guillaume Lample, Lélio Renard Lavaud, Lucile Saulnier, Marie-Anne Lachaux, Pierre Stock, Teven Le Scao, Thibaut Lavril, Thomas Wang, Timothée Lacroix, William El Sayed. +1. **[Mistral3](https://huggingface.co/docs/transformers/model_doc/mistral3)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team. +1. **[Mistral4](https://huggingface.co/docs/transformers/model_doc/mistral4)** (from Mistral AI) by The [Mistral AI](https://mistral.ai) team. +1. **[MMS](https://huggingface.co/docs/transformers/model_doc/mms)** (from Facebook) released with the paper [Scaling Speech Technology to 1,000+ Languages](https://huggingface.co/papers/2305.13516) by Vineel Pratap, Andros Tjandra, Bowen Shi, Paden Tomasello, Arun Babu, Sayani Kundu, Ali Elkahky, Zhaoheng Ni, Apoorv Vyas, Maryam Fazel-Zarandi, Alexei Baevski, Yossi Adi, Xiaohui Zhang, Wei-Ning Hsu, Alexis Conneau, Michael Auli. +1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (from CMU/Google Brain) released with the paper [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://huggingface.co/papers/2004.02984) by Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou. +1. **MobileCLIP** (from Apple) released with the paper [MobileCLIP: Fast Image-Text Models through Multi-Modal Reinforced Training](https://huggingface.co/papers/2311.17049) by Pavan Kumar Anasosalu Vasu, Hadi Pouransari, Fartash Faghri, Raviteja Vemulapalli, Oncel Tuzel. +1. **MobileLLM** (from Meta) released with the paper [MobileLLM: Optimizing Sub-billion Parameter Language Models for On-Device Use Cases](https://huggingface.co/papers/2402.14905) by Zechun Liu, Changsheng Zhao, Forrest Iandola, Chen Lai, Yuandong Tian, Igor Fedorov, Yunyang Xiong, Ernie Chang, Yangyang Shi, Raghuraman Krishnamoorthi, Liangzhen Lai, Vikas Chandra. +1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://huggingface.co/papers/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam. +1. **[MobileNetV2](https://huggingface.co/docs/transformers/model_doc/mobilenet_v2)** (from Google Inc.) released with the paper [MobileNetV2: Inverted Residuals and Linear Bottlenecks](https://huggingface.co/papers/1801.04381) by Mark Sandler, Andrew Howard, Menglong Zhu, Andrey Zhmoginov, Liang-Chieh Chen. +1. **MobileNetV3** (from Google Inc.) released with the paper [Searching for MobileNetV3](https://huggingface.co/papers/1905.02244) by Andrew Howard, Mark Sandler, Grace Chu, Liang-Chieh Chen, Bo Chen, Mingxing Tan, Weijun Wang, Yukun Zhu, Ruoming Pang, Vijay Vasudevan, Quoc V. Le, Hartwig Adam. +1. **MobileNetV4** (from Google Inc.) released with the paper [MobileNetV4 - Universal Models for the Mobile Ecosystem](https://huggingface.co/papers/2404.10518) by Danfeng Qin, Chas Leichner, Manolis Delakis, Marco Fornoni, Shixin Luo, Fan Yang, Weijun Wang, Colby Banbury, Chengxi Ye, Berkin Akin, Vaibhav Aggarwal, Tenghui Zhu, Daniele Moro, Andrew Howard. +1. **[MobileViT](https://huggingface.co/docs/transformers/model_doc/mobilevit)** (from Apple) released with the paper [MobileViT: Light-weight, General-purpose, and Mobile-friendly Vision Transformer](https://huggingface.co/papers/2110.02178) by Sachin Mehta and Mohammad Rastegari. +1. **[MobileViTV2](https://huggingface.co/docs/transformers/model_doc/mobilevitv2)** (from Apple) released with the paper [Separable Self-attention for Mobile Vision Transformers](https://huggingface.co/papers/2206.02680) by Sachin Mehta and Mohammad Rastegari. +1. **[ModernBERT](https://huggingface.co/docs/transformers/model_doc/modernbert)** (from Answer.AI and LightOn) released with the paper [Smarter, Better, Faster, Longer: A Modern Bidirectional Encoder for Fast, Memory Efficient, and Long Context Finetuning and Inference](https://huggingface.co/papers/2412.13663) by Benjamin Warner, Antoine Chaffin, Benjamin Clavié, Orion Weller, Oskar Hallström, Said Taghadouini, Alexis Gallagher, Raja Biswas, Faisal Ladhak, Tom Aarsen, Nathan Cooper, Griffin Adams, Jeremy Howard, Iacopo Poli. +1. **[ModernBERT Decoder](https://huggingface.co/docs/transformers/model_doc/modernbert-decoder)** (from Johns Hopkins University and LightOn) released with the paper [Seq vs Seq: An Open Suite of Paired Encoders and Decoders](https://huggingface.co/papers/2507.11412) by Orion Weller, Kathryn Ricci, Marc Marone, Antoine Chaffin, Dawn Lawrie, Benjamin Van Durme. +1. **Moondream1** released in the repository [moondream](https://github.com/vikhyat/moondream) by vikhyat. +1. **[Moonshine](https://huggingface.co/docs/transformers/model_doc/moonshine)** (from Useful Sensors) released with the paper [Moonshine: Speech Recognition for Live Transcription and Voice Commands](https://huggingface.co/papers/2410.15608) by Nat Jeffries, Evan King, Manjunath Kudlur, Guy Nicholson, James Wang, Pete Warden. +1. **[MPNet](https://huggingface.co/docs/transformers/model_doc/mpnet)** (from Microsoft Research) released with the paper [MPNet: Masked and Permuted Pre-training for Language Understanding](https://huggingface.co/papers/2004.09297) by Kaitao Song, Xu Tan, Tao Qin, Jianfeng Lu, Tie-Yan Liu. +1. **[MPT](https://huggingface.co/docs/transformers/model_doc/mpt)** (from MosaicML) released with the repository [llm-foundry](https://github.com/mosaicml/llm-foundry/) by the MosaicML NLP Team. +1. **[MT5](https://huggingface.co/docs/transformers/model_doc/mt5)** (from Google AI) released with the paper [mT5: A massively multilingual pre-trained text-to-text transformer](https://huggingface.co/papers/2010.11934) by Linting Xue, Noah Constant, Adam Roberts, Mihir Kale, Rami Al-Rfou, Aditya Siddhant, Aditya Barua, Colin Raffel. +1. **[NanoChat](https://huggingface.co/docs/transformers/model_doc/nanochat)** released with the repository [nanochat: The best ChatGPT that $100 can buy](https://github.com/karpathy/nanochat) by Andrej Karpathy. +1. **[Nemotron](https://huggingface.co/docs/transformers/model_doc/nemotron)** (from NVIDIA) released with the blog post [NVIDIA AI Foundation Models: Build Custom Enterprise Chatbots and Co-Pilots with Production-Ready LLMs](https://developer.nvidia.com/blog/nvidia-ai-foundation-models-build-custom-enterprise-chatbots-and-co-pilots-with-production-ready-llms/) by the Nemotron team. +1. **[Nemotron-H](https://huggingface.co/docs/transformers/model_doc/nemotron_h)** (from NVIDIA) released with the papers [Nemotron-H: A Family of Accurate and Efficient Hybrid Mamba-Transformer Models](https://huggingface.co/papers/2504.03624), [NVIDIA Nemotron 3: Efficient and Open Intelligence](https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-White-Paper.pdf), [Nemotron 3 Nano: Open, Efficient Mixture-of-Experts Hybrid Mamba-Transformer Model for Agentic Reasoning](https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Nano-Technical-Report.pdf), and [Nemotron 3 Super: Open, Efficient Mixture-of-Experts Hybrid Mamba-Transformer Model for Agentic Reasoning](https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Super-Technical-Report.pdf) by the Nemotron team. +1. **NeoBERT** (from Chandar Research Lab) released with the paper [NeoBERT: A Next-Generation BERT](https://huggingface.co/papers/2502.19587) by Lola Le Breton, Quentin Fournier, Mariam El Mezouar, John X. Morris, Sarath Chandar. +1. **[NLLB](https://huggingface.co/docs/transformers/model_doc/nllb)** (from Meta) released with the paper [No Language Left Behind: Scaling Human-Centered Machine Translation](https://huggingface.co/papers/2207.04672) by the NLLB team. +1. **[Nougat](https://huggingface.co/docs/transformers/model_doc/nougat)** (from Meta AI) released with the paper [Nougat: Neural Optical Understanding for Academic Documents](https://huggingface.co/papers/2308.13418) by Lukas Blecher, Guillem Cucurull, Thomas Scialom, Robert Stojnic. +1. **[OLMo](https://huggingface.co/docs/transformers/master/model_doc/olmo)** (from Ai2) released with the paper [OLMo: Accelerating the Science of Language Models](https://huggingface.co/papers/2402.00838) by Dirk Groeneveld, Iz Beltagy, Pete Walsh, Akshita Bhagia, Rodney Kinney, Oyvind Tafjord, Ananya Harsh Jha, Hamish Ivison, Ian Magnusson, Yizhong Wang, Shane Arora, David Atkinson, Russell Authur, Khyathi Raghavi Chandu, Arman Cohan, Jennifer Dumas, Yanai Elazar, Yuling Gu, Jack Hessel, Tushar Khot, William Merrill, Jacob Morrison, Niklas Muennighoff, Aakanksha Naik, Crystal Nam, Matthew E. Peters, Valentina Pyatkin, Abhilasha Ravichander, Dustin Schwenk, Saurabh Shah, Will Smith, Emma Strubell, Nishant Subramani, Mitchell Wortsman, Pradeep Dasigi, Nathan Lambert, Kyle Richardson, Luke Zettlemoyer, Jesse Dodge, Kyle Lo, Luca Soldaini, Noah A. Smith, Hannaneh Hajishirzi. +1. **[OLMo2](https://huggingface.co/docs/transformers/master/model_doc/olmo2)** (from Ai2) released with the blog [OLMo 2: The best fully open language model to date](https://allenai.org/blog/olmo2) by the Ai2 OLMo team. +1. **[OLMo3](https://huggingface.co/docs/transformers/master/model_doc/olmo3)** (from Ai2) released with the blog [Olmo 3: Charting a path through the model flow to lead open-source AI](https://allenai.org/blog/olmo3) by the Ai2 OLMo team. +1. **[Olmo Hybrid](https://huggingface.co/docs/transformers/master/model_doc/olmo_hybrid)** (from Ai2) released with the blog [Introducing Olmo Hybrid: Combining transformers and linear RNNs for superior scaling](https://allenai.org/blog/olmohybrid) by the Ai2 OLMo team. +1. **OpenELM** (from Apple) released with the paper [OpenELM: An Efficient Language Model Family with Open-source Training and Inference Framework](https://huggingface.co/papers/2404.14619) by Sachin Mehta, Mohammad Hossein Sekhavat, Qingqing Cao, Maxwell Horton, Yanzi Jin, Chenfan Sun, Iman Mirzadeh, Mahyar Najibi, Dmitry Belenko, Peter Zatloukal, Mohammad Rastegari. +1. **[OPT](https://huggingface.co/docs/transformers/master/model_doc/opt)** (from Meta AI) released with the paper [OPT: Open Pre-trained Transformer Language Models](https://huggingface.co/papers/2205.01068) by Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen et al. +1. **[OWL-ViT](https://huggingface.co/docs/transformers/model_doc/owlvit)** (from Google AI) released with the paper [Simple Open-Vocabulary Object Detection with Vision Transformers](https://huggingface.co/papers/2205.06230) by Matthias Minderer, Alexey Gritsenko, Austin Stone, Maxim Neumann, Dirk Weissenborn, Alexey Dosovitskiy, Aravindh Mahendran, Anurag Arnab, Mostafa Dehghani, Zhuoran Shen, Xiao Wang, Xiaohua Zhai, Thomas Kipf, and Neil Houlsby. +1. **[OWLv2](https://huggingface.co/docs/transformers/model_doc/owlv2)** (from Google AI) released with the paper [Scaling Open-Vocabulary Object Detection](https://huggingface.co/papers/2306.09683) by Matthias Minderer, Alexey Gritsenko, Neil Houlsby. +1. **[PaliGemma](https://huggingface.co/docs/transformers/main/model_doc/paligemma)** (from Google) released with the papers [PaliGemma: A versatile 3B VLM for transfer](https://huggingface.co/papers/2407.07726) and [PaliGemma 2: A Family of Versatile VLMs for Transfer](https://huggingface.co/papers/2412.03555) by the PaliGemma Google team. +1. **[Parakeet](https://huggingface.co/docs/transformers/main/model_doc/parakeet)** (from NVIDIA) released with the blog post [Introducing the Parakeet ASR family](https://developer.nvidia.com/blog/pushing-the-boundaries-of-speech-recognition-with-nemo-parakeet-asr-models/) by the NVIDIA NeMo team. +1. **[PatchTSMixer](https://huggingface.co/docs/transformers/main/model_doc/patchtsmixer)** (from IBM) released with the paper [TSMixer: Lightweight MLP-Mixer Model for Multivariate Time Series Forecasting](https://huggingface.co/papers/2306.09364) by Vijay Ekambaram, Arindam Jati, Nam Nguyen, Phanwadee Sinthong, Jayant Kalagnanam. +1. **[PatchTST](https://huggingface.co/docs/transformers/main/model_doc/patchtst)** (from Princeton University, IBM) released with the paper [A Time Series is Worth 64 Words: Long-term Forecasting with Transformers](https://huggingface.co/papers/2211.14730) by Yuqi Nie, Nam H. Nguyen, Phanwadee Sinthong, Jayant Kalagnanam. +1. **[Phi](https://huggingface.co/docs/transformers/main/model_doc/phi)** (from Microsoft) released with the papers - [Textbooks Are All You Need](https://huggingface.co/papers/2306.11644) by Suriya Gunasekar, Yi Zhang, Jyoti Aneja, Caio César Teodoro Mendes, Allie Del Giorno, Sivakanth Gopi, Mojan Javaheripi, Piero Kauffmann, Gustavo de Rosa, Olli Saarikivi, Adil Salim, Shital Shah, Harkirat Singh Behl, Xin Wang, Sébastien Bubeck, Ronen Eldan, Adam Tauman Kalai, Yin Tat Lee and Yuanzhi Li, [Textbooks Are All You Need II: phi-1.5 technical report](https://huggingface.co/papers/2309.05463) by Yuanzhi Li, Sébastien Bubeck, Ronen Eldan, Allie Del Giorno, Suriya Gunasekar and Yin Tat Lee. +1. **[Phi3](https://huggingface.co/docs/transformers/main/model_doc/phi3)** (from Microsoft) released with the paper [Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone](https://huggingface.co/papers/2404.14219v2) by Marah Abdin, Sam Ade Jacobs, Ammar Ahmad Awan, Jyoti Aneja, Ahmed Awadallah, Hany Awadalla, Nguyen Bach, Amit Bahree, Arash Bakhtiari, Harkirat Behl, Alon Benhaim, Misha Bilenko, Johan Bjorck, Sébastien Bubeck, Martin Cai, Caio César Teodoro Mendes, Weizhu Chen, Vishrav Chaudhary, Parul Chopra, Allie Del Giorno, Gustavo de Rosa, Matthew Dixon, Ronen Eldan, Dan Iter, Amit Garg, Abhishek Goswami, Suriya Gunasekar, Emman Haider, Junheng Hao, Russell J. Hewett, Jamie Huynh, Mojan Javaheripi, Xin Jin, Piero Kauffmann, Nikos Karampatziakis, Dongwoo Kim, Mahoud Khademi, Lev Kurilenko, James R. Lee, Yin Tat Lee, Yuanzhi Li, Chen Liang, Weishung Liu, Eric Lin, Zeqi Lin, Piyush Madan, Arindam Mitra, Hardik Modi, Anh Nguyen, Brandon Norick, Barun Patra, Daniel Perez-Becker, Thomas Portet, Reid Pryzant, Heyang Qin, Marko Radmilac, Corby Rosset, Sambudha Roy, Olatunji Ruwase, Olli Saarikivi, Amin Saied, Adil Salim, Michael Santacroce, Shital Shah, Ning Shang, Hiteshi Sharma, Xia Song, Masahiro Tanaka, Xin Wang, Rachel Ward, Guanhua Wang, Philipp Witte, Michael Wyatt, Can Xu, Jiahang Xu, Sonali Yadav, Fan Yang, Ziyi Yang, Donghan Yu, Chengruidong Zhang, Cyril Zhang, Jianwen Zhang, Li Lyna Zhang, Yi Zhang, Yue Zhang, Yunan Zhang, Xiren Zhou. +1. **Phi3V** (from Microsoft) released with the paper [Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone](https://huggingface.co/papers/2404.14219v4) by Marah Abdin, Jyoti Aneja, Hany Awadalla, Ahmed Awadallah, Ammar Ahmad Awan, Nguyen Bach, Amit Bahree, Arash Bakhtiari, Jianmin Bao, Harkirat Behl, Alon Benhaim, Misha Bilenko, Johan Bjorck, Sébastien Bubeck, Martin Cai, Qin Cai, Vishrav Chaudhary, Dong Chen, Dongdong Chen, Weizhu Chen, Yen-Chun Chen, Yi-Ling Chen, Hao Cheng, Parul Chopra, Xiyang Dai, Matthew Dixon, Ronen Eldan, Victor Fragoso, Jianfeng Gao, Mei Gao, Min Gao, Amit Garg, Allie Del Giorno, Abhishek Goswami, Suriya Gunasekar, Emman Haider, Junheng Hao, Russell J. Hewett, Wenxiang Hu, Jamie Huynh, Dan Iter, Sam Ade Jacobs, Mojan Javaheripi, Xin Jin, Nikos Karampatziakis, Piero Kauffmann, Mahoud Khademi, Dongwoo Kim, Young Jin Kim, Lev Kurilenko, James R. Lee, Yin Tat Lee, Yuanzhi Li, Yunsheng Li, Chen Liang, Lars Liden, Xihui Lin, Zeqi Lin, Ce Liu, Liyuan Liu, Mengchen Liu, Weishung Liu, Xiaodong Liu, Chong Luo, Piyush Madan, Ali Mahmoudzadeh, David Majercak, Matt Mazzola, Caio César Teodoro Mendes, Arindam Mitra, Hardik Modi, Anh Nguyen, Brandon Norick, Barun Patra, Daniel Perez-Becker, Thomas Portet, Reid Pryzant, Heyang Qin, Marko Radmilac, Liliang Ren, Gustavo de Rosa, Corby Rosset, Sambudha Roy, Olatunji Ruwase, Olli Saarikivi, Amin Saied, Adil Salim, Michael Santacroce, Shital Shah, Ning Shang, Hiteshi Sharma, Yelong Shen, Swadheen Shukla, Xia Song, Masahiro Tanaka, Andrea Tupini, Praneetha Vaddamanu, Chunyu Wang, Guanhua Wang, Lijuan Wang , Shuohang Wang, Xin Wang, Yu Wang, Rachel Ward, Wen Wen, Philipp Witte, Haiping Wu, Xiaoxia Wu, Michael Wyatt, Bin Xiao, Can Xu, Jiahang Xu, Weijian Xu, Jilong Xue, Sonali Yadav, Fan Yang, Jianwei Yang, Yifan Yang, Ziyi Yang, Donghan Yu, Lu Yuan, Chenruidong Zhang, Cyril Zhang, Jianwen Zhang, Li Lyna Zhang, Yi Zhang, Yue Zhang, Yunan Zhang, Xiren Zhou. +1. **[PVT](https://huggingface.co/docs/transformers/main/model_doc/pvt)** (from Nanjing University, The University of Hong Kong etc.) released with the paper [Pyramid Vision Transformer: A Versatile Backbone for Dense Prediction without Convolutions](https://huggingface.co/papers/2102.12122) by Wenhai Wang, Enze Xie, Xiang Li, Deng-Ping Fan, Kaitao Song, Ding Liang, Tong Lu, Ping Luo, Ling Shao. +1. **PyAnnote** released in the repository [pyannote/pyannote-audio](https://github.com/pyannote/pyannote-audio) by Hervé Bredin. +1. **[Qwen2](https://huggingface.co/docs/transformers/model_doc/qwen2)** (from the Qwen team, Alibaba Group) released with the papers [Qwen Technical Report](https://huggingface.co/papers/2309.16609) and [Qwen2 Technical Report](https://huggingface.co/papers/2407.10671) by Jinze Bai, Shuai Bai, Yunfei Chu, Zeyu Cui, Kai Dang, Xiaodong Deng, Yang Fan, Wenbin Ge, Yu Han, Fei Huang, Binyuan Hui, Luo Ji, Mei Li, Junyang Lin, Runji Lin, Dayiheng Liu, Gao Liu, Chengqiang Lu, Keming Lu, Jianxin Ma, Rui Men, Xingzhang Ren, Xuancheng Ren, Chuanqi Tan, Sinan Tan, Jianhong Tu, Peng Wang, Shijie Wang, Wei Wang, Shengguang Wu, Benfeng Xu, Jin Xu, An Yang, Hao Yang, Jian Yang, Shusheng Yang, Yang Yao, Bowen Yu, Hongyi Yuan, Zheng Yuan, Jianwei Zhang, Xingxuan Zhang, Yichang Zhang, Zhenru Zhang, Chang Zhou, Jingren Zhou, Xiaohuan Zhou and Tianhang Zhu. +1. **[Qwen2 MoE](https://huggingface.co/docs/transformers/model_doc/qwen2_moe)** (from the Qwen team, Alibaba Group) released with the paper [Qwen2 Technical Report](https://huggingface.co/papers/2407.10671) by Jinze Bai, Shuai Bai, Yunfei Chu, Zeyu Cui, Kai Dang, Xiaodong Deng, Yang Fan, Wenbin Ge, Yu Han, Fei Huang, Binyuan Hui, Luo Ji, Mei Li, Junyang Lin, Runji Lin, Dayiheng Liu, Gao Liu, Chengqiang Lu, Keming Lu, Jianxin Ma, Rui Men, Xingzhang Ren, Xuancheng Ren, Chuanqi Tan, Sinan Tan, Jianhong Tu, Peng Wang, Shijie Wang, Wei Wang, Shengguang Wu, Benfeng Xu, Jin Xu, An Yang, Hao Yang, Jian Yang, Shusheng Yang, Yang Yao, Bowen Yu, Hongyi Yuan, Zheng Yuan, Jianwei Zhang, Xingxuan Zhang, Yichang Zhang, Zhenru Zhang, Chang Zhou, Jingren Zhou, Xiaohuan Zhou and Tianhang Zhu. +1. **[Qwen2-VL](https://huggingface.co/docs/transformers/model_doc/qwen2_vl)** (from the Qwen team, Alibaba Group) released with the papers [Qwen-VL: A Versatile Vision-Language Model for Understanding, Localization, Text Reading, and Beyond](https://huggingface.co/papers/2308.12966) and [Qwen2-VL: Enhancing Vision-Language Model's Perception of the World at Any Resolution](https://huggingface.co/papers/2409.12191) by Peng Wang, Shuai Bai, Sinan Tan, Shijie Wang, Zhihao Fan, Jinze Bai, Keqin Chen, Xuejing Liu, Jialin Wang, Wenbin Ge, Yang Fan, Kai Dang, Mengfei Du, Xuancheng Ren, Rui Men, Dayiheng Liu, Chang Zhou, Jingren Zhou, Junyang Lin. +1. **[Qwen2.5-VL](https://huggingface.co/docs/transformers/model_doc/qwen2_5_vl)** (from the Qwen team, Alibaba Group) released with the paper [Qwen2.5-VL Technical Report](https://huggingface.co/papers/2502.13923) by Shuai Bai, Keqin Chen, Xuejing Liu, Jialin Wang, Wenbin Ge, Sibo Song, Kai Dang, Peng Wang, Shijie Wang, Jun Tang, Humen Zhong, Yuanzhi Zhu, Mingkun Yang, Zhaohai Li, Jianqiang Wan, Pengfei Wang, Wei Ding, Zheren Fu, Yiheng Xu, Jiabo Ye, Xi Zhang, Tianbao Xie, Zesen Cheng, Hang Zhang, Zhibo Yang, Haiyang Xu, Junyang Lin. +1. **[Qwen3](https://huggingface.co/docs/transformers/en/model_doc/qwen3)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3: Think Deeper, Act Faster](https://qwen.ai/blog?id=qwen3) by the Qwen team. +1. **[Qwen3 MoE](https://huggingface.co/docs/transformers/en/model_doc/qwen3_moe)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3: Think Deeper, Act Faster](https://qwen.ai/blog?id=qwen3) by the Qwen team. +1. **[Qwen3 Next](https://huggingface.co/docs/transformers/en/model_doc/qwen3_next)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3-Next: Towards Ultimate Training & Inference Efficiency](https://qwen.ai/blog?id=qwen3-next) by the Qwen team. +1. **[Qwen3-VL](https://huggingface.co/docs/transformers/en/model_doc/qwen3_vl)** (from the Qwen team, Alibaba Group) released with the paper [Qwen3-VL Technical Report](https://huggingface.co/papers/2511.21631) by Shuai Bai, Yuxuan Cai, Ruizhe Chen, Keqin Chen, Xionghui Chen, Zesen Cheng, Lianghao Deng, Wei Ding, Chang Gao, Chunjiang Ge, Wenbin Ge, Zhifang Guo, Qidong Huang, Jie Huang, Fei Huang, Binyuan Hui, Shutong Jiang, Zhaohai Li, Mingsheng Li, Mei Li, Kaixin Li, Zicheng Lin, Junyang Lin, Xuejing Liu, Jiawei Liu, Chenglong Liu, Yang Liu, Dayiheng Liu, Shixuan Liu, Dunjie Lu, Ruilin Luo, Chenxu Lv, Rui Men, Lingchen Meng, Xuancheng Ren, Xingzhang Ren, Sibo Song, Yuchong Sun, Jun Tang, Jianhong Tu, Jianqiang Wan, Peng Wang, Pengfei Wang, Qiuyue Wang, Yuxuan Wang, Tianbao Xie, Yiheng Xu, Haiyang Xu, Jin Xu, Zhibo Yang, Mingkun Yang, Jianxin Yang, An Yang, Bowen Yu, Fei Zhang, Hang Zhang, Xi Zhang, Bo Zheng, Humen Zhong, Jingren Zhou, Fan Zhou, Jing Zhou, Yuanzhi Zhu, Ke Zhu. +1. **[Qwen3-VL MoE](https://huggingface.co/docs/transformers/en/model_doc/qwen3_vl_moe)** (from the Qwen team, Alibaba Group) released with the paper [Qwen3-VL Technical Report](https://huggingface.co/papers/2511.21631) by Shuai Bai, Yuxuan Cai, Ruizhe Chen, Keqin Chen, Xionghui Chen, Zesen Cheng, Lianghao Deng, Wei Ding, Chang Gao, Chunjiang Ge, Wenbin Ge, Zhifang Guo, Qidong Huang, Jie Huang, Fei Huang, Binyuan Hui, Shutong Jiang, Zhaohai Li, Mingsheng Li, Mei Li, Kaixin Li, Zicheng Lin, Junyang Lin, Xuejing Liu, Jiawei Liu, Chenglong Liu, Yang Liu, Dayiheng Liu, Shixuan Liu, Dunjie Lu, Ruilin Luo, Chenxu Lv, Rui Men, Lingchen Meng, Xuancheng Ren, Xingzhang Ren, Sibo Song, Yuchong Sun, Jun Tang, Jianhong Tu, Jianqiang Wan, Peng Wang, Pengfei Wang, Qiuyue Wang, Yuxuan Wang, Tianbao Xie, Yiheng Xu, Haiyang Xu, Jin Xu, Zhibo Yang, Mingkun Yang, Jianxin Yang, An Yang, Bowen Yu, Fei Zhang, Hang Zhang, Xi Zhang, Bo Zheng, Humen Zhong, Jingren Zhou, Fan Zhou, Jing Zhou, Yuanzhi Zhu, Ke Zhu. +1. **[Qwen3.5](https://huggingface.co/docs/transformers/en/model_doc/qwen3_5)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3.5: Towards Native Multimodal Agents](https://qwen.ai/blog?id=qwen3.5) by the Qwen team. +1. **[Qwen3.5 Moe](https://huggingface.co/docs/transformers/en/model_doc/qwen3_5_moe)** (from the Qwen team, Alibaba Group) released with the blog post [Qwen3.5: Towards Native Multimodal Agents](https://qwen.ai/blog?id=qwen3.5) by the Qwen team. +1. **[ResNet](https://huggingface.co/docs/transformers/model_doc/resnet)** (from Microsoft Research) released with the paper [Deep Residual Learning for Image Recognition](https://huggingface.co/papers/1512.03385) by Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. +1. **[RF-DETR](https://huggingface.co/docs/transformers/model_doc/rf_detr)** (from Roboflow) released with the blog post [RF-DETR: A SOTA Real-Time Object Detection Model](https://blog.roboflow.com/rf-detr/) by Peter Robicheaux, James Gallagher, Joseph Nelson, Isaac Robinson. +1. **[RoBERTa](https://huggingface.co/docs/transformers/model_doc/roberta)** (from Facebook), released together with the paper [RoBERTa: A Robustly Optimized BERT Pretraining Approach](https://huggingface.co/papers/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. +1. **[RoFormer](https://huggingface.co/docs/transformers/model_doc/roformer)** (from ZhuiyiTechnology), released together with the paper [RoFormer: Enhanced Transformer with Rotary Position Embedding](https://huggingface.co/papers/2104.09864) by Jianlin Su and Yu Lu and Shengfeng Pan and Bo Wen and Yunfeng Liu. +1. **[RT-DETR](https://huggingface.co/docs/transformers/model_doc/rt_detr)** (from Baidu), released together with the paper [DETRs Beat YOLOs on Real-time Object Detection](https://huggingface.co/papers/2304.08069) by Yian Zhao, Wenyu Lv, Shangliang Xu, Jinman Wei, Guanzhong Wang, Qingqing Dang, Yi Liu, Jie Chen. +1. **[RT-DETRv2](https://huggingface.co/docs/transformers/model_doc/rt_detr_v2)** (from Baidu), released together with the paper [RT-DETRv2: Improved Baseline with Bag-of-Freebies for Real-Time Detection Transformer](https://huggingface.co/papers/2407.17140) by Wenyu Lv, Yian Zhao, Qinyao Chang, Kui Huang, Guanzhong Wang, Yi Liu. +1. **Sapiens** (from Meta AI) released with the paper [Sapiens: Foundation for Human Vision Models](https://huggingface.co/papers/2408.12569) by Rawal Khirodkar, Timur Bagautdinov, Julieta Martinez, Su Zhaoen, Austin James, Peter Selednik, Stuart Anderson, Shunsuke Saito. +1. **[SegFormer](https://huggingface.co/docs/transformers/model_doc/segformer)** (from NVIDIA) released with the paper [SegFormer: Simple and Efficient Design for Semantic Segmentation with Transformers](https://huggingface.co/papers/2105.15203) by Enze Xie, Wenhai Wang, Zhiding Yu, Anima Anandkumar, Jose M. Alvarez, Ping Luo. +1. **[Segment Anything](https://huggingface.co/docs/transformers/model_doc/sam)** (from Meta AI) released with the paper [Segment Anything](https://huggingface.co/papers/2304.02643v1.pdf) by Alexander Kirillov, Eric Mintun, Nikhila Ravi, Hanzi Mao, Chloe Rolland, Laura Gustafson, Tete Xiao, Spencer Whitehead, Alex Berg, Wan-Yen Lo, Piotr Dollar, Ross Girshick. +1. **[Segment Anything 2](https://huggingface.co/docs/transformers/model_doc/sam2)** (from Meta AI) released with the paper [SAM 2: Segment Anything in Images and Videos](https://huggingface.co/papers/2408.00714) by Nikhila Ravi, Valentin Gabeur, Yuan-Ting Hu, Ronghang Hu, Chaitanya Ryali, Tengyu Ma, Haitham Khedr, Roman Rädle, Chloe Rolland, Laura Gustafson, Eric Mintun, Junting Pan, Kalyan Vasudev Alwala, Nicolas Carion, Chao-Yuan Wu, Ross Girshick, Piotr Dollár, Christoph Feichtenhofer. +1. **[Segment Anything 3](https://huggingface.co/docs/transformers/model_doc/sam3)** (from Meta Superintelligence Labs) released with the paper [SAM 3: Segment Anything with Concepts](https://ai.meta.com/research/publications/sam-3-segment-anything-with-concepts/) by SAM 3D Team, Xingyu Chen, Fu-Jen Chu, Pierre Gleize, Kevin J Liang, Alexander Sax, Hao Tang, Weiyao Wang, Michelle Guo, Thibaut Hardin, Xiang Li, Aohan Lin, Jiawei Liu, Ziqi Ma, Anushka Sagar, Bowen Song, Xiaodong Wang, Jianing Yang, Bowen Zhang, Piotr Dollar, Georgia Gkioxari, Matt Feiszli, Jitendra Malik, Nicolas Carion, Laura Gustafson, Yuan-Ting Hu, Shoubhik Debnath, Ronghang Hu, Didac Suris Coll-Vinent, Chaitanya Ryali, Kalyan Vasudev Alwala, Haitham Khedr, Andrew Huang, Jie Lei, Tengyu Ma, Baishan Guo, Arpit Kalla, Markus Marks, Joseph Greer, Meng Wang, Peize Sun, Roman Rädle, Triantafyllos Afouras, Effrosyni Mavroudi, Katherine Xu, Tsung-Han Wu, Yu Zhou, Liliane Momeni, Rishi Hazra, Shuangrui Ding, Sagar Vaze, Francois Porcher, Feng Li, Siyuan Li, Aishwarya Kamath, Ho Kei Cheng, Piotr Dollar, Nikhila Ravi, Kate Saenko, Pengchuan Zhang, Christoph Feichtenhofer. +1. **[SigLIP](https://huggingface.co/docs/transformers/main/model_doc/siglip)** (from Google AI) released with the paper [Sigmoid Loss for Language Image Pre-Training](https://huggingface.co/papers/2303.15343) by Xiaohua Zhai, Basil Mustafa, Alexander Kolesnikov, Lucas Beyer. +1. **[SmolLM3](https://huggingface.co/docs/transformers/main/model_doc/smollm3) (from Hugging Face) released with the blog post [SmolLM3: smol, multilingual, long-context reasoner](https://huggingface.co/blog/smollm3) by the Hugging Face TB Research team. +1. **[SmolVLM](https://huggingface.co/docs/transformers/main/model_doc/smolvlm) (from Hugging Face) released with the blog posts [SmolVLM - small yet mighty Vision Language Model](https://huggingface.co/blog/smolvlm) and [SmolVLM Grows Smaller – Introducing the 250M & 500M Models!](https://huggingface.co/blog/smolervlm) by the Hugging Face TB Research team. +1. **[SolarOpen](https://huggingface.co/docs/transformers/main/model_doc/solar_open)** (from Upstage) released with the paper [Solar Open Technical Report](https://huggingface.co/papers/2601.07022) by Sungrae Park, Sanghoon Kim, Jungho Cho, Gyoungjin Gim, Dawoon Jung, Mikyoung Cha, Eunhae Choo, Taekgyu Hong, Minbyul Jeong, SeHwan Joo, Minsoo Khang, Eunwon Kim, Minjeong Kim, Sujeong Kim, Yunsu Kim, Hyeonju Lee, Seunghyun Lee, Sukyung Lee, Siyoung Park, Gyungin Shin, Inseo Song, Wonho Song, Seonghoon Yang, Seungyoun Yi, Sanghoon Yoon, Jeonghyun Ko, Seyoung Song, Keunwoo Choi, Hwalsuk Lee, Sunghun Kim, Du-Seong Chang, Kyunghyun Cho, Junsuk Choe, Hwaran Lee, Jae-Gil Lee, KyungTae Lim, Alice Oh. +1. **SNAC** (from Papla Media, ETH Zurich) released with the paper [SNAC: Multi-Scale Neural Audio Codec](https://huggingface.co/papers/2410.14411) by Hubert Siuzdak, Florian Grötschla, Luca A. Lanzendörfer. +1. **[SpeechT5](https://huggingface.co/docs/transformers/model_doc/speecht5)** (from Microsoft Research) released with the paper [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://huggingface.co/papers/2110.07205) by Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei. +1. **[SqueezeBERT](https://huggingface.co/docs/transformers/model_doc/squeezebert)** (from Berkeley) released with the paper [SqueezeBERT: What can computer vision teach NLP about efficient neural networks?](https://huggingface.co/papers/2006.11316) by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W. Keutzer. +1. **[StableLm](https://huggingface.co/docs/transformers/model_doc/stablelm)** (from Stability AI) released with the paper [StableLM 3B 4E1T (Technical Report)](https://stability.wandb.io/stability-llm/stable-lm/reports/StableLM-3B-4E1T--VmlldzoyMjU4?accessToken=u3zujipenkx5g7rtcj9qojjgxpconyjktjkli2po09nffrffdhhchq045vp0wyfo) by Jonathan Tow, Marco Bellagente, Dakota Mahan, Carlos Riquelme Ruiz, Duy Phung, Maksym Zhuravinskyi, Nathan Cooper, Nikhil Pinnaparaju, Reshinth Adithyan, and James Baicoianu. +1. **[Starcoder2](https://huggingface.co/docs/transformers/main/model_doc/starcoder2)** (from BigCode team) released with the paper [StarCoder 2 and The Stack v2: The Next Generation](https://huggingface.co/papers/2402.19173) by Anton Lozhkov, Raymond Li, Loubna Ben Allal, Federico Cassano, Joel Lamy-Poirier, Nouamane Tazi, Ao Tang, Dmytro Pykhtar, Jiawei Liu, Yuxiang Wei, Tianyang Liu, Max Tian, Denis Kocetkov, Arthur Zucker, Younes Belkada, Zijian Wang, Qian Liu, Dmitry Abulkhanov, Indraneil Paul, Zhuang Li, Wen-Ding Li, Megan Risdal, Jia Li, Jian Zhu, Terry Yue Zhuo, Evgenii Zheltonozhskii, Nii Osae Osae Dade, Wenhao Yu, Lucas Krauß, Naman Jain, Yixuan Su, Xuanli He, Manan Dey, Edoardo Abati, Yekun Chai, Niklas Muennighoff, Xiangru Tang, Muhtasham Oblokulov, Christopher Akiki, Marc Marone, Chenghao Mou, Mayank Mishra, Alex Gu, Binyuan Hui, Tri Dao, Armel Zebaze, Olivier Dehaene, Nicolas Patry, Canwen Xu, Julian McAuley, Han Hu, Torsten Scholak, Sebastien Paquet, Jennifer Robinson, Carolyn Jane Anderson, Nicolas Chapados, Mostofa Patwary, Nima Tajbakhsh, Yacine Jernite, Carlos Muñoz Ferrandis, Lingming Zhang, Sean Hughes, Thomas Wolf, Arjun Guha, Leandro von Werra, and Harm de Vries. +1. **StyleTTS 2** (from Columbia University) released with the paper [StyleTTS 2: Towards Human-Level Text-to-Speech through Style Diffusion and Adversarial Training with Large Speech Language Models](https://huggingface.co/papers/2306.07691) by Yinghao Aaron Li, Cong Han, Vinay S. Raghavan, Gavin Mischler, Nima Mesgarani. +1. **Supertonic** (from Supertone) released with the paper [SupertonicTTS: Towards Highly Efficient and Streamlined Text-to-Speech System](https://huggingface.co/papers/2503.23108) by Hyeongju Kim, Jinhyeok Yang, Yechan Yu, Seunghun Ji, Jacob Morton, Frederik Bous, Joon Byun, Juheon Lee. +1. **[Swin Transformer](https://huggingface.co/docs/transformers/model_doc/swin)** (from Microsoft) released with the paper [Swin Transformer: Hierarchical Vision Transformer using Shifted Windows](https://huggingface.co/papers/2103.14030) by Ze Liu, Yutong Lin, Yue Cao, Han Hu, Yixuan Wei, Zheng Zhang, Stephen Lin, Baining Guo. +1. **[Swin2SR](https://huggingface.co/docs/transformers/model_doc/swin2sr)** (from University of Würzburg) released with the paper [Swin2SR: SwinV2 Transformer for Compressed Image Super-Resolution and Restoration](https://huggingface.co/papers/2209.11345) by Marcos V. Conde, Ui-Jin Choi, Maxime Burchi, Radu Timofte. +1. **[T5](https://huggingface.co/docs/transformers/model_doc/t5)** (from Google AI) released with the paper [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://huggingface.co/papers/1910.10683) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu. +1. **[T5v1.1](https://huggingface.co/docs/transformers/model_doc/t5v1.1)** (from Google AI) released in the repository [google-research/text-to-text-transfer-transformer](https://github.com/google-research/text-to-text-transfer-transformer/blob/main/released_checkpoints.md#t511) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu. +1. **[Table Transformer](https://huggingface.co/docs/transformers/model_doc/table-transformer)** (from Microsoft Research) released with the paper [PubTables-1M: Towards Comprehensive Table Extraction From Unstructured Documents](https://huggingface.co/papers/2110.00061) by Brandon Smock, Rohith Pesala, Robin Abraham. +1. **[TrOCR](https://huggingface.co/docs/transformers/model_doc/trocr)** (from Microsoft), released together with the paper [TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models](https://huggingface.co/papers/2109.10282) by Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei. +1. **Ultravox** (from Fixie.ai) released with the repository [fixie-ai/ultravox](https://github.com/fixie-ai/ultravox) by the Fixie.ai team. +1. **[UniSpeech](https://huggingface.co/docs/transformers/model_doc/unispeech)** (from Microsoft Research) released with the paper [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled Data](https://huggingface.co/papers/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei, Michael Zeng, Xuedong Huang. +1. **[UniSpeechSat](https://huggingface.co/docs/transformers/model_doc/unispeech-sat)** (from Microsoft Research) released with the paper [UNISPEECH-SAT: UNIVERSAL SPEECH REPRESENTATION LEARNING WITH SPEAKER AWARE PRE-TRAINING](https://huggingface.co/papers/2110.05752) by Sanyuan Chen, Yu Wu, Chengyi Wang, Zhengyang Chen, Zhuo Chen, Shujie Liu, Jian Wu, Yao Qian, Furu Wei, Jinyu Li, Xiangzhan Yu. +1. **[VaultGemma](https://huggingface.co/docs/transformers/main/model_doc/vaultgemma)** (from Google) released with the technical report [VaultGemma: A Differentially Private Gemma Model](https://services.google.com/fh/files/blogs/vaultgemma_tech_report.pdf) by the VaultGemma Google team. +1. **[Vision Transformer (ViT)](https://huggingface.co/docs/transformers/model_doc/vit)** (from Google AI) released with the paper [An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale](https://huggingface.co/papers/2010.11929) by Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, Georg Heigold, Sylvain Gelly, Jakob Uszkoreit, Neil Houlsby. +1. **[ViTMAE](https://huggingface.co/docs/transformers/model_doc/vit_mae)** (from Meta AI) released with the paper [Masked Autoencoders Are Scalable Vision Learners](https://huggingface.co/papers/2111.06377) by Kaiming He, Xinlei Chen, Saining Xie, Yanghao Li, Piotr Dollár, Ross Girshick. +1. **[ViTMatte](https://huggingface.co/docs/transformers/model_doc/vitmatte)** (from HUST-VL) released with the paper [ViTMatte: Boosting Image Matting with Pretrained Plain Vision Transformers](https://huggingface.co/papers/2305.15272) by Jingfeng Yao, Xinggang Wang, Shusheng Yang, Baoyuan Wang. +1. **[ViTMSN](https://huggingface.co/docs/transformers/model_doc/vit_msn)** (from Meta AI) released with the paper [Masked Siamese Networks for Label-Efficient Learning](https://huggingface.co/papers/2204.07141) by Mahmoud Assran, Mathilde Caron, Ishan Misra, Piotr Bojanowski, Florian Bordes, Pascal Vincent, Armand Joulin, Michael Rabbat, Nicolas Ballas. +1. **[ViTPose](https://huggingface.co/docs/transformers/model_doc/vitpose)** (from The University of Sydney) released with the paper [ViTPose: Simple Vision Transformer Baselines for Human Pose Estimation](https://huggingface.co/papers/2204.12484) by Yufei Xu, Jing Zhang, Qiming Zhang, Dacheng Tao. +1. **[VITS](https://huggingface.co/docs/transformers/model_doc/vits)** (from Kakao Enterprise) released with the paper [Conditional Variational Autoencoder with Adversarial Learning for End-to-End Text-to-Speech](https://huggingface.co/papers/2106.06103) by Jaehyeon Kim, Jungil Kong, Juhee Son. +1. **[Voxtral](https://huggingface.co/docs/transformers/model_doc/voxtral)** (from Mistral AI) released with the paper [Voxtral](https://huggingface.co/papers/2507.13264) by Alexander H. Liu, Andy Ehrenberg, Andy Lo, Clément Denoix, Corentin Barreau, Guillaume Lample, Jean-Malo Delignon, Khyathi Raghavi Chandu, Patrick von Platen, Pavankumar Reddy Muddireddy, Sanchit Gandhi, Soham Ghosh, Srijan Mishra, Thomas Foubert, Abhinav Rastogi, Adam Yang, Albert Q. Jiang, Alexandre Sablayrolles, Amélie Héliou, Amélie Martin, Anmol Agarwal, Antoine Roux, Arthur Darcet, Arthur Mensch, Baptiste Bout, Baptiste Rozière, Baudouin De Monicault, Chris Bamford, Christian Wallenwein, Christophe Renaudin, Clémence Lanfranchi, Darius Dabert, Devendra Singh Chaplot, Devon Mizelle, Diego de las Casas, Elliot Chane-Sane, Emilien Fugier, Emma Bou Hanna, Gabrielle Berrada, Gauthier Delerce, Gauthier Guinet, Georgii Novikov, Guillaume Martin, Himanshu Jaju, Jan Ludziejewski, Jason Rute, Jean-Hadrien Chabran, Jessica Chudnovsky, Joachim Studnia, Joep Barmentlo, Jonas Amar, Josselin Somerville Roberts, Julien Denize, Karan Saxena, Karmesh Yadav, Kartik Khandelwal, Kush Jain, Lélio Renard Lavaud, Léonard Blier, Lingxiao Zhao, Louis Martin, Lucile Saulnier, Luyu Gao, Marie Pellat, Mathilde Guillaumin, Mathis Felardos, Matthieu Dinot, Maxime Darrin, Maximilian Augustin, Mickaël Seznec, Neha Gupta, Nikhil Raghuraman, Olivier Duchenne, Patricia Wang, Patryk Saffer, Paul Jacob, Paul Wambergue, Paula Kurylowicz, Philomène Chagniot, Pierre Stock, Pravesh Agrawal, Rémi Delacourt, Romain Sauvestre, Roman Soletskyi, Sagar Vaze, Sandeep Subramanian, Saurabh Garg, Shashwat Dalal, Siddharth Gandhi, Sumukh Aithal, Szymon Antoniak, Teven Le Scao, Thibault Schueller, Thibaut Lavril, Thomas Robert, Thomas Wang, Timothée Lacroix, Tom Bewley, Valeriia Nemychnikova, Victor Paltz , Virgile Richard, Wen-Ding Li, William Marshall, Xuanyu Zhang, Yihan Wan, Yunhao Tang. +1. **[Voxtral Realtime](https://huggingface.co/docs/transformers/model_doc/voxtral_realtime)** (from Mistral AI) released with the paper [Voxtral Realtime](https://huggingface.co/papers/2602.11298) by Alexander H. Liu, Andy Ehrenberg, Andy Lo, Chen-Yo Sun, Guillaume Lample, Jean-Malo Delignon, Khyathi Raghavi Chandu, Patrick von Platen, Pavankumar Reddy Muddireddy, Rohin Arora, Sanchit Gandhi, Sandeep Subramanian, Soham Ghosh, Srijan Mishra, Abhinav Rastogi, Adrien Sadé, Alan Jeffares, Albert Jiang, Alexandre Cahill, Alexandre Gavaudan, Alexandre Sablayrolles, Amélie Héliou, Amos You, Andrew Bai, Angele Lenglemetz, Anmol Agarwal, Anton Eliseev, Antonia Calvi, Arjun Majumdar, Avi Sooriyarachchi, Baptiste Bout, Baptiste Rozière, Baudouin De Monicault, Benjamin Tibi, Charlotte Cronjäger, Clémence Lanfranchi, Connor Chen, Corentin Barreau, Corentin Sautier, Cyprien Courtot, Darius Dabert, Diego de las Casas, Elizaveta Demyanenko, Elliot Chane-Sane, Enguerrand Paquin, Etienne Goffinet, Fabien Niel, Faruk Ahmed, Federico Baldassarre, Gabrielle Berrada, Gaëtan Ecrepont, Gauthier Guinet, Genevieve Hayes, Georgii Novikov, Giada Pistilli, Guillaume Kunsch, Guillaume Martin, Guillaume Raille, Gunjan Dhanuka, Gunshi Gupta, Han Zhou, Harshil Shah, Hope McGovern, Hugo Thimonier, Indraneel Mukherjee, Irene Zhang, Jaeyoung Kim, Jan Ludziejewski, Jason Rute, Joachim Studnia, John Harvill, Jonas Amar, Joséphine Delas, Josselin Somerville Roberts, Julien Tauran, Karmesh Yadav, Kartik Khandelwal, Kilian Tep, Kush Jain, Laurence Aitchison, Laurent Fainsin, Léonard Blier, Lingxiao Zhao, Louis Martin, Lucile Saulnier, Luyu Gao, Maarten Buyl, Manan Sharma, Margaret Jennings, Marie Pellat, Mark Prins, Martin Alexandre, Mathieu Poirée, Mathilde Guillaumin, Matthieu Dinot, Matthieu Futeral, Maxime Darrin, Maximilian Augustin, Mert Unsal, Mia Chiquier , Minh-Quang Pham, Nathan Grinsztajn, Neha Gupta, Olivier Bousquet, Olivier Duchenne, Patricia Wang, Paul Jacob, Paul Wambergue, Paula Kurylowicz, Philippe Pinel, Philomène Chagniot, Pierre Stock, Piotr Miłoś, Prateek Gupta, Pravesh Agrawal, Quentin Torroba, Ram Ramrakhya, Rishi Shah, Romain Sauvestre, Roman Soletskyi, Rosalie Millner, Rupert Menneer, Sagar Vaze, Samuel Barry, Samuel Humeau, Sean Cha, Shashwat Verma, Siddhant Waghjale, Siddharth Gandhi, Simon Lepage, Sumukh Aithal, Szymon Antoniak, Teven Le Scao, Théo Cachet, Theo Simon Sorg, Thibaut Lavril, Thomas Chabal, Thomas Foubert, Thomas Robert, Thomas Wang, Tim Lawson, Tom Bewley, Tom Edwards, Tyler Wang, Umar Jamil, Umberto Tomasini, Valeriia Nemychnikova, Van Phung, Vedant Nanda, Victor Jouault, Vincent Maladière, Virgile Richard, Vladislav Bataev, Wassim Bouaziz, Wen-Ding Li, William Havard, William Marshall, Xinghui Li, Xingran Guo, Xinyu Yang, Yannic Neuhaus, Yassine El Ouahidi, Yassir Bendou, Yihan Wang, Yimu Pan, Zaccharie Ramzi, Zhenlin Xu. +1. **[Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2)** (from Facebook AI) released with the paper [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations](https://huggingface.co/papers/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael Auli. +1. **[Wav2Vec2-BERT](https://huggingface.co/docs/transformers/main/model_doc/wav2vec2-bert)** (from Meta AI) released with the paper [Seamless: Multilingual Expressive and Streaming Speech Translation](https://ai.meta.com/research/publications/seamless-multilingual-expressive-and-streaming-speech-translation/) by the Seamless Communication team. +1. **[WavLM](https://huggingface.co/docs/transformers/model_doc/wavlm)** (from Microsoft Research) released with the paper [WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://huggingface.co/papers/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei. +1. **[Whisper](https://huggingface.co/docs/transformers/model_doc/whisper)** (from OpenAI) released with the paper [Robust Speech Recognition via Large-Scale Weak Supervision](https://cdn.openai.com/papers/whisper.pdf) by Alec Radford, Jong Wook Kim, Tao Xu, Greg Brockman, Christine McLeavey, Ilya Sutskever. +1. **[XLM](https://huggingface.co/docs/transformers/model_doc/xlm)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://huggingface.co/papers/1901.07291) by Guillaume Lample and Alexis Conneau. +1. **[XLM-RoBERTa](https://huggingface.co/docs/transformers/model_doc/xlm-roberta)** (from Facebook AI), released together with the paper [Unsupervised Cross-lingual Representation Learning at Scale](https://huggingface.co/papers/1911.02116) by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov. +1. **[YOLOS](https://huggingface.co/docs/transformers/model_doc/yolos)** (from Huazhong University of Science & Technology) released with the paper [You Only Look at One Sequence: Rethinking Transformer in Vision through Object Detection](https://huggingface.co/papers/2106.00666) by Yuxin Fang, Bencheng Liao, Xinggang Wang, Jiemin Fang, Jiyang Qi, Rui Wu, Jianwei Niu, Wenyu Liu. +1. **[Youtu-LLM](https://huggingface.co/docs/transformers/model_doc/youtu)** (from the Tencent Youtu Team) released with the paper [Youtu-LLM: Unlocking the Native Agentic Potential for Lightweight Large Language Models](https://huggingface.co/papers/2512.24618) by Junru Lu, Jiarui Qin, Lingfeng Qiao, Yinghui Li, Xinyi Dai, Bo Ke, Jianfeng He, Ruizhi Qiao, Di Yin, Xing Sun, Yunsheng Wu, Yinsong Liu, Shuangyin Liu, Mingkong Tang, Haodong Lin, Jiayi Kuang, Fanxu Meng, Xiaojuan Tang, Yunjia Xi, Junjie Huang, Haotong Yang, Zhenyi Shen, Yangning Li, Qianwen Zhang, Yifei Yu, Siyu An, Junnan Dong, Qiufeng Wang, Jie Wang, Keyu Chen, Wei Wen, Taian Guo, Zhifeng Shen, Daohai Yu, Jiahao Li, Ke Li, Zongyi Li, Xiaoyu Tan. +1. **Zaya** (from Zyphra) released with the paper [ZAYA1-8B Technical Report](https://huggingface.co/papers/2605.05365) by Robert Washbourne, Rishi Iyer, Tomas Figliolia, Henry Zheng, Ryan Lorig-Roach, Sungyeon Yang, Pritish Yuvraj, Quentin Anthony, Yury Tokpanov, Xiao Yang, Ganesh Nanduru, Stephen Ebert, Praneeth Medepalli, Skyler Szot, Srivatsan Rajagopal, Alex Ong, Bhavana Mehta, Beren Millidge. diff --git a/packages/transformers/docs/source/_toctree.yml b/packages/transformers/docs/source/_toctree.yml new file mode 100644 index 0000000..76d7792 --- /dev/null +++ b/packages/transformers/docs/source/_toctree.yml @@ -0,0 +1,97 @@ +- local: index + title: 🤗 Transformers.js +- sections: + - local: installation + title: Installation + - local: pipelines + title: The pipeline API + - local: custom_usage + title: Custom usage + title: Get started +- sections: + - local: tutorials/vanilla-js + title: Building a Vanilla JS Application + - local: tutorials/react + title: Building a React Application + - local: tutorials/next + title: Building a Next.js Application + - local: tutorials/browser-extension + title: Building a Browser Extension + - local: tutorials/electron + title: Building an Electron Application + - local: tutorials/node + title: Server-side Inference in Node.js + - local: tutorials/next-ai-sdk + title: Building a Next.js AI Chatbot with Vercel AI SDK + title: Tutorials +- sections: + - local: guides/webgpu + title: Running models on WebGPU + - local: guides/dtypes + title: Using quantized models (dtypes) + - local: guides/private + title: Accessing Private/Gated Models + - local: guides/node-audio-processing + title: Server-side Audio Processing + title: Developer Guides +- sections: + - local: integrations/vercel-ai-sdk + title: Vercel AI SDK + title: Integrations +- sections: + - local: api/transformers + title: Index + - local: api/pipelines + title: Pipelines + - local: api/models + title: Models + - local: api/tokenizers + title: Tokenizers + - local: api/processors + title: Processors + - local: api/configs + title: Configs + - local: api/env + title: Environment variables + - sections: + - local: api/backends/onnx + title: ONNX + title: Backends + isExpanded: false + - sections: + - local: api/generation/parameters + title: Parameters + - local: api/generation/configuration_utils + title: Configuration + - local: api/generation/logits_process + title: Logits Processors + - local: api/generation/logits_sampler + title: Logits Samplers + - local: api/generation/stopping_criteria + title: Stopping Criteria + - local: api/generation/streamers + title: Streamers + title: Generation + isExpanded: false + - sections: + - local: api/utils/core + title: Core + - local: api/utils/hub + title: Hub + - local: api/utils/model_registry + title: Model Registry + - local: api/utils/image + title: Image + - local: api/utils/audio + title: Audio + - local: api/utils/tensor + title: Tensor + - local: api/utils/maths + title: Maths + - local: api/utils/logger + title: Logger + - local: api/utils/random + title: Random + title: Utilities + isExpanded: false + title: API Reference diff --git a/packages/transformers/docs/source/api/.gitignore b/packages/transformers/docs/source/api/.gitignore new file mode 100644 index 0000000..d4d8fcf --- /dev/null +++ b/packages/transformers/docs/source/api/.gitignore @@ -0,0 +1,3 @@ +# These docs are automatically generated from the JSDoc in the source code. +* +!.gitignore diff --git a/packages/transformers/docs/source/custom_usage.md b/packages/transformers/docs/source/custom_usage.md new file mode 100644 index 0000000..8761585 --- /dev/null +++ b/packages/transformers/docs/source/custom_usage.md @@ -0,0 +1,7 @@ +# Use custom models + + +{ + "path": "../snippets/3_custom-usage.snippet" +} + diff --git a/packages/transformers/docs/source/guides/dtypes.md b/packages/transformers/docs/source/guides/dtypes.md new file mode 100644 index 0000000..b3e2472 --- /dev/null +++ b/packages/transformers/docs/source/guides/dtypes.md @@ -0,0 +1,156 @@ +# Using quantized models (dtypes) + +Before Transformers.js v3, we used the `quantized` option to specify whether to use a quantized (q8) or full-precision (fp32) variant of the model by setting `quantized` to `true` or `false`, respectively. Now, we've added the ability to select from a much larger list with the `dtype` parameter. + +The list of available quantizations depends on the model, but some common ones are: full-precision (`"fp32"`), half-precision (`"fp16"`), 8-bit (`"q8"`, `"int8"`, `"uint8"`), and 4-bit (`"q4"`, `"bnb4"`, `"q4f16"`). + +

+ + + + Available dtypes for mixedbread-ai/mxbai-embed-xsmall-v1 + + (e.g., mixedbread-ai/mxbai-embed-xsmall-v1) +

+ +## Basic usage + +**Example:** Run Qwen2.5-0.5B-Instruct in 4-bit quantization ([demo](https://v2.scrimba.com/s0dlcpv0ci)) + +```js +import { pipeline } from "@huggingface/transformers"; + +// Create a text generation pipeline +const generator = await pipeline( + "text-generation", + "onnx-community/Qwen2.5-0.5B-Instruct", + { dtype: "q4", device: "webgpu" }, +); + +// Define the list of messages +const messages = [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Tell me a funny joke." }, +]; + +// Generate a response +const output = await generator(messages, { max_new_tokens: 128 }); +console.log(output[0].generated_text.at(-1).content); +``` + +## Detecting available dtypes + +Not sure which quantizations a model offers? Use `ModelRegistry.get_available_dtypes()` to probe the repository and find out: + +```js +import { ModelRegistry } from "@huggingface/transformers"; + +const dtypes = await ModelRegistry.get_available_dtypes("onnx-community/all-MiniLM-L6-v2-ONNX"); +console.log(dtypes); // e.g., [ 'fp32', 'fp16', 'int8', 'uint8', 'q8', 'q4' ] +``` + +This checks which ONNX files exist on the Hugging Face Hub for each dtype. For multi-session models (e.g., encoder-decoder), a dtype is only listed if **all** required session files are present. + +You can use this to build UIs that let users pick a quantization level, or to automatically select the smallest available dtype: + +```js +const dtypes = await ModelRegistry.get_available_dtypes("onnx-community/Qwen3-0.6B-ONNX"); + +// Pick the smallest available quantization, falling back to fp32 +const preferred = ["q4", "q8", "fp16", "fp32"]; +const dtype = preferred.find((d) => dtypes.includes(d)) ?? "fp32"; + +const generator = await pipeline("text-generation", "onnx-community/Qwen3-0.6B-ONNX", { dtype }); +``` + +## Per-module dtypes + +Some encoder-decoder models, like Whisper or Florence-2, are extremely sensitive to quantization settings: especially of the encoder. For this reason, we added the ability to select per-module dtypes, which can be done by providing a mapping from module name to dtype. + +**Example:** Run Florence-2 on WebGPU ([demo](https://v2.scrimba.com/s0pdm485fo)) + +```js +import { Florence2ForConditionalGeneration } from "@huggingface/transformers"; + +const model = await Florence2ForConditionalGeneration.from_pretrained( + "onnx-community/Florence-2-base-ft", + { + dtype: { + embed_tokens: "fp16", + vision_encoder: "fp16", + encoder_model: "q4", + decoder_model_merged: "q4", + }, + device: "webgpu", + }, +); +``` + +

+ Florence-2 running on WebGPU +

+ +
+ +See full code example + + +```js +import { + Florence2ForConditionalGeneration, + AutoProcessor, + AutoTokenizer, + RawImage, +} from "@huggingface/transformers"; + +// Load model, processor, and tokenizer +const model_id = "onnx-community/Florence-2-base-ft"; +const model = await Florence2ForConditionalGeneration.from_pretrained( + model_id, + { + dtype: { + embed_tokens: "fp16", + vision_encoder: "fp16", + encoder_model: "q4", + decoder_model_merged: "q4", + }, + device: "webgpu", + }, +); +const processor = await AutoProcessor.from_pretrained(model_id); +const tokenizer = await AutoTokenizer.from_pretrained(model_id); + +// Load image and prepare vision inputs +const url = + "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg"; +const image = await RawImage.fromURL(url); +const vision_inputs = await processor(image); + +// Specify task and prepare text inputs +const task = ""; +const prompts = processor.construct_prompts(task); +const text_inputs = tokenizer(prompts); + +// Generate text +const generated_ids = await model.generate({ + ...text_inputs, + ...vision_inputs, + max_new_tokens: 100, +}); + +// Decode generated text +const generated_text = tokenizer.batch_decode(generated_ids, { + skip_special_tokens: false, +})[0]; + +// Post-process the generated text +const result = processor.post_process_generation( + generated_text, + task, + image.size, +); +console.log(result); +// { '': 'A green car is parked in front of a tan building. The building has a brown door and two brown windows. The car is a two door and the door is closed. The green car has black tires.' } +``` + +
diff --git a/packages/transformers/docs/source/guides/node-audio-processing.md b/packages/transformers/docs/source/guides/node-audio-processing.md new file mode 100644 index 0000000..2f82d78 --- /dev/null +++ b/packages/transformers/docs/source/guides/node-audio-processing.md @@ -0,0 +1,112 @@ +# Server-side Audio Processing in Node.js + +A major benefit of writing code for the web is that you can access the multitude of APIs that are available in modern browsers. Unfortunately, when writing server-side code, we are not afforded such luxury, so we have to find another way. In this tutorial, we will design a simple Node.js application that uses Transformers.js for speech recognition with [Whisper](https://huggingface.co/Xenova/whisper-tiny.en), and in the process, learn how to process audio on the server. + +The main problem we need to solve is that the [Web Audio API](https://developer.mozilla.org/en-US/docs/Web/API/Web_Audio_API) is not available in Node.js, meaning we can't use the [`AudioContext`](https://developer.mozilla.org/en-US/docs/Web/API/AudioContext) class to process audio. So, we will need to install third-party libraries to obtain the raw audio data. For this example, we will only consider `.wav` files, but the same principles apply to other audio formats. + + + +This tutorial will be written as an ES module, but you can easily adapt it to use CommonJS instead. For more information, see the [node tutorial](https://huggingface.co/docs/transformers.js/tutorials/node). + + + +**Useful links:** + +- [Source code](https://github.com/huggingface/transformers.js/tree/main/examples/node-audio-processing) +- [Documentation](https://huggingface.co/docs/transformers.js) + +## Prerequisites + +- [Node.js](https://nodejs.org/en/) version 18+ +- [npm](https://www.npmjs.com/) version 9+ + +## Getting started + +Let's start by creating a new Node.js project and installing Transformers.js via [NPM](https://www.npmjs.com/package/@huggingface/transformers): + +```bash +npm init -y +npm i @huggingface/transformers +``` + + + +Remember to add `"type": "module"` to your `package.json` to indicate that your project uses ECMAScript modules. + + + +Next, let's install the [`wavefile`](https://www.npmjs.com/package/wavefile) package, which we will use for loading `.wav` files: + +```bash +npm i wavefile +``` + +## Creating the application + +Start by creating a new file called `index.js`, which will be the entry point for our application. Let's also import the necessary modules: + +```js +import { pipeline } from "@huggingface/transformers"; +import wavefile from "wavefile"; +``` + +For this tutorial, we will use the `Xenova/whisper-tiny.en` model, but feel free to choose one of the other whisper models from the [Hugging Face Hub](https://huggingface.co/models?library=transformers.js&search=whisper). Let's create our pipeline with: + +```js +let transcriber = await pipeline( + "automatic-speech-recognition", + "Xenova/whisper-tiny.en", +); +``` + +Next, let's load an audio file and convert it to the format required by Transformers.js: + +```js +// Load audio data +let url = + "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav"; +let buffer = Buffer.from(await fetch(url).then((x) => x.arrayBuffer())); + +// Read .wav file and convert it to required format +let wav = new wavefile.WaveFile(buffer); +wav.toBitDepth("32f"); // Pipeline expects input as a Float32Array +wav.toSampleRate(16000); // Whisper expects audio with a sampling rate of 16000 +let audioData = wav.getSamples(); +if (Array.isArray(audioData)) { + if (audioData.length > 1) { + const SCALING_FACTOR = Math.sqrt(2); + + // Merge channels (into first channel to save memory) + for (let i = 0; i < audioData[0].length; ++i) { + audioData[0][i] = + (SCALING_FACTOR * (audioData[0][i] + audioData[1][i])) / 2; + } + } + + // Select first channel + audioData = audioData[0]; +} +``` + +Finally, let's run the model and measure execution duration. + +```js +let start = performance.now(); +let output = await transcriber(audioData); +let end = performance.now(); +console.log(`Execution duration: ${(end - start) / 1000} seconds`); +console.log(output); +``` + +You can now run the application with `node index.js`. Note that when running the script for the first time, it may take a while to download and cache the model. Subsequent requests will use the cached model, and model loading will be much faster. + +You should see output similar to: + +``` +Execution duration: 0.6460317999720574 seconds +{ + text: ' And so my fellow Americans ask not what your country can do for you. Ask what you can do for your country.' +} +``` + +That's it! You've successfully created a Node.js application that uses Transformers.js for speech recognition with Whisper. You can now use this as a starting point for your own applications. diff --git a/packages/transformers/docs/source/guides/private.md b/packages/transformers/docs/source/guides/private.md new file mode 100644 index 0000000..1470cd7 --- /dev/null +++ b/packages/transformers/docs/source/guides/private.md @@ -0,0 +1,63 @@ +# Accessing Private/Gated Models + + + +Due to the possibility of leaking access tokens to users of your website or web application, we only support accessing private/gated models from server-side environments (e.g., Node.js) that have access to the process' environment variables. + + + +## Step 1: Generating a User Access Token + +[User Access Tokens](https://huggingface.co/docs/hub/security-tokens) are the preferred way to authenticate an application to Hugging Face services. + +To generate an access token, navigate to the [Access Tokens tab](https://huggingface.co/settings/tokens) in your settings and click on the **New token** button. Choose a name for your token and click **Generate a token** (we recommend keeping the "Role" as read-only). You can then click the **Copy** button next to your newly-created token to copy it to your clipboard. + +
+ + +
+ +To delete or refresh User Access Tokens, you can click the **Manage** button. + +## Step 2: Using the access token in Transformers.js + +Transformers.js will attach an Authorization header to requests made to the Hugging Face Hub when the `HF_TOKEN` environment variable is set and visible to the process. + +One way to do this is to call your program with the environment variable set. For example, let's say you have a file called `llama.js` with the following code: + +```js +import { AutoTokenizer } from "@huggingface/transformers"; + +// Load tokenizer for a gated repository. +const tokenizer = await AutoTokenizer.from_pretrained( + "meta-llama/Llama-2-7b-hf", +); + +// Encode text. +const text = "Hello world!"; +const encoded = tokenizer.encode(text); +console.log(encoded); +``` + +You can then use the following command to set the `HF_TOKEN` environment variable and run the file: + +```bash +HF_TOKEN=hf_... node tests/llama.js +``` + +(remember to replace `hf_...` with your actual access token). + +If done correctly, you should see the following output: + +```bash +[ 1, 15043, 3186, 29991 ] +``` + +Alternatively, you can set the environment variable directly in your code: + +```js +// Set access token (NB: Keep this private!) +process.env.HF_TOKEN = "hf_..."; + +// ... rest of your code +``` diff --git a/packages/transformers/docs/source/guides/webgpu.md b/packages/transformers/docs/source/guides/webgpu.md new file mode 100644 index 0000000..3d6c2ad --- /dev/null +++ b/packages/transformers/docs/source/guides/webgpu.md @@ -0,0 +1,88 @@ +# Running models on WebGPU + +WebGPU is a new web standard for accelerated graphics and compute. The [API](https://developer.mozilla.org/en-US/docs/Web/API/WebGPU_API) enables web developers to use the underlying system's GPU to carry out high-performance computations directly in the browser. WebGPU is the successor to [WebGL](https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API) and provides significantly better performance, because it allows for more direct interaction with modern GPUs. Lastly, it supports general-purpose GPU computations, which makes it just perfect for machine learning! + +> [!WARNING] +> As of October 2024, global WebGPU support is around 70% (according to [caniuse.com](https://caniuse.com/webgpu)), meaning some users may not be able to use the API. +> +> If the following demos do not work in your browser, you may need to enable it using a feature flag: +> +> - Firefox: with the `dom.webgpu.enabled` flag (see [here](https://developer.mozilla.org/en-US/docs/Mozilla/Firefox/Experimental_features#:~:text=tested%20by%20Firefox.-,WebGPU%20API,-The%20WebGPU%20API)). +> - Safari: with the `WebGPU` feature flag (see [here](https://webkit.org/blog/14879/webgpu-now-available-for-testing-in-safari-technology-preview/)). +> - Older Chromium browsers (on Windows, macOS, Linux): with the `enable-unsafe-webgpu` flag (see [here](https://developer.chrome.com/docs/web-platform/webgpu/troubleshooting-tips)). + +## Usage in Transformers.js v3 + +Thanks to our collaboration with [ONNX Runtime Web](https://www.npmjs.com/package/onnxruntime-web), enabling WebGPU acceleration is as simple as setting `device: 'webgpu'` when loading a model. Let's see some examples! + +**Example:** Compute text embeddings on WebGPU ([demo](https://v2.scrimba.com/s06a2smeej)) + +```js +import { pipeline } from "@huggingface/transformers"; + +// Create a feature-extraction pipeline +const extractor = await pipeline( + "feature-extraction", + "mixedbread-ai/mxbai-embed-xsmall-v1", + { device: "webgpu" }, +); + +// Compute embeddings +const texts = ["Hello world!", "This is an example sentence."]; +const embeddings = await extractor(texts, { pooling: "mean", normalize: true }); +console.log(embeddings.tolist()); +// [ +// [-0.016986183822155, 0.03228696808218956, -0.0013630966423079371, ... ], +// [0.09050482511520386, 0.07207386940717697, 0.05762749910354614, ... ], +// ] +``` + +**Example:** Perform automatic speech recognition with OpenAI whisper on WebGPU ([demo](https://v2.scrimba.com/s0oi76h82g)) + +```js +import { pipeline } from "@huggingface/transformers"; + +// Create automatic speech recognition pipeline +const transcriber = await pipeline( + "automatic-speech-recognition", + "onnx-community/whisper-tiny.en", + { device: "webgpu" }, +); + +// Transcribe audio from a URL +const url = + "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav"; +const output = await transcriber(url); +console.log(output); +// { text: ' And so my fellow Americans ask not what your country can do for you, ask what you can do for your country.' } +``` + +**Example:** Perform image classification with MobileNetV4 on WebGPU ([demo](https://v2.scrimba.com/s0fv2uab1t)) + +```js +import { pipeline } from "@huggingface/transformers"; + +// Create image classification pipeline +const classifier = await pipeline( + "image-classification", + "onnx-community/mobilenetv4_conv_small.e2400_r224_in1k", + { device: "webgpu" }, +); + +// Classify an image from a URL +const url = + "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/tiger.jpg"; +const output = await classifier(url); +console.log(output); +// [ +// { label: 'tiger, Panthera tigris', score: 0.6149784922599792 }, +// { label: 'tiger cat', score: 0.30281734466552734 }, +// { label: 'tabby, tabby cat', score: 0.0019135422771796584 }, +// { label: 'lynx, catamount', score: 0.0012161266058683395 }, +// { label: 'Egyptian cat', score: 0.0011465961579233408 } +// ] +``` + +## Reporting bugs and providing feedback + +Due to the experimental nature of WebGPU, especially in non-Chromium browsers, you may experience issues when trying to run a model (even if it can run in WASM). If you do, please open [an issue on GitHub](https://github.com/huggingface/transformers.js/issues/new?title=[WebGPU]%20Error%20running%20MODEL_GOES_HERE&assignees=&labels=bug,webgpu&projects=&template=1_bug-report.yml) and we'll do our best to address it. Thanks! diff --git a/packages/transformers/docs/source/index.md b/packages/transformers/docs/source/index.md new file mode 100644 index 0000000..f8530ba --- /dev/null +++ b/packages/transformers/docs/source/index.md @@ -0,0 +1,46 @@ +# Transformers.js + + +{ + "path": "../snippets/0_introduction.snippet" +} + + +## Quick tour + + +{ + "path": "../snippets/1_quick-tour.snippet" +} + + +## Contents + +The documentation is organized into 5 sections: + +1. **GET STARTED** provides a quick tour of the library and installation instructions to get up and running. +2. **TUTORIALS** are a great place to start if you're a beginner! We also include sample applications for you to play around with! +3. **DEVELOPER GUIDES** show you how to use the library to achieve a specific goal. +4. **INTEGRATIONS** show you how to use Transformers.js with other popular libraries and frameworks. +5. **API REFERENCE** describes all classes and functions, as well as their available parameters and types. + +## Supported tasks/models + +Here is the list of all tasks and architectures currently supported by Transformers.js. +If you don't see your task/model listed here or it is not yet supported, feel free +to open up a feature request [here](https://github.com/huggingface/transformers.js/issues/new/choose). + +To find compatible models on the Hub, select the "transformers.js" library tag in the filter menu (or visit [this link](https://huggingface.co/models?library=transformers.js)). +You can refine your search by selecting the task you're interested in (e.g., [text-classification](https://huggingface.co/models?pipeline_tag=text-classification&library=transformers.js)). + + +{ + "path": "../snippets/4_supported-tasks.snippet" +} + + + +{ + "path": "../snippets/5_supported-models.snippet" +} + diff --git a/packages/transformers/docs/source/installation.md b/packages/transformers/docs/source/installation.md new file mode 100644 index 0000000..0327f58 --- /dev/null +++ b/packages/transformers/docs/source/installation.md @@ -0,0 +1,7 @@ +# Installation + + +{ + "path": "../snippets/2_installation.snippet" +} + diff --git a/packages/transformers/docs/source/integrations/vercel-ai-sdk.md b/packages/transformers/docs/source/integrations/vercel-ai-sdk.md new file mode 100644 index 0000000..9f5d541 --- /dev/null +++ b/packages/transformers/docs/source/integrations/vercel-ai-sdk.md @@ -0,0 +1,286 @@ +# Using Transformers.js with the Vercel AI SDK + +[Vercel AI SDK](https://ai-sdk.dev/) is a popular toolkit for building AI-powered applications. With [`@browser-ai/transformers-js`](https://www.browser-ai.dev/docs/ai-sdk-v6/transformers-js), you can use Transformers.js as a model provider for the AI SDK, enabling in-browser (and server-side) inference with a clean, declarative API. + +This guide covers the core concepts and API patterns. For a full step-by-step project walkthrough, see the [Building a Next.js AI Chatbot](../tutorials/next-ai-sdk) tutorial. + +## Why use the Vercel AI SDK with Transformers.js? + +The `@browser-ai/transformers-js` provider builds on top of `@huggingface/transformers` to give you a standard AI SDK interface — handling Web Worker setup, message passing, progress tracking, streaming, interrupt handling, and state management, so you can use the same `streamText`, `generateText`, and `useChat` APIs you'd use with any other AI SDK provider. +Read more about this [here](https://www.browser-ai.dev/docs/ai-sdk-v6/transformers-js/why). + +## Installation + +```bash +npm install @browser-ai/transformers-js @huggingface/transformers ai @ai-sdk/react +``` + +| @browser-ai/transformers-js | AI SDK | Notes | +|---|---|---| +| v2.0.0+ | v6.x | Current stable | +| v1.0.0 | v5.x | Legacy | + +## Text generation + +### Streaming text + +```js +import { streamText } from "ai"; +import { transformersJS } from "@browser-ai/transformers-js"; + +const result = streamText({ + model: transformersJS("HuggingFaceTB/SmolLM2-360M-Instruct"), + prompt: "Invent a new holiday and describe its traditions.", +}); + +for await (const textPart of result.textStream) { + console.log(textPart); +} +``` + +### Non-streaming text + +```js +import { generateText } from "ai"; +import { transformersJS } from "@browser-ai/transformers-js"; + +const result = await generateText({ + model: transformersJS("HuggingFaceTB/SmolLM2-360M-Instruct"), + prompt: "Invent a new holiday and describe its traditions.", +}); +console.log(result.text); +``` + +## Text embeddings + +```js +import { embed, embedMany } from "ai"; +import { transformersJS } from "@browser-ai/transformers-js"; + +// Single embedding +const { embedding } = await embed({ + model: transformersJS.embedding("Supabase/gte-small"), + value: "Hello, world!", +}); + +// Multiple embeddings +const { embeddings } = await embedMany({ + model: transformersJS.embedding("Supabase/gte-small"), + values: ["Hello", "World", "AI"], +}); +``` + +## Audio transcription + +```js +import { experimental_transcribe as transcribe } from "ai"; +import { transformersJS } from "@browser-ai/transformers-js"; + +const transcript = await transcribe({ + model: transformersJS.transcription("Xenova/whisper-base"), + audio: audioFile, +}); +console.log(transcript.text); +console.log(transcript.segments); // segments with timestamps +``` + +## Vision models + +```js +import { streamText } from "ai"; +import { transformersJS } from "@browser-ai/transformers-js"; + +const result = streamText({ + model: transformersJS("HuggingFaceTB/SmolVLM-256M-Instruct", { + isVisionModel: true, + device: "webgpu", + }), + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Describe this image" }, + { type: "image", image: someImageBlobOrUrl }, + ], + }, + ], +}); + +for await (const chunk of result.textStream) { + console.log(chunk); +} +``` + +## Web Worker offloading + +For better performance, run model inference off the main thread with a Web Worker. + +**1. Create `worker.ts`:** + +```typescript +import { TransformersJSWorkerHandler } from "@browser-ai/transformers-js"; + +const handler = new TransformersJSWorkerHandler(); +self.onmessage = (msg: MessageEvent) => { + handler.onmessage(msg); +}; +``` + +**2. Pass the worker when creating the model:** + +```js +import { streamText } from "ai"; +import { transformersJS } from "@browser-ai/transformers-js"; + +const model = transformersJS("HuggingFaceTB/SmolLM2-360M-Instruct", { + device: "webgpu", + worker: new Worker(new URL("./worker.ts", import.meta.url), { + type: "module", + }), +}); + +const result = streamText({ + model, + messages: [{ role: "user", content: "Hello!" }], +}); +``` + +## Download progress tracking + +Models are downloaded on first use. Track progress to provide a better UX: + +```js +import { streamText } from "ai"; +import { transformersJS } from "@browser-ai/transformers-js"; + +const model = transformersJS("HuggingFaceTB/SmolLM2-360M-Instruct"); +const availability = await model.availability(); + +if (availability === "unavailable") { + console.log("Browser doesn't support Transformers.js"); +} else if (availability === "downloadable") { + await model.createSessionWithProgress(({ progress }) => { + console.log(`Download progress: ${Math.round(progress * 100)}%`); + }); +} + +// Model is ready +const result = streamText({ model, prompt: "Hello!" }); +``` + +## Tool calling + + + +For best tool calling results, use reasoning models like Qwen3 which handle multi-step reasoning well. + + + +```js +import { streamText, tool, stepCountIs } from "ai"; +import { transformersJS } from "@browser-ai/transformers-js"; +import { z } from "zod"; + +const result = await streamText({ + model: transformersJS("onnx-community/Qwen3-0.6B-ONNX"), + messages: [{ role: "user", content: "What's the weather in San Francisco?" }], + tools: { + weather: tool({ + description: "Get the weather in a location", + inputSchema: z.object({ + location: z.string().describe("The location to get the weather for"), + }), + execute: async ({ location }) => ({ + location, + temperature: 72 + Math.floor(Math.random() * 21) - 10, + }), + }), + }, + stopWhen: stepCountIs(5), +}); +``` + +Tool calling also supports [tool execution approval (`needsApproval`)](https://ai-sdk.dev/docs/ai-sdk-core/tools-and-tool-calling#tool-execution-approval) for human-in-the-loop workflows. + +## `useChat` with custom transport + +When using the `useChat` hook, you create a [custom transport](https://ai-sdk.dev/docs/ai-sdk-ui/transport) to handle client-side inference. Here's a minimal example: + +```typescript +import { + ChatTransport, UIMessageChunk, streamText, + convertToModelMessages, ChatRequestOptions, +} from "ai"; +import { + TransformersJSLanguageModel, + TransformersUIMessage, +} from "@browser-ai/transformers-js"; + +export class TransformersChatTransport + implements ChatTransport +{ + constructor(private readonly model: TransformersJSLanguageModel) {} + + async sendMessages( + options: { + chatId: string; + messages: TransformersUIMessage[]; + abortSignal: AbortSignal | undefined; + } & { + trigger: "submit-message" | "submit-tool-result" | "regenerate-message"; + messageId: string | undefined; + } & ChatRequestOptions, + ): Promise> { + const prompt = await convertToModelMessages(options.messages); + const result = streamText({ + model: this.model, + messages: prompt, + abortSignal: options.abortSignal, + }); + return result.toUIMessageStream(); + } + + async reconnectToStream(): Promise | null> { + return null; // client-side AI doesn't support stream reconnection + } +} +``` + +Then use it in your component: + +```typescript +import { useChat } from "@ai-sdk/react"; +import { transformersJS, TransformersUIMessage } from "@browser-ai/transformers-js"; + +const model = transformersJS("HuggingFaceTB/SmolLM2-360M-Instruct", { + device: "webgpu", + worker: new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }), +}); + +const { sendMessage, messages, stop } = useChat({ + transport: new TransformersChatTransport(model), +}); +``` + +## Browser compatibility fallback + +If the device doesn't support in-browser inference, you can fall back to a server-side model: + +```typescript +import { + transformersJS, TransformersUIMessage, + doesBrowserSupportTransformersJS, +} from "@browser-ai/transformers-js"; + +const { sendMessage, messages, stop } = useChat({ + transport: doesBrowserSupportTransformersJS() + ? new TransformersChatTransport(model) + : new DefaultChatTransport({ api: "/api/chat" }), +}); +``` + +## Further reading + +- [Building a Next.js AI Chatbot](../tutorials/next-ai-sdk) — a step-by-step tutorial building a full chatbot with tool calling +- [`@browser-ai/transformers-js` documentation](https://www.browser-ai.dev/docs/ai-sdk-v6/transformers-js) +- [Vercel AI SDK documentation](https://ai-sdk.dev/) diff --git a/packages/transformers/docs/source/pipelines.md b/packages/transformers/docs/source/pipelines.md new file mode 100644 index 0000000..024f51c --- /dev/null +++ b/packages/transformers/docs/source/pipelines.md @@ -0,0 +1,252 @@ +# The `pipeline` API + +Just like the [transformers Python library](https://github.com/huggingface/transformers), Transformers.js provides users with a simple way to leverage the power of transformers. The `pipeline()` function is the easiest and fastest way to use a pretrained model for inference. + + + +For the full list of available tasks/pipelines, check out [this table](#available-tasks). + + + +## The basics + +Start by creating an instance of `pipeline()` and specifying a task you want to use it for. For example, to create a sentiment analysis pipeline, you can do: + +```javascript +import { pipeline } from "@huggingface/transformers"; + +const classifier = await pipeline("sentiment-analysis"); +``` + +When running for the first time, the `pipeline` will download and cache the default pretrained model associated with the task. This can take a while, but subsequent calls will be much faster. + + + +By default, models will be downloaded from the [Hugging Face Hub](https://huggingface.co/models) and stored in [browser cache](https://developer.mozilla.org/en-US/docs/Web/API/Cache), but there are ways to specify custom models and cache locations. For more information see [here](./custom_usage). + + + +You can now use the classifier on your target text by calling it as a function: + +```javascript +const result = await classifier("I love transformers!"); +// [{'label': 'POSITIVE', 'score': 0.9998}] +``` + +If you have multiple inputs, you can pass them as an array: + +```javascript +const result = await classifier([ + "I love transformers!", + "I hate transformers!", +]); +// [{'label': 'POSITIVE', 'score': 0.9998}, {'label': 'NEGATIVE', 'score': 0.9982}] +``` + +You can also specify a different model to use for the pipeline by passing it as the second argument to the `pipeline()` function. For example, to use a different model for sentiment analysis (like one trained to predict sentiment of a review as a number of stars between 1 and 5), you can do: + + + +```javascript +const reviewer = await pipeline( + "sentiment-analysis", + "Xenova/bert-base-multilingual-uncased-sentiment", +); + +const result = await reviewer( + "The Shawshank Redemption is a true masterpiece of cinema.", +); +// [{label: '5 stars', score: 0.8167929649353027}] +``` + +Transformers.js supports loading any model hosted on the Hugging Face Hub, provided it has ONNX weights (located in a subfolder called `onnx`). For more information on how to convert your PyTorch, TensorFlow, or JAX model to ONNX, see the [conversion section](./custom_usage#convert-your-models-to-onnx). + +The `pipeline()` function is a great way to quickly use a pretrained model for inference, as it takes care of all the preprocessing and postprocessing for you. For example, if you want to perform Automatic Speech Recognition (ASR) using OpenAI's Whisper model, you can do: + + + +```javascript +// Create a pipeline for Automatic Speech Recognition +const transcriber = await pipeline( + "automatic-speech-recognition", + "Xenova/whisper-small.en", +); + +// Transcribe an audio file, loaded from a URL. +const result = await transcriber( + "https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/mlk.flac", +); +// {text: ' I have a dream that one day this nation will rise up and live out the true meaning of its creed.'} +``` + +## Pipeline options + +### Loading + +We offer a variety of options to control how models are loaded from the Hugging Face Hub (or locally). +By default, when running in-browser, a _quantized_ version of the model is used, which is smaller and faster, +but usually less accurate. To override this behaviour (i.e., use the unquantized model), you can use a custom +`PretrainedOptions` object as the third parameter to the `pipeline` function: + +```javascript +// Create a pipeline for feature extraction, using the full-precision model (fp32) +const pipe = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2", { + dtype: "fp32", +}); +``` + +Check out the section on [quantization](./guides/dtypes) to learn more. + +You can also specify which revision of the model to use, by passing a `revision` parameter. +Since the Hugging Face Hub uses a git-based versioning system, you can use any valid git revision specifier (e.g., branch name or commit hash). + +```javascript +const transcriber = await pipeline( + "automatic-speech-recognition", + "Xenova/whisper-tiny.en", + { + revision: "output_attentions", + }, +); +``` + +For the full list of options, check out the [PretrainedOptions](./api/utils/hub#module_utils/hub..PretrainedOptions) documentation. + +### Running + +Many pipelines have additional options that you can specify. For example, when using a model that does multilingual translation, you can specify the source and target languages like this: + + + +```javascript +// Create a pipeline for translation +const translator = await pipeline( + "translation", + "Xenova/nllb-200-distilled-600M", +); + +// Translate from English to Greek +const result = await translator("I like to walk my dog.", { + src_lang: "eng_Latn", + tgt_lang: "ell_Grek", +}); +// [ { translation_text: 'Μου αρέσει να περπατάω το σκυλί μου.' } ] + +// Translate back to English +const result2 = await translator(result[0].translation_text, { + src_lang: "ell_Grek", + tgt_lang: "eng_Latn", +}); +// [ { translation_text: 'I like to walk my dog.' } ] +``` + +When using models that support auto-regressive generation, you can specify generation parameters like the number of new tokens, sampling methods, temperature, repetition penalty, and much more. For a full list of available parameters, see to the [GenerationConfig](./api/utils/generation#module_utils/generation.GenerationConfig) class. + +For example, to generate a poem using `LaMini-Flan-T5-783M`, you can do: + + + +```javascript +// Create a pipeline for text2text-generation +const poet = await pipeline( + "text2text-generation", + "Xenova/LaMini-Flan-T5-783M", +); +const result = await poet("Write me a love poem about cheese.", { + max_new_tokens: 200, + temperature: 0.9, + repetition_penalty: 2.0, + no_repeat_ngram_size: 3, +}); +``` + +Logging `result[0].generated_text` to the console gives: + +``` +Cheese, oh cheese! You're the perfect comfort food. +Your texture so smooth and creamy you can never get old. +With every bite it melts in your mouth like buttery delights +that make me feel right at home with this sweet treat of mine. + +From classic to bold flavor combinations, +I love how versatile you are as an ingredient too? +Cheddar is my go-to for any occasion or mood; +It adds depth and richness without being overpowering its taste buds alone +``` + +### Streaming + +Some pipelines such as `text-generation` or `automatic-speech-recognition` support streaming output. This is achieved using the `TextStreamer` class. For example, when using a chat model like `Qwen2.5-Coder-0.5B-Instruct`, you can specify a callback function that will be called with each generated token text (if unset, new tokens will be printed to the console). + +```js +import { pipeline, TextStreamer } from "@huggingface/transformers"; + +// Create a text generation pipeline +const generator = await pipeline( + "text-generation", + "onnx-community/Qwen2.5-Coder-0.5B-Instruct", + { dtype: "q4" }, +); + +// Define the list of messages +const messages = [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Write a quick sort algorithm." }, +]; + +// Create text streamer +const streamer = new TextStreamer(generator.tokenizer, { + skip_prompt: true, + // Optionally, do something with the text (e.g., write to a textbox) + // callback_function: (text) => { /* Do something with text */ }, +}); + +// Generate a response +const result = await generator(messages, { + max_new_tokens: 512, + do_sample: false, + streamer, +}); +``` + +Logging `result[0].generated_text` to the console gives: + +
+Click to view the console output +
+Here's a simple implementation of the quick sort algorithm in Python:
+```python
+def quick_sort(arr):
+    if len(arr) <= 1:
+        return arr
+    pivot = arr[len(arr) // 2]
+    left = [x for x in arr if x < pivot]
+    middle = [x for x in arr if x == pivot]
+    right = [x for x in arr if x > pivot]
+    return quick_sort(left) + middle + quick_sort(right)
+# Example usage:
+arr = [3, 6, 8, 10, 1, 2]
+sorted_arr = quick_sort(arr)
+print(sorted_arr)
+```
+### Explanation:
+- **Base Case**: If the array has less than or equal to one element (i.e., `len(arr)` is less than or equal to `1`), it is already sorted and can be returned as is.
+- **Pivot Selection**: The pivot is chosen as the middle element of the array.
+- **Partitioning**: The array is partitioned into three parts: elements less than the pivot (`left`), elements equal to the pivot (`middle`), and elements greater than the pivot (`right`). These partitions are then recursively sorted.
+- **Recursive Sorting**: The subarrays are sorted recursively using `quick_sort`.
+This approach ensures that each recursive call reduces the problem size by half until it reaches a base case.
+
+
+ +This streaming feature allows you to process the output as it is generated, rather than waiting for the entire output to be generated before processing it. + +For more information on the available options for each pipeline, refer to the [API Reference](./api/pipelines). +If you would like more control over the inference process, you can use the [`AutoModel`](./api/models), [`AutoTokenizer`](./api/tokenizers), or [`AutoProcessor`](./api/processors) classes instead. + +## Available tasks + + +{ + "path": "../snippets/4_supported-tasks.snippet" +} + diff --git a/packages/transformers/docs/source/tutorials/browser-extension.md b/packages/transformers/docs/source/tutorials/browser-extension.md new file mode 100644 index 0000000..faf28e2 --- /dev/null +++ b/packages/transformers/docs/source/tutorials/browser-extension.md @@ -0,0 +1,3 @@ +# Building a browser extension + +_Full tutorial coming soon..._ In the meantime, check out the example application: https://github.com/huggingface/transformers.js-examples/tree/main/browser-extension diff --git a/packages/transformers/docs/source/tutorials/electron.md b/packages/transformers/docs/source/tutorials/electron.md new file mode 100644 index 0000000..06d5705 --- /dev/null +++ b/packages/transformers/docs/source/tutorials/electron.md @@ -0,0 +1,3 @@ +# Building an Electron application + +_Full tutorial coming soon..._ In the meantime, check out the example application: https://github.com/huggingface/transformers.js-examples/tree/main/electron diff --git a/packages/transformers/docs/source/tutorials/next-ai-sdk.md b/packages/transformers/docs/source/tutorials/next-ai-sdk.md new file mode 100644 index 0000000..09f534a --- /dev/null +++ b/packages/transformers/docs/source/tutorials/next-ai-sdk.md @@ -0,0 +1,437 @@ +# Building a Next.js AI Chatbot with Vercel AI SDK + +In this tutorial, we'll build an in-browser AI chatbot using Next.js, Transformers.js, and the Vercel AI SDK v6. The chatbot runs entirely client-side with WebGPU acceleration — and supports tool calling with human approval. + +Useful links: +- [Source code](https://github.com/huggingface/transformers.js-examples/tree/main/next-vercel-ai-sdk-v6-tool-calling) +- [`@browser-ai/transformers-js` docs](https://www.browser-ai.dev/docs/ai-sdk-v6/transformers-js) +- [Vercel AI SDK docs](https://ai-sdk.dev/) + +## Prerequisites + +- [Node.js](https://nodejs.org/en/) version 18+ +- [npm](https://www.npmjs.com/) version 9+ +- A browser with WebGPU support (Chrome 113+, Edge 113+, or Firefox/Safari with flags enabled) + +## Step 1: Create the project + +Create a new Next.js application: + +```bash +npx create-next-app@latest next-ai-chatbot +cd next-ai-chatbot +``` + +Install the AI and Transformers.js dependencies: + +```bash +npm install ai @ai-sdk/react @browser-ai/transformers-js @huggingface/transformers zod +``` + +## Step 2: Configure Next.js for browser inference + +Transformers.js uses ONNX Runtime under the hood for both browser and server-side (Node.js) inference. In our case we only need the browser runtime so we can tell Next.js to exclude the Node.js-specific packages when bundling for the browser. Update `next.config.ts` + +```typescript +import type { NextConfig } from "next"; + +const nextConfig: NextConfig = { + output: "export", // optional: export as a static site + turbopack: {}, + webpack: (config) => { + config.resolve.alias = { + ...config.resolve.alias, + sharp$: false, + "onnxruntime-node$": false, + }; + return config; + }, +}; + +export default nextConfig; +``` + +## Step 3: Create the Web Worker + +Running model inference on the main thread would block the UI. The `@browser-ai/transformers-js` package provides a ready-made worker handler that handles all the complexity for you. + +Create `src/app/worker.ts`: + +```typescript +import { TransformersJSWorkerHandler } from "@browser-ai/transformers-js"; + +const handler = new TransformersJSWorkerHandler(); +self.onmessage = (msg: MessageEvent) => { + handler.onmessage(msg); +}; +``` + +That's it — the handler takes care of model loading, inference, streaming, and communication with the main thread. + +## Step 4: Define the model configuration + +Create `src/app/models.ts` to define which models are available. These are ONNX-format models from Hugging Face: + +```typescript +import { WorkerLoadOptions } from "@browser-ai/transformers-js"; + +export interface ModelConfig extends Omit { + id: string; + name: string; + supportsWorker?: boolean; +} + +export const MODELS: ModelConfig[] = [ + { + id: "onnx-community/Qwen3-0.6B-ONNX", + name: "Qwen3 0.6B", + device: "webgpu", + dtype: "q4f16", + supportsWorker: true, + }, + { + id: "onnx-community/granite-4.0-350m-ONNX-web", + name: "Granite 4.0 350M", + device: "webgpu", + dtype: "fp16", + supportsWorker: true, + }, +]; +``` + + + +For tool calling, use reasoning models like Qwen3 which handle multi-step reasoning well, or fine-tuned model specifically for tool-calling capabilities. The `supportsWorker` flag controls whether the model is loaded in a Web Worker for better performance. + + + +## Step 5: Define tools + +Create `src/app/tools.ts` with tools the model can call. Each tool uses [Zod](https://zod.dev/) for input validation: + +```typescript +import { tool } from "ai"; +import z from "zod"; + +export const createTools = () => ({ + getCurrentTime: tool({ + description: "Get the current date and time.", + inputSchema: z.object({}), + execute: async () => { + const now = new Date(); + return { + timestamp: now.toISOString(), + date: now.toLocaleDateString("en-US", { + weekday: "long", year: "numeric", month: "long", day: "numeric", + }), + time: now.toLocaleTimeString("en-US", { + hour: "2-digit", minute: "2-digit", second: "2-digit", hour12: true, + }), + timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, + }; + }, + }), + randomNumber: tool({ + description: "Generate a random integer between min and max (inclusive).", + inputSchema: z.object({ + min: z.number().describe("The minimum value (inclusive)"), + max: z.number().describe("The maximum value (inclusive)"), + }), + execute: async ({ min, max }) => { + return Math.floor(Math.random() * (Math.floor(max) - Math.ceil(min) + 1)) + Math.ceil(min); + }, + }), + getLocation: tool({ + description: "Get the user's current geographic location.", + inputSchema: z.object({}), + needsApproval: true, // requires user confirmation before executing + execute: async () => { + return new Promise((resolve, reject) => { + navigator.geolocation.getCurrentPosition( + (pos) => resolve({ + latitude: pos.coords.latitude, + longitude: pos.coords.longitude, + }), + (err) => reject(err.message), + ); + }); + }, + }), +}); +``` + +The `getLocation` tool uses `needsApproval: true`, which means the AI SDK will pause execution and wait for the user to approve or reject the tool call before running it. + +## Step 6: Create the chat transport + +The Vercel AI SDK's `useChat` hook needs a [transport](https://ai-sdk.dev/docs/ai-sdk-ui/transport) that handles communication with the model. For client-side inference, we implement a custom `ChatTransport`. + +Create `src/app/chat-transport.ts`: + +```typescript +import { + ChatTransport, UIMessageChunk, streamText, + convertToModelMessages, ChatRequestOptions, + createUIMessageStream, stepCountIs, +} from "ai"; +import { + TransformersJSLanguageModel, + TransformersUIMessage, + transformersJS, +} from "@browser-ai/transformers-js"; +import { MODELS } from "./models"; +import { createTools } from "./tools"; + +export class TransformersChatTransport + implements ChatTransport +{ + private model: TransformersJSLanguageModel; + private tools: ReturnType; + + constructor() { + const config = MODELS[0]; + this.model = transformersJS(config.id, { + device: config.device, + dtype: config.dtype, + ...(config.supportsWorker + ? { + worker: new Worker(new URL("./worker.ts", import.meta.url), { + type: "module", + }), + } + : {}), + }); + this.tools = createTools(); + } + + async sendMessages( + options: { + chatId: string; + messages: TransformersUIMessage[]; + abortSignal: AbortSignal | undefined; + } & { + trigger: "submit-message" | "submit-tool-result" | "regenerate-message"; + messageId: string | undefined; + } & ChatRequestOptions, + ): Promise> { + const { messages, abortSignal } = options; + const prompt = await convertToModelMessages(messages); + + return createUIMessageStream({ + execute: async ({ writer }) => { + // Track download progress if the model hasn't been downloaded yet + let downloadProgressId: string | undefined; + const availability = await this.model.availability(); + + if (availability !== "available") { + await this.model.createSessionWithProgress( + (progress: number) => { + const percent = Math.round(progress * 100); + + if (progress >= 1) { + if (downloadProgressId) { + writer.write({ + type: "data-modelDownloadProgress", + id: downloadProgressId, + data: { + status: "complete", progress: 100, + message: "Model ready!", + }, + }); + } + return; + } + + if (!downloadProgressId) { + downloadProgressId = `download-${Date.now()}`; + } + + writer.write({ + type: "data-modelDownloadProgress", + id: downloadProgressId, + data: { + status: "downloading", progress: percent, + message: `Downloading model... ${percent}%`, + }, + }); + }, + ); + } + + const result = streamText({ + model: this.model, + tools: this.tools, + stopWhen: stepCountIs(5), + messages: prompt, + abortSignal, + }); + + writer.merge(result.toUIMessageStream({ sendStart: false })); + }, + }); + } + + async reconnectToStream(): Promise | null> { + return null; + } +} +``` + +Key parts of the transport: +- **Availability check**: Determines if the model needs downloading before inference. +- **Progress streaming**: Sends download progress as custom data parts (`data-modelDownloadProgress`) that the UI can render as a progress bar. +- **Tool support**: Passes the tools to `streamText()` so the model can call them. +- **Step limiting**: `stopWhen: stepCountIs(5)` prevents infinite tool-calling loops. + +## Step 7: Build the chat UI + +Now wire everything together in your page component. Create `src/app/page.tsx`: + +```tsx +"use client"; + +import { useState } from "react"; +import { useChat } from "@ai-sdk/react"; +import { TransformersUIMessage } from "@browser-ai/transformers-js"; +import { lastAssistantMessageIsCompleteWithApprovalResponses } from "ai"; +import { TransformersChatTransport } from "./chat-transport"; + +export default function ChatPage() { + const [input, setInput] = useState(""); + + const { + messages, + sendMessage, + status, + stop, + addToolApprovalResponse, + } = useChat({ + transport: new TransformersChatTransport(), + experimental_throttle: 75, + // Automatically resumes after tool approval responses are submitted + sendAutomaticallyWhen: lastAssistantMessageIsCompleteWithApprovalResponses, + }); + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + if (input.trim() && status === "ready") { + sendMessage({ text: input }); + setInput(""); + } + }; + + return ( +
+

AI Chatbot

+ +
+ {messages.map((message) => ( +
+ {message.role === "user" ? "You" : "Assistant"}: + {message.parts.map((part, i) => { + switch (part.type) { + case "text": + return

{part.text}

; + + case "data-modelDownloadProgress": + if (!part.data.message) return null; + return ( +
+

{part.data.message}

+ {part.data.status === "downloading" && ( + + )} +
+ ); + + default: + // Handle tool parts + if (part.type.startsWith("tool-") && "state" in part) { + if ( + part.state === "approval-requested" && + "approval" in part + ) { + return ( +
+

Tool {part.type.replace("tool-", "")} wants to run.

+ + +
+ ); + } + if ("output" in part && part.output) { + return ( + 
+                          {JSON.stringify(part.output, null, 2)}
+
+ ); + } + } + return null; + } + })} +
+ ))} +
+ + {status === "submitted" &&

Thinking...

} + +
+ setInput(e.target.value)} + placeholder="Ask something..." + style={{ width: "100%", padding: 8 }} + /> +
+ {status === "streaming" ? ( + + ) : ( + + )} +
+
+
+ ); +} +``` + +The component renders message parts based on their `type`: +- `text` — standard text output from the model. +- `data-modelDownloadProgress` — custom data parts sent by the transport during model download. +- `tool-*` — tool call parts with states like `approval-requested`, `output-available`, etc. + +The `sendAutomaticallyWhen: lastAssistantMessageIsCompleteWithApprovalResponses` option tells `useChat` to automatically resume generation after the user responds to a tool approval request. + +## Step 8: Run the application + +Start the development server: + +```bash +npm run dev +``` + +Open your browser and navigate to the URL shown in the terminal. The first time you send a message, the model will be downloaded and cached in the browser. Subsequent visits will load the cached model. + +Try prompts like: +- "What time is it?" +- "Generate a random number between 1 and 100" +- "Where am I located?" (this will trigger a tool approval prompt) + +## Next steps + +- Add more models and a model selector — see the [full example source](https://github.com/huggingface/transformers.js-examples/tree/main/next-vercel-ai-sdk-v6-tool-calling) for a multi-model implementation with Zustand state management. +- Add a browser compatibility check with `doesBrowserSupportTransformersJS()` and fall back to a server-side route if WebGPU is unavailable. +- Explore the [Vercel AI SDK agents documentation](https://ai-sdk.dev/docs/agents/overview) for more complex agent patterns. +- See the [Vercel AI SDK guide](../integrations/vercel-ai-sdk) for a reference of all supported features (embeddings, vision, transcription, etc.). diff --git a/packages/transformers/docs/source/tutorials/next.md b/packages/transformers/docs/source/tutorials/next.md new file mode 100644 index 0000000..ff0dbf9 --- /dev/null +++ b/packages/transformers/docs/source/tutorials/next.md @@ -0,0 +1,432 @@ +# Building a Next.js application + +In this tutorial, we'll build a simple Next.js application that performs sentiment analysis using Transformers.js! +Since Transformers.js can run in the browser or in Node.js, you can choose whether you want to perform inference [client-side](#client-side-inference) or [server-side](#server-side-inference) (we'll show you how to do both). In either case, we will be developing with the new [App Router](https://nextjs.org/docs/app) paradigm. +The final product will look something like this: + +![Demo](https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/next-demo.gif) + +Useful links: + +- Demo site: [client-side](https://huggingface.co/spaces/Xenova/next-example-app) or [server-side](https://huggingface.co/spaces/Xenova/next-server-example-app) +- Source code: [client-side](https://github.com/huggingface/transformers.js/tree/main/examples/next-client) or [server-side](https://github.com/huggingface/transformers.js/tree/main/examples/next-server) + +## Prerequisites + +- [Node.js](https://nodejs.org/en/) version 18+ +- [npm](https://www.npmjs.com/) version 9+ + +## Client-side inference + +### Step 1: Initialise the project + +Start by creating a new Next.js application using `create-next-app`: + +```bash +npx create-next-app@latest +``` + +On installation, you'll see various prompts. For this demo, we'll be selecting those shown below in bold: + +
√ What is your project named? ... next
+√ Would you like to use TypeScript? ... No / Yes
+√ Would you like to use ESLint? ... No / Yes
+√ Would you like to use Tailwind CSS? ... No / Yes
+√ Would you like to use `src/` directory? ... No / Yes
+√ Would you like to use App Router? (recommended) ... No / Yes
+√ Would you like to customize the default import alias? ... No / Yes
+
+ +### Step 2: Install and configure Transformers.js + +You can install Transformers.js from [NPM](https://www.npmjs.com/package/@huggingface/transformers) with the following command: + +```bash +npm i @huggingface/transformers +``` + +We also need to update the `next.config.js` file to ignore node-specific modules when bundling for the browser: + +```js +/** @type {import('next').NextConfig} */ +const nextConfig = { + // (Optional) Export as a static site + // See https://nextjs.org/docs/pages/building-your-application/deploying/static-exports#configuration + output: "export", // Feel free to modify/remove this option + + // Override the default webpack configuration + webpack: (config) => { + // See https://webpack.js.org/configuration/resolve/#resolvealias + config.resolve.alias = { + ...config.resolve.alias, + sharp$: false, + "onnxruntime-node$": false, + }; + return config; + }, +}; + +module.exports = nextConfig; +``` + +Next, we'll create a new [Web Worker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) script where we'll place all ML-related code. This is to ensure that the main thread is not blocked while the model is loading and performing inference. For this application, we'll be using [`Xenova/distilbert-base-uncased-finetuned-sst-2-english`](https://huggingface.co/Xenova/distilbert-base-uncased-finetuned-sst-2-english), a ~67M parameter model finetuned on the [Stanford Sentiment Treebank](https://huggingface.co/datasets/sst) dataset. Add the following code to `./src/app/worker.js`: + +```js +import { pipeline, env } from "@huggingface/transformers"; + +// Skip local model check +env.allowLocalModels = false; + +// Use the Singleton pattern to enable lazy construction of the pipeline. +class PipelineSingleton { + static task = "text-classification"; + static model = "Xenova/distilbert-base-uncased-finetuned-sst-2-english"; + static instance = null; + + static async getInstance(progress_callback = null) { + if (this.instance === null) { + this.instance = pipeline(this.task, this.model, { progress_callback }); + } + return this.instance; + } +} + +// Listen for messages from the main thread +self.addEventListener("message", async (event) => { + // Retrieve the classification pipeline. When called for the first time, + // this will load the pipeline and save it for future use. + let classifier = await PipelineSingleton.getInstance((x) => { + // We also add a progress callback to the pipeline so that we can + // track model loading. + self.postMessage(x); + }); + + // Actually perform the classification + let output = await classifier(event.data.text); + + // Send the output back to the main thread + self.postMessage({ + status: "complete", + output: output, + }); +}); +``` + +### Step 3: Design the user interface + +We'll now modify the default `./src/app/page.js` file so that it connects to our worker thread. Since we'll only be performing in-browser inference, we can opt-in to Client components using the [`'use client'` directive](https://nextjs.org/docs/getting-started/react-essentials#the-use-client-directive). + +```jsx +'use client' + +import { useState, useEffect, useRef, useCallback } from 'react' + +export default function Home() { + /* TODO: Add state variables */ + + // Create a reference to the worker object. + const worker = useRef(null); + + // We use the `useEffect` hook to set up the worker as soon as the `App` component is mounted. + useEffect(() => { + if (!worker.current) { + // Create the worker if it does not yet exist. + worker.current = new Worker(new URL('./worker.js', import.meta.url), { + type: 'module' + }); + } + + // Create a callback function for messages from the worker thread. + const onMessageReceived = (e) => { /* TODO: See below */}; + + // Attach the callback function as an event listener. + worker.current.addEventListener('message', onMessageReceived); + + // Define a cleanup function for when the component is unmounted. + return () => worker.current.removeEventListener('message', onMessageReceived); + }); + + const classify = useCallback((text) => { + if (worker.current) { + worker.current.postMessage({ text }); + } + }, []); + + return ( /* TODO: See below */ ) +} +``` + +Initialise the following state variables at the beginning of the `Home` component: + +```jsx +// Keep track of the classification result and the model loading status. +const [result, setResult] = useState(null); +const [ready, setReady] = useState(null); +``` + +and fill in the `onMessageReceived` function to update these variables when the worker thread sends a message: + +```js +const onMessageReceived = (e) => { + switch (e.data.status) { + case "initiate": + setReady(false); + break; + case "ready": + setReady(true); + break; + case "complete": + setResult(e.data.output[0]); + break; + } +}; +``` + +Finally, we can add a simple UI to the `Home` component, consisting of an input textbox and a preformatted text element to display the classification result: + +```jsx +
+

Transformers.js

+

Next.js template

+ + { + classify(e.target.value); + }} + /> + + {ready !== null && ( + 
+      {!ready || !result ? "Loading..." : JSON.stringify(result, null, 2)}
+
+ )} +
+``` + +You can now run your application using the following command: + +```bash +npm run dev +``` + +Visit the URL shown in the terminal (e.g., [http://localhost:3000/](http://localhost:3000/)) to see your application in action! + +### (Optional) Step 4: Build and deploy + +To build your application, simply run: + +```bash +npm run build +``` + +This will bundle your application and output the static files to the `out` folder. + +For this demo, we will deploy our application as a static [Hugging Face Space](https://huggingface.co/docs/hub/spaces), but you can deploy it anywhere you like! If you haven't already, you can create a free Hugging Face account [here](https://huggingface.co/join). + +1. Visit [https://huggingface.co/new-space](https://huggingface.co/new-space) and fill in the form. Remember to select "Static" as the space type. +2. Click the "Create space" button at the bottom of the page. +3. Go to "Files" → "Add file" → "Upload files". Drag the files from the `out` folder into the upload box and click "Upload". After they have uploaded, scroll down to the button and click "Commit changes to main". + +**That's it!** Your application should now be live at `https://huggingface.co/spaces//`! + +## Server-side inference + +While there are many different ways to perform server-side inference, the simplest (which we will discuss in this tutorial) is using the new [Route Handlers](https://nextjs.org/docs/app/building-your-application/routing/router-handlers) feature. + +### Step 1: Initialise the project + +Start by creating a new Next.js application using `create-next-app`: + +```bash +npx create-next-app@latest +``` + +On installation, you'll see various prompts. For this demo, we'll be selecting those shown below in bold: + +
√ What is your project named? ... next
+√ Would you like to use TypeScript? ... No / Yes
+√ Would you like to use ESLint? ... No / Yes
+√ Would you like to use Tailwind CSS? ... No / Yes
+√ Would you like to use `src/` directory? ... No / Yes
+√ Would you like to use App Router? (recommended) ... No / Yes
+√ Would you like to customize the default import alias? ... No / Yes
+
+ +### Step 2: Install and configure Transformers.js + +You can install Transformers.js from [NPM](https://www.npmjs.com/package/@huggingface/transformers) with the following command: + +```bash +npm i @huggingface/transformers +``` + +We also need to update the `next.config.js` file to prevent Webpack from bundling certain packages: + +```js +/** @type {import('next').NextConfig} */ +const nextConfig = { + // (Optional) Export as a standalone site + // See https://nextjs.org/docs/pages/api-reference/next-config-js/output#automatically-copying-traced-files + output: "standalone", // Feel free to modify/remove this option + + // Indicate that these packages should not be bundled by webpack + experimental: { + serverComponentsExternalPackages: ["sharp", "onnxruntime-node"], + }, +}; + +module.exports = nextConfig; +``` + +Next, let's set up our Route Handler. We can do this by creating two files in a new `./src/app/classify/` directory: + +1. `pipeline.js` - to handle the construction of our pipeline. + + ```js + import { pipeline } from "@huggingface/transformers"; + + // Use the Singleton pattern to enable lazy construction of the pipeline. + // NOTE: We wrap the class in a function to prevent code duplication (see below). + const P = () => + class PipelineSingleton { + static task = "text-classification"; + static model = "Xenova/distilbert-base-uncased-finetuned-sst-2-english"; + static instance = null; + + static async getInstance(progress_callback = null) { + if (this.instance === null) { + this.instance = pipeline(this.task, this.model, { + progress_callback, + }); + } + return this.instance; + } + }; + + let PipelineSingleton; + if (process.env.NODE_ENV !== "production") { + // When running in development mode, attach the pipeline to the + // global object so that it's preserved between hot reloads. + // For more information, see https://vercel.com/guides/nextjs-prisma-postgres + if (!global.PipelineSingleton) { + global.PipelineSingleton = P(); + } + PipelineSingleton = global.PipelineSingleton; + } else { + PipelineSingleton = P(); + } + export default PipelineSingleton; + ``` + +2. `route.js` - to process requests made to the `/classify` route. + + ```js + import { NextResponse } from "next/server"; + import PipelineSingleton from "./pipeline.js"; + + export async function GET(request) { + const text = request.nextUrl.searchParams.get("text"); + if (!text) { + return NextResponse.json( + { + error: "Missing text parameter", + }, + { status: 400 }, + ); + } + // Get the classification pipeline. When called for the first time, + // this will load the pipeline and cache it for future use. + const classifier = await PipelineSingleton.getInstance(); + + // Actually perform the classification + const result = await classifier(text); + + return NextResponse.json(result); + } + ``` + +### Step 3: Design the user interface + +We'll now modify the default `./src/app/page.js` file to make requests to our newly-created Route Handler. + +```jsx +"use client"; + +import { useState } from "react"; + +export default function Home() { + // Keep track of the classification result and the model loading status. + const [result, setResult] = useState(null); + const [ready, setReady] = useState(null); + + const classify = async (text) => { + if (!text) return; + if (ready === null) setReady(false); + + // Make a request to the /classify route on the server. + const result = await fetch(`/classify?text=${encodeURIComponent(text)}`); + + // If this is the first time we've made a request, set the ready flag. + if (!ready) setReady(true); + + const json = await result.json(); + setResult(json); + }; + return ( +
+

Transformers.js

+

+ Next.js template (server-side) +

+ { + classify(e.target.value); + }} + /> + + {ready !== null && ( + 
+          {!ready || !result ? "Loading..." : JSON.stringify(result, null, 2)}
+
+ )} +
+ ); +} +``` + +You can now run your application using the following command: + +```bash +npm run dev +``` + +Visit the URL shown in the terminal (e.g., [http://localhost:3000/](http://localhost:3000/)) to see your application in action! + +### (Optional) Step 4: Build and deploy + +For this demo, we will build and deploy our application to [Hugging Face Spaces](https://huggingface.co/docs/hub/spaces). If you haven't already, you can create a free Hugging Face account [here](https://huggingface.co/join). + +1. Create a new `Dockerfile` in your project's root folder. You can use our [example Dockerfile](https://github.com/huggingface/transformers.js/blob/main/examples/next-server/Dockerfile) as a template. +2. Visit [https://huggingface.co/new-space](https://huggingface.co/new-space) and fill in the form. Remember to select "Docker" as the space type (you can choose the "Blank" Docker template). +3. Click the "Create space" button at the bottom of the page. +4. Go to "Files" → "Add file" → "Upload files". Drag the files from your project folder (excluding `node_modules` and `.next`, if present) into the upload box and click "Upload". After they have uploaded, scroll down to the button and click "Commit changes to main". +5. Add the following lines to the top of your `README.md`: + ``` + --- + title: Next Server Example App + emoji: 🔥 + colorFrom: yellow + colorTo: red + sdk: docker + pinned: false + app_port: 3000 + --- + ``` + +**That's it!** Your application should now be live at `https://huggingface.co/spaces//`! diff --git a/packages/transformers/docs/source/tutorials/node.md b/packages/transformers/docs/source/tutorials/node.md new file mode 100644 index 0000000..dec6f3f --- /dev/null +++ b/packages/transformers/docs/source/tutorials/node.md @@ -0,0 +1,214 @@ +# Server-side Inference in Node.js + +Although Transformers.js was originally designed to be used in the browser, it's also able to run inference on the server. In this tutorial, we will design a simple Node.js API that uses Transformers.js for sentiment analysis. + +We'll also show you how to use the library in both CommonJS and ECMAScript modules, so you can choose the module system that works best for your project: + +- [ECMAScript modules (ESM)](#ecmascript-modules-esm) - The official standard format + to package JavaScript code for reuse. It's the default module system in modern + browsers, with modules imported using `import` and exported using `export`. + Fortunately, starting with version 13.2.0, Node.js has stable support of ES modules. +- [CommonJS](#commonjs) - The default module system in Node.js. In this system, + modules are imported using `require()` and exported using `module.exports`. + + + +Although you can always use the [Python library](https://github.com/huggingface/transformers) for server-side inference, using Transformers.js means that you can write all of your code in JavaScript (instead of having to set up and communicate with a separate Python process). + + + +**Useful links:** + +- Source code ([ESM](https://github.com/huggingface/transformers.js/tree/main/examples/node/esm/app.js) or [CommonJS](https://github.com/huggingface/transformers.js/tree/main/examples/node/commonjs/app.js)) +- [Documentation](https://huggingface.co/docs/transformers.js) + +## Prerequisites + +- [Node.js](https://nodejs.org/en/) version 18+ +- [npm](https://www.npmjs.com/) version 9+ + +## Getting started + +Let's start by creating a new Node.js project and installing Transformers.js via [NPM](https://www.npmjs.com/package/@huggingface/transformers): + +```bash +npm init -y +npm i @huggingface/transformers +``` + +Next, create a new file called `app.js`, which will be the entry point for our application. Depending on whether you're using [ECMAScript modules](#ecmascript-modules-esm) or [CommonJS](#commonjs), you will need to do some things differently (see below). + +We'll also create a helper class called `MyClassificationPipeline` control the loading of the pipeline. It uses the [singleton pattern](https://en.wikipedia.org/wiki/Singleton_pattern) to lazily create a single instance of the pipeline when `getInstance` is first called, and uses this pipeline for all subsequent calls: + +### ECMAScript modules (ESM) + +To indicate that your project uses ECMAScript modules, you need to add `"type": "module"` to your `package.json`: + +```json +{ + ... + "type": "module", + ... +} +``` + +Next, you will need to add the following imports to the top of `app.js`: + +```javascript +import http from "http"; +import querystring from "querystring"; +import url from "url"; +``` + +Following that, let's import Transformers.js and define the `MyClassificationPipeline` class. + +```javascript +import { pipeline, env } from "@huggingface/transformers"; + +class MyClassificationPipeline { + static task = "text-classification"; + static model = "Xenova/distilbert-base-uncased-finetuned-sst-2-english"; + static instance = null; + + static async getInstance(progress_callback = null) { + if (this.instance === null) { + // NOTE: Uncomment this to change the cache directory + // env.cacheDir = './.cache'; + + this.instance = pipeline(this.task, this.model, { progress_callback }); + } + + return this.instance; + } +} +``` + +### CommonJS + +Start by adding the following imports to the top of `app.js`: + +```javascript +const http = require("http"); +const querystring = require("querystring"); +const url = require("url"); +``` + +Following that, let's import Transformers.js and define the `MyClassificationPipeline` class. Since Transformers.js is an ESM module, we will need to dynamically import the library using the [`import()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/import) function: + +```javascript +class MyClassificationPipeline { + static task = "text-classification"; + static model = "Xenova/distilbert-base-uncased-finetuned-sst-2-english"; + static instance = null; + + static async getInstance(progress_callback = null) { + if (this.instance === null) { + // Dynamically import the Transformers.js library + let { pipeline, env } = await import("@huggingface/transformers"); + + // NOTE: Uncomment this to change the cache directory + // env.cacheDir = './.cache'; + + this.instance = pipeline(this.task, this.model, { progress_callback }); + } + + return this.instance; + } +} +``` + +## Creating a basic HTTP server + +Next, let's create a basic server with the built-in [HTTP](https://nodejs.org/api/http.html#http) module. We will listen for requests made to the server (using the `/classify` endpoint), extract the `text` query parameter, and run this through the pipeline. + +```javascript +// Define the HTTP server +const server = http.createServer(); +const hostname = "127.0.0.1"; +const port = 3000; + +// Listen for requests made to the server +server.on("request", async (req, res) => { + // Parse the request URL + const parsedUrl = url.parse(req.url); + + // Extract the query parameters + const { text } = querystring.parse(parsedUrl.query); + + // Set the response headers + res.setHeader("Content-Type", "application/json"); + + let response; + if (parsedUrl.pathname === "/classify" && text) { + const classifier = await MyClassificationPipeline.getInstance(); + response = await classifier(text); + res.statusCode = 200; + } else { + response = { error: "Bad request" }; + res.statusCode = 400; + } + + // Send the JSON response + res.end(JSON.stringify(response)); +}); + +server.listen(port, hostname, () => { + console.log(`Server running at http://${hostname}:${port}/`); +}); +``` + + + +Since we use lazy loading, the first request made to the server will also be responsible for loading the pipeline. If you would like to begin loading the pipeline as soon as the server starts running, you can add the following line of code after defining `MyClassificationPipeline`: + +```javascript +MyClassificationPipeline.getInstance(); +``` + + + +To start the server, run the following command: + +```bash +node app.js +``` + +The server should be live at http://127.0.0.1:3000/, which you can visit in your web browser. You should see the following message: + +```json +{ "error": "Bad request" } +``` + +This is because we aren't targeting the `/classify` endpoint with a valid `text` query parameter. Let's try again, this time with a valid request. For example, you can visit http://127.0.0.1:3000/classify?text=I%20love%20Transformers.js and you should see: + +```json +[{ "label": "POSITIVE", "score": 0.9996721148490906 }] +``` + +Great! We've successfully created a basic HTTP server that uses Transformers.js to classify text. + +## (Optional) Customization + +### Model caching + +By default, the first time you run the application, it will download the model files and cache them on your file system (in `./node_modules/@huggingface/transformers/.cache/`). All subsequent requests will then use this model. You can change the location of the cache by setting `env.cacheDir`. For example, to cache the model in the `.cache` directory in the current working directory, you can add: + +```javascript +env.cacheDir = "./.cache"; +``` + +### Use local models + +If you want to use local model files, you can set `env.localModelPath` as follows: + +```javascript +// Specify a custom location for models (defaults to '/models/'). +env.localModelPath = "/path/to/models/"; +``` + +You can also disable loading of remote models by setting `env.allowRemoteModels` to `false`: + +```javascript +// Disable the loading of remote models from the Hugging Face Hub: +env.allowRemoteModels = false; +``` diff --git a/packages/transformers/docs/source/tutorials/react.md b/packages/transformers/docs/source/tutorials/react.md new file mode 100644 index 0000000..dba6b0f --- /dev/null +++ b/packages/transformers/docs/source/tutorials/react.md @@ -0,0 +1,524 @@ +# Building a React application + +In this tutorial, we'll be building a simple React application that performs multilingual translation using Transformers.js! The final product will look something like this: + +![Demo](https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/react-translator-demo.gif) + +Useful links: + +- [Demo site](https://huggingface.co/spaces/Xenova/react-translator) +- [Source code](https://github.com/huggingface/transformers.js-examples/tree/main/react-translator) + +## Prerequisites + +- [Node.js](https://nodejs.org/en/) version 18+ +- [npm](https://www.npmjs.com/) version 9+ + +## Step 1: Initialise the project + +For this tutorial, we will use [Vite](https://vitejs.dev/) to initialise our project. Vite is a build tool that allows us to quickly set up a React application with minimal configuration. Run the following command in your terminal: + +```bash +npm create vite@latest react-translator -- --template react +``` + +If prompted to install `create-vite`, type y and press Enter. + +Next, enter the project directory and install the necessary development dependencies: + +```bash +cd react-translator +npm install +``` + +To test that our application is working, we can run the following command: + +```bash +npm run dev +``` + +Visiting the URL shown in the terminal (e.g., [http://localhost:5173/](http://localhost:5173/)) should show the default "React + Vite" landing page. +You can stop the development server by pressing Ctrl + C in the terminal. + +## Step 2: Install and configure Transformers.js + +Now we get to the fun part: adding machine learning to our application! First, install Transformers.js from [NPM](https://www.npmjs.com/package/@huggingface/transformers) with the following command: + +```bash +npm install @huggingface/transformers +``` + +For this application, we will use the [Xenova/nllb-200-distilled-600M](https://huggingface.co/Xenova/nllb-200-distilled-600M) model, which can perform multilingual translation among 200 languages. Before we start, there are 2 things we need to take note of: + +1. ML inference can be quite computationally intensive, so it's better to load and run the models in a separate thread from the main (UI) thread. +2. Since the model is quite large (>1 GB), we don't want to download it until the user clicks the "Translate" button. + +We can achieve both of these goals by using a [Web Worker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) and some [React hooks](https://react.dev/reference/react). + +1. Create a file called `worker.js` in the `src` directory. This script will do all the heavy-lifting for us, including loading and running of the translation pipeline. To ensure the model is only loaded once, we will create the `MyTranslationPipeline` class which use the [singleton pattern](https://en.wikipedia.org/wiki/Singleton_pattern) to lazily create a single instance of the pipeline when `getInstance` is first called, and use this pipeline for all subsequent calls: + + ```javascript + import { pipeline, TextStreamer } from "@huggingface/transformers"; + + class MyTranslationPipeline { + static task = "translation"; + static model = "Xenova/nllb-200-distilled-600M"; + static instance = null; + + static async getInstance(progress_callback = null) { + this.instance ??= pipeline(this.task, this.model, { progress_callback }); + return this.instance; + } + } + ``` + +2. Modify `App.jsx` in the `src` directory. This file is automatically created when initializing our React project, and will contain some boilerplate code. Inside the `App` function, let's create the web worker and store a reference to it using the `useRef` hook: + + ```jsx + // Remember to import the relevant hooks + import { useEffect, useRef, useState } from 'react' + import './App.css' + + function App() { + // Create a reference to the worker object. + const worker = useRef(null); + + // We use the `useEffect` hook to setup the worker as soon as the `App` component is mounted. + useEffect(() => { + // Create the worker if it does not yet exist. + worker.current ??= new Worker(new URL('./worker.js', import.meta.url), { + type: 'module' + }); + + // Create a callback function for messages from the worker thread. + const onMessageReceived = (e) => { + // TODO: Will fill in later + }; + + // Attach the callback function as an event listener. + worker.current.addEventListener('message', onMessageReceived); + + // Define a cleanup function for when the component is unmounted. + return () => worker.current.removeEventListener('message', onMessageReceived); + }); + + return ( + // TODO: Rest of our app goes here... + ) + } + + export default App + + ``` + +## Step 3: Design the user interface + + + +We recommend starting the development server again with `npm run dev` +(if not already running) so that you can see your changes in real-time. + + + +First, let's define our components. Create a folder called `components` in the `src` directory, and create the following files: + +1. `LanguageSelector.jsx`: This component will allow the user to select the input and output languages. Check out the full list of languages [here](https://github.com/huggingface/transformers.js-examples/tree/main/react-translator/src/components/LanguageSelector.jsx). + + ```jsx + const LANGUAGES = { + "Acehnese (Arabic script)": "ace_Arab", + "Acehnese (Latin script)": "ace_Latn", + "Afrikaans": "afr_Latn", + ... + "Zulu": "zul_Latn", + } + + export default function LanguageSelector({ type, onChange, defaultLanguage }) { + return ( +
+ + +
+ ) + } + ``` + +2. `Progress.jsx`: This component will display the progress for downloading each model file. + ```jsx + export default function Progress({ text, percentage }) { + percentage = percentage ?? 0; + return ( +
+
+ {text} ({`${percentage.toFixed(2)}%`}) +
+
+ ); + } + ``` + +We can now use these components in `App.jsx` by adding these imports to the top of the file: + +```jsx +import LanguageSelector from "./components/LanguageSelector"; +import Progress from "./components/Progress"; +``` + +Let's also add some state variables to keep track of a few things in our application, like model loading, languages, input text, and output text. Add the following code to the beginning of the `App` function in `src/App.jsx`: + +```jsx +function App() { + // Model loading + const [ready, setReady] = useState(null); + const [disabled, setDisabled] = useState(false); + const [progressItems, setProgressItems] = useState([]); + + // Inputs and outputs + const [input, setInput] = useState("I love walking my dog."); + const [sourceLanguage, setSourceLanguage] = useState("eng_Latn"); + const [targetLanguage, setTargetLanguage] = useState("fra_Latn"); + const [output, setOutput] = useState(""); + + // rest of the code... +} +``` + +Next, we can add our custom components to the main `App` component. We will also add two `textarea` elements for input and output text, and a `button` to trigger the translation. Modify the `return` statement to look like this: + +```jsx +return ( + <> +

Transformers.js

+

ML-powered multilingual translation in React!

+ +
+
+ setSourceLanguage(x.target.value)} + /> + setTargetLanguage(x.target.value)} + /> +
+ +
+ + +
+
+ + + +
+ {ready === false && } + {progressItems.map((data) => ( +
+ +
+ ))} +
+ +); +``` + +Don't worry about the `translate` function for now. We will define it in the next section. + +Finally, we can add some CSS to make our app look a little nicer. Modify the following files in the `src` directory: + +1. `index.css`: +
+ View code + + ```css + :root { + font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif; + line-height: 1.5; + font-weight: 400; + color: #213547; + background-color: #ffffff; + + font-synthesis: none; + text-rendering: optimizeLegibility; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + -webkit-text-size-adjust: 100%; + } + + body { + margin: 0; + display: flex; + place-items: center; + min-width: 320px; + min-height: 100vh; + } + + h1 { + font-size: 3.2em; + line-height: 1; + } + + h1, + h2 { + margin: 8px; + } + + select { + padding: 0.3em; + cursor: pointer; + } + + textarea { + padding: 0.6em; + } + + button { + padding: 0.6em 1.2em; + cursor: pointer; + font-weight: 500; + } + + button[disabled] { + cursor: not-allowed; + } + + select, + textarea, + button { + border-radius: 8px; + border: 1px solid transparent; + font-size: 1em; + font-family: inherit; + background-color: #f9f9f9; + transition: border-color 0.25s; + } + + select:hover, + textarea:hover, + button:not([disabled]):hover { + border-color: #646cff; + } + + select:focus, + select:focus-visible, + textarea:focus, + textarea:focus-visible, + button:focus, + button:focus-visible { + outline: 4px auto -webkit-focus-ring-color; + } + ``` + +
+ +1. `App.css` +
+ View code + + ```css + #root { + max-width: 1280px; + margin: 0 auto; + padding: 2rem; + text-align: center; + } + + .language-container { + display: flex; + gap: 20px; + } + + .textbox-container { + display: flex; + justify-content: center; + gap: 20px; + width: 800px; + } + + .textbox-container > textarea, + .language-selector { + width: 50%; + } + + .language-selector > select { + width: 150px; + } + + .progress-container { + position: relative; + font-size: 14px; + color: white; + background-color: #e9ecef; + border: solid 1px; + border-radius: 8px; + text-align: left; + overflow: hidden; + } + + .progress-bar { + padding: 0 4px; + z-index: 0; + top: 0; + width: 1%; + overflow: hidden; + background-color: #007bff; + white-space: nowrap; + } + + .progress-text { + z-index: 2; + } + + .selector-container { + display: flex; + gap: 20px; + } + + .progress-bars-container { + padding: 8px; + height: 140px; + } + + .container { + margin: 25px; + display: flex; + flex-direction: column; + gap: 10px; + } + ``` + +
+ +## Step 4: Connecting everything together + +Now that we have a basic user interface set up, we can finally connect everything together. + +First, let's define the `translate` function, which will be called when the user clicks the `Translate` button. This sends a message (containing the input text, source language, and target language) to the worker thread for processing. We will also disable the button so the user doesn't click it multiple times. Add the following code just before the `return` statement in the `App` function: + +```jsx +const translate = () => { + setDisabled(true); + setOutput(""); + worker.current.postMessage({ + text: input, + src_lang: sourceLanguage, + tgt_lang: targetLanguage, + }); +}; +``` + +Now, let's add an event listener in `src/worker.js` to listen for messages from the main thread. We will send back messages (e.g., for model loading progress and text streaming) to the main thread with `self.postMessage`. + +```javascript +// Listen for messages from the main thread +self.addEventListener("message", async (event) => { + // Retrieve the translation pipeline. When called for the first time, + // this will load the pipeline and save it for future use. + const translator = await MyTranslationPipeline.getInstance((x) => { + // We also add a progress callback to the pipeline so that we can + // track model loading. + self.postMessage(x); + }); + + // Capture partial output as it streams from the pipeline + const streamer = new TextStreamer(translator.tokenizer, { + skip_prompt: true, + skip_special_tokens: true, + callback_function: function (text) { + self.postMessage({ + status: "update", + output: text, + }); + }, + }); + + // Actually perform the translation + const output = await translator(event.data.text, { + tgt_lang: event.data.tgt_lang, + src_lang: event.data.src_lang, + + // Allows for partial output to be captured + streamer, + }); + + // Send the output back to the main thread + self.postMessage({ + status: "complete", + output, + }); +}); +``` + +Finally, let's fill in our `onMessageReceived` function in `src/App.jsx`, which will update the application state in response to messages from the worker thread. Add the following code inside the `useEffect` hook we defined earlier: + +```jsx +const onMessageReceived = (e) => { + switch (e.data.status) { + case "initiate": + // Model file start load: add a new progress item to the list. + setReady(false); + setProgressItems((prev) => [...prev, e.data]); + break; + + case "progress": + // Model file progress: update one of the progress items. + setProgressItems((prev) => + prev.map((item) => { + if (item.file === e.data.file) { + return { ...item, progress: e.data.progress }; + } + return item; + }), + ); + break; + + case "done": + // Model file loaded: remove the progress item from the list. + setProgressItems((prev) => + prev.filter((item) => item.file !== e.data.file), + ); + break; + + case "ready": + // Pipeline ready: the worker is ready to accept messages. + setReady(true); + break; + + case "update": + // Generation update: update the output text. + setOutput((o) => o + e.data.output); + break; + + case "complete": + // Generation complete: re-enable the "Translate" button + setDisabled(false); + break; + } +}; +``` + +You can now run the application with `npm run dev` and perform multilingual translation directly in your browser! + +## (Optional) Step 5: Build and deploy + +To build your application, simply run `npm run build`. This will bundle your application and output the static files to the `dist` folder. + +For this demo, we will deploy our application as a static [Hugging Face Space](https://huggingface.co/docs/hub/spaces), but you can deploy it anywhere you like! If you haven't already, you can create a free Hugging Face account [here](https://huggingface.co/join). + +1. Visit [https://huggingface.co/new-space](https://huggingface.co/new-space) and fill in the form. Remember to select "Static" as the space type. +2. Go to "Files" → "Add file" → "Upload files". Drag the `index.html` file and `public/` folder from the `dist` folder into the upload box and click "Upload". After they have uploaded, scroll down to the button and click "Commit changes to main". + +**That's it!** Your application should now be live at `https://huggingface.co/spaces//`! diff --git a/packages/transformers/docs/source/tutorials/vanilla-js.md b/packages/transformers/docs/source/tutorials/vanilla-js.md new file mode 100644 index 0000000..049d085 --- /dev/null +++ b/packages/transformers/docs/source/tutorials/vanilla-js.md @@ -0,0 +1,302 @@ +# Building a Vanilla JavaScript Application + +In this tutorial, you’ll build a simple web application that detects objects in images using Transformers.js! To follow along, all you need is a code editor, a browser, and a simple server (e.g., VS Code Live Server). + +Here's how it works: the user clicks “Upload image” and selects an image using an input dialog. After analysing the image with an object detection model, the predicted bounding boxes are overlaid on top of the image, like this: + +![Demo](https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/js-detection-interence-zebra.png) + +Useful links: + +- [Demo site](https://huggingface.co/spaces/Scrimba/vanilla-js-object-detector) +- [Source code](https://github.com/huggingface/transformers.js-examples/tree/main/vanilla-js) + +## Step 1: HTML and CSS setup + +Before we start building with Transformers.js, we first need to lay the groundwork with some markup and styling. Create an `index.html` file with a basic HTML skeleton, and add the following `
` tag to the ``: + +```html +
+ +
+

+
+``` + +
+ +Click here to see a breakdown of this markup. + +We’re adding an `` element with `type="file"` that accepts images. This allows the user to select an image from their local file system using a popup dialog. The default styling for this element looks quite bad, so let's add some styling. The easiest way to achieve this is to wrap the `` element in a `
+ +Next, add the following CSS rules in a `style.css` file and link it to the HTML: + +```css +html, +body { + font-family: Arial, Helvetica, sans-serif; +} + +.container { + margin: 40px auto; + width: max(50vw, 400px); + display: flex; + flex-direction: column; + align-items: center; +} + +.custom-file-upload { + display: flex; + align-items: center; + gap: 10px; + border: 2px solid black; + padding: 8px 16px; + cursor: pointer; + border-radius: 6px; +} + +#file-upload { + display: none; +} + +.upload-icon { + width: 30px; +} + +#image-container { + width: 100%; + margin-top: 20px; + position: relative; +} + +#image-container > img { + width: 100%; +} +``` + +Here's how the UI looks at this point: + +![Demo](https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/js-detection-btn.png) + +## Step 2: JavaScript setup + +With the _boring_ part out of the way, let's start writing some JavaScript code! Create a file called `index.js` and link to it in `index.html` by adding the following to the end of the ``: + +```html + +``` + + + +The `type="module"` attribute is important, as it turns our file into a [JavaScript module](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Modules), meaning that we’ll be able to use imports and exports. + + + +Moving into `index.js`, let's import Transformers.js by adding the following line to the top of the file: + +```js +import { + pipeline, + env, +} from "https://cdn.jsdelivr.net/npm/@huggingface/transformers"; +``` + +Since we will be downloading the model from the Hugging Face Hub, we can skip the local model check by setting: + +```js +env.allowLocalModels = false; +``` + +Next, let's create references to the various DOM elements we will access later: + +```js +const fileUpload = document.getElementById("file-upload"); +const imageContainer = document.getElementById("image-container"); +const status = document.getElementById("status"); +``` + +## Step 3: Create an object detection pipeline + +We’re finally ready to create our object detection pipeline! As a reminder, a [pipeline](../pipelines). is a high-level interface provided by the library to perform a specific task. In our case, we will instantiate an object detection pipeline with the `pipeline()` helper function. + +Since this can take some time (especially the first time when we have to download the ~40MB model), we first update the `status` paragraph so that the user knows that we’re about to load the model. + +```js +status.textContent = "Loading model..."; +``` + + + +To keep this tutorial simple, we'll be loading and running the model in the main (UI) thread. This is not recommended for production applications, since the UI will freeze when we're performing these actions. This is because JavaScript is a single-threaded language. To overcome this, you can use a [web worker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) to download and run the model in the background. However, we’re not going to do cover that in this tutorial... + + + +We can now call the `pipeline()` function that we imported at the top of our file, to create our object detection pipeline: + +```js +const detector = await pipeline("object-detection", "Xenova/detr-resnet-50"); +``` + +We’re passing two arguments into the `pipeline()` function: (1) task and (2) model. + +1. The first tells Transformers.js what kind of task we want to perform. In our case, that is `object-detection`, but there are many other tasks that the library supports, including `text-generation`, `sentiment-analysis`, `summarization`, or `automatic-speech-recognition`. See [here](https://huggingface.co/docs/transformers.js/pipelines#tasks) for the full list. + +2. The second argument specifies which model we would like to use to solve the given task. We will use [`Xenova/detr-resnet-50`](https://huggingface.co/Xenova/detr-resnet-50), as it is a relatively small (~40MB) but powerful model for detecting objects in an image. + +Once the function returns, we’ll tell the user that the app is ready to be used. + +```js +status.textContent = "Ready"; +``` + +## Step 4: Create the image uploader + +The next step is to support uploading/selection of images. To achieve this, we will listen for "change" events from the `fileUpload` element. In the callback function, we use a `FileReader()` to read the contents of the image if one is selected (and nothing otherwise). + +```js +fileUpload.addEventListener("change", function (e) { + const file = e.target.files[0]; + if (!file) { + return; + } + + const reader = new FileReader(); + + // Set up a callback when the file is loaded + reader.onload = function (e2) { + imageContainer.innerHTML = ""; + const image = document.createElement("img"); + image.src = e2.target.result; + imageContainer.appendChild(image); + // detect(image); // Uncomment this line to run the model + }; + reader.readAsDataURL(file); +}); +``` + +Once the image has been loaded into the browser, the `reader.onload` callback function will be invoked. In it, we append the new `` element to the `imageContainer` to be displayed to the user. + +Don’t worry about the `detect(image)` function call (which is commented out) - we will explain it later! For now, try to run the app and upload an image to the browser. You should see your image displayed under the button like this: + +![Demo](https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/js-detection-btn-img.png) + +## Step 5: Run the model + +We’re finally ready to start interacting with Transformers.js! Let’s uncomment the `detect(image)` function call from the snippet above. Then we’ll define the function itself: + +```js +async function detect(img) { + status.textContent = "Analysing..."; + const output = await detector(img.src, { + threshold: 0.5, + percentage: true, + }); + status.textContent = ""; + console.log("output", output); + // ... +} +``` + + + +NOTE: The `detect` function needs to be asynchronous, since we’ll `await` the result of the the model. + + + +Once we’ve updated the `status` to "Analysing", we’re ready to perform _inference_, which simply means to run the model with some data. This is done via the `detector()` function that was returned from `pipeline()`. The first argument we’re passing is the image data (`img.src`). + +The second argument is an options object: + +- We set the `threshold` property to `0.5`. This means that we want the model to be at least 50% confident before claiming it has detected an object in the image. The lower the threshold, the more objects it'll detect (but may misidentify objects); the higher the threshold, the fewer objects it'll detect (but may miss objects in the scene). +- We also specify `percentage: true`, which means that we want the bounding box for the objects to be returned as percentages (instead of pixels). + +If you now try to run the app and upload an image, you should see the following output logged to the console: + +![Demo](https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/js-detection-console.png) + +In the example above, we uploaded an image of two elephants, so the `output` variable holds an array with two objects, each containing a `label` (the string “elephant”), a `score` (indicating the model's confidence in its prediction) and a `box` object (representing the bounding box of the detected entity). + +## Step 6: Render the boxes + +The final step is to display the `box` coordinates as rectangles around each of the elephants. + +At the end of our `detect()` function, we’ll run the `renderBox` function on each object in the `output` array, using `.forEach()`. + +```js +output.forEach(renderBox); +``` + +Here’s the code for the `renderBox()` function with comments to help you understand what’s going on: + +```js +// Render a bounding box and label on the image +function renderBox({ box, label }) { + const { xmax, xmin, ymax, ymin } = box; + + // Generate a random color for the box + const color = + "#" + + Math.floor(Math.random() * 0xffffff) + .toString(16) + .padStart(6, 0); + + // Draw the box + const boxElement = document.createElement("div"); + boxElement.className = "bounding-box"; + Object.assign(boxElement.style, { + borderColor: color, + left: 100 * xmin + "%", + top: 100 * ymin + "%", + width: 100 * (xmax - xmin) + "%", + height: 100 * (ymax - ymin) + "%", + }); + + // Draw the label + const labelElement = document.createElement("span"); + labelElement.textContent = label; + labelElement.className = "bounding-box-label"; + labelElement.style.backgroundColor = color; + + boxElement.appendChild(labelElement); + imageContainer.appendChild(boxElement); +} +``` + +The bounding box and label span also need some styling, so add the following to the `style.css` file: + +```css +.bounding-box { + position: absolute; + box-sizing: border-box; + border-width: 2px; + border-style: solid; +} + +.bounding-box-label { + color: white; + position: absolute; + font-size: 12px; + margin-top: -16px; + margin-left: -2px; + padding: 1px; +} +``` + +**And that’s it!** + +You've now built your own fully-functional AI application that detects objects in images, which runs completely in your browser: no external server, APIs, or build tools. Pretty cool! 🥳 + +![Demo](https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/js-detection-inference-elephant.png) + +The app is live at the following URL: [https://huggingface.co/spaces/Scrimba/vanilla-js-object-detector](https://huggingface.co/spaces/Scrimba/vanilla-js-object-detector) diff --git a/packages/transformers/jest.config.mjs b/packages/transformers/jest.config.mjs new file mode 100644 index 0000000..03c923c --- /dev/null +++ b/packages/transformers/jest.config.mjs @@ -0,0 +1,192 @@ +/* + * For a detailed explanation regarding each configuration property, visit: + * https://jestjs.io/docs/configuration + */ + +/** @type {import('jest').Config} */ +export default { + // All imported modules in your tests should be mocked automatically + // automock: false, + + // Stop running tests after `n` failures + // bail: 0, + + // Automatically clear mock calls, instances, contexts and results before every test + clearMocks: true, + + // Indicates whether the coverage information should be collected while executing the test + collectCoverage: true, + + // An array of glob patterns indicating a set of files for which coverage information should be collected + // collectCoverageFrom: undefined, + + // The directory where Jest should output its coverage files + coverageDirectory: "coverage", + + // An array of regexp pattern strings used to skip coverage collection + coveragePathIgnorePatterns: ["node_modules", "tests"], + + // Indicates which provider should be used to instrument code for coverage + coverageProvider: "v8", + + // A list of reporter names that Jest uses when writing coverage reports + // coverageReporters: [ + // "json", + // "text", + // "lcov", + // "clover" + // ], + + // An object that configures minimum threshold enforcement for coverage results + // coverageThreshold: undefined, + + // A path to a custom dependency extractor + // dependencyExtractor: undefined, + + // Make calling deprecated APIs throw helpful error messages + // errorOnDeprecated: false, + + // The default configuration for fake timers + // fakeTimers: { + // "enableGlobally": false + // }, + + // Force coverage collection from ignored files using an array of glob patterns + // forceCoverageMatch: [], + + // A path to a module which exports an async function that is triggered once before all test suites + // globalSetup: undefined, + + // A path to a module which exports an async function that is triggered once after all test suites + // globalTeardown: undefined, + + // A set of global variables that need to be available in all test environments + // globals: {}, + + // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers. + // maxWorkers: "50%", + + // An array of directory names to be searched recursively up from the requiring module's location + // moduleDirectories: [ + // "node_modules" + // ], + + // An array of file extensions your modules use + // moduleFileExtensions: [ + // "js", + // "mjs", + // "cjs", + // "jsx", + // "ts", + // "tsx", + // "json", + // "node" + // ], + + // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module + // moduleNameMapper: {}, + + // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader + // modulePathIgnorePatterns: [], + + // Activates notifications for test results + // notify: false, + + // An enum that specifies notification mode. Requires { notify: true } + // notifyMode: "failure-change", + + // A preset that is used as a base for Jest's configuration + // preset: undefined, + + // Run tests from one or more projects + // projects: undefined, + + // Use this configuration option to add custom reporters to Jest + // reporters: undefined, + + // Automatically reset mock state before every test + // resetMocks: false, + + // Reset the module registry before running each individual test + // resetModules: false, + + // A path to a custom resolver + // resolver: undefined, + + // Automatically restore mock state and implementation before every test + // restoreMocks: false, + + // The root directory that Jest should scan for tests and modules within + // rootDir: undefined, + + // A list of paths to directories that Jest should use to search for files in + roots: ["./tests/"], + + // Allows you to use a custom runner instead of Jest's default test runner + // runner: "jest-runner", + + // The paths to modules that run some code to configure or set up the testing environment before each test + // setupFiles: [], + + // A list of paths to modules that run some code to configure or set up the testing framework before each test + // setupFilesAfterEnv: [], + + // The number of seconds after which a test is considered as slow and reported as such in the results. + // slowTestThreshold: 5, + + // Timeout in milliseconds for each test. + testTimeout: 32000, + + // A list of paths to snapshot serializer modules Jest should use for snapshot testing + // snapshotSerializers: [], + + // The test environment that will be used for testing + // testEnvironment: "jest-environment-node", + + // Options that will be passed to the testEnvironment + // testEnvironmentOptions: {}, + + // Adds a location field to test results + // testLocationInResults: false, + + // The glob patterns Jest uses to detect test files + // testMatch: [ + // "**/__tests__/**/*.[jt]s?(x)", + // "**/?(*.)+(spec|test).[tj]s?(x)" + // ], + + // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped + // testPathIgnorePatterns: [ + // "\\\\node_modules\\\\" + // ], + + // The regexp pattern or array of patterns that Jest uses to detect test files + // testRegex: [], + + // This option allows the use of a custom results processor + // testResultsProcessor: undefined, + + // This option allows use of a custom test runner + // testRunner: "jest-circus/runner", + + // A map from regular expressions to paths to transformers + transform: {}, + + // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation + // transformIgnorePatterns: [ + // "\\\\node_modules\\\\", + // "\\.pnp\\.[^\\\\]+$" + // ], + + // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them + // unmockedModulePathPatterns: undefined, + + // Indicates whether each individual test should be reported during the run + // verbose: undefined, + + // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode + // watchPathIgnorePatterns: [], + + // Whether to use watchman for file crawling + // watchman: true, +}; diff --git a/packages/transformers/package.json b/packages/transformers/package.json new file mode 100644 index 0000000..8ea694b --- /dev/null +++ b/packages/transformers/package.json @@ -0,0 +1,87 @@ +{ + "name": "@huggingface/transformers", + "version": "4.2.0", + "description": "State-of-the-art Machine Learning for the web. Run 🤗 Transformers directly in your browser, with no need for a server!", + "main": "./dist/transformers.node.cjs", + "types": "./types/transformers.d.ts", + "type": "module", + "exports": { + "node": { + "import": { + "types": "./types/transformers.d.ts", + "default": "./dist/transformers.node.mjs" + }, + "require": { + "types": "./types/transformers.d.ts", + "default": "./dist/transformers.node.cjs" + } + }, + "default": { + "types": "./types/transformers.d.ts", + "default": "./dist/transformers.web.js" + } + }, + "scripts": { + "format": "prettier --write . --ignore-path ../../.prettierignore", + "format:check": "prettier --check . --ignore-path ../../.prettierignore", + "typegen": "tsc --build", + "dev": "node scripts/dev.mjs", + "build": "node scripts/build.mjs && pnpm typegen", + "test": "node --experimental-vm-modules --expose-gc node_modules/jest/bin/jest.js --verbose --logHeapUsage", + "readme": "python ./docs/scripts/build_readme.py", + "docs-api": "node ./docs/scripts/generate.js", + "docs-build": "doc-builder build transformers.js ./docs/source/ --not_python_module --build_dir ./docs/build/", + "docs-preview": "doc-builder preview transformers.js ./docs/source/ --not_python_module" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/huggingface/transformers.js.git" + }, + "keywords": [ + "transformers", + "transformers.js", + "huggingface", + "hugging face", + "machine learning", + "deep learning", + "artificial intelligence", + "AI", + "ML" + ], + "author": "Hugging Face", + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/huggingface/transformers.js/issues" + }, + "homepage": "https://github.com/huggingface/transformers.js#readme", + "dependencies": { + "@huggingface/jinja": "^0.5.6", + "@huggingface/tokenizers": "^0.1.3", + "onnxruntime-node": "1.24.3", + "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c", + "sharp": "^0.34.5" + }, + "devDependencies": { + "@types/jest": "^30.0.0", + "@types/node": "^24.1.0", + "@webgpu/types": "^0.1.69", + "esbuild": "^0.27.2", + "jest": "^30.2.0", + "jest-environment-node": "^30.2.0", + "jsdoc-to-markdown": "^9.1.3", + "typescript": "5.9.3" + }, + "files": [ + "src", + "dist", + "types", + "README.md", + "LICENSE", + "!**/*.tsbuildinfo" + ], + "publishConfig": { + "access": "public" + }, + "jsdelivr": "./dist/transformers.min.js", + "unpkg": "./dist/transformers.min.js" +} diff --git a/packages/transformers/scripts/build.mjs b/packages/transformers/scripts/build.mjs new file mode 100644 index 0000000..3b27e1e --- /dev/null +++ b/packages/transformers/scripts/build.mjs @@ -0,0 +1,25 @@ +import { OUT_DIR } from "./build/constants.mjs"; +import prepareOutDir from "../../../scripts/prepareOutDir.mjs"; +import { colors, createLogger } from "../../../scripts/logger.mjs"; +import { buildAll } from "./build/buildAll.mjs"; + +const log = createLogger("transformers"); + +log.section("BUILD"); +log.info("Building transformers.js with esbuild...\n"); + +const startTime = performance.now(); + +try { + prepareOutDir(OUT_DIR); + + await buildAll(log); + + const endTime = performance.now(); + const duration = (endTime - startTime).toFixed(2); + log.success(`All builds completed in ${colors.bright}${duration}ms${colors.reset}\n`); +} catch (error) { + log.error(`Build failed: ${error.message}`); + console.error(error); + process.exit(1); +} diff --git a/packages/transformers/scripts/build/buildAll.mjs b/packages/transformers/scripts/build/buildAll.mjs new file mode 100644 index 0000000..8cfc895 --- /dev/null +++ b/packages/transformers/scripts/build/buildAll.mjs @@ -0,0 +1,75 @@ +import { build as esbuild } from "esbuild"; +import path from "node:path"; +import { stripNodePrefixPlugin } from "./plugins/stripNodePrefixPlugin.mjs"; +import { ignoreModulesPlugin } from "./plugins/ignoreModulesPlugin.mjs"; +import { postBuildPlugin } from "./plugins/postBuildPlugin.mjs"; +import { externalNodeBuiltinsPlugin } from "./plugins/externalNodeBuiltinsPlugin.mjs"; +import { OUT_DIR, ROOT_DIR, getEsbuildProdConfig } from "./constants.mjs"; +import { reportSize } from "../../../../scripts/reportSize.mjs"; +import { colors } from "../../../../scripts/logger.mjs"; +import { BUILD_TARGETS } from "./targets.mjs"; + +/** + * Helper function to create build configurations. + */ +async function buildTarget( + { + name = "", + suffix = ".js", + format = "esm", // 'esm' | 'cjs' + ignoreModules = [], + externalModules = [], + usePostBuild = false, + }, + log, +) { + const platform = format === "cjs" ? "node" : "neutral"; + + const regularFile = `transformers${name}${suffix}`; + const minFile = `transformers${name}.min${suffix}`; + + const plugins = []; + // Add ignoreModulesPlugin FIRST so it can catch modules before stripNodePrefixPlugin marks them as external + if (ignoreModules.length > 0) { + plugins.push(ignoreModulesPlugin(ignoreModules)); + } + plugins.push(stripNodePrefixPlugin()); + plugins.push(externalNodeBuiltinsPlugin()); + if (usePostBuild) { + plugins.push(postBuildPlugin(OUT_DIR, ROOT_DIR)); + } + + log.build(`Building ${colors.bright}${regularFile}${colors.reset}...`); + await esbuild({ + ...getEsbuildProdConfig(ROOT_DIR), + platform, + format, + outfile: path.join(OUT_DIR, regularFile), + external: externalModules, + plugins, + }); + reportSize(path.join(OUT_DIR, regularFile), log); + + log.build(`Building ${colors.bright}${minFile}${colors.reset}...`); + await esbuild({ + ...getEsbuildProdConfig(ROOT_DIR), + platform, + format, + outfile: path.join(OUT_DIR, minFile), + minify: true, + external: externalModules, + plugins, + legalComments: "none", + }); + reportSize(path.join(OUT_DIR, minFile), log); +} + +/** + * Build all targets for production + */ +export async function buildAll(log) { + for (const target of BUILD_TARGETS) { + log.section(target.name); + await buildTarget(target.config, log); + } +} diff --git a/packages/transformers/scripts/build/buildAllWithWatch.mjs b/packages/transformers/scripts/build/buildAllWithWatch.mjs new file mode 100644 index 0000000..05f6aa8 --- /dev/null +++ b/packages/transformers/scripts/build/buildAllWithWatch.mjs @@ -0,0 +1,60 @@ +import { context } from "esbuild"; +import path from "node:path"; +import { stripNodePrefixPlugin } from "./plugins/stripNodePrefixPlugin.mjs"; +import { ignoreModulesPlugin } from "./plugins/ignoreModulesPlugin.mjs"; +import { postBuildPlugin } from "./plugins/postBuildPlugin.mjs"; +import { externalNodeBuiltinsPlugin } from "./plugins/externalNodeBuiltinsPlugin.mjs"; +import { rebuildPlugin } from "../../../../scripts/rebuildPlugin.mjs"; +import { OUT_DIR, ROOT_DIR, getEsbuildDevConfig } from "./constants.mjs"; +import { BUILD_TARGETS } from "./targets.mjs"; + +/** + * Create an esbuild context for a single build target + */ +async function createBuildContext(targetName, targetConfig, log) { + const { name, suffix, format, ignoreModules, externalModules, usePostBuild } = targetConfig; + const platform = format === "cjs" ? "node" : "neutral"; + const outputFile = `transformers${name}${suffix}`; + + const plugins = []; + // Add ignoreModulesPlugin FIRST so it can catch modules before stripNodePrefixPlugin marks them as external + if (ignoreModules.length > 0) { + plugins.push(ignoreModulesPlugin(ignoreModules)); + } + plugins.push(stripNodePrefixPlugin()); + plugins.push(externalNodeBuiltinsPlugin()); + if (usePostBuild) { + plugins.push(postBuildPlugin(OUT_DIR, ROOT_DIR)); + } + plugins.push(rebuildPlugin(targetName, log)); + + return context({ + ...getEsbuildDevConfig(ROOT_DIR), + platform, + format, + outfile: path.join(OUT_DIR, outputFile), + external: externalModules, + plugins, + }); +} + +/** + * Build all targets in watch mode for development + * @returns {Promise} Array of esbuild contexts that can be disposed later + */ +export async function buildAllWithWatch(log) { + log.dim("Creating build contexts...\n"); + + // Create contexts for all targets + const contexts = await Promise.all(BUILD_TARGETS.map((target) => createBuildContext(target.name, target.config, log))); + + log.dim("Starting initial build...\n"); + + // Wait for the initial builds to complete + await Promise.all(contexts.map((ctx) => ctx.rebuild())); + + // Start watching all targets + await Promise.all(contexts.map((ctx) => ctx.watch())); + + return contexts; +} diff --git a/packages/transformers/scripts/build/constants.mjs b/packages/transformers/scripts/build/constants.mjs new file mode 100644 index 0000000..f8b2a33 --- /dev/null +++ b/packages/transformers/scripts/build/constants.mjs @@ -0,0 +1,38 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +export const DIST_FOLDER = "dist"; +export const NODE_IGNORE_MODULES = ["onnxruntime-web"]; +export const NODE_EXTERNAL_MODULES = [ + "onnxruntime-common", + "onnxruntime-node", + "sharp", + // node:* modules are handled by externalNodeBuiltinsPlugin +]; + +export const WEB_IGNORE_MODULES = ["onnxruntime-node", "sharp", "fs", "path", "url", "stream", "stream/promises"]; +export const WEB_EXTERNAL_MODULES = ["onnxruntime-common", "onnxruntime-web"]; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +export const ROOT_DIR = path.join(__dirname, "../.."); +export const OUT_DIR = path.join(ROOT_DIR, DIST_FOLDER); + +export const getEsbuildDevConfig = (rootDir) => ({ + bundle: true, + treeShaking: true, + logLevel: "info", + entryPoints: [path.join(rootDir, "src/transformers.js")], + platform: "neutral", + format: "esm", + sourcemap: true, + logOverride: { + // Suppress import.meta warning for CJS builds - it's handled gracefully in the code + "empty-import-meta": "silent", + }, +}); + +export const getEsbuildProdConfig = (rootDir) => ({ + ...getEsbuildDevConfig(rootDir), + logLevel: "warning", + sourcemap: false, +}); diff --git a/packages/transformers/scripts/build/plugins/externalNodeBuiltinsPlugin.mjs b/packages/transformers/scripts/build/plugins/externalNodeBuiltinsPlugin.mjs new file mode 100644 index 0000000..ce0b28c --- /dev/null +++ b/packages/transformers/scripts/build/plugins/externalNodeBuiltinsPlugin.mjs @@ -0,0 +1,14 @@ +/** + * Plugin to automatically mark all node:* imports as external. + * This prevents having to manually list all Node.js built-in modules. + */ +export const externalNodeBuiltinsPlugin = () => ({ + name: "external-node-builtins", + setup(build) { + // Mark all node:* imports as external + build.onResolve({ filter: /^node:/ }, (args) => ({ + path: args.path, + external: true, + })); + }, +}); diff --git a/packages/transformers/scripts/build/plugins/ignoreModulesPlugin.mjs b/packages/transformers/scripts/build/plugins/ignoreModulesPlugin.mjs new file mode 100644 index 0000000..4900cf3 --- /dev/null +++ b/packages/transformers/scripts/build/plugins/ignoreModulesPlugin.mjs @@ -0,0 +1,49 @@ +/** + * Plugin to ignore/exclude certain modules by returning an empty module. + * Equivalent to webpack's resolve.alias with false value. + */ +export const ignoreModulesPlugin = (modules = []) => ({ + name: "ignore-modules", + setup(build) { + // Escape special regex characters in module names + const escapeRegex = (str) => str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const escapedModules = modules.map(escapeRegex); + + // Match both "module" and "node:module" patterns + const patterns = escapedModules.flatMap((mod) => [mod, `node:${mod}`]); + const filter = new RegExp(`^(${patterns.join("|")})$`); + + build.onResolve({ filter }, (args) => { + return { path: args.path, namespace: "ignore-modules" }; + }); + build.onLoad({ filter: /.*/, namespace: "ignore-modules" }, (args) => { + switch (args.path) { + case "node:stream": + return { + contents: ` + const noop = () => {}; + export default {}; + export const Readable = { fromWeb: noop }; + `, + }; + case "node:stream/promises": + return { + contents: ` + const noop = () => {}; + export default {}; + export const pipeline = noop; + `, + }; + case "node:fs": + case "node:path": + case "node:url": + case "sharp": + case "onnxruntime-node": + default: + return { + contents: `export default {};`, + }; + } + }); + }, +}); diff --git a/packages/transformers/scripts/build/plugins/postBuildPlugin.mjs b/packages/transformers/scripts/build/plugins/postBuildPlugin.mjs new file mode 100644 index 0000000..769b02e --- /dev/null +++ b/packages/transformers/scripts/build/plugins/postBuildPlugin.mjs @@ -0,0 +1,43 @@ +import path from "node:path"; +import { copyFileSync, unlinkSync, existsSync } from "node:fs"; +import { colors, createLogger } from "../../../../../scripts/logger.mjs"; + +const log = createLogger("transformers"); + +/** + * Plugin to post-process build files. + * Equivalent to webpack's PostBuildPlugin. + */ +export const postBuildPlugin = (distDir, rootDir) => { + // it should copy the files only once. In watch mode for example it should not rerun every time + let completed = false; + + return { + name: "post-build", + setup(build) { + build.onEnd(() => { + if (completed) return; + completed = true; + + const ORT_JSEP_FILE = "ort-wasm-simd-threaded.jsep.mjs"; + const ORT_BUNDLE_FILE = "ort.bundle.min.mjs"; + + // 1. Remove unnecessary files + const file = path.join(distDir, ORT_BUNDLE_FILE); + if (existsSync(file)) unlinkSync(file); + + // 2. Copy unbundled JSEP file + const ORT_SOURCE_DIR = path.join(rootDir, "node_modules/onnxruntime-web/dist"); + const src = path.join(ORT_SOURCE_DIR, ORT_JSEP_FILE); + + if (existsSync(src)) { + const dest = path.join(distDir, ORT_JSEP_FILE); + copyFileSync(src, dest); + log.success(`${colors.gray}Copied ${ORT_JSEP_FILE}${colors.reset}`); + } else { + log.warning(`Could not find ${ORT_JSEP_FILE} in node_modules`); + } + }); + }, + }; +}; diff --git a/packages/transformers/scripts/build/plugins/stripNodePrefixPlugin.mjs b/packages/transformers/scripts/build/plugins/stripNodePrefixPlugin.mjs new file mode 100644 index 0000000..1a4cab9 --- /dev/null +++ b/packages/transformers/scripts/build/plugins/stripNodePrefixPlugin.mjs @@ -0,0 +1,15 @@ +/** + * Plugin to strip the "node:" prefix from module requests. + * Equivalent to webpack's StripNodePrefixPlugin. + */ +export const stripNodePrefixPlugin = () => ({ + name: "strip-node-prefix", + setup(build) { + build.onResolve({ filter: /^node:/ }, (args) => { + return { + path: args.path.replace(/^node:/, ""), + external: true, + }; + }); + }, +}); diff --git a/packages/transformers/scripts/build/targets.mjs b/packages/transformers/scripts/build/targets.mjs new file mode 100644 index 0000000..fcbf438 --- /dev/null +++ b/packages/transformers/scripts/build/targets.mjs @@ -0,0 +1,51 @@ +import { NODE_IGNORE_MODULES, NODE_EXTERNAL_MODULES, WEB_IGNORE_MODULES, WEB_EXTERNAL_MODULES } from "./constants.mjs"; + +/** + * Build target configuration + * Each target defines a specific build variant + */ +export const BUILD_TARGETS = [ + { + name: "Bundle Build (ESM)", + config: { + name: "", + suffix: ".js", + format: "esm", + ignoreModules: WEB_IGNORE_MODULES, + usePostBuild: true, + }, + }, + { + name: "Web Build (ESM)", + config: { + name: ".web", + suffix: ".js", + format: "esm", + ignoreModules: WEB_IGNORE_MODULES, + externalModules: WEB_EXTERNAL_MODULES, + usePostBuild: false, + }, + }, + { + name: "Node Build (ESM)", + config: { + name: ".node", + suffix: ".mjs", + format: "esm", + ignoreModules: NODE_IGNORE_MODULES, + externalModules: NODE_EXTERNAL_MODULES, + usePostBuild: false, + }, + }, + { + name: "Node Build (CJS)", + config: { + name: ".node", + suffix: ".cjs", + format: "cjs", + ignoreModules: NODE_IGNORE_MODULES, + externalModules: NODE_EXTERNAL_MODULES, + usePostBuild: false, + }, + }, +]; diff --git a/packages/transformers/scripts/dev.mjs b/packages/transformers/scripts/dev.mjs new file mode 100644 index 0000000..3da4ff6 --- /dev/null +++ b/packages/transformers/scripts/dev.mjs @@ -0,0 +1,113 @@ +import { spawn } from "node:child_process"; +import { unlinkSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { OUT_DIR } from "./build/constants.mjs"; +import prepareOutDir from "../../../scripts/prepareOutDir.mjs"; +import { colors, createLogger } from "../../../scripts/logger.mjs"; +import { buildAllWithWatch } from "./build/buildAllWithWatch.mjs"; + +const log = createLogger("transformers"); +const startTime = performance.now(); + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT_DIR = path.join(__dirname, ".."); + +prepareOutDir(OUT_DIR); + +// Remove tsbuildinfo to force TypeScript to rebuild type declarations +try { + unlinkSync(`${ROOT_DIR}/types/tsconfig.tsbuildinfo`); +} catch (err) { + // File doesn't exist, that's fine +} + +log.section("BUILD"); +log.info("Building transformers.js with esbuild in watch mode..."); + +// Build all targets with watch mode +const contexts = await buildAllWithWatch(log); + +const endTime = performance.now(); +const duration = (endTime - startTime).toFixed(2); +log.success(`All builds completed in ${colors.bright}${duration}ms${colors.reset}`); + +// Generate initial TypeScript declarations, then start watch mode +log.section("TYPES"); +log.info("Generating initial type declarations..."); + +await new Promise((resolve, reject) => { + const tscBuild = spawn("tsc", ["--build"], { + cwd: ROOT_DIR, + stdio: "pipe", + shell: true, + }); + + tscBuild.stdout.on("data", (data) => { + const output = data.toString().trim(); + if (output && output.includes("error")) { + log.dim(output); + } + }); + + tscBuild.stderr.on("data", (data) => { + const output = data.toString().trim(); + if (output) { + log.error(output); + } + }); + + tscBuild.on("exit", (code) => { + if (code === 0) { + log.done("Type declarations generated"); + resolve(); + } else { + reject(new Error(`TypeScript build failed with code ${code}`)); + } + }); +}); + +log.info("Starting TypeScript watch mode...\n"); + +const tscWatch = spawn("tsc", ["--build", "--watch", "--preserveWatchOutput"], { + cwd: ROOT_DIR, + stdio: "pipe", + shell: true, +}); + +tscWatch.stdout.on("data", (data) => { + const output = data.toString().trim(); + if (output) { + output.split("\n").forEach((line) => { + // Filter out verbose output, only show important messages + if ( + line.includes("error") || + line.includes("File change detected") || + line.includes("Found 0 errors") || + (line.includes("Found") && line.includes("error")) + ) { + log.dim(line); + } + }); + } +}); + +tscWatch.stderr.on("data", (data) => { + const output = data.toString().trim(); + if (output) { + output.split("\n").forEach((line) => { + log.error(line); + }); + } +}); + +log.dim(`Watching for changes...\n`); + +// Keep process alive and cleanup +process.on("SIGINT", async () => { + log.warning(`\nStopping watch mode...`); + tscWatch.kill(); + await Promise.all(contexts.map((ctx) => ctx.dispose())); + log.dim("Goodbye!"); + process.exit(0); +}); diff --git a/packages/transformers/src/backends/onnx.js b/packages/transformers/src/backends/onnx.js new file mode 100644 index 0000000..963f6ff --- /dev/null +++ b/packages/transformers/src/backends/onnx.js @@ -0,0 +1,391 @@ +/** + * @file Handler file for choosing the correct version of ONNX Runtime, based on the environment. + * Ideally, we could import the `onnxruntime-web` and `onnxruntime-node` packages only when needed, + * but dynamic imports don't seem to work with the current webpack version and/or configuration. + * This is possibly due to the experimental nature of top-level await statements. + * So, we just import both packages, and use the appropriate one based on the environment: + * - When running in node, we use `onnxruntime-node`. + * - When running in the browser, we use `onnxruntime-web` (`onnxruntime-node` is not bundled). + * + * This module is not directly exported, but can be accessed through the environment variables: + * ```javascript + * import { env } from '@huggingface/transformers'; + * console.log(env.backends.onnx); + * ``` + * + * @module backends/onnx + */ + +import { env, apis, LogLevel } from '../env.js'; + +// NOTE: Import order matters here. We need to import `onnxruntime-node` before `onnxruntime-web`. +// In either case, we select the default export if it exists, otherwise we use the named export. +import * as ONNX_NODE from 'onnxruntime-node'; +import * as ONNX_WEB from 'onnxruntime-web/webgpu'; +import { loadWasmBinary, loadWasmFactory } from './utils/cacheWasm.js'; +import { isBlobURL, toAbsoluteURL } from '../utils/hub/utils.js'; +import { logger } from '../utils/logger.js'; +export { Tensor } from 'onnxruntime-common'; + +/** + * @typedef {import('onnxruntime-common').InferenceSession.ExecutionProviderConfig} ONNXExecutionProviders + */ + +/** @type {Record} */ +const DEVICE_TO_EXECUTION_PROVIDER_MAPPING = Object.freeze({ + auto: null, // Auto-detect based on device and environment + gpu: null, // Auto-detect GPU + cpu: 'cpu', // CPU + wasm: 'wasm', // WebAssembly + webgpu: 'webgpu', // WebGPU + cuda: 'cuda', // CUDA + dml: 'dml', // DirectML + coreml: 'coreml', // CoreML + + webnn: { name: 'webnn', deviceType: 'cpu' }, // WebNN (default) + 'webnn-npu': { name: 'webnn', deviceType: 'npu' }, // WebNN NPU + 'webnn-gpu': { name: 'webnn', deviceType: 'gpu' }, // WebNN GPU + 'webnn-cpu': { name: 'webnn', deviceType: 'cpu' }, // WebNN CPU +}); + +/** + * Converts any LogLevel value to ONNX Runtime's numeric severity level (0-4). + * This handles both standard LogLevel values (10, 20, 30, 40, 50) and custom intermediate values. + * + * @param {number} logLevel - The LogLevel value to convert + * @returns {number} ONNX Runtime severity level (0-4) + */ +function getOnnxLogSeverityLevel(logLevel) { + // ONNX Runtime's log severity levels are defined as follows: + // (0) ORT_LOGGING_LEVEL_VERBOSE: Print all log messages. + // (1) ORT_LOGGING_LEVEL_INFO: Print info and higher level log messages. + // (2) ORT_LOGGING_LEVEL_WARNING: Print warning and higher level log messages. + // (3) ORT_LOGGING_LEVEL_ERROR: Print error log messages. + // (4) ORT_LOGGING_LEVEL_FATAL: Print only fatal log messages. + // + // In practice, ONNX Runtime's logging is extremely verbose (especially on session creation). + // For this reason, we map multiple LogLevel values to the same ONNX severity level to avoid + // overwhelming users with logs. + if (logLevel <= LogLevel.DEBUG) { + return 0; // ORT_LOGGING_LEVEL_VERBOSE + } else if (logLevel <= LogLevel.INFO) { + return 2; // ORT_LOGGING_LEVEL_WARNING + } else if (logLevel <= LogLevel.WARNING) { + return 3; // ORT_LOGGING_LEVEL_ERROR + } else if (logLevel <= LogLevel.ERROR) { + return 3; // ORT_LOGGING_LEVEL_ERROR + } else { + return 4; // ORT_LOGGING_LEVEL_FATAL + } +} + +/** + * Maps ONNX Runtime numeric severity levels to string log levels. + * @type {Record<0 | 1 | 2 | 3 | 4, 'verbose' | 'info' | 'warning' | 'error' | 'fatal'>} + */ +const ONNX_LOG_LEVEL_NAMES = { + 0: 'verbose', + 1: 'info', + 2: 'warning', + 3: 'error', + 4: 'fatal', +}; + +/** + * The list of supported devices, sorted by priority/performance. + * @type {import("../utils/devices.js").DeviceType[]} + */ +const supportedDevices = []; + +/** @type {ONNXExecutionProviders[]} */ +let defaultDevices; +let ONNX; +const ORT_SYMBOL = Symbol.for('onnxruntime'); + +if (ORT_SYMBOL in globalThis) { + // If the JS runtime exposes their own ONNX runtime, use it + ONNX = globalThis[ORT_SYMBOL]; +} else if (apis.IS_NODE_ENV) { + ONNX = ONNX_NODE; + + // Updated as of ONNX Runtime 1.23.0-dev.20250612-70f14d7670 + // The following table lists the supported versions of ONNX Runtime Node.js binding provided with pre-built binaries. + // | EPs/Platforms | Windows x64 | Windows arm64 | Linux x64 | Linux arm64 | MacOS x64 | MacOS arm64 | + // | --------------------- | ------------------ | ------------------ | ------------------ | ------------------ | ------------------ | ------------------ | + // | CPU | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | + // | WebGPU (experimental) | ✔️ | ✔️ | ✔️ | ❌ | ✔️ | ✔️ | + // | DirectML | ✔️ | ✔️ | ❌ | ❌ | ❌ | ❌ | + // | CUDA | ❌ | ❌ | ✔️ (CUDA v12) | ❌ | ❌ | ❌ | + // | CoreML | ❌ | ❌ | ❌ | ❌ | ✔️ | ✔️ | + switch (process.platform) { + case 'win32': // Windows x64 and Windows arm64 + supportedDevices.push('dml'); + break; + case 'linux': // Linux x64 and Linux arm64 + if (process.arch === 'x64') { + supportedDevices.push('cuda'); + } + break; + case 'darwin': // MacOS x64 and MacOS arm64 + supportedDevices.push('coreml'); + break; + } + + supportedDevices.push('webgpu'); + supportedDevices.push('cpu'); + defaultDevices = ['cpu']; +} else { + ONNX = ONNX_WEB; + + if (apis.IS_WEBNN_AVAILABLE) { + // TODO: Only push supported providers (depending on available hardware) + supportedDevices.push('webnn-npu', 'webnn-gpu', 'webnn-cpu', 'webnn'); + } + + if (apis.IS_WEBGPU_AVAILABLE) { + supportedDevices.push('webgpu'); + } + + supportedDevices.push('wasm'); + defaultDevices = ['wasm']; +} + +// @ts-ignore +const InferenceSession = ONNX.InferenceSession; + +/** + * Map a device to the execution providers to use for the given device. + * @param {import("../utils/devices.js").DeviceType|"auto"|null} [device=null] (Optional) The device to run the inference on. + * @returns {ONNXExecutionProviders[]} The execution providers to use for the given device. + */ +export function deviceToExecutionProviders(device = null) { + // Use the default execution providers if the user hasn't specified anything + if (!device) return defaultDevices; + + // Handle overloaded cases + switch (device) { + case 'auto': + return supportedDevices; + case 'gpu': + return supportedDevices.filter((x) => ['webgpu', 'cuda', 'dml', 'webnn-gpu'].includes(x)); + } + + if (supportedDevices.includes(device)) { + return [DEVICE_TO_EXECUTION_PROVIDER_MAPPING[device] ?? device]; + } + + throw new Error(`Unsupported device: "${device}". Should be one of: ${supportedDevices.join(', ')}.`); +} + +/** + * Currently, Transformers.js doesn't support simultaneous loading of sessions in WASM/WebGPU. + * For this reason, we need to chain the loading calls. + * @type {Promise} + */ +let webInitChain = Promise.resolve(); + +/** + * Promise that resolves when WASM binary has been loaded (if caching is enabled). + * This ensures we only attempt to load the WASM binary once. + * @type {Promise|null} + */ +let wasmLoadPromise = null; + +/** + * Ensures the WASM binary is loaded and cached before creating an inference session. + * Only runs once, even if called multiple times. + * + * @returns {Promise} + */ +async function ensureWasmLoaded() { + // If already loading or loaded, return the existing promise + if (wasmLoadPromise) { + return wasmLoadPromise; + } + + // Check if we should load the WASM binary + const shouldUseWasmCache = + env.useWasmCache && + typeof ONNX_ENV?.wasm?.wasmPaths === 'object' && + ONNX_ENV?.wasm?.wasmPaths?.wasm && + ONNX_ENV?.wasm?.wasmPaths?.mjs; + + if (!shouldUseWasmCache) { + // In Deno's web runtime, the WASM factory must be loaded via blob URL so that Node.js detection + // can be patched out (see loadWasmFactory). Without caching, the factory is imported directly + // from its URL and Deno would crash trying to use Node.js APIs. useWasmCache defaults to true + // in this environment, so this only happens if the user explicitly disables it. + if (apis.IS_DENO_WEB_RUNTIME) { + throw new Error( + "env.useWasmCache=false is not supported in Deno's web runtime. Remove the useWasmCache override.", + ); + } + wasmLoadPromise = Promise.resolve(); + return wasmLoadPromise; + } + + // Start loading the WASM binary + wasmLoadPromise = (async () => { + // At this point, we know wasmPaths is an object (not a string) because + // shouldUseWasmCache checks for wasmPaths.wasm and wasmPaths.mjs + const urls = /** @type {{ wasm: string, mjs: string }} */ (ONNX_ENV.wasm.wasmPaths); + + // Load both in parallel; the .mjs blob URL is only kept if wasmBinary succeeded. + // ORT only sets locateFile when wasmBinary is provided (onnxruntime PR https://github.com/microsoft/onnxruntime/pull/27411), which + // prevents new URL(fileName, import.meta.url) from failing inside a blob URL factory. + let wasmBinaryLoaded = false; + await Promise.all([ + // Load and cache the WASM binary + urls.wasm && !isBlobURL(urls.wasm) + ? (async () => { + try { + const wasmBinary = await loadWasmBinary(toAbsoluteURL(urls.wasm)); + if (wasmBinary) { + ONNX_ENV.wasm.wasmBinary = wasmBinary; + wasmBinaryLoaded = true; + } + } catch (err) { + logger.warn('Failed to pre-load WASM binary:', err); + } + })() + : Promise.resolve(), + + // Load and cache the WASM factory as a blob URL + urls.mjs && !isBlobURL(urls.mjs) + ? (async () => { + try { + const wasmFactoryBlob = await loadWasmFactory(toAbsoluteURL(urls.mjs)); + if (wasmFactoryBlob) { + // @ts-ignore + ONNX_ENV.wasm.wasmPaths.mjs = wasmFactoryBlob; + } + } catch (err) { + logger.warn('Failed to pre-load WASM factory:', err); + } + })() + : Promise.resolve(), + ]); + + // If wasmBinary failed to load, revert wasmPaths.mjs to the original URL (factory can only be loaded from blob if ONNX_ENV.wasm.wasmBinary is set. @see ORT PR #27411) + if (!wasmBinaryLoaded) { + // @ts-ignore + ONNX_ENV.wasm.wasmPaths.mjs = urls.mjs; + } + })(); + + return wasmLoadPromise; +} + +/** + * Create an ONNX inference session. + * @param {Uint8Array|string} buffer_or_path The ONNX model buffer or path. + * @param {import('onnxruntime-common').InferenceSession.SessionOptions} session_options ONNX inference session options. + * @param {Object} session_config ONNX inference session configuration. + * @returns {Promise} The ONNX inference session. + */ +export async function createInferenceSession(buffer_or_path, session_options, session_config) { + await ensureWasmLoaded(); + const logSeverityLevel = getOnnxLogSeverityLevel(env.logLevel ?? LogLevel.WARNING); + const load = () => + InferenceSession.create(buffer_or_path, { + // Set default log severity level, but allow overriding through session options + logSeverityLevel, + ...session_options, + }); + const session = await (apis.IS_WEB_ENV ? (webInitChain = webInitChain.then(load)) : load()); + session.config = session_config; + return session; +} + +/** + * Currently, Transformers.js doesn't support simultaneous execution of sessions in WASM/WebGPU. + * For this reason, we need to chain the inference calls (otherwise we get "Error: Session already started"). + * @type {Promise} + */ +let webInferenceChain = Promise.resolve(); + +/** + * Run an inference session. + * @param {import('onnxruntime-common').InferenceSession} session The ONNX inference session. + * @param {Record} ortFeed The input tensors. + * @returns {Promise>} The output tensors. + */ +export async function runInferenceSession(session, ortFeed) { + const run = () => session.run(ortFeed); + return apis.IS_WEB_ENV ? (webInferenceChain = webInferenceChain.then(run)) : run(); +} + +/** + * Check if an object is an ONNX tensor. + * @param {any} x The object to check + * @returns {boolean} Whether the object is an ONNX tensor. + */ +export function isONNXTensor(x) { + return x instanceof ONNX.Tensor; +} +/** @type {import('onnxruntime-common').Env} */ +const ONNX_ENV = ONNX?.env; + +/** + * Check if ONNX's WASM backend is being proxied. + * @returns {boolean} Whether ONNX's WASM backend is being proxied. + */ +export function isONNXProxy() { + // TODO: Update this when allowing non-WASM backends. + return ONNX_ENV?.wasm?.proxy; +} + +if (ONNX_ENV) { + if (ONNX_ENV.wasm) { + // Initialize wasm backend with suitable default settings. + + // (Optional) Set path to wasm files. This will override the default path search behavior of onnxruntime-web. + // By default, we only do this if we are not in a service worker and the wasmPaths are not already set. + if ( + // @ts-ignore Cannot find name 'ServiceWorkerGlobalScope'.ts(2304) + !(typeof ServiceWorkerGlobalScope !== 'undefined' && self instanceof ServiceWorkerGlobalScope) && + ONNX_ENV.versions?.web && + !ONNX_ENV.wasm.wasmPaths + ) { + const wasmPathPrefix = `https://cdn.jsdelivr.net/npm/onnxruntime-web@${ONNX_ENV.versions.web}/dist/`; + + let wasmPathSuffix = '.asyncify'; // Default to asyncify WASM build + if (apis.IS_SAFARI_BELOW_26 && !apis.IS_WEBGPU_AVAILABLE) { + // Disable asyncify for Safari below 26 when WebGPU is not available + wasmPathSuffix = ''; + } + + ONNX_ENV.wasm.wasmPaths = { + mjs: `${wasmPathPrefix}ort-wasm-simd-threaded${wasmPathSuffix}.mjs`, + wasm: `${wasmPathPrefix}ort-wasm-simd-threaded${wasmPathSuffix}.wasm`, + }; + } + + // Users may wish to proxy the WASM backend to prevent the UI from freezing, + // However, this is not necessary when using WebGPU, so we default to false. + ONNX_ENV.wasm.proxy = false; + } + + if (ONNX_ENV.webgpu) { + ONNX_ENV.webgpu.powerPreference = 'high-performance'; + } + + /** + * A function to map Transformers.js log levels to ONNX Runtime log severity + * levels, and set the log level environment variable in ONNX Runtime. + * @param {number} logLevel The log level to set. + */ + function setLogLevel(logLevel) { + const severityLevel = getOnnxLogSeverityLevel(logLevel); + ONNX_ENV.logLevel = ONNX_LOG_LEVEL_NAMES[severityLevel]; + } + + // Set the initial log level to be the default Transformers.js log level. + setLogLevel(env.logLevel ?? LogLevel.WARNING); + + // Expose ONNX environment variables to `env.backends.onnx` + env.backends.onnx = { + ...ONNX_ENV, + setLogLevel, + }; +} diff --git a/packages/transformers/src/backends/utils/cacheWasm.js b/packages/transformers/src/backends/utils/cacheWasm.js new file mode 100644 index 0000000..a0c0965 --- /dev/null +++ b/packages/transformers/src/backends/utils/cacheWasm.js @@ -0,0 +1,101 @@ +import { apis, env } from '../../env.js'; +import { getCache } from '../../utils/cache.js'; +import { logger } from '../../utils/logger.js'; + +/** + * Loads and caches a file from the given URL. + * @param {string} url The URL of the file to load. + * @returns {Promise} The response object, or null if loading failed. + */ +async function loadAndCacheFile(url) { + const fileName = url.split('/').pop(); + + /** @type {import('../../utils/cache.js').CacheInterface|undefined} */ + let cache; + try { + cache = await getCache(); + + // Try to get from cache first + if (cache) { + const result = await cache.match(url); + if (result) { + return result; + } + } + } catch (error) { + logger.warn(`Failed to load ${fileName} from cache:`, error); + } + + // If not in cache, fetch it + const response = await env.fetch(url); + + if (!response.ok) { + throw new Error(`Failed to fetch ${fileName}: ${response.status} ${response.statusText}`); + } + + // Cache the response for future use + if (cache) { + try { + await cache.put(url, response.clone()); + } catch (e) { + logger.warn(`Failed to cache ${fileName}:`, e); + } + } + + return response; +} + +/** + * Loads and caches the WASM binary for ONNX Runtime. + * @param {string} wasmURL The URL of the WASM file to load. + * @returns {Promise} The WASM binary as an ArrayBuffer, or null if loading failed. + */ + +export async function loadWasmBinary(wasmURL) { + const response = await loadAndCacheFile(wasmURL); + if (!response || typeof response === 'string') return null; + + try { + return await response.arrayBuffer(); + } catch (error) { + logger.warn('Failed to read WASM binary:', error); + return null; + } +} + +/** + * Loads and caches the WASM Factory (.mjs file) for ONNX Runtime. + * Creates a blob URL from cached content (when safe) to bridge Cache API with dynamic imports used in ORT. + * @param {string} libURL The URL of the WASM Factory to load. + * @returns {Promise} The blob URL (if enabled), original URL (if disabled), or null if loading failed. + */ +export async function loadWasmFactory(libURL) { + // We can't use Blob URLs in some environments (Service Workers, Chrome extensions) due to security restrictions on dynamic import() of blob URLs. + // In such cases, just return the original URL and don't bother caching since dynamic import() won't use the Cache API anyway. + // See https://github.com/huggingface/transformers.js/issues/1532. + if (apis.IS_SERVICE_WORKER_ENV || apis.IS_CHROME_AVAILABLE) { + return libURL; + } + + // Fetch from cache or network, then create blob URL + const response = await loadAndCacheFile(libURL); + if (!response || typeof response === 'string') return null; + + try { + let code = await response.text(); + + // Handle the case where we are importing the bundled version of the library in Deno (e.g., via CDN or local file), + // where we need to patch out Node.js detection in the factory. Without this, Deno (which exposes globalThis.process.versions.node) + // would enter the Node.js branch and try to use Node.js APIs (worker_threads, fs, etc.) that aren't used in the bundled web version. + // Only needed for the asyncify (single-threaded) variant loaded via blob URL. The module-level pthread auto-start code is unreachable since asyncify never spawns workers. + // See https://github.com/huggingface/transformers.js/pull/1546/ for more information. + // + // NOTE: This does not affect default usage via Deno (i.e., imported via npm: prefix), since we'll be using onnxruntime-node (Native) instead of onnxruntime-web (WASM). + code = code.replaceAll('globalThis.process?.versions?.node', 'false'); + const blob = new Blob([code], { type: 'text/javascript' }); + return URL.createObjectURL(blob); + } catch (error) { + logger.warn('Failed to read WASM factory:', error); + return null; + } +} diff --git a/packages/transformers/src/cache_utils.js b/packages/transformers/src/cache_utils.js new file mode 100644 index 0000000..4fb1699 --- /dev/null +++ b/packages/transformers/src/cache_utils.js @@ -0,0 +1,82 @@ +import { Tensor } from './utils/tensor.js'; + +/** + * A cache class that stores past key values as named tensors. + */ +class _DynamicCache { + /** + * Create a DynamicCache, optionally pre-populated with entries. + * @param {Record} [entries] Initial name→Tensor mappings. + */ + constructor(entries) { + if (!entries) return; + for (const key in entries) { + if (key in this) { + throw new TypeError(`Key "${key}" conflicts with an existing property on DynamicCache`); + } + const value = entries[key]; + if (!(value instanceof Tensor)) { + throw new TypeError(`Expected a Tensor for key "${key}", got ${typeof value}`); + } + this[key] = value; + } + } + + /** + * Get the cached sequence length. This requires at least one attention cache entry to be present. + * @returns {number} The past sequence length. + */ + get_seq_length() { + /** @type {Record} */ + const self = /** @type {any} */ (this); + + if (Object.keys(self).length === 0) { + return 0; + } + + for (const name in self) { + if (name.startsWith('past_key_values.')) { + return self[name].dims.at(-2); + } + } + throw new Error('Unable to determine sequence length from the cache.'); + } + + /** + * Update the cache in-place with new entries, disposing replaced GPU tensors. + * @param {Record} newEntries The new name → Tensor mappings. + */ + update(newEntries) { + for (const key in newEntries) { + const oldValue = this[key]; + const newValue = newEntries[key]; + if (oldValue && oldValue !== newValue && oldValue.location === 'gpu-buffer') { + oldValue.dispose(); + } + this[key] = newValue; + } + } + + /** + * Dispose all contained tensors whose data resides on the GPU. + * Returns a promise that resolves when all disposals are complete. + * @returns {Promise} Promise that resolves when all GPU tensors are disposed. + */ + async dispose() { + const promises = []; + for (const t of /** @type {Tensor[]} */ (Object.values(this))) { + if (t.location === 'gpu-buffer') { + promises.push(t.dispose()); + } + } + await Promise.all(promises); + } +} + +/** + * @typedef {Record & _DynamicCache} DynamicCache + */ + +export const DynamicCache = /** @type {new (entries?: Record) => DynamicCache} */ ( + /** @type {unknown} */ (_DynamicCache) +); diff --git a/packages/transformers/src/configs.js b/packages/transformers/src/configs.js new file mode 100644 index 0000000..68a29d8 --- /dev/null +++ b/packages/transformers/src/configs.js @@ -0,0 +1,581 @@ +/** + * @file Helper module for using model configs. For more information, see the corresponding + * [Python documentation](https://huggingface.co/docs/transformers/main/en/model_doc/auto#transformers.AutoConfig). + * + * **Example:** Load an `AutoConfig`. + * + * ```javascript + * import { AutoConfig } from '@huggingface/transformers'; + * const config = await AutoConfig.from_pretrained('bert-base-uncased'); + * console.log(config); + * // PretrainedConfig { + * // "model_type": "bert", + * // "is_encoder_decoder": false, + * // "architectures": [ + * // "BertForMaskedLM" + * // ], + * // "vocab_size": 30522 + * // "num_attention_heads": 12, + * // "num_hidden_layers": 12, + * // "hidden_size": 768, + * // "max_position_embeddings": 512, + * // ... + * // } + * ``` + * + * @module configs + */ + +import { pick } from './utils/core.js'; +import { getModelJSON } from './utils/hub.js'; + +/** + * @typedef {import('./utils/hub.js').PretrainedOptions} PretrainedOptions + */ + +/** + * @typedef {import('./utils/core.js').ProgressCallback} ProgressCallback + */ + +/** + * @typedef {import('./utils/core.js').ProgressInfo} ProgressInfo + */ + +/** + * Loads a config from the specified path. + * @param {string} pretrained_model_name_or_path The path to the config directory. + * @param {PretrainedOptions} options Additional options for loading the config. + * @returns {Promise} A promise that resolves with information about the loaded config. + */ +async function loadConfig(pretrained_model_name_or_path, options) { + return await getModelJSON(pretrained_model_name_or_path, 'config.json', true, options); +} + +/** + * + * @param {PretrainedConfig} config + * @returns {Object} The normalized configuration. + */ +function getNormalizedConfig(config) { + const mapping = {}; + + let init_normalized_config = {}; + switch (config.model_type) { + // Sub-configs + case 'llava': + case 'paligemma': + case 'gemma3': + case 'florence2': + case 'llava_onevision': + case 'idefics3': + case 'granite_speech': + case 'ultravox': + case 'voxtral': + case 'voxtral_realtime': + case 'smolvlm': + case 'gemma3n': + case 'gemma4': + case 'lfm2_vl': + case 'chatterbox': + case 'lighton_ocr': + case 'glm_ocr': + case 'mistral3': + case 'qwen2_5_vl': + case 'qwen3_vl': + case 'qwen3_vl_moe': + // @ts-expect-error TS2339 + init_normalized_config = getNormalizedConfig(config.text_config); + break; + case 'moondream1': + // @ts-expect-error TS2339 + init_normalized_config = getNormalizedConfig(config.phi_config); + break; + case 'musicgen': + // @ts-expect-error TS2339 + init_normalized_config = getNormalizedConfig(config.decoder); + break; + case 'multi_modality': + // @ts-expect-error TS2339 + init_normalized_config = getNormalizedConfig(config.language_config); + break; + + // Decoder-only models + case 'gpt2': + case 'gptj': + case 'jais': + case 'codegen': + case 'gpt_bigcode': + mapping['num_heads'] = 'n_head'; + mapping['num_layers'] = 'n_layer'; + mapping['hidden_size'] = 'n_embd'; + break; + case 'gpt_neox': + case 'stablelm': + case 'opt': + case 'falcon': + case 'modernbert-decoder': + mapping['num_heads'] = 'num_attention_heads'; + mapping['num_layers'] = 'num_hidden_layers'; + mapping['hidden_size'] = 'hidden_size'; + break; + case 'gpt_oss': + case 'llama': + case 'llama4_text': + case 'nanochat': + case 'apertus': + case 'arcee': + case 'afmoe': + case 'lfm2': + case 'lfm2_moe': + case 'smollm3': + case 'olmo': + case 'olmo2': + case 'olmo3': + case 'mobilellm': + case 'granite': + case 'granitemoehybrid': + case 'cohere': + case 'cohere2': + case 'mistral': + case 'voxtral_realtime_text': + case 'voxtral_realtime_encoder': + case 'starcoder2': + case 'qwen2': + case 'qwen2_moe': + case 'qwen2_vl': + case 'qwen2_vl_text': + case 'qwen2_5_vl_text': + case 'qwen3_moe': + case 'qwen3_vl_text': + case 'qwen3_vl_moe_text': + case 'phi': + case 'phi3': + case 'phi3_v': + case 'llava_qwen2': + mapping['num_heads'] = 'num_key_value_heads'; + mapping['num_layers'] = 'num_hidden_layers'; + mapping['hidden_size'] = 'hidden_size'; + mapping['num_attention_heads'] = 'num_attention_heads'; + mapping['dim_kv'] = 'head_dim'; + break; + case 'qwen3': + case 'solar_open': + case 'glm_ocr_text': + case 'gemma': + case 'gemma2': + case 'vaultgemma': + case 'gemma3_text': + case 'gemma3n_text': + case 'gemma4_text': + case 'glm': + case 'helium': + case 'ernie4_5': + case 'hunyuan_v1_dense': + case 'falcon_h1': + case 'nemotron_h': + case 'ministral': + case 'ministral3': + mapping['num_heads'] = 'num_key_value_heads'; + mapping['num_layers'] = 'num_hidden_layers'; + mapping['dim_kv'] = 'head_dim'; + break; + case 'openelm': + mapping['num_heads'] = 'num_kv_heads'; + mapping['num_layers'] = 'num_transformer_layers'; + mapping['dim_kv'] = 'head_dim'; + break; + case 'gpt_neo': + case 'donut-swin': + mapping['num_heads'] = 'num_heads'; + mapping['num_layers'] = 'num_layers'; + mapping['hidden_size'] = 'hidden_size'; + break; + case 'bloom': + mapping['num_heads'] = 'n_head'; + mapping['num_layers'] = 'n_layer'; + mapping['hidden_size'] = 'hidden_size'; + break; + case 'mpt': + mapping['num_heads'] = 'n_heads'; + mapping['num_layers'] = 'n_layers'; + mapping['hidden_size'] = 'd_model'; + break; + case 'exaone': + mapping['num_heads'] = 'num_key_value_heads'; + mapping['num_layers'] = 'num_layers'; + mapping['dim_kv'] = 'head_dim'; + mapping['num_attention_heads'] = 'num_attention_heads'; + break; + case 'youtu': + case 'deepseek_v3': + case 'deepseek_v4': + case 'glm_moe_dsa': + case 'mistral4': + mapping['num_heads'] = 'num_key_value_heads'; + mapping['num_layers'] = 'num_hidden_layers'; + mapping['dim_kv'] = config.model_type === 'deepseek_v4' ? 'head_dim' : 'qk_head_dim'; + mapping['num_attention_heads'] = 'num_attention_heads'; + break; + case 'zaya': + mapping['num_heads'] = 'num_key_value_heads'; + mapping['num_layers'] = 'num_hidden_layers'; + mapping['hidden_size'] = 'hidden_size'; + mapping['dim_kv'] = 'head_dim'; + mapping['num_attention_heads'] = 'num_attention_heads'; + break; + case 'hrm_text': + mapping['num_heads'] = 'num_key_value_heads'; + mapping['num_layers'] = 'num_hidden_layers'; + mapping['hidden_size'] = 'hidden_size'; + mapping['dim_kv'] = 'head_dim'; + mapping['num_attention_heads'] = 'num_attention_heads'; + break; + + // Encoder-decoder models + case 't5': + case 'mt5': + case 'longt5': + mapping['num_decoder_layers'] = 'num_decoder_layers'; + mapping['num_decoder_heads'] = 'num_heads'; + mapping['decoder_dim_kv'] = 'd_kv'; + mapping['num_encoder_layers'] = 'num_layers'; + mapping['num_encoder_heads'] = 'num_heads'; + mapping['encoder_dim_kv'] = 'd_kv'; + break; + case 'bart': + case 'mbart': + case 'marian': + case 'whisper': + case 'lite-whisper': + case 'm2m_100': + case 'blenderbot': + case 'blenderbot-small': + case 'florence2_language': + mapping['num_decoder_layers'] = 'decoder_layers'; + mapping['num_decoder_heads'] = 'decoder_attention_heads'; + mapping['decoder_hidden_size'] = 'd_model'; + mapping['num_encoder_layers'] = 'encoder_layers'; + mapping['num_encoder_heads'] = 'encoder_attention_heads'; + mapping['encoder_hidden_size'] = 'd_model'; + break; + case 'speecht5': + mapping['num_decoder_layers'] = 'decoder_layers'; + mapping['num_decoder_heads'] = 'decoder_attention_heads'; + mapping['decoder_hidden_size'] = 'hidden_size'; + mapping['num_encoder_layers'] = 'encoder_layers'; + mapping['num_encoder_heads'] = 'encoder_attention_heads'; + mapping['encoder_hidden_size'] = 'hidden_size'; + break; + case 'trocr': + mapping['num_encoder_layers'] = mapping['num_decoder_layers'] = 'decoder_layers'; + mapping['num_encoder_heads'] = mapping['num_decoder_heads'] = 'decoder_attention_heads'; + mapping['encoder_hidden_size'] = mapping['decoder_hidden_size'] = 'd_model'; + break; + case 'musicgen_decoder': + mapping['num_encoder_layers'] = mapping['num_decoder_layers'] = 'num_hidden_layers'; + mapping['num_encoder_heads'] = mapping['num_decoder_heads'] = 'num_attention_heads'; + mapping['encoder_hidden_size'] = mapping['decoder_hidden_size'] = 'hidden_size'; + break; + case 'moonshine': + mapping['num_decoder_layers'] = 'decoder_num_hidden_layers'; + mapping['num_decoder_heads'] = 'decoder_num_key_value_heads'; + mapping['num_encoder_layers'] = 'encoder_num_hidden_layers'; + mapping['num_encoder_heads'] = 'encoder_num_key_value_heads'; + mapping['encoder_hidden_size'] = mapping['decoder_hidden_size'] = 'hidden_size'; + break; + case 'cohere_asr': + mapping['num_decoder_layers'] = 'num_hidden_layers'; + mapping['num_decoder_heads'] = 'num_key_value_heads'; + mapping['decoder_hidden_size'] = 'hidden_size'; + mapping['decoder_dim_kv'] = 'head_dim'; + const { + num_hidden_layers: num_encoder_layers, + num_attention_heads: num_encoder_heads, + hidden_size: encoder_hidden_size, + } = /** @type {any} */ (config).encoder_config; + init_normalized_config = { + num_encoder_layers, + num_encoder_heads, + encoder_hidden_size, + // @ts-expect-error TS2339 + encoder_dim_kv: config.head_dim, + }; + break; + case 'vision-encoder-decoder': + // @ts-expect-error TS2339 + const decoderConfig = getNormalizedConfig(config.decoder); + + const add_encoder_pkv = 'num_decoder_layers' in decoderConfig; + const result = pick(config, ['model_type', 'is_encoder_decoder']); + if (add_encoder_pkv) { + // Decoder is part of an encoder-decoder model + result.num_decoder_layers = decoderConfig.num_decoder_layers; + result.num_decoder_heads = decoderConfig.num_decoder_heads; + result.decoder_hidden_size = decoderConfig.decoder_hidden_size; + + result.num_encoder_layers = decoderConfig.num_encoder_layers; + result.num_encoder_heads = decoderConfig.num_encoder_heads; + result.encoder_hidden_size = decoderConfig.encoder_hidden_size; + } else { + // Decoder is a decoder-only model + result.num_layers = decoderConfig.num_layers; + result.num_heads = decoderConfig.num_heads; + result.hidden_size = decoderConfig.hidden_size; + } + return result; + } + + // NOTE: If `num_attention_heads` is not set, it is assumed to be equal to `num_heads` + const normalized_config = { + ...init_normalized_config, + ...pick(config, ['model_type', 'multi_query', 'is_encoder_decoder']), + }; + for (const key in mapping) { + normalized_config[key] = config[mapping[key]]; + } + return normalized_config; +} + +/** + * @param {PretrainedConfig} config + * @param {{ prefix?: string, session_name?: string }} [options] + * @returns {Set} + */ +export function getCacheNames(config, options) { + if (!(config instanceof PretrainedConfig)) { + config = new PretrainedConfig(config); + } + + const pkv_prefix = options?.prefix ?? 'past_key_values'; + const cache_prefix = pkv_prefix === 'present' ? 'present' : 'past'; + /** @type {Set} */ + const names = new Set(); + + if (['lfm2', 'lfm2_moe'].includes(config.model_type)) { + const { layer_types } = /** @type {any} */ (config); + for (let i = 0; i < layer_types.length; ++i) { + if (layer_types[i] === 'full_attention') { + names.add(`${pkv_prefix}.${i}.key`); + names.add(`${pkv_prefix}.${i}.value`); + } else if (layer_types[i] === 'conv') { + names.add(`${cache_prefix}_conv.${i}`); + } else { + throw new Error(`Unsupported layer type: ${layer_types[i]}`); + } + } + return names; + } else if (['granitemoehybrid', 'falcon_h1', 'nemotron_h'].includes(config.model_type)) { + const c = /** @type {any} */ (config); + const layer_types = c.layer_types ?? c.layers_block_type; + const num_layers = c.num_hidden_layers ?? layer_types?.length; + + for (let i = 0; i < num_layers; ++i) { + if (!layer_types || layer_types[i] === 'mamba') { + names.add(`${cache_prefix}_conv.${i}`); + names.add(`${cache_prefix}_ssm.${i}`); + } + if (!layer_types || layer_types[i] === 'attention') { + names.add(`${pkv_prefix}.${i}.key`); + names.add(`${pkv_prefix}.${i}.value`); + } + } + return names; + } else if (['qwen3_next', 'qwen3_5_text', 'qwen3_5_moe_text', 'olmo_hybrid'].includes(config.model_type)) { + const { layer_types } = /** @type {any} */ (config); + for (let i = 0; i < layer_types.length; ++i) { + if (layer_types[i] === 'full_attention') { + names.add(`${pkv_prefix}.${i}.key`); + names.add(`${pkv_prefix}.${i}.value`); + } else if (layer_types[i] === 'linear_attention') { + if (config.model_type === 'olmo_hybrid') { + names.add(`${cache_prefix}_conv.${i}.key`); + names.add(`${cache_prefix}_conv.${i}.value`); + names.add(`${cache_prefix}_conv.${i}.query`); + } else { + names.add(`${cache_prefix}_conv.${i}`); + } + names.add(`${cache_prefix}_recurrent.${i}`); + } else { + throw new Error(`Unsupported layer type: ${layer_types[i]}`); + } + } + return names; + } else if (['gemma4', 'gemma4_text'].includes(config.model_type)) { + const c = /** @type {any} */ ( + config.model_type === 'gemma4' ? /** @type {any} */ (config).text_config : config + ); + const num_hidden_layers = c.num_hidden_layers; + const num_kv_shared_layers = c.num_kv_shared_layers ?? 0; + const num_kv_layers = num_hidden_layers - num_kv_shared_layers; + + for (let i = 0; i < num_kv_layers; ++i) { + names.add(`${pkv_prefix}.${i}.key`); + names.add(`${pkv_prefix}.${i}.value`); + } + return names; + } else if (config.model_type === 'deepseek_v4') { + const { layer_types, num_hidden_layers } = /** @type {any} */ (config); + + for (let i = 0; i < num_hidden_layers; ++i) { + names.add(`${pkv_prefix}.${i}.key`); + names.add(`${pkv_prefix}.${i}.value`); + + const layer_type = layer_types[i]; + if (layer_type === 'compressed_sparse_attention') { + names.add(`${cache_prefix}_compressor.${i}.kv`); + names.add(`${cache_prefix}_compressor.${i}.gate`); + names.add(`${cache_prefix}_indexer.${i}.kv`); + names.add(`${cache_prefix}_indexer.${i}.gate`); + } else if (layer_type === 'heavily_compressed_attention') { + names.add(`${cache_prefix}_compressor.${i}.kv`); + names.add(`${cache_prefix}_compressor.${i}.gate`); + } else if (layer_type && layer_type !== 'sliding_attention') { + throw new Error(`Unsupported layer type: ${layer_type}`); + } + } + return names; + } else if (config.model_type === 'zaya') { + const { num_hidden_layers, cca_time1 } = /** @type {any} */ (config); + const stride = cca_time1 ?? 1; + for (let i = 0; i < num_hidden_layers; i += stride) { + names.add(`${pkv_prefix}.${i}.key`); + names.add(`${pkv_prefix}.${i}.value`); + names.add(`${pkv_prefix}.${i}.conv_state`); + names.add(`${pkv_prefix}.${i}.shift_state`); + } + return names; + } else if (['lfm2_vl', 'qwen3_5', 'qwen3_5_moe', 'voxtral_realtime'].includes(config.model_type)) { + let subConfig; + if (config.model_type === 'voxtral_realtime' && options?.session_name === 'audio_encoder') { + subConfig = /** @type {any} */ (config).audio_config; + } else { + subConfig = /** @type {any} */ (config).text_config; + } + return getCacheNames(subConfig, options); + } + + return getKeyValueNames(config, { prefix: pkv_prefix }); +} + +/** + * @param {PretrainedConfig} config + * @param {{ prefix?: string }} [options] + * @returns {Set} + */ +function getKeyValueNames(config, { prefix = 'past_key_values' } = {}) { + /** @type {Set} */ + const names = new Set(); + const normalized_config = config.normalized_config; + + if ( + normalized_config.is_encoder_decoder && + 'num_encoder_heads' in normalized_config && + 'num_decoder_heads' in normalized_config + ) { + for (let i = 0; i < normalized_config.num_decoder_layers; ++i) { + names.add(`${prefix}.${i}.encoder.key`); + names.add(`${prefix}.${i}.encoder.value`); + names.add(`${prefix}.${i}.decoder.key`); + names.add(`${prefix}.${i}.decoder.value`); + } + } else if (normalized_config.multi_query) { + // e.g., for `gpt_bigcode` + for (let i = 0; i < normalized_config.num_layers; ++i) { + names.add(`${prefix}.${i}.key_value`); + } + } else { + for (let i = 0; i < normalized_config.num_layers; ++i) { + names.add(`${prefix}.${i}.key`); + names.add(`${prefix}.${i}.value`); + } + } + + return names; +} +/** + * Base class for all configuration classes. For more information, see the corresponding + * [Python documentation](https://huggingface.co/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig). + */ +export class PretrainedConfig { + // NOTE: Typo in original + + /** @type {string|null} */ + model_type = null; + + /** @type {boolean} */ + is_encoder_decoder = false; + + /** @type {number} */ + max_position_embeddings; + + /** @type {TransformersJSConfig} */ + 'transformers.js_config'; + + /** + * Create a new PreTrainedTokenizer instance. + * @param {Object} configJSON The JSON of the config. + */ + constructor(configJSON) { + Object.assign(this, configJSON); + this.normalized_config = getNormalizedConfig(this); + } + + /** + * Loads a pre-trained config from the given `pretrained_model_name_or_path`. + * + * @param {string} pretrained_model_name_or_path The path to the pre-trained config. + * @param {PretrainedOptions} options Additional options for loading the config. + * @throws {Error} Throws an error if the config.json is not found in the `pretrained_model_name_or_path`. + * + * @returns {Promise} A new instance of the `PretrainedConfig` class. + */ + static async from_pretrained( + pretrained_model_name_or_path, + { progress_callback = null, config = null, cache_dir = null, local_files_only = false, revision = 'main' } = {}, + ) { + if (config && !(config instanceof PretrainedConfig)) { + config = new PretrainedConfig(config); + } + + const data = + config ?? + (await loadConfig(pretrained_model_name_or_path, { + progress_callback, + config, + cache_dir, + local_files_only, + revision, + })); + return new this(data); + } +} + +/** + * Helper class which is used to instantiate pretrained configs with the `from_pretrained` function. + * + * @example + * const config = await AutoConfig.from_pretrained('Xenova/bert-base-uncased'); + */ +export class AutoConfig { + /** @type {typeof PretrainedConfig.from_pretrained} */ + static async from_pretrained(...args) { + return PretrainedConfig.from_pretrained(...args); + } +} + +/** + * Transformers.js-specific configuration, possibly present in config.json under the key `transformers.js_config`. + * @typedef {Object} TransformersJSConfig + * @property {Record} [device_config] Device-specific configurations. + * @property {Record} [free_dimension_overrides] Override the free dimensions of the model. + * See https://onnxruntime.ai/docs/tutorials/web/env-flags-and-session-options.html#freedimensionoverrides + * for more information. + * @property {import('./utils/devices.js').DeviceType} [device] The default device to use for the model. + * @property {import('./utils/dtypes.js').DataType|Record} [dtype] The default data type to use for the model. + * @property {import('./utils/hub.js').ExternalData|Record} [use_external_data_format=false] Whether to load the model using the external data format (used for models >= 2GB in size). + */ + +/** + * Device-specific configuration options. + * @typedef {Omit} DeviceConfig + */ diff --git a/packages/transformers/src/env.js b/packages/transformers/src/env.js new file mode 100644 index 0000000..bea3ef0 --- /dev/null +++ b/packages/transformers/src/env.js @@ -0,0 +1,293 @@ +/** + * @file Module used to configure Transformers.js. + * + * **Example:** Disable remote models. + * ```javascript + * import { env } from '@huggingface/transformers'; + * env.allowRemoteModels = false; + * ``` + * + * **Example:** Set local model path. + * ```javascript + * import { env } from '@huggingface/transformers'; + * env.localModelPath = '/path/to/local/models/'; + * ``` + * + * **Example:** Set cache directory. + * ```javascript + * import { env } from '@huggingface/transformers'; + * env.cacheDir = '/path/to/cache/directory/'; + * ``` + * + * @module env + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import url from 'node:url'; + +const VERSION = '4.2.0'; + +const HAS_SELF = typeof self !== 'undefined'; + +const IS_FS_AVAILABLE = !isEmpty(fs); +const IS_PATH_AVAILABLE = !isEmpty(path); +const IS_WEB_CACHE_AVAILABLE = HAS_SELF && 'caches' in self; + +// Runtime detection +const IS_DENO_RUNTIME = typeof globalThis.Deno !== 'undefined'; +const IS_BUN_RUNTIME = typeof globalThis.Bun !== 'undefined'; + +const IS_DENO_WEB_RUNTIME = IS_DENO_RUNTIME && IS_WEB_CACHE_AVAILABLE && !IS_FS_AVAILABLE; + +const IS_PROCESS_AVAILABLE = typeof process !== 'undefined'; +const IS_NODE_ENV = IS_PROCESS_AVAILABLE && process?.release?.name === 'node' && !IS_DENO_WEB_RUNTIME; + +// Check if various APIs are available (depends on environment) +const IS_BROWSER_ENV = typeof window !== 'undefined' && typeof window.document !== 'undefined'; +const IS_WEBWORKER_ENV = + HAS_SELF && + ['DedicatedWorkerGlobalScope', 'ServiceWorkerGlobalScope', 'SharedWorkerGlobalScope'].includes( + self.constructor?.name, + ); +const IS_WEB_ENV = IS_BROWSER_ENV || IS_WEBWORKER_ENV || IS_DENO_WEB_RUNTIME; + +const IS_WEBGPU_AVAILABLE = IS_NODE_ENV || (typeof navigator !== 'undefined' && 'gpu' in navigator); +const IS_WEBNN_AVAILABLE = typeof navigator !== 'undefined' && 'ml' in navigator; +const IS_CRYPTO_AVAILABLE = typeof crypto !== 'undefined' && typeof crypto.getRandomValues === 'function'; + +const IS_CHROME_AVAILABLE = + // @ts-ignore - chrome may not exist in all environments + typeof chrome !== 'undefined' && typeof chrome.runtime !== 'undefined' && typeof chrome.runtime.id === 'string'; + +const IS_SERVICE_WORKER_ENV = + // @ts-ignore - ServiceWorkerGlobalScope may not exist in all environments + typeof ServiceWorkerGlobalScope !== 'undefined' && HAS_SELF && self instanceof ServiceWorkerGlobalScope; + +/** + * Check whether the current environment is a Safari browser older than version 26 (Safari >= 26 enables WebGPU by default). + * @returns {boolean} Whether the current environment is Safari older than version 26. + */ +const isSafariBelow26 = () => { + if (typeof navigator === 'undefined') { + return false; + } + + const userAgent = navigator.userAgent; + + // Safari has "Apple" in its vendor string. Exclude Chrome on iOS (CriOS), + // Firefox on iOS (FxiOS), Edge on iOS (EdgiOS), and other browsers. + const isSafari = + (navigator.vendor || '').indexOf('Apple') > -1 && + !userAgent.match(/CriOS|FxiOS|EdgiOS|OPiOS|mercury|brave/i) && + !userAgent.includes('Chrome') && + !userAgent.includes('Android'); + if (!isSafari) { + return false; + } + + // Safari reports its version via the "Version/." token in the user agent. + // If the version is missing/unparseable, assume a modern Safari (i.e. not below 26). + const match = userAgent.match(/Version\/(\d+)/); + return match ? parseInt(match[1], 10) < 26 : false; +}; +const IS_SAFARI_BELOW_26 = isSafariBelow26(); + +/** + * A read-only object containing information about the APIs available in the current environment. + */ +export const apis = Object.freeze({ + /** Whether we are running in a browser environment (and not a web worker) */ + IS_BROWSER_ENV, + + /** Whether we are running in a web worker environment */ + IS_WEBWORKER_ENV, + + /** Whether we are running in a web-like environment (browser, web worker, or Deno web runtime) */ + IS_WEB_ENV, + + /** Whether we are running in a service worker environment */ + IS_SERVICE_WORKER_ENV, + + /** Whether we are running in Deno's web runtime (CDN imports, Cache API available, no filesystem) */ + IS_DENO_WEB_RUNTIME, + + /** Whether the Cache API is available */ + IS_WEB_CACHE_AVAILABLE, + + /** Whether the WebGPU API is available */ + IS_WEBGPU_AVAILABLE, + + /** Whether the WebNN API is available */ + IS_WEBNN_AVAILABLE, + + /** Whether we are running in a Safari browser older than version 26. */ + IS_SAFARI_BELOW_26, + + /** Whether the Node.js process API is available */ + IS_PROCESS_AVAILABLE, + + /** Whether we are running in a Node.js-like environment (node, deno, bun) */ + IS_NODE_ENV, + + /** Whether the filesystem API is available */ + IS_FS_AVAILABLE, + + /** Whether the path API is available */ + IS_PATH_AVAILABLE, + + /** Whether the crypto API is available */ + IS_CRYPTO_AVAILABLE, + + /** Whether the Chrome runtime API is available */ + IS_CHROME_AVAILABLE, +}); + +const RUNNING_LOCALLY = IS_FS_AVAILABLE && IS_PATH_AVAILABLE; + +let dirname__ = './'; +if (RUNNING_LOCALLY) { + // NOTE: We wrap `import.meta` in a call to `Object` to prevent Webpack from trying to bundle it in CommonJS. + // Although we get the warning: "Accessing import.meta directly is unsupported (only property access or destructuring is supported)", + // it is safe to ignore since the bundled value (`{}`) isn't used for CommonJS environments (we use __dirname instead). + const _import_meta_url = Object(import.meta).url; + + if (_import_meta_url) { + dirname__ = path.dirname(path.dirname(url.fileURLToPath(_import_meta_url))); // ESM + } else if (typeof __dirname !== 'undefined') { + dirname__ = path.dirname(__dirname); // CommonJS + } +} + +// Only used for environments with access to file system +const DEFAULT_CACHE_DIR = RUNNING_LOCALLY ? path.join(dirname__, '/.cache/') : null; + +// Set local model path, based on available APIs +const DEFAULT_LOCAL_MODEL_PATH = '/models/'; +const localModelPath = RUNNING_LOCALLY ? path.join(dirname__, DEFAULT_LOCAL_MODEL_PATH) : DEFAULT_LOCAL_MODEL_PATH; + +// Ensure default fetch is called with the correct receiver in browser environments. +const DEFAULT_FETCH = typeof globalThis.fetch === 'function' ? globalThis.fetch.bind(globalThis) : undefined; + +/** + * Log levels for controlling output verbosity. + * + * Each level is represented by a number, where higher numbers include all lower level messages. + * Use these values to set `env.logLevel`. + * + * @example + * import { env, LogLevel } from '@huggingface/transformers'; + * + * // Set log level to show only errors + * env.logLevel = LogLevel.ERROR; + * + * // Set log level to show errors, warnings, and info + * env.logLevel = LogLevel.INFO; + * + * // Disable all logging + * env.logLevel = LogLevel.NONE; + * + */ +export const LogLevel = Object.freeze({ + /** All messages including debug output (value: 10) */ + DEBUG: 10, + /** Errors, warnings, and info messages (value: 20) */ + INFO: 20, + /** Errors and warnings (value: 30) */ + WARNING: 30, + /** Only error messages (value: 40) */ + ERROR: 40, + /** No logging output (value: 50) */ + NONE: 50, +}); + +/** + * Global variable given visible to users to control execution. This provides users a simple way to configure Transformers.js. + * @typedef {Object} TransformersEnvironment + * @property {string} version This version of Transformers.js. + * @property {{onnx: Partial & { setLogLevel?: (logLevel: number) => void }}} backends Expose environment variables of different backends, + * allowing users to set these variables if they want to. + * @property {number} logLevel The logging level. Use LogLevel enum values. Defaults to LogLevel.ERROR. + * @property {boolean} allowRemoteModels Whether to allow loading of remote files, defaults to `true`. + * If set to `false`, it will have the same effect as setting `local_files_only=true` when loading pipelines, models, tokenizers, processors, etc. + * @property {string} remoteHost Host URL to load models from. Defaults to the Hugging Face Hub. + * @property {string} remotePathTemplate Path template to fill in and append to `remoteHost` when loading models. + * @property {boolean} allowLocalModels Whether to allow loading of local files, defaults to `false` if running in-browser, and `true` otherwise. + * If set to `false`, it will skip the local file check and try to load the model from the remote host. + * @property {string} localModelPath Path to load local models from. Defaults to `/models/`. + * @property {boolean} useFS Whether to use the file system to load files. By default, it is `true` if available. + * @property {boolean} useBrowserCache Whether to use Cache API to cache models. By default, it is `true` if available. + * @property {boolean} useFSCache Whether to use the file system to cache files. By default, it is `true` if available. + * @property {string|null} cacheDir The directory to use for caching files with the file system. By default, it is `./.cache`. + * @property {boolean} useCustomCache Whether to use a custom cache system (defined by `customCache`), defaults to `false`. + * @property {import('./utils/cache.js').CacheInterface|null} customCache The custom cache to use. Defaults to `null`. Note: this must be an object which + * implements the `match` and `put` functions of the Web Cache API. For more information, see https://developer.mozilla.org/en-US/docs/Web/API/Cache. + * @property {boolean} useWasmCache Whether to pre-load and cache WASM binaries and the WASM factory (.mjs) for ONNX Runtime. + * Defaults to `true` when cache is available. This can improve performance and enables offline usage by avoiding repeated downloads. + * @property {string} cacheKey The cache key to use for storing models and WASM binaries. Defaults to 'transformers-cache'. + * @property {boolean} experimental_useCrossOriginStorage Whether to use the Cross-Origin Storage API to cache model files + * across origins, allowing different sites to share the same cached model weights. Defaults to `false`. + * Requires the Cross-Origin Storage Chrome extension: {@link https://chromewebstore.google.com/detail/cross-origin-storage/denpnpcgjgikjpoglpjefakmdcbmlgih}. + * The `experimental_` prefix indicates that the underlying browser API is not yet standardised and may change or be + * removed without a major version bump. For more information, see {@link https://github.com/WICG/cross-origin-storage}. + * @property {(input: string | URL, init?: any) => Promise} fetch The fetch function to use. Defaults to `fetch`. + */ + +let logLevel = LogLevel.WARNING; // Default log level +/** @type {TransformersEnvironment} */ +export const env = { + version: VERSION, + + /////////////////// Backends settings /////////////////// + // NOTE: These will be populated later by the backends themselves. + backends: { + // onnxruntime-web/onnxruntime-node + onnx: {}, + }, + + /////////////////// Logging settings /////////////////// + get logLevel() { + return logLevel; + }, + set logLevel(level) { + logLevel = level; + + // invoke hook to set ONNX Runtime log level when Transformers.js log level changes + env.backends.onnx?.setLogLevel?.(level); + }, + /////////////////// Model settings /////////////////// + allowRemoteModels: true, + remoteHost: 'https://huggingface.co/', + remotePathTemplate: '{model}/resolve/{revision}/', + + allowLocalModels: !(IS_BROWSER_ENV || IS_WEBWORKER_ENV || IS_DENO_WEB_RUNTIME), // Default to true for non-web environments, false for web environments + localModelPath: localModelPath, + useFS: IS_FS_AVAILABLE, + + /////////////////// Cache settings /////////////////// + useBrowserCache: IS_WEB_CACHE_AVAILABLE, + + useFSCache: IS_FS_AVAILABLE, + cacheDir: DEFAULT_CACHE_DIR, + + useCustomCache: false, + customCache: null, + + useWasmCache: IS_WEB_CACHE_AVAILABLE || IS_FS_AVAILABLE, + cacheKey: 'transformers-cache', + + experimental_useCrossOriginStorage: false, + + /////////////////// Custom fetch ///////////////////// + fetch: DEFAULT_FETCH, + + ////////////////////////////////////////////////////// +}; + +/** + * @param {Object} obj + * @private + */ +function isEmpty(obj) { + return Object.keys(obj).length === 0; +} diff --git a/packages/transformers/src/feature_extraction_utils.js b/packages/transformers/src/feature_extraction_utils.js new file mode 100644 index 0000000..eed1064 --- /dev/null +++ b/packages/transformers/src/feature_extraction_utils.js @@ -0,0 +1,53 @@ +import { FEATURE_EXTRACTOR_NAME } from './utils/constants.js'; +import { Callable } from './utils/generic.js'; +import { getModelJSON } from './utils/hub.js'; + +/** + * Base class for feature extractors. + */ +export class FeatureExtractor extends Callable { + /** + * Constructs a new FeatureExtractor instance. + * + * @param {Object} config The configuration for the feature extractor. + */ + constructor(config) { + super(); + this.config = config; + } + + /** + * Instantiate one of the feature extractor classes of the library from a pretrained model. + * + * The feature extractor class to instantiate is selected based on the `feature_extractor_type` property of + * the config object (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible) + * + * @param {string} pretrained_model_name_or_path The name or path of the pretrained model. Can be either: + * - A string, the *model id* of a pretrained feature_extractor hosted inside a model repo on huggingface.co. + * Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a + * user or organization name, like `dbmdz/bert-base-german-cased`. + * - A path to a *directory* containing feature_extractor files, e.g., `./my_model_directory/`. + * @param {import('./utils/hub.js').PretrainedOptions} options Additional options for loading the feature_extractor. + * + * @returns {Promise} A new instance of the Feature Extractor class. + */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + const config = await getModelJSON(pretrained_model_name_or_path, FEATURE_EXTRACTOR_NAME, true, options); + return new this(config); + } +} + +/** + * Helper function to validate audio inputs. + * @param {any} audio The audio data. + * @param {string} feature_extractor The name of the feature extractor. + * @private + */ +export function validate_audio_inputs(audio, feature_extractor) { + if (!(audio instanceof Float32Array || audio instanceof Float64Array)) { + throw new Error( + `${feature_extractor} expects input to be a Float32Array or a Float64Array, but got ${audio?.constructor?.name ?? typeof audio} instead. ` + + `If using the feature extractor directly, remember to use \`read_audio(url, sampling_rate)\` to obtain the raw audio data of the file/url.`, + ); + } +} diff --git a/packages/transformers/src/generation/configuration_utils.js b/packages/transformers/src/generation/configuration_utils.js new file mode 100644 index 0000000..da27891 --- /dev/null +++ b/packages/transformers/src/generation/configuration_utils.js @@ -0,0 +1,386 @@ +/** + * @module generation/configuration_utils + */ + +import { pick } from '../utils/core.js'; + +/** + * Class that holds a configuration for a generation task. + */ +export class GenerationConfig { + // Parameters that control the length of the output + /** + * The maximum length the generated tokens can have. + * Corresponds to the length of the input prompt + `max_new_tokens`. + * Its effect is overridden by `max_new_tokens`, if also set. + * @type {number} + * @default 20 + */ + max_length = 20; + + /** + * The maximum numbers of tokens to generate, ignoring the number of tokens in the prompt. + * @type {number} + * @default null + */ + max_new_tokens = null; + + /** + * The minimum length of the sequence to be generated. + * Corresponds to the length of the input prompt + `min_new_tokens`. + * Its effect is overridden by `min_new_tokens`, if also set. + * @type {number} + * @default 0 + */ + min_length = 0; + + /** + * The minimum numbers of tokens to generate, ignoring the number of tokens in the prompt. + * @type {number} + * @default null + */ + min_new_tokens = null; + + /** + * Controls the stopping condition for beam-based methods, like beam-search. It accepts the following values: + * - `true`, where the generation stops as soon as there are `num_beams` complete candidates; + * - `false`, where an heuristic is applied and the generation stops when is it very unlikely to find better candidates; + * - `"never"`, where the beam search procedure only stops when there cannot be better candidates (canonical beam search algorithm). + * @type {boolean|"never"} + * @default false + */ + early_stopping = false; + + /** + * The maximum amount of time you allow the computation to run for in seconds. + * Generation will still finish the current pass after allocated time has been passed. + * @type {number} + * @default null + */ + max_time = null; + + // Parameters that control the generation strategy used + /** + * Whether or not to use sampling; use greedy decoding otherwise. + * @type {boolean} + * @default false + */ + do_sample = false; + + /** + * Number of beams for beam search. 1 means no beam search. + * @type {number} + * @default 1 + */ + num_beams = 1; + + /** + * Number of groups to divide `num_beams` into in order to ensure diversity among different groups of beams. + * See [this paper](https://huggingface.co/papers/1610.02424) for more details. + * @type {number} + * @default 1 + */ + num_beam_groups = 1; + + /** + * The values balance the model confidence and the degeneration penalty in contrastive search decoding. + * @type {number} + * @default null + */ + penalty_alpha = null; + + /** + * Whether or not the model should use the past last key/values attentions (if applicable to the model) to speed up decoding. + * @type {boolean} + * @default true + */ + use_cache = true; + + // Parameters for manipulation of the model output logits + /** + * The value used to modulate the next token probabilities. + * @type {number} + * @default 1.0 + */ + temperature = 1.0; + + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + * @type {number} + * @default 50 + */ + top_k = 50; + + /** + * If set to float < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or higher are kept for generation. + * @type {number} + * @default 1.0 + */ + top_p = 1.0; + + /** + * Local typicality measures how similar the conditional probability of predicting a target token next is to the expected conditional probability of predicting a random token next, given the partial text already generated. + * If set to float < 1, the smallest set of the most locally typical tokens with probabilities that add up to `typical_p` or higher are kept for generation. + * See [this paper](https://huggingface.co/papers/2202.00666) for more details. + * @type {number} + * @default 1.0 + */ + typical_p = 1.0; + + /** + * If set to float strictly between 0 and 1, only tokens with a conditional probability greater than `epsilon_cutoff` will be sampled. + * In the paper, suggested values range from 3e-4 to 9e-4, depending on the size of the model. + * See [Truncation Sampling as Language Model Desmoothing](https://huggingface.co/papers/2210.15191) for more details. + * @type {number} + * @default 0.0 + */ + epsilon_cutoff = 0.0; + + /** + * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. + * If set to float strictly between 0 and 1, a token is only considered if it is greater than either `eta_cutoff` or `sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits)))`. + * The latter term is intuitively the expected next token probability, scaled by `sqrt(eta_cutoff)`. In the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + * See [Truncation Sampling as Language Model Desmoothing](https://huggingface.co/papers/2210.15191) for more details. + * @type {number} + * @default 0.0 + */ + eta_cutoff = 0.0; + + /** + * This value is subtracted from a beam's score if it generates a token same as any beam from other group at a particular time. + * Note that `diversity_penalty` is only effective if `group beam search` is enabled. + * @type {number} + * @default 0.0 + */ + diversity_penalty = 0.0; + + /** + * The parameter for repetition penalty. 1.0 means no penalty. + * See [this paper](https://huggingface.co/papers/1909.05858) for more details. + * @type {number} + * @default 1.0 + */ + repetition_penalty = 1.0; + + /** + * The paramater for encoder_repetition_penalty. + * An exponential penalty on sequences that are not in the original input. + * 1.0 means no penalty. + * @type {number} + * @default 1.0 + */ + encoder_repetition_penalty = 1.0; + + /** + * Exponential penalty to the length that is used with beam-based generation. + * It is applied as an exponent to the sequence length, which in turn is used to divide the score of the sequence. + * Since the score is the log likelihood of the sequence (i.e. negative), `length_penalty` > 0.0 promotes longer sequences, while `length_penalty` < 0.0 encourages shorter sequences. + * @type {number} + * @default 1.0 + */ + length_penalty = 1.0; + + /** + * If set to int > 0, all ngrams of that size can only occur once. + * @type {number} + * @default 0 + */ + no_repeat_ngram_size = 0; + + /** + * List of token ids that are not allowed to be generated. + * In order to get the token ids of the words that should not appear in the generated text, use + * `tokenizer(bad_words, { add_prefix_space: true, add_special_tokens: false }).input_ids`. + * @type {number[][]} + * @default null + */ + bad_words_ids = null; + + /** + * List of token ids that must be generated. + * If given a `number[][]`, this is treated as a simple list of words that must be included, the opposite to `bad_words_ids`. + * If given `number[][][]`, this triggers a [disjunctive constraint](https://github.com/huggingface/transformers/issues/14081), where one can allow different forms of each word. + * @type {number[][]|number[][][]} + * @default null + */ + force_words_ids = null; + + /** + * Whether to renormalize the logits after applying all the logits processors or warpers (including the custom ones). + * It's highly recommended to set this flag to `true` as the search algorithms suppose the score logits are normalized but some logit processors or warpers break the normalization. + * @type {boolean} + * @default false + */ + renormalize_logits = false; + + /** + * Custom constraints that can be added to the generation to ensure that the output will contain the use of certain tokens as defined by `Constraint` objects, in the most sensible way possible. + * @type {Object[]} + * @default null + */ + constraints = null; + + /** + * The id of the token to force as the first generated token after the `decoder_start_token_id`. + * Useful for multilingual models like mBART where the first generated token needs to be the target language token. + * @type {number} + * @default null + */ + forced_bos_token_id = null; + + /** + * The id of the token to force as the last generated token when `max_length` is reached. + * Optionally, use a list to set multiple *end-of-sequence* tokens. + * @type {number|number[]} + * @default null + */ + forced_eos_token_id = null; + + /** + * Whether to remove possible *nan* and *inf* outputs of the model to prevent the generation method to crash. Note that using `remove_invalid_values` can slow down generation. + * @type {boolean} + */ + remove_invalid_values = false; + + /** + * This Tuple adds an exponentially increasing length penalty, after a certain amount of tokens have been generated. + * The tuple shall consist of: `(start_index, decay_factor)` where `start_index` indicates where penalty starts and `decay_factor` represents the factor of exponential decay. + * @type {[number, number]} + * @default null + */ + exponential_decay_length_penalty = null; + + /** + * A list of tokens that will be suppressed at generation. + * The `SuppressTokens` logit processor will set their log probs to `-inf` so that they are not sampled. + * @type {number[]} + * @default null + */ + suppress_tokens = null; + + /** + * A streamer that will be used to stream the generation. + * @type {import('./streamers.js').TextStreamer} + * @default null + */ + streamer = null; + + /** + * A list of tokens that will be suppressed at the beginning of the generation. + * The `SuppressBeginTokens` logit processor will set their log probs to `-inf` so that they are not sampled. + * @type {number[]} + * @default null + */ + begin_suppress_tokens = null; + + /** + * A list of pairs of integers which indicates a mapping from generation indices to token indices that will be forced before sampling. + * For example, `[[1, 123]]` means the second generated token will always be a token of index 123. + * @type {[number, number][]} + * @default null + */ + forced_decoder_ids = null; + + /** + * The guidance scale for classifier free guidance (CFG). CFG is enabled by setting `guidance_scale > 1`. + * Higher guidance scale encourages the model to generate samples that are more closely linked to the input + * prompt, usually at the expense of poorer quality. + * @type {number} + * @default null + */ + guidance_scale = null; + + // Parameters that define the output variables of `generate` + /** + * The number of independently computed returned sequences for each element in the batch. + * @type {number} + * @default 1 + */ + num_return_sequences = 1; + + /** + * Whether or not to return the attentions tensors of all attention layers. + * See `attentions` under returned tensors for more details. + * @type {boolean} + * @default false + */ + output_attentions = false; + + /** + * Whether or not to return the hidden states of all layers. + * See `hidden_states` under returned tensors for more details. + * @type {boolean} + * @default false + */ + output_hidden_states = false; + + /** + * Whether or not to return the prediction scores. + * See `scores` under returned tensors for more details. + * @type {boolean} + * @default false + */ + output_scores = false; + + /** + * Whether or not to return a `ModelOutput` instead of a plain tuple. + * @type {boolean} + * @default false + */ + return_dict_in_generate = false; + + // Special tokens that can be used at generation time + /** + * The id of the *padding* token. + * @type {number} + * @default null + */ + pad_token_id = null; + + /** + * The id of the *beginning-of-sequence* token. + * @type {number} + * @default null + */ + bos_token_id = null; + + /** + * The id of the *end-of-sequence* token. + * Optionally, use a list to set multiple *end-of-sequence* tokens. + * @type {number|number[]} + * @default null + */ + eos_token_id = null; + + // Generation parameters exclusive to encoder-decoder models + /** + * If set to int > 0, all ngrams of that size that occur in the `encoder_input_ids` cannot occur in the `decoder_input_ids`. + * @type {number} + * @default 0 + */ + encoder_no_repeat_ngram_size = 0; + + /** + * If an encoder-decoder model starts decoding with a different token than *bos*, the id of that token. + * @type {number} + * @default null + */ + decoder_start_token_id = null; + + // Wild card + /** + * Additional generation kwargs will be forwarded to the `generate` function of the model. + * Kwargs that are not present in `generate`'s signature will be used in the model forward pass. + * @type {Object} + * @default {} + */ + generation_kwargs = {}; + + /** + * + * @param {GenerationConfig|import('../configs.js').PretrainedConfig} config + */ + constructor(config) { + Object.assign(this, pick(config, Object.getOwnPropertyNames(this))); + } +} diff --git a/packages/transformers/src/generation/logits_process.js b/packages/transformers/src/generation/logits_process.js new file mode 100644 index 0000000..647a308 --- /dev/null +++ b/packages/transformers/src/generation/logits_process.js @@ -0,0 +1,751 @@ +/** + * @module generation/logits_process + */ + +import { Callable } from '../utils/generic.js'; +import { Tensor } from '../utils/tensor.js'; + +import { max, log_softmax } from '../utils/maths.js'; + +/** + * Abstract base class for all logit processors that can be applied during generation. + */ +export class LogitsProcessor extends Callable { + /** + * Apply the processor to the input logits. + * + * @abstract + * @param {bigint[][]} input_ids The input ids. + * @param {Tensor} logits The logits to process. + * @throws {Error} Throws an error if `_call` is not implemented in the subclass. + */ + _call(input_ids, logits) { + throw Error('`_call` should be implemented in a subclass'); + } +} + +/** + * Abstract base class for all logit warpers that can be applied during generation with multinomial sampling. + */ +export class LogitsWarper extends Callable { + /** + * Apply the processor to the input logits. + * + * @abstract + * @param {bigint[][]} input_ids The input ids. + * @param {Tensor} logits The logits to process. + * @throws {Error} Throws an error if `_call` is not implemented in the subclass. + */ + _call(input_ids, logits) { + throw Error('`_call` should be implemented in a subclass'); + } +} + +/** + * A class representing a list of logits processors. A logits processor is a function that modifies the logits + * output of a language model. This class provides methods for adding new processors and applying all processors to a + * batch of logits. + */ +export class LogitsProcessorList extends Callable { + /** + * Constructs a new instance of `LogitsProcessorList`. + */ + constructor() { + super(); + this.processors = []; + } + + /** + * Adds a new logits processor to the list. + * + * @param {LogitsProcessor} item The logits processor function to add. + */ + push(item) { + this.processors.push(item); + } + + /** + * Adds multiple logits processors to the list. + * + * @param {LogitsProcessor[]} items The logits processor functions to add. + */ + extend(items) { + this.processors.push(...items); + } + + /** + * Applies all logits processors in the list to a batch of logits, modifying them in-place. + * + * @param {bigint[][]} input_ids The input IDs for the language model. + * @param {Tensor} logits + */ + _call(input_ids, logits) { + let toReturn = logits; + // NOTE: Most processors modify logits inplace + for (const processor of this.processors) { + toReturn = processor(input_ids, toReturn); + } + return toReturn; + } + + [Symbol.iterator]() { + return this.processors.values(); + } +} + +// DEPRECATED: https://github.com/huggingface/transformers/pull/29485 +// /** +// * A logits processor that forces a specific token to be generated by the decoder. +// */ +// export class ForceTokensLogitsProcessor extends LogitsProcessor { +// /** +// * Constructs a new instance of `ForceTokensLogitsProcessor`. +// * +// * @param {[number, number][]} forced_decoder_ids The ids of tokens that should be forced. +// */ +// constructor(forced_decoder_ids) { +// super(); +// // TODO: convert to `new Map(forced_decoder_ids)` +// this.force_token_map = Object.fromEntries(forced_decoder_ids ?? []); +// } + +// /** +// * Apply the processor to the input logits. +// * +// * @param {bigint[][]} input_ids The input ids. +// * @param {Tensor} logits The logits to process. +// * @returns {Tensor} The processed logits. +// */ +// _call(input_ids, logits) { +// console.log('this.force_token_map', this.force_token_map) +// console.log('call ForceTokensLogitsProcessor', input_ids, logits) +// console.log('input_ids.length', input_ids.length) +// let map = this.force_token_map[input_ids.length]; +// if (map) { // There exists a mapping +// logits.data.fill(-Infinity) +// logits.data[map] = 0; +// } +// console.log('map', map) +// // throw Error("Not implemented") +// return logits; +// } +// } + +/** + * A LogitsProcessor that forces a BOS token at the beginning of the generated sequence. + */ +export class ForcedBOSTokenLogitsProcessor extends LogitsProcessor { + /** + * Create a ForcedBOSTokenLogitsProcessor. + * @param {number} bos_token_id The ID of the beginning-of-sequence token to be forced. + */ + constructor(bos_token_id) { + super(); + this.bos_token_id = bos_token_id; + } + + /** + * Apply the BOS token forcing to the logits. + * @param {bigint[][]} input_ids The input IDs. + * @param {Tensor} logits The logits. + * @returns {Tensor} The logits with BOS token forcing. + */ + _call(input_ids, logits) { + for (let i = 0; i < input_ids.length; ++i) { + if (input_ids[i].length === 1) { + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); + batch_logits_data.fill(-Infinity); + batch_logits_data[this.bos_token_id] = 0; + } + } + return logits; + } +} + +/** + * A logits processor that enforces the specified token as the last generated token when `max_length` is reached. + */ +export class ForcedEOSTokenLogitsProcessor extends LogitsProcessor { + /** + * Create a ForcedEOSTokenLogitsProcessor. + * @param {number} max_length The maximum length of the sequence to be generated. + * @param {number|number[]} eos_token_id The id(s) of the *end-of-sequence* token. + */ + constructor(max_length, eos_token_id) { + super(); + this.max_length = max_length; + this.eos_token_id = Array.isArray(eos_token_id) ? eos_token_id : [eos_token_id]; + } + + /** + * Apply the processor to input_ids and logits. + * + * @param {bigint[][]} input_ids The input ids. + * @param {Tensor} logits The logits tensor. + */ + _call(input_ids, logits) { + for (let i = 0; i < input_ids.length; ++i) { + if (input_ids[i].length === this.max_length - 1) { + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); + batch_logits_data.fill(-Infinity); + for (const eos_token of this.eos_token_id) { + batch_logits_data[eos_token] = 0; + } + } + } + return logits; + } +} + +/** + * A LogitsProcessor that suppresses a list of tokens throughout generation. + * Sets their log probs to `-inf` so that they are not generated. + */ +export class SuppressTokensLogitsProcessor extends LogitsProcessor { + /** + * Create a SuppressTokensLogitsProcessor. + * @param {number[]} suppress_tokens The IDs of the tokens to suppress. + */ + constructor(suppress_tokens) { + super(); + this.suppress_tokens = suppress_tokens; + } + + /** + * Suppress the specified tokens by setting their logits to -Infinity. + * @param {bigint[][]} input_ids The input IDs. + * @param {Tensor} logits The logits. + * @returns {Tensor} The modified logits. + */ + _call(input_ids, logits) { + for (let i = 0; i < input_ids.length; ++i) { + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); + for (const token_id of this.suppress_tokens) { + batch_logits_data[token_id] = -Infinity; + } + } + return logits; + } +} + +/** + * A LogitsProcessor that suppresses a list of tokens as soon as the `generate` function starts + * generating using `begin_index` tokens. This should ensure that the tokens defined by + * `begin_suppress_tokens` at not sampled at the begining of the generation. + */ +export class SuppressTokensAtBeginLogitsProcessor extends LogitsProcessor { + /** + * Create a SuppressTokensAtBeginLogitsProcessor. + * @param {number[]} begin_suppress_tokens The IDs of the tokens to suppress. + * @param {number} begin_index The number of tokens to generate before suppressing tokens. + */ + constructor(begin_suppress_tokens, begin_index) { + super(); + this.begin_suppress_tokens = begin_suppress_tokens; + this.begin_index = begin_index; + } + + /** + * Apply the BOS token forcing to the logits. + * @param {bigint[][]} input_ids The input IDs. + * @param {Tensor} logits The logits. + * @returns {Tensor} The logits with BOS token forcing. + */ + _call(input_ids, logits) { + for (let i = 0; i < input_ids.length; ++i) { + if (input_ids[i].length === this.begin_index) { + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); + for (const token_id of this.begin_suppress_tokens) { + batch_logits_data[token_id] = -Infinity; + } + } + } + return logits; + } +} + +/** + * A LogitsProcessor that handles adding timestamps to generated text. + */ +export class WhisperTimeStampLogitsProcessor extends LogitsProcessor { + /** + * Constructs a new WhisperTimeStampLogitsProcessor. + * @param {import('../models/whisper/generation_whisper.js').WhisperGenerationConfig} generate_config The config object passed to the `generate()` method of a transformer model. + * @param {number[]} init_tokens The initial tokens of the input sequence. + */ + constructor(generate_config, init_tokens) { + super(); + this.eos_token_id = Array.isArray(generate_config.eos_token_id) + ? generate_config.eos_token_id[0] + : generate_config.eos_token_id; + + this.no_timestamps_token_id = generate_config.no_timestamps_token_id; + this.timestamp_begin = this.no_timestamps_token_id + 1; + + this.begin_index = init_tokens.length; + if (init_tokens.at(-1) === this.no_timestamps_token_id) { + this.begin_index -= 1; + } + this.max_initial_timestamp_index = generate_config.max_initial_timestamp_index; + } + + /** + * Modify the logits to handle timestamp tokens. + * @param {bigint[][]} input_ids The input sequence of tokens. + * @param {Tensor} logits The logits output by the model. + * @returns {Tensor} The modified logits. + */ + _call(input_ids, logits) { + for (let i = 0; i < input_ids.length; ++i) { + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); + + // suppress <|notimestamps|> which is handled by without_timestamps + batch_logits_data[this.no_timestamps_token_id] = -Infinity; + + if (input_ids[i].length === this.begin_index) { + batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity); + continue; + } + + // timestamps have to appear in pairs, except directly before eos_token; mask logits accordingly + const seq = input_ids[i].slice(this.begin_index); + const last_was_timestamp = seq.length >= 1 && seq[seq.length - 1] >= this.timestamp_begin; + const penultimate_was_timestamp = seq.length < 2 || seq[seq.length - 2] >= this.timestamp_begin; + + if (last_was_timestamp) { + if (penultimate_was_timestamp) { + // has to be non-timestamp + batch_logits_data.subarray(this.timestamp_begin).fill(-Infinity); + } else { + // cannot be normal text tokens + batch_logits_data.subarray(0, this.eos_token_id).fill(-Infinity); + } + } + + // apply the `max_initial_timestamp` option + if (input_ids[i].length === this.begin_index && this.max_initial_timestamp_index !== null) { + const last_allowed = this.timestamp_begin + this.max_initial_timestamp_index; + batch_logits_data.subarray(last_allowed + 1).fill(-Infinity); + } + + // if sum of probability over timestamps is above any other token, sample timestamp + const logprobs = log_softmax(batch_logits_data); + const timestamp_logprob = Math.log( + logprobs + .subarray(this.timestamp_begin) + .map(Math.exp) + .reduce((a, b) => a + b), + ); + const max_text_token_logprob = max(logprobs.subarray(0, this.timestamp_begin))[0]; + + if (timestamp_logprob > max_text_token_logprob) { + batch_logits_data.subarray(0, this.timestamp_begin).fill(-Infinity); + } + } + + return logits; + } +} + +/** + * A logits processor that disallows ngrams of a certain size to be repeated. + */ +export class NoRepeatNGramLogitsProcessor extends LogitsProcessor { + /** + * Create a NoRepeatNGramLogitsProcessor. + * @param {number} no_repeat_ngram_size The no-repeat-ngram size. All ngrams of this size can only occur once. + */ + constructor(no_repeat_ngram_size) { + super(); + this.no_repeat_ngram_size = no_repeat_ngram_size; + } + + /** + * Generate n-grams from a sequence of token ids. + * @param {bigint[]} prevInputIds List of previous input ids + * @returns {Map} Map of generated n-grams + */ + getNgrams(prevInputIds) { + const curLen = prevInputIds.length; + + /**@type {number[][]} */ + const ngrams = []; + for (let j = 0; j < curLen + 1 - this.no_repeat_ngram_size; ++j) { + const ngram = []; + for (let k = 0; k < this.no_repeat_ngram_size; ++k) { + ngram.push(prevInputIds[j + k]); + } + ngrams.push(ngram.map(Number)); + } + + /** @type {Map} */ + const generatedNgram = new Map(); + for (const ngram of ngrams) { + const prevNgram = ngram.slice(0, ngram.length - 1); + const prevNgramKey = JSON.stringify(prevNgram); + const prevNgramValue = generatedNgram.get(prevNgramKey) ?? []; + prevNgramValue.push(ngram[ngram.length - 1]); + generatedNgram.set(prevNgramKey, prevNgramValue); + } + return generatedNgram; + } + + /** + * Generate n-grams from a sequence of token ids. + * @param {Map} bannedNgrams Map of banned n-grams + * @param {bigint[]} prevInputIds List of previous input ids + * @returns {number[]} Map of generated n-grams + */ + getGeneratedNgrams(bannedNgrams, prevInputIds) { + const ngramIdx = prevInputIds.slice(prevInputIds.length + 1 - this.no_repeat_ngram_size, prevInputIds.length); + const banned = bannedNgrams.get(JSON.stringify(ngramIdx.map(Number))) ?? []; + return banned; + } + + /** + * Calculate banned n-gram tokens + * @param {bigint[]} prevInputIds List of previous input ids + * @returns {number[]} Map of generated n-grams + */ + calcBannedNgramTokens(prevInputIds) { + const bannedTokens = []; + if (prevInputIds.length + 1 < this.no_repeat_ngram_size) { + // return no banned tokens if we haven't generated no_repeat_ngram_size tokens yet + return bannedTokens; + } else { + const generatedNgrams = this.getNgrams(prevInputIds); + const bannedTokens = this.getGeneratedNgrams(generatedNgrams, prevInputIds); + return bannedTokens; + } + } + + /** + * Apply the no-repeat-ngram processor to the logits. + * @param {bigint[][]} input_ids The input IDs. + * @param {Tensor} logits The logits. + * @returns {Tensor} The logits with no-repeat-ngram processing. + */ + _call(input_ids, logits) { + for (let i = 0; i < input_ids.length; ++i) { + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); + const bannedTokens = this.calcBannedNgramTokens(input_ids[i]); + for (const token of bannedTokens) { + batch_logits_data[token] = -Infinity; + } + } + return logits; + } +} + +/** + * A logits processor that prevents the repetition of previous tokens through a penalty. + * This penalty is applied at most once per token. Note that, for decoder-only models like most LLMs, + * the considered tokens include the prompt. + * + * In the original [paper](https://huggingface.co/papers/1909.05858), the authors suggest the use of a + * penalty of around 1.2 to achieve a good balance between truthful generation and lack of repetition. + * To penalize and reduce repetition, use `penalty` values above 1.0, where a higher value penalizes + * more strongly. To reward and encourage repetition, use `penalty` values between 0.0 and 1.0, where + * a lower value rewards more strongly. + */ +export class RepetitionPenaltyLogitsProcessor extends LogitsProcessor { + /** + * Create a RepetitionPenaltyLogitsProcessor. + * @param {number} penalty The parameter for repetition penalty. + * - 1.0 means no penalty. Above 1.0 penalizes previously generated tokens. + * - Between 0.0 and 1.0 rewards previously generated tokens. + */ + constructor(penalty) { + super(); + this.penalty = penalty; + } + + /** + * Apply the repetition penalty to the logits. + * @param {bigint[][]} input_ids The input IDs. + * @param {Tensor} logits The logits. + * @returns {Tensor} The logits with repetition penalty processing. + */ + _call(input_ids, logits) { + for (let i = 0; i < input_ids.length; ++i) { + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); + for (const input_id of new Set(input_ids[i])) { + const token = Number(input_id); + if (batch_logits_data[token] < 0) { + batch_logits_data[token] *= this.penalty; + } else { + batch_logits_data[token] /= this.penalty; + } + } + } + + return logits; + } +} + +/** + * A logits processor that enforces a minimum number of tokens. + */ +export class MinLengthLogitsProcessor extends LogitsProcessor { + /** + * Create a MinLengthLogitsProcessor. + * @param {number} min_length The minimum length below which the score of `eos_token_id` is set to negative infinity. + * @param {number|number[]} eos_token_id The ID/IDs of the end-of-sequence token. + */ + constructor(min_length, eos_token_id) { + super(); + this.min_length = min_length; + this.eos_token_id = Array.isArray(eos_token_id) ? eos_token_id : [eos_token_id]; + } + + /** + * Apply logit processor. + * @param {bigint[][]} input_ids The input IDs. + * @param {Tensor} logits The logits. + * @returns {Tensor} The processed logits. + */ + _call(input_ids, logits) { + for (let i = 0; i < input_ids.length; ++i) { + if (input_ids[i].length < this.min_length) { + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); + + for (const eos_token of this.eos_token_id) { + batch_logits_data[eos_token] = -Infinity; + } + } + } + + return logits; + } +} + +/** + * A logits processor that enforces a minimum number of new tokens. + */ +export class MinNewTokensLengthLogitsProcessor extends LogitsProcessor { + /** + * Create a MinNewTokensLengthLogitsProcessor. + * @param {number} prompt_length_to_skip The input tokens length. + * @param {number} min_new_tokens The minimum *new* tokens length below which the score of `eos_token_id` is set to negative infinity. + * @param {number|number[]} eos_token_id The ID/IDs of the end-of-sequence token. + */ + constructor(prompt_length_to_skip, min_new_tokens, eos_token_id) { + super(); + this.prompt_length_to_skip = prompt_length_to_skip; + this.min_new_tokens = min_new_tokens; + this.eos_token_id = Array.isArray(eos_token_id) ? eos_token_id : [eos_token_id]; + } + + /** + * Apply logit processor. + * @param {bigint[][]} input_ids The input IDs. + * @param {Tensor} logits The logits. + * @returns {Tensor} The processed logits. + */ + _call(input_ids, logits) { + for (let i = 0; i < input_ids.length; ++i) { + const new_tokens_length = input_ids[i].length - this.prompt_length_to_skip; + if (new_tokens_length < this.min_new_tokens) { + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); + + for (const eos_token of this.eos_token_id) { + batch_logits_data[eos_token] = -Infinity; + } + } + } + return logits; + } +} + +export class NoBadWordsLogitsProcessor extends LogitsProcessor { + /** + * Create a `NoBadWordsLogitsProcessor`. + * @param {number[][]} bad_words_ids List of list of token ids that are not allowed to be generated. + * @param {number|number[]} eos_token_id The id of the *end-of-sequence* token. Optionally, use a list to set multiple *end-of-sequence* tokens. + */ + constructor(bad_words_ids, eos_token_id) { + super(); + this.bad_words_ids = bad_words_ids; + this.eos_token_id = Array.isArray(eos_token_id) ? eos_token_id : [eos_token_id]; + } + + /** + * Apply logit processor. + * @param {bigint[][]} input_ids The input IDs. + * @param {Tensor} logits The logits. + * @returns {Tensor} The processed logits. + */ + _call(input_ids, logits) { + for (let i = 0; i < input_ids.length; ++i) { + const batch_logits_data = /** @type {Float32Array} */ (logits[i].data); + const ids = input_ids[i]; + for (const bad_word_ids of this.bad_words_ids) { + // There aren't enough tokens to match the banned sequence + if (ids.length < bad_word_ids.length - 1) continue; + + // Whether to modify the logits of the last token in the bad word id sequence + let mark = true; + + // For each bad word in the list, if the current sequence of input ids ends with this sequence (excluding the last), + // then we set the logits of the last bad word id to -Infinity. + for (let j = 1; j <= bad_word_ids.length - 1; ++j) { + // NOTE: We use != instead of !== to compare bigint and number + // @ts-ignore + if (bad_word_ids.at(-j - 1) != ids.at(-j)) { + // We have found a mismatch + mark = false; + break; + } + } + if (mark) { + batch_logits_data[bad_word_ids.at(-1)] = -Infinity; + } + } + } + return logits; + } +} + +/** + * [`LogitsProcessor`] for classifier free guidance (CFG). The scores are split over the batch dimension, + * where the first half correspond to the conditional logits (predicted from the input prompt) and the second half + * correspond to the unconditional logits (predicted from an empty or 'null' prompt). The processor computes a + * weighted average across the conditional and unconditional logits, parameterised by the `guidance_scale`. + * + * See [the paper](https://huggingface.co/papers/2306.05284) for more information. + */ +export class ClassifierFreeGuidanceLogitsProcessor extends LogitsProcessor { + /** + * Create a `ClassifierFreeGuidanceLogitsProcessor`. + * @param {number} guidance_scale The guidance scale for classifier free guidance (CFG). CFG is enabled by setting `guidance_scale > 1`. + * Higher guidance scale encourages the model to generate samples that are more closely linked to the input + * prompt, usually at the expense of poorer quality. + */ + constructor(guidance_scale) { + super(); + if (guidance_scale <= 1) { + throw new Error( + `Require guidance scale >1 to use the classifier free guidance processor, got guidance scale ${guidance_scale}.`, + ); + } + this.guidance_scale = guidance_scale; + } + + /** + * Apply logit processor. + * @param {bigint[][]} input_ids The input IDs. + * @param {Tensor} logits The logits. + * @returns {Tensor} The processed logits. + */ + _call(input_ids, logits) { + if (logits.dims[0] !== 2 * input_ids.length) { + throw new Error( + `Logits should have twice the batch size of the input ids, the first half of batches corresponding to ` + + `the conditional inputs, and the second half of batches corresponding to the unconditional inputs. Got ` + + `batch size ${logits.dims[0]} for the logits and ${input_ids.length} for the input ids.`, + ); + } + + const unguided_bsz = input_ids.length; + const cond_logits = logits.slice([0, unguided_bsz], null); + const uncond_logits = logits.slice([unguided_bsz, logits.dims[0]], null); + + // Merge into uncond_logits (to save memory). This is equivalent to the following: + // scores = uncond_logits + (cond_logits - uncond_logits) * guidance_scale + for (let i = 0; i < uncond_logits.data.length; ++i) { + uncond_logits.data[i] += (cond_logits.data[i] - uncond_logits.data[i]) * this.guidance_scale; + } + + return uncond_logits; + } +} + +/** + * [`LogitsWarper`] for temperature (exponential scaling output probability distribution), which effectively means + * that it can control the randomness of the predicted tokens. Often used together with [`TopPLogitsWarper`] and [`TopKLogitsWarper`]. + */ +export class TemperatureLogitsWarper extends LogitsWarper { + /** + * Create a `TemperatureLogitsWarper`. + * @param {number} temperature Strictly positive float value used to modulate the logits distribution. + * A value smaller than `1` decreases randomness (and vice versa), with `0` being equivalent to shifting + * all probability mass to the most likely token. + */ + constructor(temperature) { + super(); + + if (typeof temperature !== 'number' || temperature <= 0) { + let errorMessage = `\`temperature\` (=${temperature}) must be a strictly positive float, otherwise your next token scores will be invalid.`; + + if (temperature === 0) { + errorMessage += " If you're looking for greedy decoding strategies, set `do_sample=false`."; + } + } + this.temperature = temperature; + } + + /** + * Apply logit warper. + * @param {bigint[][]} input_ids The input IDs. + * @param {Tensor} logits The logits. + * @returns {Tensor} The processed logits. + */ + _call(input_ids, logits) { + const batch_logits_data = /** @type {Float32Array} */ (logits.data); + for (let i = 0; i < batch_logits_data.length; ++i) { + batch_logits_data[i] /= this.temperature; + } + return logits; + } +} + +/** + * [`LogitsWarper`] that performs top-p, i.e. restricting to top tokens summing to prob_cut_off <= prob_cut_off. + * Often used together with [`TemperatureLogitsWarper`] and [`TopKLogitsWarper`]. + */ +export class TopPLogitsWarper extends LogitsWarper { + /** + * Create a `TopPLogitsWarper`. + * @param {number} top_p If set to < 1, only the smallest set of most probable tokens with + * probabilities that add up to `top_p` or higher are kept for generation. + * @param {Object} options Additional options for the top-p sampling. + * @param {number} [options.filter_value=-Infinity] All filtered values will be set to this float value. + * @param {number} [options.min_tokens_to_keep=1] Minimum number of tokens that cannot be filtered. + */ + constructor(top_p, { filter_value = -Infinity, min_tokens_to_keep = 1 } = {}) { + super(); + if (top_p < 0 || top_p > 1.0) { + throw new Error(`\`top_p\` must be a float > 0 and < 1, but is ${top_p}`); + } + if (!Number.isInteger(min_tokens_to_keep) || min_tokens_to_keep < 1) { + throw new Error(`\`min_tokens_to_keep\` must be a positive integer, but is ${min_tokens_to_keep}`); + } + + this.top_p = top_p; + this.filter_value = filter_value; + this.min_tokens_to_keep = min_tokens_to_keep; + } +} + +/** + * [`LogitsWarper`] that performs top-k, i.e. restricting to the k highest probability elements. + * Often used together with [`TemperatureLogitsWarper`] and [`TopPLogitsWarper`]. + */ +export class TopKLogitsWarper extends LogitsWarper { + /** + * Create a `TopKLogitsWarper`. + * @param {number} top_k If set to > 0, only the top `top_k` tokens are kept for generation. + * @param {Object} options Additional options for the top-k sampling. + * @param {number} [options.filter_value=-Infinity] All filtered values will be set to this float value. + * @param {number} [options.min_tokens_to_keep=1] Minimum number of tokens that cannot be filtered. + */ + constructor(top_k, { filter_value = -Infinity, min_tokens_to_keep = 1 } = {}) { + super(); + if (!Number.isInteger(top_k) || top_k < 0) { + throw new Error(`\`top_k\` must be a positive integer, but is ${top_k}`); + } + + this.top_k = Math.max(top_k, min_tokens_to_keep); + this.filter_value = filter_value; + } +} diff --git a/packages/transformers/src/generation/logits_sampler.js b/packages/transformers/src/generation/logits_sampler.js new file mode 100644 index 0000000..a0418c0 --- /dev/null +++ b/packages/transformers/src/generation/logits_sampler.js @@ -0,0 +1,183 @@ +/** + * @module generation/logits_sampler + */ + +import { Callable } from '../utils/generic.js'; +import { Tensor, topk } from '../utils/tensor.js'; + +import { max, softmax } from '../utils/maths.js'; +import { _weightedIndex } from '../utils/random.js'; +import { GenerationConfig } from '../generation/configuration_utils.js'; + +/** + * Sampler is a base class for all sampling methods used for text generation. + */ +export class LogitsSampler extends Callable { + /** + * Creates a new Sampler object with the specified generation config. + * @param {GenerationConfig} generation_config The generation config. + */ + constructor(generation_config) { + super(); + this.generation_config = generation_config; + } + + /** + * Executes the sampler, using the specified logits. + * @param {Tensor} logits + * @returns {Promise<[bigint, number][]>} + */ + async _call(logits) { + // Sample from logits, of dims [batch, sequence_length, vocab_size]. + // If index is specified, sample from [batch, index, vocab_size]. + return this.sample(logits); + } + + /** + * Abstract method for sampling the logits. + * @param {Tensor} logits + * @throws {Error} If not implemented in subclass. + * @returns {Promise<[bigint, number][]>} + */ + async sample(logits) { + throw Error('sample should be implemented in subclasses.'); + } + + /** + * Returns the specified logits as an array, with temperature applied. + * @param {Tensor} logits + * @param {number} index + * @returns {Float32Array} + */ + getLogits(logits, index) { + let vocabSize = logits.dims.at(-1); + + let logs = /** @type {Float32Array} */ (logits.data); + + if (index === -1) { + logs = logs.slice(-vocabSize); + } else { + let startIndex = index * vocabSize; + logs = logs.slice(startIndex, startIndex + vocabSize); + } + return logs; + } + + /** + * Selects an item randomly based on the specified probabilities. + * @param {Float32Array} probabilities An array of probabilities to use for selection. + * @returns {number} The index of the selected item. + */ + randomSelect(probabilities) { + return _weightedIndex(probabilities); + } + + /** + * Returns a Sampler object based on the specified options. + * @param {GenerationConfig} generation_config An object containing options for the sampler. + * @returns {LogitsSampler} A Sampler object. + */ + static getSampler(generation_config) { + // - *greedy decoding*: `num_beams=1` and `do_sample=False` + // - *contrastive search*: `penalty_alpha>0` and `top_k>1` + // - *multinomial sampling*: `num_beams=1` and `do_sample=True` + // - *beam-search decoding*: `num_beams>1` and `do_sample=False` + // - *beam-search multinomial sampling*: `num_beams>1` and `do_sample=True` + // - *diverse beam-search decoding*: `num_beams>1` and `num_beam_groups>1` + // - *constrained beam-search decoding*: `constraints!=None` or `force_words_ids!=None` + + // NOTE: beam search is implemented directly into the generation function + if (generation_config.do_sample) { + return new MultinomialSampler(generation_config); + } else if (generation_config.num_beams > 1) { + return new BeamSearchSampler(generation_config); + } else { + if (generation_config.num_return_sequences > 1) { + throw Error( + `num_return_sequences has to be 1 when doing greedy search, but is ${generation_config.num_return_sequences}.`, + ); + } + return new GreedySampler(generation_config); + } + } +} + +/** + * Class representing a Greedy Sampler. + */ +class GreedySampler extends LogitsSampler { + /** + * Sample the maximum probability of a given logits tensor. + * @param {Tensor} logits + * @returns {Promise<[bigint, number][]>} An array with a single tuple, containing the index of the maximum value and a meaningless score (since this is a greedy search). + */ + async sample(logits) { + // NOTE: no need to do log_softmax here since we only take the maximum + const argmax = max(logits.data)[1]; + + // Note: score is meaningless in this context, since we are performing + // greedy search (p = 1 => log(p) = 0) + return [[BigInt(argmax), 0]]; + } +} + +/** + * Class representing a MultinomialSampler. + */ +class MultinomialSampler extends LogitsSampler { + /** + * Sample from the logits. + * @param {Tensor} logits + * @returns {Promise<[bigint, number][]>} + */ + async sample(logits) { + let k = logits.dims.at(-1); // defaults to vocab size + if (this.generation_config.top_k > 0) { + k = Math.min(this.generation_config.top_k, k); + } + + // Get top k tokens + const [v, i] = await topk(logits, k); + + // Compute softmax over logits + const probabilities = softmax(/** @type {Float32Array} */ (v.data)); + + return Array.from({ length: this.generation_config.num_beams }, () => { + const sampledIndex = this.randomSelect(probabilities); + return [ + i.data[sampledIndex], // token id + Math.log(probabilities[sampledIndex]), // score + ]; + }); + } +} + +/** + * Class representing a BeamSearchSampler. + */ +class BeamSearchSampler extends LogitsSampler { + /** + * Sample from the logits. + * @param {Tensor} logits + * @returns {Promise<[bigint, number][]>} + */ + async sample(logits) { + let k = logits.dims.at(-1); // defaults to vocab size + if (this.generation_config.top_k > 0) { + k = Math.min(this.generation_config.top_k, k); + } + + // Get top k tokens + const [v, i] = await topk(logits, k); + + // Compute softmax over logits + const probabilities = softmax(/** @type {Float32Array} */ (v.data)); + + return Array.from({ length: this.generation_config.num_beams }, (_, x) => { + return [ + i.data[x], // token id + Math.log(probabilities[x]), // score + ]; + }); + } +} diff --git a/packages/transformers/src/generation/parameters.js b/packages/transformers/src/generation/parameters.js new file mode 100644 index 0000000..bc0b734 --- /dev/null +++ b/packages/transformers/src/generation/parameters.js @@ -0,0 +1,42 @@ +/** + * @module generation/parameters + */ + +/** + * @typedef {Object} GenerationFunctionParametersBase + * @property {import('../utils/tensor.js').Tensor} [inputs=null] (`Tensor` of varying shape depending on the modality, *optional*): + * The sequence used as a prompt for the generation or as model inputs to the encoder. If `null` the + * method initializes it with `bos_token_id` and a batch size of 1. For decoder-only models `inputs` + * should be in the format of `input_ids`. For encoder-decoder models *inputs* can represent any of + * `input_ids`, `input_values`, `input_features`, or `pixel_values`. + * @property {import('./configuration_utils.js').GenerationConfig} [generation_config=null] (`GenerationConfig`, *optional*): + * The generation configuration to be used as base parametrization for the generation call. + * `**kwargs` passed to generate matching the attributes of `generation_config` will override them. + * If `generation_config` is not provided, the default will be used, which has the following loading + * priority: + * - (1) from the `generation_config.json` model file, if it exists; + * - (2) from the model configuration. Please note that unspecified parameters will inherit [`GenerationConfig`]'s + * default values, whose documentation should be checked to parameterize generation. + * @property {import('./logits_process.js').LogitsProcessorList} [logits_processor=null] (`LogitsProcessorList`, *optional*): + * Custom logits processors that complement the default logits processors built from arguments and + * generation config. If a logit processor is passed that is already created with the arguments or a + * generation config an error is thrown. This feature is intended for advanced users. + * @property {import('./stopping_criteria.js').StoppingCriteria|import('./stopping_criteria.js').StoppingCriteria[]|import('./stopping_criteria.js').StoppingCriteriaList} [stopping_criteria=null] (`StoppingCriteriaList`, *optional*): + * Custom stopping criteria that complements the default stopping criteria built from arguments and a + * generation config. If a stopping criteria is passed that is already created with the arguments or a + * generation config an error is thrown. This feature is intended for advanced users. + * @property {import('./streamers.js').BaseStreamer} [streamer=null] (`BaseStreamer`, *optional*): + * Streamer object that will be used to stream the generated sequences. Generated tokens are passed + * through `streamer.put(token_ids)` and the streamer is responsible for any further processing. + * @property {number[]|import('../utils/tensor.js').Tensor} [decoder_input_ids=null] (`number[]` or `Tensor`, *optional*): + * If the model is an encoder-decoder model, this argument is used to pass the `decoder_input_ids`. + * @property {import('../cache_utils.js').DynamicCache | null} [past_key_values=null] (`DynamicCache`, *optional*): + * A cache object that stores previously computed key/value states. When provided, the model will + * use these cached states to avoid recomputing them, significantly speeding up sequential generation. + */ + +/** + * @typedef {GenerationFunctionParametersBase & Partial & {[key: string]: unknown}} GenerationFunctionParameters + */ + +export {}; // Ensure this file is treated as a module diff --git a/packages/transformers/src/generation/stopping_criteria.js b/packages/transformers/src/generation/stopping_criteria.js new file mode 100644 index 0000000..6451f6c --- /dev/null +++ b/packages/transformers/src/generation/stopping_criteria.js @@ -0,0 +1,153 @@ +/** + * @module generation/stopping_criteria + */ + +import { Callable } from '../utils/generic.js'; + +// NOTE: +// Stopping Criteria returns a list of `batch_size` booleans, indicating whether each sequence in the batch should be stopped. + +/** + * Abstract base class for all stopping criteria that can be applied during generation. + */ +export class StoppingCriteria extends Callable { + /** + * + * @param {number[][]} input_ids (`number[][]` of shape `(batch_size, sequence_length)`): + * Indices of input sequence tokens in the vocabulary. + * @param {number[][]} scores scores (`number[][]` of shape `(batch_size, config.vocab_size)`): + * Prediction scores of a language modeling head. These can be scores for each vocabulary token before SoftMax + * or scores for each vocabulary token after SoftMax. + * @returns {boolean[]} A list of booleans indicating whether each sequence should be stopped. + */ + _call(input_ids, scores) { + throw Error('StoppingCriteria needs to be subclassed'); + } +} +/** + */ +export class StoppingCriteriaList extends Callable { + /** + * Constructs a new instance of `StoppingCriteriaList`. + */ + constructor() { + super(); + this.criteria = []; + } + + /** + * Adds a new stopping criterion to the list. + * + * @param {StoppingCriteria} item The stopping criterion to add. + */ + push(item) { + this.criteria.push(item); + } + + /** + * Adds multiple stopping criteria to the list. + * + * @param {StoppingCriteria|StoppingCriteriaList|StoppingCriteria[]} items The stopping criteria to add. + */ + extend(items) { + if (items instanceof StoppingCriteriaList) { + items = items.criteria; + } else if (items instanceof StoppingCriteria) { + items = [items]; + } + this.criteria.push(...items); + } + + _call(input_ids, scores) { + const is_done = new Array(input_ids.length).fill(false); + for (const criterion of this.criteria) { + const criterion_done = criterion(input_ids, scores); + for (let i = 0; i < is_done.length; ++i) { + is_done[i] ||= criterion_done[i]; + } + } + return is_done; + } + + [Symbol.iterator]() { + return this.criteria.values(); + } +} + +/** + * This class can be used to stop generation whenever the full generated number of tokens exceeds `max_length`. + * Keep in mind for decoder-only type of transformers, this will include the initial prompted tokens. + */ +export class MaxLengthCriteria extends StoppingCriteria { + /** + * + * @param {number} max_length The maximum length that the output sequence can have in number of tokens. + * @param {number} [max_position_embeddings=null] The maximum model length, as defined by the model's `config.max_position_embeddings` attribute. + */ + constructor(max_length, max_position_embeddings = null) { + super(); + this.max_length = max_length; + this.max_position_embeddings = max_position_embeddings; + } + + _call(input_ids) { + return input_ids.map((ids) => ids.length >= this.max_length); + } +} + +// TODO: add MaxTimeCriteria + +/** + * This class can be used to stop generation whenever the "end-of-sequence" token is generated. + * By default, it uses the `model.generation_config.eos_token_id`. + */ +export class EosTokenCriteria extends StoppingCriteria { + /** + * + * @param {number|number[]} eos_token_id The id of the *end-of-sequence* token. + * Optionally, use a list to set multiple *end-of-sequence* tokens. + */ + constructor(eos_token_id) { + super(); + if (!Array.isArray(eos_token_id)) { + eos_token_id = [eos_token_id]; + } + this.eos_token_id = eos_token_id; + } + + /** + * + * @param {number[][]} input_ids + * @param {number[][]} scores + * @returns {boolean[]} + */ + _call(input_ids, scores) { + return input_ids.map((ids) => { + const last = ids.at(-1); + // NOTE: We use == instead of === to allow for number/bigint comparison + return this.eos_token_id.some((eos_id) => last == eos_id); + }); + } +} + +/** + * This class can be used to stop generation whenever the user interrupts the process. + */ +export class InterruptableStoppingCriteria extends StoppingCriteria { + constructor() { + super(); + this.interrupted = false; + } + + interrupt() { + this.interrupted = true; + } + + reset() { + this.interrupted = false; + } + + _call(input_ids, scores) { + return new Array(input_ids.length).fill(this.interrupted); + } +} diff --git a/packages/transformers/src/generation/streamers.js b/packages/transformers/src/generation/streamers.js new file mode 100644 index 0000000..813dd95 --- /dev/null +++ b/packages/transformers/src/generation/streamers.js @@ -0,0 +1,258 @@ +/** + * @module generation/streamers + */ + +import { mergeArrays } from '../utils/core.js'; +import { apis } from '../env.js'; + +const is_chinese_char = (cp) => + (cp >= 0x4e00 && cp <= 0x9fff) || + (cp >= 0x3400 && cp <= 0x4dbf) || + (cp >= 0x20000 && cp <= 0x2a6df) || + (cp >= 0x2a700 && cp <= 0x2b73f) || + (cp >= 0x2b740 && cp <= 0x2b81f) || + (cp >= 0x2b820 && cp <= 0x2ceaf) || + (cp >= 0xf900 && cp <= 0xfaff) || + (cp >= 0x2f800 && cp <= 0x2fa1f); + +export class BaseStreamer { + /** + * Function that is called by `.generate()` to push new tokens + * @param {bigint[][]} value + */ + put(value) { + throw Error('Not implemented'); + } + + /** + * Function that is called by `.generate()` to signal the end of generation + */ + end() { + throw Error('Not implemented'); + } +} + +const stdout_write = apis.IS_PROCESS_AVAILABLE ? (x) => process.stdout.write(x) : (x) => console.log(x); + +/** + * Simple text streamer that prints the token(s) to stdout as soon as entire words are formed. + */ +export class TextStreamer extends BaseStreamer { + /** + * + * @param {import('../tokenization_utils.js').PreTrainedTokenizer} tokenizer + * @param {Object} options + * @param {boolean} [options.skip_prompt=false] Whether to skip the prompt tokens + * @param {boolean} [options.skip_special_tokens=true] Whether to skip special tokens when decoding + * @param {function(string): void} [options.callback_function=null] Function to call when a piece of text is ready to display + * @param {function(bigint[]): void} [options.token_callback_function=null] Function to call when a new token is generated + * @param {Object} [options.decode_kwargs={}] Additional keyword arguments to pass to the tokenizer's decode method + */ + constructor( + tokenizer, + { + skip_prompt = false, + callback_function = null, + token_callback_function = null, + skip_special_tokens = true, + decode_kwargs = {}, + ...kwargs + } = {}, + ) { + super(); + this.tokenizer = tokenizer; + this.skip_prompt = skip_prompt; + this.callback_function = callback_function ?? stdout_write; + this.token_callback_function = token_callback_function; + this.decode_kwargs = { skip_special_tokens, ...decode_kwargs, ...kwargs }; + + // variables used in the streaming process + this.token_cache = []; + this.print_len = 0; + this.next_tokens_are_prompt = true; + + // Track special token IDs for special handling during streaming. + this.special_ids = new Set(this.tokenizer.all_special_ids.map(BigInt)); + } + + /** + * Receives tokens, decodes them, and prints them to stdout as soon as they form entire words. + * @param {bigint[][]} value + */ + put(value) { + if (value.length > 1) { + throw Error('TextStreamer only supports batch size of 1'); + } + + const is_prompt = this.next_tokens_are_prompt; + if (is_prompt) { + this.next_tokens_are_prompt = false; + if (this.skip_prompt) return; + } + + const tokens = value[0]; + this.token_callback_function?.(tokens); + + // Handle special tokens: flush any existing text, then print or skip them + if (tokens.length === 1 && this.special_ids.has(tokens[0])) { + if (this.decode_kwargs.skip_special_tokens) return; + + // Flush any existing cached text first + if (this.token_cache.length > 0) { + const text = this.tokenizer.decode(this.token_cache, this.decode_kwargs); + const printable_text = text.slice(this.print_len); + this.on_finalized_text(printable_text, false); + this.token_cache = []; + this.print_len = 0; + } + // Print the special token immediately + const special_text = this.tokenizer.decode(tokens, this.decode_kwargs); + this.on_finalized_text(special_text, false); + return; + } + + // Add the new token to the cache and decodes the entire thing. + this.token_cache = mergeArrays(this.token_cache, tokens); + const text = this.tokenizer.decode(this.token_cache, this.decode_kwargs); + + let printable_text; + if (is_prompt || text.endsWith('\n')) { + // After the symbol for a new line, we flush the cache. + printable_text = text.slice(this.print_len); + this.token_cache = []; + this.print_len = 0; + } else if (text.length > 0 && is_chinese_char(text.charCodeAt(text.length - 1))) { + // If the last token is a CJK character, we print the characters. + printable_text = text.slice(this.print_len); + this.print_len += printable_text.length; + } else { + // Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words, + // which may change with the subsequent token -- there are probably smarter ways to do this!) + printable_text = text.slice(this.print_len, text.lastIndexOf(' ') + 1); + this.print_len += printable_text.length; + } + + this.on_finalized_text(printable_text, false); + } + + /** + * Flushes any remaining cache and prints a newline to stdout. + */ + end() { + let printable_text; + if (this.token_cache.length > 0) { + const text = this.tokenizer.decode(this.token_cache, this.decode_kwargs); + printable_text = text.slice(this.print_len); + this.token_cache = []; + this.print_len = 0; + } else { + printable_text = ''; + } + this.next_tokens_are_prompt = true; + this.on_finalized_text(printable_text, true); + } + + /** + * Prints the new text to stdout. If the stream is ending, also prints a newline. + * @param {string} text + * @param {boolean} stream_end + */ + on_finalized_text(text, stream_end) { + if (text.length > 0) { + this.callback_function?.(text); + } + if (stream_end && this.callback_function === stdout_write && apis.IS_PROCESS_AVAILABLE) { + this.callback_function?.('\n'); + } + } +} + +/** + * Utility class to handle streaming of tokens generated by whisper speech-to-text models. + * Callback functions are invoked when each of the following events occur: + * - A new chunk starts (on_chunk_start) + * - A new token is generated (callback_function) + * - A chunk ends (on_chunk_end) + * - The stream is finalized (on_finalize) + */ +export class WhisperTextStreamer extends TextStreamer { + /** + * @param {import('../models/whisper/tokenization_whisper.js').WhisperTokenizer} tokenizer + * @param {Object} options + * @param {boolean} [options.skip_prompt=false] Whether to skip the prompt tokens + * @param {function(string): void} [options.callback_function=null] Function to call when a piece of text is ready to display + * @param {function(bigint[]): void} [options.token_callback_function=null] Function to call when a new token is generated + * @param {function(number): void} [options.on_chunk_start=null] Function to call when a new chunk starts + * @param {function(number): void} [options.on_chunk_end=null] Function to call when a chunk ends + * @param {function(): void} [options.on_finalize=null] Function to call when the stream is finalized + * @param {number} [options.time_precision=0.02] Precision of the timestamps + * @param {boolean} [options.skip_special_tokens=true] Whether to skip special tokens when decoding + * @param {Object} [options.decode_kwargs={}] Additional keyword arguments to pass to the tokenizer's decode method + */ + constructor( + tokenizer, + { + skip_prompt = false, + callback_function = null, + token_callback_function = null, + on_chunk_start = null, + on_chunk_end = null, + on_finalize = null, + time_precision = 0.02, + skip_special_tokens = true, + decode_kwargs = {}, + } = {}, + ) { + super(tokenizer, { + skip_prompt, + skip_special_tokens, + callback_function, + token_callback_function, + decode_kwargs, + }); + this.timestamp_begin = tokenizer.timestamp_begin; + + this.on_chunk_start = on_chunk_start; + this.on_chunk_end = on_chunk_end; + this.on_finalize = on_finalize; + + this.time_precision = time_precision; + + this.waiting_for_timestamp = false; + } + + /** + * @param {bigint[][]} value + */ + put(value) { + if (value.length > 1) { + throw Error('WhisperTextStreamer only supports batch size of 1'); + } + const tokens = value[0]; + + // Check if the token is a timestamp + if (tokens.length === 1) { + const offset = Number(tokens[0]) - this.timestamp_begin; + if (offset >= 0) { + const time = offset * this.time_precision; + if (this.waiting_for_timestamp) { + this.on_chunk_end?.(time); + } else { + this.on_chunk_start?.(time); + } + this.waiting_for_timestamp = !this.waiting_for_timestamp; // Toggle + + // NOTE: Timestamp tokens should not be printed. Although, since they + // aren't classified as "special tokens", we need to handle them here. + this.token_callback_function?.(tokens); + return; + } + } + return super.put(value); + } + + end() { + super.end(); + this.on_finalize?.(); + } +} diff --git a/packages/transformers/src/image_processors_utils.js b/packages/transformers/src/image_processors_utils.js new file mode 100644 index 0000000..b296010 --- /dev/null +++ b/packages/transformers/src/image_processors_utils.js @@ -0,0 +1,1094 @@ +import { Callable } from './utils/generic.js'; +import { Tensor, interpolate, stack } from './utils/tensor.js'; +import { bankers_round, max, min, softmax } from './utils/maths.js'; +import { RawImage } from './utils/image.js'; +import { calculateReflectOffset } from './utils/core.js'; +import { getModelJSON } from './utils/hub.js'; +import { IMAGE_PROCESSOR_NAME } from './utils/constants.js'; +import { logger } from './utils/logger.js'; + +/** + * Named tuple to indicate the order we are using is (height x width), + * even though the Graphics' industry standard is (width x height). + * @typedef {[height: number, width: number]} HeightWidth + */ + +/** + * @typedef {Object} ImageProcessorResult + * @property {Tensor} pixel_values The pixel values of the batched preprocessed images. + * @property {HeightWidth[]} original_sizes Array of two-dimensional tuples like [[480, 640]]. + * @property {HeightWidth[]} reshaped_input_sizes Array of two-dimensional tuples like [[1000, 1330]]. + */ + +/** + * Helper function to constrain a value to be a multiple of a number. + * @param {number} val The value to constrain. + * @param {number} multiple The number to constrain to. + * @param {number} [minVal=0] The minimum value to constrain to. + * @param {number} [maxVal=null] The maximum value to constrain to. + * @returns {number} The constrained value. + * @private + */ +function constraint_to_multiple_of(val, multiple, minVal = 0, maxVal = null) { + const a = val / multiple; + let x = bankers_round(a) * multiple; + + if (maxVal !== null && x > maxVal) { + x = Math.floor(a) * multiple; + } + + if (x < minVal) { + x = Math.ceil(a) * multiple; + } + + return x; +} + +/** + * Rounds the height and width down to the closest multiple of size_divisibility + * @param {[number, number]} size The size of the image + * @param {number} divisor The divisor to use. + * @returns {[number, number]} The rounded size. + */ +function enforce_size_divisibility([width, height], divisor) { + return [Math.max(Math.floor(width / divisor), 1) * divisor, Math.max(Math.floor(height / divisor), 1) * divisor]; +} + +// Helper functions + +/** + * Converts bounding boxes from center format to corners format. + * + * @param {number[]} arr The coordinate for the center of the box and its width, height dimensions (center_x, center_y, width, height) + * @returns {number[]} The coodinates for the top-left and bottom-right corners of the box (top_left_x, top_left_y, bottom_right_x, bottom_right_y) + */ +export function center_to_corners_format([centerX, centerY, width, height]) { + return [centerX - width / 2, centerY - height / 2, centerX + width / 2, centerY + height / 2]; +} + +/** + * Post-processes the outputs of the model (for object detection). + * @param {Object} outputs The outputs of the model that must be post-processed + * @param {Tensor} outputs.logits The logits + * @param {Tensor} outputs.pred_boxes The predicted boxes. + * @param {number} [threshold=0.5] The threshold to use for the scores. + * @param {[number, number][]} [target_sizes=null] The sizes of the original images. + * @param {boolean} [is_zero_shot=false] Whether zero-shot object detection was performed. + * @return {Object[]} An array of objects containing the post-processed outputs. + */ +export function post_process_object_detection(outputs, threshold = 0.5, target_sizes = null, is_zero_shot = false) { + const out_logits = outputs.logits; + const out_bbox = outputs.pred_boxes; + const [batch_size, num_boxes, num_classes] = out_logits.dims; + + if (target_sizes !== null && target_sizes.length !== batch_size) { + throw Error('Make sure that you pass in as many target sizes as the batch dimension of the logits'); + } + let toReturn = []; + for (let i = 0; i < batch_size; ++i) { + let target_size = target_sizes !== null ? target_sizes[i] : null; + let info = { + boxes: [], + classes: [], + scores: [], + }; + let logits = out_logits[i]; + let bbox = out_bbox[i]; + + for (let j = 0; j < num_boxes; ++j) { + let logit = logits[j]; + + let indices = []; + let probs; + if (is_zero_shot) { + // Get indices of classes with high enough probability + probs = logit.sigmoid().data; + for (let k = 0; k < probs.length; ++k) { + if (probs[k] > threshold) { + indices.push(k); + } + } + } else { + // Get most probable class + let maxIndex = max(logit.data)[1]; + + if (maxIndex === num_classes - 1) { + // This is the background class, skip it + continue; + } + // Compute softmax over classes + probs = softmax(logit.data); + + if (probs[maxIndex] < threshold) { + continue; + } + indices.push(maxIndex); + } + + for (const index of indices) { + // Some class has a high enough probability + /** @type {number[]} */ + let box = bbox[j].data; + + // convert to [x0, y0, x1, y1] format + box = center_to_corners_format(box); + if (target_size !== null) { + box = box.map((x, i) => x * target_size[(i + 1) % 2]); + } + + info.boxes.push(box); + info.classes.push(index); + info.scores.push(probs[index]); + } + } + toReturn.push(info); + } + return toReturn; +} + +/** + * Post-processes the outputs of the model (for semantic segmentation). + * @param {*} outputs Raw outputs of the model. + * @param {[number, number][]} [target_sizes=null] List of tuples corresponding to the requested final size + * (height, width) of each prediction. If unset, predictions will not be resized. + * @returns {{segmentation: Tensor; labels: number[]}[]} The semantic segmentation maps. + */ +export function post_process_semantic_segmentation(outputs, target_sizes = null) { + const logits = outputs.logits; + const batch_size = logits.dims[0]; + + if (target_sizes !== null && target_sizes.length !== batch_size) { + throw Error('Make sure that you pass in as many target sizes as the batch dimension of the logits'); + } + + const toReturn = []; + for (let i = 0; i < batch_size; ++i) { + const target_size = target_sizes !== null ? target_sizes[i] : null; + + let data = logits[i]; + + // 1. If target_size is not null, we need to resize the masks to the target size + if (target_size !== null) { + // resize the masks to the target size + data = interpolate(data, target_size, 'bilinear', false); + } + const [height, width] = target_size ?? data.dims.slice(-2); + + const segmentation = new Tensor('int32', new Int32Array(height * width), [height, width]); + + // Buffer to store current largest value + const buffer = data[0].data; + const segmentation_data = segmentation.data; + for (let j = 1; j < data.dims[0]; ++j) { + const row = data[j].data; + for (let k = 0; k < row.length; ++k) { + if (row[k] > buffer[k]) { + buffer[k] = row[k]; + segmentation_data[k] = j; + } + } + } + + // Store which objects have labels + // This is much more efficient that creating a set of the final values + const hasLabel = new Array(data.dims[0]); + for (let j = 0; j < segmentation_data.length; ++j) { + const index = segmentation_data[j]; + hasLabel[index] = index; + } + /** + * The unique list of labels that were detected + * @type {number[]} + */ + const labels = hasLabel.filter((x) => x !== undefined); + + toReturn.push({ segmentation, labels }); + } + return toReturn; +} + +/** + * Binarize the given masks using `object_mask_threshold`, it returns the associated values of `masks`, `scores` and `labels`. + * @param {Tensor} class_logits The class logits. + * @param {Tensor} mask_logits The mask logits. + * @param {number} object_mask_threshold A number between 0 and 1 used to binarize the masks. + * @param {number} num_labels The number of labels. + * @returns {[Tensor[], number[], number[]]} The binarized masks, the scores, and the labels. + * @private + */ +function remove_low_and_no_objects(class_logits, mask_logits, object_mask_threshold, num_labels) { + const mask_probs_item = []; + const pred_scores_item = []; + const pred_labels_item = []; + + for (let j = 0; j < class_logits.dims[0]; ++j) { + const cls = class_logits[j]; + const mask = mask_logits[j]; + + const pred_label = max(cls.data)[1]; + if (pred_label === num_labels) { + // Is the background, so we ignore it + continue; + } + + const scores = softmax(cls.data); + const pred_score = scores[pred_label]; + if (pred_score > object_mask_threshold) { + mask_probs_item.push(mask); + pred_scores_item.push(pred_score); + pred_labels_item.push(pred_label); + } + } + + return [mask_probs_item, pred_scores_item, pred_labels_item]; +} + +/** + * Checks whether the segment is valid or not. + * @param {Int32Array} mask_labels Labels for each pixel in the mask. + * @param {Tensor[]} mask_probs Probabilities for each pixel in the masks. + * @param {number} k The class id of the segment. + * @param {number} mask_threshold The mask threshold. + * @param {number} overlap_mask_area_threshold The overlap mask area threshold. + * @returns {[boolean, number[]]} Whether the segment is valid or not, and the indices of the valid labels. + * @private + */ +function check_segment_validity(mask_labels, mask_probs, k, mask_threshold = 0.5, overlap_mask_area_threshold = 0.8) { + // mask_k is a 1D array of indices, indicating where the mask is equal to k + const mask_k = []; + let mask_k_area = 0; + let original_area = 0; + + const mask_probs_k_data = mask_probs[k].data; + + // Compute the area of all the stuff in query k + for (let i = 0; i < mask_labels.length; ++i) { + if (mask_labels[i] === k) { + mask_k.push(i); + ++mask_k_area; + } + + if (mask_probs_k_data[i] >= mask_threshold) { + ++original_area; + } + } + let mask_exists = mask_k_area > 0 && original_area > 0; + + // Eliminate disconnected tiny segments + if (mask_exists) { + // Perform additional check + let area_ratio = mask_k_area / original_area; + mask_exists = area_ratio > overlap_mask_area_threshold; + } + + return [mask_exists, mask_k]; +} + +/** + * Computes the segments. + * @param {Tensor[]} mask_probs The mask probabilities. + * @param {number[]} pred_scores The predicted scores. + * @param {number[]} pred_labels The predicted labels. + * @param {number} mask_threshold The mask threshold. + * @param {number} overlap_mask_area_threshold The overlap mask area threshold. + * @param {Set} label_ids_to_fuse The label ids to fuse. + * @param {number[]} target_size The target size of the image. + * @returns {[Tensor, Array<{id: number, label_id: number, score: number}>]} The computed segments. + * @private + */ +function compute_segments( + mask_probs, + pred_scores, + pred_labels, + mask_threshold, + overlap_mask_area_threshold, + label_ids_to_fuse = null, + target_size = null, +) { + const [height, width] = target_size ?? mask_probs[0].dims; + + const segmentation = new Tensor('int32', new Int32Array(height * width), [height, width]); + const segments = []; + + // 1. If target_size is not null, we need to resize the masks to the target size + if (target_size !== null) { + // resize the masks to the target size + for (let i = 0; i < mask_probs.length; ++i) { + mask_probs[i] = interpolate(mask_probs[i], target_size, 'bilinear', false); + } + } + + // 2. Weigh each mask by its prediction score + // NOTE: `mask_probs` is updated in-place + // + // Temporary storage for the best label/scores for each pixel ([height, width]): + const mask_labels = new Int32Array(mask_probs[0].data.length); + const bestScores = new Float32Array(mask_probs[0].data.length); + + for (let i = 0; i < mask_probs.length; ++i) { + let score = pred_scores[i]; + + const mask_probs_i_data = mask_probs[i].data; + + for (let j = 0; j < mask_probs_i_data.length; ++j) { + mask_probs_i_data[j] *= score; + if (mask_probs_i_data[j] > bestScores[j]) { + mask_labels[j] = i; + bestScores[j] = mask_probs_i_data[j]; + } + } + } + + let current_segment_id = 0; + + // let stuff_memory_list = {} + const segmentation_data = segmentation.data; + for (let k = 0; k < pred_labels.length; ++k) { + const pred_class = pred_labels[k]; + + // TODO add `should_fuse` + // let should_fuse = pred_class in label_ids_to_fuse + + // Check if mask exists and large enough to be a segment + const [mask_exists, mask_k] = check_segment_validity( + mask_labels, + mask_probs, + k, + mask_threshold, + overlap_mask_area_threshold, + ); + + if (!mask_exists) { + // Nothing to see here + continue; + } + + // TODO + // if (pred_class in stuff_memory_list) { + // current_segment_id = stuff_memory_list[pred_class] + // } else { + // current_segment_id += 1; + // } + ++current_segment_id; + + // Add current object segment to final segmentation map + for (const index of mask_k) { + segmentation_data[index] = current_segment_id; + } + + segments.push({ + id: current_segment_id, + label_id: pred_class, + // was_fused: should_fuse, TODO + score: pred_scores[k], + }); + + // TODO + // if(should_fuse){ + // stuff_memory_list[pred_class] = current_segment_id + // } + } + + return [segmentation, segments]; +} + +/** + * Rescales the image so that the following conditions are met: + * + * 1. Both dimensions (height and width) are divisible by 'factor'. + * 2. The total number of pixels is within the range ['min_pixels', 'max_pixels']. + * 3. The aspect ratio of the image is maintained as closely as possible. + * + * @param {number} height The height of the image. + * @param {number} width The width of the image. + * @param {number} [factor=28] The factor to use for resizing. + * @param {number} [min_pixels=56*56] The minimum number of pixels. + * @param {number} [max_pixels=14*14*4*1280] The maximum number of pixels. + * @param {number} [temporal_factor=1] The temporal factor to include in the pixel budget (e.g. temporal_patch_size for video/3D models). + * @returns {[number, number]} The new width and height of the image. + * @throws {Error} If the height or width is smaller than the factor. + */ +export function smart_resize( + height, + width, + factor = 28, + min_pixels = 56 * 56, + max_pixels = 14 * 14 * 4 * 1280, + temporal_factor = 1, +) { + if (height < factor || width < factor) { + const scale = Math.max(factor / height, factor / width); + height = Math.round(height * scale); + width = Math.round(width * scale); + } + if (Math.max(height, width) / Math.min(height, width) > 200) { + throw new Error( + `absolute aspect ratio must be smaller than 200, got ${Math.max(height, width) / Math.min(height, width)}`, + ); + } + + let h_bar = Math.round(height / factor) * factor; + let w_bar = Math.round(width / factor) * factor; + + if (temporal_factor * h_bar * w_bar > max_pixels) { + const beta = Math.sqrt((temporal_factor * height * width) / max_pixels); + h_bar = Math.max(factor, Math.floor(height / beta / factor) * factor); + w_bar = Math.max(factor, Math.floor(width / beta / factor) * factor); + } else if (temporal_factor * h_bar * w_bar < min_pixels) { + const beta = Math.sqrt(min_pixels / (temporal_factor * height * width)); + h_bar = Math.ceil((height * beta) / factor) * factor; + w_bar = Math.ceil((width * beta) / factor) * factor; + } + + return [w_bar, h_bar]; +} + +/** + * Post-process the model output to generate the final panoptic segmentation. + * @param {*} outputs The model output to post process + * @param {number} [threshold=0.5] The probability score threshold to keep predicted instance masks. + * @param {number} [mask_threshold=0.5] Threshold to use when turning the predicted masks into binary values. + * @param {number} [overlap_mask_area_threshold=0.8] The overlap mask area threshold to merge or discard small disconnected parts within each binary instance mask. + * @param {Set} [label_ids_to_fuse=null] The labels in this state will have all their instances be fused together. + * @param {[number, number][]} [target_sizes=null] The target sizes to resize the masks to. + * @returns {Array<{ segmentation: Tensor, segments_info: Array<{id: number, label_id: number, score: number}>}>} + */ +export function post_process_panoptic_segmentation( + outputs, + threshold = 0.5, + mask_threshold = 0.5, + overlap_mask_area_threshold = 0.8, + label_ids_to_fuse = null, + target_sizes = null, +) { + if (label_ids_to_fuse === null) { + logger.warn('`label_ids_to_fuse` unset. No instance will be fused.'); + label_ids_to_fuse = new Set(); + } + + const class_queries_logits = outputs.class_queries_logits ?? outputs.logits; // [batch_size, num_queries, num_classes+1] + const masks_queries_logits = outputs.masks_queries_logits ?? outputs.pred_masks; // [batch_size, num_queries, height, width] + + const mask_probs = masks_queries_logits.sigmoid(); // [batch_size, num_queries, height, width] + + let [batch_size, num_queries, num_labels] = class_queries_logits.dims; + num_labels -= 1; // Remove last class (background) + + if (target_sizes !== null && target_sizes.length !== batch_size) { + throw Error('Make sure that you pass in as many target sizes as the batch dimension of the logits'); + } + + let toReturn = []; + for (let i = 0; i < batch_size; ++i) { + let target_size = target_sizes !== null ? target_sizes[i] : null; + + let class_logits = class_queries_logits[i]; + let mask_logits = mask_probs[i]; + + let [mask_probs_item, pred_scores_item, pred_labels_item] = remove_low_and_no_objects( + class_logits, + mask_logits, + threshold, + num_labels, + ); + + if (pred_labels_item.length === 0) { + // No mask found + let [height, width] = target_size ?? mask_logits.dims.slice(-2); + + let segmentation = new Tensor('int32', new Int32Array(height * width).fill(-1), [height, width]); + toReturn.push({ + segmentation: segmentation, + segments_info: [], + }); + continue; + } + + // Get segmentation map and segment information of batch item + let [segmentation, segments] = compute_segments( + mask_probs_item, + pred_scores_item, + pred_labels_item, + mask_threshold, + overlap_mask_area_threshold, + label_ids_to_fuse, + target_size, + ); + + toReturn.push({ + segmentation: segmentation, + segments_info: segments, + }); + } + + return toReturn; +} + +/** + * Post-processes the outputs of the model (for instance segmentation). + * @param {*} outputs Raw outputs of the model. + * @param {number} [threshold=0.5] The probability score threshold to keep predicted instance masks. + * @param {[number, number][]} [target_sizes=null] List of tuples corresponding to the requested final size + * (height, width) of each prediction. If unset, predictions will not be resized. + * @returns {Array<{ segmentation: Tensor, segments_info: Array<{id: number, label_id: number, score: number}>}>} + */ +export function post_process_instance_segmentation(outputs, threshold = 0.5, target_sizes = null) { + throw new Error('`post_process_instance_segmentation` is not yet implemented.'); +} + +/** + * @typedef {Object} ImageProcessorConfig A configuration object used to create an image processor. + * @property {function} [progress_callback=null] If specified, this function will be called during model construction, to provide the user with progress updates. + * @property {number[]} [image_mean] The mean values for image normalization. + * @property {number[]} [image_std] The standard deviation values for image normalization. + * @property {boolean} [do_rescale] Whether to rescale the image pixel values to the [0,1] range. + * @property {number} [rescale_factor] The factor to use for rescaling the image pixel values. + * @property {boolean} [do_normalize] Whether to normalize the image pixel values. + * @property {boolean} [do_resize] Whether to resize the image. + * @property {number} [resample] What method to use for resampling. + * @property {number|Object} [size] The size to resize the image to. + * @property {number|Object} [image_size] The size to resize the image to (same as `size`). + * @property {boolean} [do_flip_channel_order=false] Whether to flip the color channels from RGB to BGR. + * Can be overridden by the `do_flip_channel_order` parameter in the `preprocess` method. + * @property {boolean} [do_center_crop] Whether to center crop the image to the specified `crop_size`. + * Can be overridden by `do_center_crop` in the `preprocess` method. + * @property {boolean} [do_thumbnail] Whether to resize the image using thumbnail method. + * @property {boolean} [keep_aspect_ratio] If `true`, the image is resized to the largest possible size such that the aspect ratio is preserved. + * Can be overidden by `keep_aspect_ratio` in `preprocess`. + * @property {number} [ensure_multiple_of] If `do_resize` is `true`, the image is resized to a size that is a multiple of this value. + * Can be overidden by `ensure_multiple_of` in `preprocess`. + * + * @property {number[]} [mean] The mean values for image normalization (same as `image_mean`). + * @property {number[]} [std] The standard deviation values for image normalization (same as `image_std`). + */ + +export class ImageProcessor extends Callable { + /** + * Constructs a new `ImageProcessor`. + * @param {ImageProcessorConfig} config The configuration object. + */ + constructor(config) { + super(); + + this.image_mean = config.image_mean ?? config.mean; + this.image_std = config.image_std ?? config.std; + + this.resample = config.resample ?? 2; // 2 => bilinear + this.do_rescale = config.do_rescale ?? true; + this.rescale_factor = config.rescale_factor ?? 1 / 255; + this.do_normalize = config.do_normalize; + + this.do_thumbnail = config.do_thumbnail; + this.size = config.size ?? config.image_size; + this.do_resize = config.do_resize ?? this.size !== undefined; + // @ts-expect-error TS2339 + this.size_divisibility = config.size_divisibility ?? config.size_divisor; + + this.do_center_crop = config.do_center_crop; + // @ts-expect-error TS2339 + this.crop_size = config.crop_size; + // @ts-expect-error TS2339 + this.do_convert_rgb = config.do_convert_rgb ?? true; + // @ts-expect-error TS2339 + this.do_crop_margin = config.do_crop_margin; + + // @ts-expect-error TS2339 + this.pad_size = config.pad_size; + // @ts-expect-error TS2339 + this.do_pad = config.do_pad; + // @ts-expect-error TS2339 + this.min_pixels = config.min_pixels; + // @ts-expect-error TS2339 + this.max_pixels = config.max_pixels; + + if ( + this.do_pad && + !this.pad_size && + !this.size_divisibility && + this.size && + this.size.width !== undefined && + this.size.height !== undefined + ) { + // Should pad, but no pad size specified + // We infer the pad size from the resize size + this.pad_size = this.size; + } + + this.do_flip_channel_order = config.do_flip_channel_order ?? false; + + this.config = config; + } + + /** + * Resize the image to make a thumbnail. The image is resized so that no dimension is larger than any + * corresponding dimension of the specified size. + * @param {RawImage} image The image to be resized. + * @param {{height:number, width:number}} size The size `{"height": h, "width": w}` to resize the image to. + * @param {string | 0 | 1 | 2 | 3 | 4 | 5} [resample=2] The resampling filter to use. + * @returns {Promise} The resized image. + */ + async thumbnail(image, size, resample = 2) { + const input_height = image.height; + const input_width = image.width; + + const output_height = size.height; + const output_width = size.width; + + // We always resize to the smallest of either the input or output size. + let height = Math.min(input_height, output_height); + let width = Math.min(input_width, output_width); + + if (height === input_height && width === input_width) { + return image; + } + if (input_height > input_width) { + width = Math.floor((input_width * height) / input_height); + } else if (input_width > input_height) { + height = Math.floor((input_height * width) / input_width); + } + return await image.resize(width, height, { resample }); + } + + /** + * Crops the margin of the image. Gray pixels are considered margin (i.e., pixels with a value below the threshold). + * @param {RawImage} image The image to be cropped. + * @param {number} gray_threshold Value below which pixels are considered to be gray. + * @returns {Promise} The cropped image. + */ + async crop_margin(image, gray_threshold = 200) { + const gray_image = image.clone().grayscale(); + + const minValue = min(gray_image.data)[0]; + const maxValue = max(gray_image.data)[0]; + const diff = maxValue - minValue; + + if (diff === 0) { + return image; + } + + const threshold = gray_threshold / 255; + + let x_min = gray_image.width, + y_min = gray_image.height, + x_max = 0, + y_max = 0; + const gray_image_data = gray_image.data; + for (let j = 0; j < gray_image.height; ++j) { + const row = j * gray_image.width; + for (let i = 0; i < gray_image.width; ++i) { + if ((gray_image_data[row + i] - minValue) / diff < threshold) { + // We have a non-zero pixel, so we update the min/max values accordingly + x_min = Math.min(x_min, i); + y_min = Math.min(y_min, j); + x_max = Math.max(x_max, i); + y_max = Math.max(y_max, j); + } + } + } + + image = await image.crop([x_min, y_min, x_max, y_max]); + return image; + } + + /** + * Pad the image by a certain amount. + * @param {Float32Array} pixelData The pixel data to pad. + * @param {number[]} imgDims The dimensions of the image (height, width, channels). + * @param {{width:number; height:number}|number|'square'} padSize The dimensions of the padded image. + * @param {Object} options The options for padding. + * @param {'constant'|'symmetric'} [options.mode='constant'] The type of padding to add. + * @param {boolean} [options.center=false] Whether to center the image. + * @param {number|number[]} [options.constant_values=0] The constant value to use for padding. + * @returns {[Float32Array, number[]]} The padded pixel data and image dimensions. + */ + pad_image(pixelData, imgDims, padSize, { mode = 'constant', center = false, constant_values = 0 } = {}) { + const [imageHeight, imageWidth, imageChannels] = imgDims; + + let paddedImageWidth, paddedImageHeight; + if (typeof padSize === 'number') { + paddedImageWidth = padSize; + paddedImageHeight = padSize; + } else if (padSize === 'square') { + paddedImageWidth = paddedImageHeight = Math.max(imageHeight, imageWidth); + } else { + paddedImageWidth = padSize.width; + paddedImageHeight = padSize.height; + } + + // Only add padding if there is a difference in size + if (paddedImageWidth !== imageWidth || paddedImageHeight !== imageHeight) { + const paddedPixelData = new Float32Array(paddedImageWidth * paddedImageHeight * imageChannels); + if (Array.isArray(constant_values)) { + // Fill with constant values, cycling through the array + for (let i = 0; i < paddedPixelData.length; ++i) { + paddedPixelData[i] = constant_values[i % imageChannels]; + } + } else if (constant_values !== 0) { + paddedPixelData.fill(constant_values); + } + + const [left, top] = center + ? [Math.floor((paddedImageWidth - imageWidth) / 2), Math.floor((paddedImageHeight - imageHeight) / 2)] + : [0, 0]; + + // Copy the original image into the padded image + for (let i = 0; i < imageHeight; ++i) { + const a = (i + top) * paddedImageWidth; + const b = i * imageWidth; + for (let j = 0; j < imageWidth; ++j) { + const c = (a + j + left) * imageChannels; + const d = (b + j) * imageChannels; + for (let k = 0; k < imageChannels; ++k) { + paddedPixelData[c + k] = pixelData[d + k]; + } + } + } + + if (mode === 'symmetric') { + if (center) { + throw new Error('`center` padding is not supported when `mode` is set to `symmetric`.'); + // TODO: Implement this + } + const h1 = imageHeight - 1; + const w1 = imageWidth - 1; + for (let i = 0; i < paddedImageHeight; ++i) { + const a = i * paddedImageWidth; + const b = calculateReflectOffset(i, h1) * imageWidth; + + for (let j = 0; j < paddedImageWidth; ++j) { + if (i < imageHeight && j < imageWidth) continue; // Do not overwrite original image + const c = (a + j) * imageChannels; + const d = (b + calculateReflectOffset(j, w1)) * imageChannels; + + // Copy channel-wise + for (let k = 0; k < imageChannels; ++k) { + paddedPixelData[c + k] = pixelData[d + k]; + } + } + } + } + + // Update pixel data and image dimensions + pixelData = paddedPixelData; + imgDims = [paddedImageHeight, paddedImageWidth, imageChannels]; + } + return [pixelData, imgDims]; + } + + /** + * Rescale the image' pixel values by `this.rescale_factor`. + * @param {Float32Array} pixelData The pixel data to rescale. + * @returns {void} + */ + rescale(pixelData) { + for (let i = 0; i < pixelData.length; ++i) { + pixelData[i] = this.rescale_factor * pixelData[i]; + } + } + + /** + * Find the target (width, height) dimension of the output image after + * resizing given the input image and the desired size. + * @param {RawImage} image The image to resize. + * @param {any} size The size to use for resizing the image. + * @returns {[number, number]} The target (width, height) dimension of the output image after resizing. + */ + get_resize_output_image_size(image, size) { + // `size` comes in many forms, so we need to handle them all here: + // 1. `size` is an integer, in which case we resize the image to be a square + + const [srcWidth, srcHeight] = image.size; + + let shortest_edge; + let longest_edge; + + if (this.do_thumbnail) { + // NOTE: custom logic for `Donut` models + const { height, width } = size; + shortest_edge = Math.min(height, width); + } + // Support both formats for backwards compatibility + else if (Number.isInteger(size)) { + shortest_edge = size; + // @ts-expect-error TS2339 + longest_edge = this.config.max_size ?? shortest_edge; + } else if (size !== undefined) { + // Extract known properties from `size` + shortest_edge = size.shortest_edge; + longest_edge = size.longest_edge; + } + + // If `longest_edge` and `shortest_edge` are set, maintain aspect ratio and resize to `shortest_edge` + // while keeping the largest dimension <= `longest_edge` + if (shortest_edge !== undefined || longest_edge !== undefined) { + // http://opensourcehacker.com/2011/12/01/calculate-aspect-ratio-conserving-resize-for-images-in-javascript/ + // Try resize so that shortest edge is `shortest_edge` (target) + const shortResizeFactor = + shortest_edge === undefined + ? 1 // If `shortest_edge` is not set, don't upscale + : Math.max(shortest_edge / srcWidth, shortest_edge / srcHeight); + + const newWidth = srcWidth * shortResizeFactor; + const newHeight = srcHeight * shortResizeFactor; + + // The new width and height might be greater than `longest_edge`, so + // we downscale again to ensure the largest dimension is `longest_edge` + const longResizeFactor = + longest_edge === undefined + ? 1 // If `longest_edge` is not set, don't downscale + : Math.min(longest_edge / newWidth, longest_edge / newHeight); + + // To avoid certain floating point precision issues, we round to 2 decimal places + let finalWidth = Math.floor(Number((newWidth * longResizeFactor).toFixed(2))); + let finalHeight = Math.floor(Number((newHeight * longResizeFactor).toFixed(2))); + + if (this.size_divisibility !== undefined) { + [finalWidth, finalHeight] = enforce_size_divisibility( + [finalWidth, finalHeight], + this.size_divisibility, + ); + } + return [finalWidth, finalHeight]; + } else if (size !== undefined && size.width !== undefined && size.height !== undefined) { + // If `width` and `height` are set, resize to those dimensions + + let newWidth = size.width; + let newHeight = size.height; + + // Custom for DPT models + if (this.config.keep_aspect_ratio && this.config.ensure_multiple_of) { + // determine new height and width + let scale_height = newHeight / srcHeight; + let scale_width = newWidth / srcWidth; + + // scale as little as possible + if (Math.abs(1 - scale_width) < Math.abs(1 - scale_height)) { + // fit width + scale_height = scale_width; + } else { + // fit height + scale_width = scale_height; + } + + newHeight = constraint_to_multiple_of(scale_height * srcHeight, this.config.ensure_multiple_of); + newWidth = constraint_to_multiple_of(scale_width * srcWidth, this.config.ensure_multiple_of); + } + + return [newWidth, newHeight]; + } else if (this.size_divisibility !== undefined) { + return enforce_size_divisibility([srcWidth, srcHeight], this.size_divisibility); + } else { + throw new Error( + `Could not resize image due to unsupported \`this.size\` option in config: ${JSON.stringify(size)}`, + ); + } + } + + /** + * Resizes the image. + * @param {RawImage} image The image to resize. + * @returns {Promise} The resized image. + */ + async resize(image) { + const [newWidth, newHeight] = this.get_resize_output_image_size(image, this.size); + return await image.resize(newWidth, newHeight, { + // @ts-expect-error TS2322 + resample: this.resample, + }); + } + + /** + * @typedef {Object} PreprocessedImage + * @property {HeightWidth} original_size The original size of the image. + * @property {HeightWidth} reshaped_input_size The reshaped input size of the image. + * @property {Tensor} pixel_values The pixel values of the preprocessed image. + */ + + /** + * Preprocesses the given image. + * + * @param {RawImage} image The image to preprocess. + * @param {Object} overrides The overrides for the preprocessing options. + * @returns {Promise} The preprocessed image. + */ + async preprocess( + image, + { + do_normalize = null, + do_pad = null, + do_convert_rgb = null, + do_convert_grayscale = null, + do_flip_channel_order = null, + } = {}, + ) { + if (this.do_crop_margin) { + // NOTE: Specific to nougat processors. This is done before resizing, + // and can be interpreted as a pre-preprocessing step. + image = await this.crop_margin(image); + } + + const [srcWidth, srcHeight] = image.size; // original image size + + // Convert image to RGB if specified in config. + if (do_convert_rgb ?? this.do_convert_rgb) { + image = image.rgb(); + } else if (do_convert_grayscale) { + image = image.grayscale(); + } + + // TODO: + // For efficiency reasons, it might be best to merge the resize and center crop operations into one. + + // Resize all images + if (this.do_resize) { + image = await this.resize(image); + } + + // Resize the image using thumbnail method. + if (this.do_thumbnail) { + // @ts-expect-error TS2345 + image = await this.thumbnail(image, this.size, this.resample); + } + + if (this.do_center_crop) { + let crop_width; + let crop_height; + if (Number.isInteger(this.crop_size)) { + crop_width = this.crop_size; + crop_height = this.crop_size; + } else { + crop_width = this.crop_size.width; + crop_height = this.crop_size.height; + } + + image = await image.center_crop(crop_width, crop_height); + } + + /** @type {HeightWidth} */ + const reshaped_input_size = [image.height, image.width]; + + // NOTE: All pixel-level manipulation (i.e., modifying `pixelData`) + // occurs with data in the hwc format (height, width, channels), + // to emulate the behavior of the original Python code (w/ numpy). + /** @type {Float32Array} */ + let pixelData = Float32Array.from(image.data); + let imgDims = [image.height, image.width, image.channels]; + + if (this.do_rescale) { + this.rescale(pixelData); + } + + if (do_normalize ?? this.do_normalize) { + let image_mean = this.image_mean; + if (!Array.isArray(this.image_mean)) { + image_mean = new Array(image.channels).fill(image_mean); + } + + let image_std = this.image_std; + if (!Array.isArray(this.image_std)) { + image_std = new Array(image.channels).fill(image_std); + } + + if (image_mean.length !== image.channels || image_std.length !== image.channels) { + throw new Error( + `When set to arrays, the length of \`image_mean\` (${image_mean.length}) and \`image_std\` (${image_std.length}) must match the number of channels in the image (${image.channels}).`, + ); + } + + for (let i = 0; i < pixelData.length; i += image.channels) { + for (let j = 0; j < image.channels; ++j) { + pixelData[i + j] = (pixelData[i + j] - image_mean[j]) / image_std[j]; + } + } + } + + // do padding after rescaling/normalizing + if (do_pad ?? this.do_pad) { + if (this.pad_size) { + const padded = this.pad_image(pixelData, [image.height, image.width, image.channels], this.pad_size); + [pixelData, imgDims] = padded; // Update pixel data and image dimensions + } else if (this.size_divisibility) { + const paddedWidth = Math.ceil(imgDims[1] / this.size_divisibility) * this.size_divisibility; + const paddedHeight = Math.ceil(imgDims[0] / this.size_divisibility) * this.size_divisibility; + [pixelData, imgDims] = this.pad_image(pixelData, imgDims, { width: paddedWidth, height: paddedHeight }); + } + } + + if (do_flip_channel_order ?? this.do_flip_channel_order) { + if (imgDims[2] !== 3) { + throw new Error('Flipping channel order is only supported for RGB images.'); + } + // Convert RGB to BGR + for (let i = 0; i < pixelData.length; i += 3) { + const temp = pixelData[i]; + pixelData[i] = pixelData[i + 2]; + pixelData[i + 2] = temp; + } + } + + const pixel_values = new Tensor('float32', pixelData, imgDims).permute(2, 0, 1); // convert to channel dimension format (hwc -> chw) + + return { + original_size: [srcHeight, srcWidth], + reshaped_input_size: reshaped_input_size, + pixel_values, + }; + } + + /** + * Calls the feature extraction process on an array of images, + * preprocesses each image, and concatenates the resulting + * features into a single Tensor. + * @param {RawImage[]} images The image(s) to extract features from. + * @param {...any} args Additional arguments. + * @returns {Promise} An object containing the concatenated pixel values (and other metadata) of the preprocessed images. + */ + async _call(images, ...args) { + if (!Array.isArray(images)) { + images = [images]; + } + /** @type {PreprocessedImage[]} */ + const imageData = await Promise.all(images.map((x) => this.preprocess(x))); + + // Stack pixel values + const pixel_values = stack( + imageData.map((x) => x.pixel_values), + 0, + ); + + return { + pixel_values, + + // Original sizes of images + original_sizes: imageData.map((x) => x.original_size), + + // Reshaped sizes of images, before padding or cropping + reshaped_input_sizes: imageData.map((x) => x.reshaped_input_size), + }; + } + + /** + * Instantiate one of the processor classes of the library from a pretrained model. + * + * The processor class to instantiate is selected based on the `image_processor_type` (or `feature_extractor_type`; legacy) + * property of the config object (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible) + * + * @param {string} pretrained_model_name_or_path The name or path of the pretrained model. Can be either: + * - A string, the *model id* of a pretrained processor hosted inside a model repo on huggingface.co. + * Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a + * user or organization name, like `dbmdz/bert-base-german-cased`. + * - A path to a *directory* containing processor files, e.g., `./my_model_directory/`. + * @param {import('./utils/hub.js').PretrainedOptions} options Additional options for loading the processor. + * + * @returns {Promise} A new instance of the Processor class. + */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + const preprocessorConfig = await getModelJSON( + pretrained_model_name_or_path, + IMAGE_PROCESSOR_NAME, + true, + options, + ); + return new this(preprocessorConfig); + } +} diff --git a/packages/transformers/src/models/afmoe/modeling_afmoe.js b/packages/transformers/src/models/afmoe/modeling_afmoe.js new file mode 100644 index 0000000..fc77806 --- /dev/null +++ b/packages/transformers/src/models/afmoe/modeling_afmoe.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class AfmoePreTrainedModel extends PreTrainedModel {} +export class AfmoeModel extends AfmoePreTrainedModel {} +export class AfmoeForCausalLM extends AfmoePreTrainedModel {} diff --git a/packages/transformers/src/models/albert/modeling_albert.js b/packages/transformers/src/models/albert/modeling_albert.js new file mode 100644 index 0000000..e32e3f2 --- /dev/null +++ b/packages/transformers/src/models/albert/modeling_albert.js @@ -0,0 +1,38 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { MaskedLMOutput, QuestionAnsweringModelOutput, SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class AlbertPreTrainedModel extends PreTrainedModel {} +export class AlbertModel extends AlbertPreTrainedModel {} +export class AlbertForSequenceClassification extends AlbertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} +export class AlbertForQuestionAnswering extends AlbertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} +export class AlbertForMaskedLM extends AlbertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/albert/tokenization_albert.js b/packages/transformers/src/models/albert/tokenization_albert.js new file mode 100644 index 0000000..3dda1f0 --- /dev/null +++ b/packages/transformers/src/models/albert/tokenization_albert.js @@ -0,0 +1,5 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class AlbertTokenizer extends PreTrainedTokenizer { + return_token_type_ids = true; +} diff --git a/packages/transformers/src/models/apertus/modeling_apertus.js b/packages/transformers/src/models/apertus/modeling_apertus.js new file mode 100644 index 0000000..4eb603d --- /dev/null +++ b/packages/transformers/src/models/apertus/modeling_apertus.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class ApertusPreTrainedModel extends PreTrainedModel {} +export class ApertusModel extends ApertusPreTrainedModel {} +export class ApertusForCausalLM extends ApertusPreTrainedModel {} diff --git a/packages/transformers/src/models/arcee/modeling_arcee.js b/packages/transformers/src/models/arcee/modeling_arcee.js new file mode 100644 index 0000000..3438cb4 --- /dev/null +++ b/packages/transformers/src/models/arcee/modeling_arcee.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class ArceePreTrainedModel extends PreTrainedModel {} +export class ArceeModel extends ArceePreTrainedModel {} +export class ArceeForCausalLM extends ArceePreTrainedModel {} diff --git a/packages/transformers/src/models/audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.js b/packages/transformers/src/models/audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.js new file mode 100644 index 0000000..ce5e575 --- /dev/null +++ b/packages/transformers/src/models/audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.js @@ -0,0 +1,82 @@ +import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { Tensor } from '../../utils/tensor.js'; +import { mel_filter_bank, spectrogram, window_function } from '../../utils/audio.js'; + +export class ASTFeatureExtractor extends FeatureExtractor { + constructor(config) { + super(config); + + const sampling_rate = this.config.sampling_rate; + const mel_filters = mel_filter_bank( + 257, // num_frequency_bins + this.config.num_mel_bins, // num_mel_filters + 20, // min_frequency + Math.floor(sampling_rate / 2), // max_frequency + sampling_rate, // sampling_rate + null, // norm + 'kaldi', // mel_scale + true, // triangularize_in_mel_space + ); + this.mel_filters = mel_filters; + + this.window = window_function(400, 'hann', { + periodic: false, + }); + + this.mean = this.config.mean; + this.std = this.config.std; + } + + /** + * Computes the log-Mel spectrogram of the provided audio waveform. + * @param {Float32Array|Float64Array} waveform The audio waveform to process. + * @param {number} max_length The maximum number of frames to return. + * @returns {Promise} An object containing the log-Mel spectrogram data as a Float32Array and its dimensions as an array of numbers. + */ + async _extract_fbank_features(waveform, max_length) { + // NOTE: We don't pad/truncate since that is passed in as `max_num_frames` + return spectrogram( + waveform, + this.window, // window + 400, // frame_length + 160, // hop_length + { + fft_length: 512, + power: 2.0, + center: false, + preemphasis: 0.97, + mel_filters: this.mel_filters, + log_mel: 'log', + mel_floor: 1.192092955078125e-7, + remove_dc_offset: true, + + // Custom + max_num_frames: max_length, + transpose: true, + }, + ); + } + + /** + * Asynchronously extracts features from a given audio using the provided configuration. + * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array. + * @returns {Promise<{ input_values: Tensor }>} A Promise resolving to an object containing the extracted input features as a Tensor. + */ + async _call(audio) { + validate_audio_inputs(audio, 'ASTFeatureExtractor'); + + const features = await this._extract_fbank_features(audio, this.config.max_length); + if (this.config.do_normalize) { + // Normalize the input audio spectrogram to have mean=0, std=0.5 + const denom = this.std * 2; + const features_data = features.data; + for (let i = 0; i < features_data.length; ++i) { + features_data[i] = (features_data[i] - this.mean) / denom; + } + } + + return { + input_values: features.unsqueeze_(0), + }; + } +} diff --git a/packages/transformers/src/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.js b/packages/transformers/src/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.js new file mode 100644 index 0000000..1815ee5 --- /dev/null +++ b/packages/transformers/src/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.js @@ -0,0 +1,14 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class ASTPreTrainedModel extends PreTrainedModel {} + +/** + * The bare AST Model transformer outputting raw hidden-states without any specific head on top. + */ +export class ASTModel extends ASTPreTrainedModel {} + +/** + * Audio Spectrogram Transformer model with an audio classification head on top + * (a linear layer on top of the pooled output) e.g. for datasets like AudioSet, Speech Commands v2. + */ +export class ASTForAudioClassification extends ASTPreTrainedModel {} diff --git a/packages/transformers/src/models/auto/feature_extraction_auto.js b/packages/transformers/src/models/auto/feature_extraction_auto.js new file mode 100644 index 0000000..4765962 --- /dev/null +++ b/packages/transformers/src/models/auto/feature_extraction_auto.js @@ -0,0 +1,27 @@ +import { FEATURE_EXTRACTOR_NAME, GITHUB_ISSUE_URL } from '../../utils/constants.js'; +import { getModelJSON } from '../../utils/hub.js'; +import { FeatureExtractor } from '../../feature_extraction_utils.js'; +import * as AllFeatureExtractors from '../feature_extractors.js'; + +export class AutoFeatureExtractor { + /** @type {typeof FeatureExtractor.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + const preprocessorConfig = await getModelJSON( + pretrained_model_name_or_path, + FEATURE_EXTRACTOR_NAME, + true, + options, + ); + + // Determine feature extractor class + const key = preprocessorConfig.feature_extractor_type; + const feature_extractor_class = AllFeatureExtractors[key]; + + if (!feature_extractor_class) { + throw new Error(`Unknown feature_extractor_type: '${key}'. Please report this at ${GITHUB_ISSUE_URL}.`); + } + + // Instantiate feature extractor + return new feature_extractor_class(preprocessorConfig); + } +} diff --git a/packages/transformers/src/models/auto/image_processing_auto.js b/packages/transformers/src/models/auto/image_processing_auto.js new file mode 100644 index 0000000..820dfac --- /dev/null +++ b/packages/transformers/src/models/auto/image_processing_auto.js @@ -0,0 +1,34 @@ +import { getModelJSON } from '../../utils/hub.js'; +import { ImageProcessor } from '../../image_processors_utils.js'; +import * as AllImageProcessors from '../image_processors.js'; +import { GITHUB_ISSUE_URL, IMAGE_PROCESSOR_NAME } from '../../utils/constants.js'; +import { logger } from '../../utils/logger.js'; + +export class AutoImageProcessor { + /** @type {typeof ImageProcessor.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + const preprocessorConfig = await getModelJSON( + pretrained_model_name_or_path, + IMAGE_PROCESSOR_NAME, + true, + options, + ); + + // Determine image processor class + const key = preprocessorConfig.image_processor_type ?? preprocessorConfig.feature_extractor_type; + let image_processor_class = AllImageProcessors[key?.replace(/Fast$/, '')]; + + if (!image_processor_class) { + if (key !== undefined) { + // Only log a warning if the class is not found and the key is set. + logger.warn( + `Image processor type '${key}' not found, assuming base ImageProcessor. Please report this at ${GITHUB_ISSUE_URL}.`, + ); + } + image_processor_class = ImageProcessor; + } + + // Instantiate image processor + return new image_processor_class(preprocessorConfig); + } +} diff --git a/packages/transformers/src/models/auto/modeling_auto.js b/packages/transformers/src/models/auto/modeling_auto.js new file mode 100644 index 0000000..030c51a --- /dev/null +++ b/packages/transformers/src/models/auto/modeling_auto.js @@ -0,0 +1,384 @@ +/** + * @file Definitions of all models available in Transformers.js. + * + * **Example:** Load and run an `AutoModel`. + * + * ```javascript + * import { AutoModel, AutoTokenizer } from '@huggingface/transformers'; + * + * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/bert-base-uncased'); + * const model = await AutoModel.from_pretrained('Xenova/bert-base-uncased'); + * + * const inputs = await tokenizer('I love transformers!'); + * const { logits } = await model(inputs); + * // Tensor { + * // data: Float32Array(183132) [-7.117443084716797, -7.107812881469727, -7.092104911804199, ...] + * // dims: (3) [1, 6, 30522], + * // type: "float32", + * // size: 183132, + * // } + * ``` + * + * We also provide other `AutoModel`s (listed below), which you can use in the same way as the Python library. For example: + * + * **Example:** Load and run an `AutoModelForSeq2SeqLM`. + * ```javascript + * import { AutoModelForSeq2SeqLM, AutoTokenizer } from '@huggingface/transformers'; + * + * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/t5-small'); + * const model = await AutoModelForSeq2SeqLM.from_pretrained('Xenova/t5-small'); + * + * const { input_ids } = await tokenizer('translate English to German: I love transformers!'); + * const outputs = await model.generate(input_ids); + * const decoded = tokenizer.decode(outputs[0], { skip_special_tokens: true }); + * // 'Ich liebe Transformatoren!' + * ``` + * + * @module models + */ + +import { AutoConfig } from '../../configs.js'; +import { PreTrainedModel } from '../modeling_utils.js'; + +import { CUSTOM_ARCHITECTURES, MODEL_CLASS_TYPE_MAPPING, MODEL_MAPPINGS } from '../registry.js'; + +import * as ALL_MODEL_FILES from '../models.js'; +import { logger } from '../../utils/logger.js'; + +/** + * Base class of all AutoModels. Contains the `from_pretrained` function + * which is used to instantiate pretrained models. + */ +class PretrainedMixin { + /** + * Mapping from model type to model class. + * @type {Map[]} + */ + static MODEL_CLASS_MAPPINGS = null; + + /** + * Whether to attempt to instantiate the base class (`PretrainedModel`) if + * the model type is not found in the mapping. + */ + static BASE_IF_FAIL = false; + + /** + * Check whether this AutoModel class supports a given model type. + * @param {string} model_type The model type from config (e.g., 'bert', 'whisper'). + * @returns {boolean} Whether this class can handle the given model type. + */ + static supports(model_type) { + if (!this.MODEL_CLASS_MAPPINGS) return false; + for (const mapping of this.MODEL_CLASS_MAPPINGS) { + if (mapping.has(model_type)) return true; + } + return this.BASE_IF_FAIL; + } + + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained( + pretrained_model_name_or_path, + { + progress_callback = null, + config = null, + cache_dir = null, + local_files_only = false, + revision = 'main', + model_file_name = null, + subfolder = 'onnx', + device = null, + dtype = null, + use_external_data_format = null, + session_options = {}, + } = {}, + ) { + const options = { + progress_callback, + config, + cache_dir, + local_files_only, + revision, + model_file_name, + subfolder, + device, + dtype, + use_external_data_format, + session_options, + }; + options.config = await AutoConfig.from_pretrained(pretrained_model_name_or_path, options); + + if (!this.MODEL_CLASS_MAPPINGS) { + throw new Error('`MODEL_CLASS_MAPPINGS` not implemented for this type of `AutoClass`: ' + this.name); + } + const { model_type } = options.config; + for (const MODEL_CLASS_MAPPING of this.MODEL_CLASS_MAPPINGS) { + let modelInfo = MODEL_CLASS_MAPPING.get(model_type); + if (!modelInfo) { + // As a fallback, we check if model_type is specified as the exact class + for (const cls of MODEL_CLASS_MAPPING.values()) { + if (cls[0] === model_type) { + modelInfo = cls; + break; + } + } + if (!modelInfo) continue; // Item not found in this mapping + } + return await ALL_MODEL_FILES[modelInfo].from_pretrained(pretrained_model_name_or_path, options); + } + + if (this.BASE_IF_FAIL) { + if (!CUSTOM_ARCHITECTURES.has(model_type)) { + logger.warn(`Unknown model class "${model_type}", attempting to construct from base class.`); + } + return await PreTrainedModel.from_pretrained(pretrained_model_name_or_path, options); + } else { + throw Error(`Unsupported model type: ${model_type}`); + } + } +} + +/** + * Helper class which is used to instantiate pretrained models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModel.from_pretrained('Xenova/bert-base-uncased'); + */ +export class AutoModel extends PretrainedMixin { + /** @type {Map[]} */ + // @ts-ignore + static MODEL_CLASS_MAPPINGS = MODEL_CLASS_TYPE_MAPPING.map((x) => x[0]); + static BASE_IF_FAIL = true; +} + +/** + * Helper class which is used to instantiate pretrained sequence classification models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForSequenceClassification.from_pretrained('Xenova/distilbert-base-uncased-finetuned-sst-2-english'); + */ +export class AutoModelForSequenceClassification extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES]; +} + +/** + * Helper class which is used to instantiate pretrained token classification models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForTokenClassification.from_pretrained('Xenova/distilbert-base-multilingual-cased-ner-hrl'); + */ +export class AutoModelForTokenClassification extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES]; +} + +/** + * Helper class which is used to instantiate pretrained sequence-to-sequence models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForSeq2SeqLM.from_pretrained('Xenova/t5-small'); + */ +export class AutoModelForSeq2SeqLM extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES]; +} + +/** + * Helper class which is used to instantiate pretrained sequence-to-sequence speech-to-text models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForSpeechSeq2Seq.from_pretrained('openai/whisper-tiny.en'); + */ +export class AutoModelForSpeechSeq2Seq extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES]; +} + +/** + * Helper class which is used to instantiate pretrained sequence-to-sequence text-to-spectrogram models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForTextToSpectrogram.from_pretrained('microsoft/speecht5_tts'); + */ +export class AutoModelForTextToSpectrogram extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES]; +} + +/** + * Helper class which is used to instantiate pretrained text-to-waveform models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForTextToSpectrogram.from_pretrained('facebook/mms-tts-eng'); + */ +export class AutoModelForTextToWaveform extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES]; +} + +/** + * Helper class which is used to instantiate pretrained causal language models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForCausalLM.from_pretrained('Xenova/gpt2'); + */ +export class AutoModelForCausalLM extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES]; +} + +/** + * Helper class which is used to instantiate pretrained masked language models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForMaskedLM.from_pretrained('Xenova/bert-base-uncased'); + */ +export class AutoModelForMaskedLM extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_MASKED_LM_MAPPING_NAMES]; +} + +/** + * Helper class which is used to instantiate pretrained question answering models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForQuestionAnswering.from_pretrained('Xenova/distilbert-base-cased-distilled-squad'); + */ +export class AutoModelForQuestionAnswering extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES]; +} + +/** + * Helper class which is used to instantiate pretrained vision-to-sequence models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForVision2Seq.from_pretrained('Xenova/vit-gpt2-image-captioning'); + */ +export class AutoModelForVision2Seq extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES]; +} + +/** + * Helper class which is used to instantiate pretrained image classification models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForImageClassification.from_pretrained('Xenova/vit-base-patch16-224'); + */ +export class AutoModelForImageClassification extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES]; +} + +/** + * Helper class which is used to instantiate pretrained image segmentation models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForImageSegmentation.from_pretrained('Xenova/detr-resnet-50-panoptic'); + */ +export class AutoModelForImageSegmentation extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES]; +} + +/** + * Helper class which is used to instantiate pretrained image segmentation models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForSemanticSegmentation.from_pretrained('nvidia/segformer-b3-finetuned-cityscapes-1024-1024'); + */ +export class AutoModelForSemanticSegmentation extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES]; +} + +/** + * Helper class which is used to instantiate pretrained universal image segmentation models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForUniversalSegmentation.from_pretrained('hf-internal-testing/tiny-random-MaskFormerForInstanceSegmentation'); + */ +export class AutoModelForUniversalSegmentation extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING_NAMES]; +} + +/** + * Helper class which is used to instantiate pretrained object detection models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForObjectDetection.from_pretrained('Xenova/detr-resnet-50'); + */ +export class AutoModelForObjectDetection extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES]; +} + +export class AutoModelForZeroShotObjectDetection extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES]; +} + +/** + * Helper class which is used to instantiate pretrained mask generation models with the `from_pretrained` function. + * The chosen model class is determined by the type specified in the model config. + * + * @example + * const model = await AutoModelForMaskGeneration.from_pretrained('Xenova/sam-vit-base'); + */ +export class AutoModelForMaskGeneration extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_MASK_GENERATION_MAPPING_NAMES]; +} + +export class AutoModelForCTC extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_CTC_MAPPING_NAMES]; +} + +export class AutoModelForAudioClassification extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES]; +} + +export class AutoModelForXVector extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES]; +} + +export class AutoModelForAudioFrameClassification extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES]; +} + +export class AutoModelForDocumentQuestionAnswering extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES]; +} + +export class AutoModelForImageMatting extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_IMAGE_MATTING_MAPPING_NAMES]; +} + +export class AutoModelForImageToImage extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_IMAGE_TO_IMAGE_MAPPING_NAMES]; +} + +export class AutoModelForDepthEstimation extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES]; +} + +export class AutoModelForNormalEstimation extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_NORMAL_ESTIMATION_MAPPING_NAMES]; +} + +export class AutoModelForPoseEstimation extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_POSE_ESTIMATION_MAPPING_NAMES]; +} + +export class AutoModelForImageFeatureExtraction extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_IMAGE_FEATURE_EXTRACTION_MAPPING_NAMES]; +} + +export class AutoModelForImageTextToText extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES]; +} + +export class AutoModelForAudioTextToText extends PretrainedMixin { + static MODEL_CLASS_MAPPINGS = [MODEL_MAPPINGS.MODEL_FOR_AUDIO_TEXT_TO_TEXT_MAPPING_NAMES]; +} diff --git a/packages/transformers/src/models/auto/processing_auto.js b/packages/transformers/src/models/auto/processing_auto.js new file mode 100644 index 0000000..c0f444e --- /dev/null +++ b/packages/transformers/src/models/auto/processing_auto.js @@ -0,0 +1,90 @@ +import { IMAGE_PROCESSOR_NAME } from '../../utils/constants.js'; +import { getModelJSON } from '../../utils/hub.js'; +import { Processor } from '../../processing_utils.js'; + +import * as AllProcessors from '../processors.js'; +import * as AllImageProcessors from '../image_processors.js'; +import * as AllFeatureExtractors from '../feature_extractors.js'; + +/** + * @typedef {import('../../processing_utils.js').PretrainedProcessorOptions} PretrainedProcessorOptions + */ + +/** + * Helper class which is used to instantiate pretrained processors with the `from_pretrained` function. + * The chosen processor class is determined by the type specified in the processor config. + * + * **Example:** Load a processor using `from_pretrained`. + * ```javascript + * let processor = await AutoProcessor.from_pretrained('openai/whisper-tiny.en'); + * ``` + * + * **Example:** Run an image through a processor. + * ```javascript + * let processor = await AutoProcessor.from_pretrained('Xenova/clip-vit-base-patch16'); + * let image = await RawImage.read('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg'); + * let image_inputs = await processor(image); + * // { + * // "pixel_values": { + * // "dims": [ 1, 3, 224, 224 ], + * // "type": "float32", + * // "data": Float32Array [ -1.558687686920166, -1.558687686920166, -1.5440893173217773, ... ], + * // "size": 150528 + * // }, + * // "original_sizes": [ + * // [ 533, 800 ] + * // ], + * // "reshaped_input_sizes": [ + * // [ 224, 224 ] + * // ] + * // } + * ``` + */ +export class AutoProcessor { + /** @type {typeof Processor.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + // TODO: first check for processor.json + const preprocessorConfig = await getModelJSON( + pretrained_model_name_or_path, + IMAGE_PROCESSOR_NAME, + true, + options, + ); + + const { image_processor_type, feature_extractor_type, processor_class } = preprocessorConfig; + if (processor_class && AllProcessors[processor_class]) { + return AllProcessors[processor_class].from_pretrained(pretrained_model_name_or_path, options); + } + + if (!image_processor_type && !feature_extractor_type) { + throw new Error('No `image_processor_type` or `feature_extractor_type` found in the config.'); + } + + const components = {}; + if (image_processor_type) { + // Some image processors are saved with the "Fast" suffix, so we remove that if present. + const image_processor_class = AllImageProcessors[image_processor_type.replace(/Fast$/, '')]; + if (!image_processor_class) { + throw new Error(`Unknown image_processor_type: '${image_processor_type}'.`); + } + components.image_processor = new image_processor_class(preprocessorConfig); + } + + if (feature_extractor_type) { + const image_processor_class = AllImageProcessors[feature_extractor_type]; + if (image_processor_class) { + // Handle legacy case where image processors were specified as feature extractors + components.image_processor = new image_processor_class(preprocessorConfig); + } else { + const feature_extractor_class = AllFeatureExtractors[feature_extractor_type]; + if (!feature_extractor_class) { + throw new Error(`Unknown feature_extractor_type: '${feature_extractor_type}'.`); + } + components.feature_extractor = new feature_extractor_class(preprocessorConfig); + } + } + + const config = {}; + return new Processor(config, components, null); + } +} diff --git a/packages/transformers/src/models/auto/tokenization_auto.js b/packages/transformers/src/models/auto/tokenization_auto.js new file mode 100644 index 0000000..62f3c61 --- /dev/null +++ b/packages/transformers/src/models/auto/tokenization_auto.js @@ -0,0 +1,63 @@ +import { PreTrainedTokenizer, loadTokenizer } from '../../tokenization_utils.js'; +import * as AllTokenizers from '../tokenizers.js'; +import { logger } from '../../utils/logger.js'; + +/** + * Helper class which is used to instantiate pretrained tokenizers with the `from_pretrained` function. + * The chosen tokenizer class is determined by the type specified in the tokenizer config. + * + * **Example:** Create an `AutoTokenizer` and use it to tokenize a sentence. + * This will automatically detect the tokenizer type based on the tokenizer class defined in `tokenizer_config.json`. + * + * ```javascript + * import { AutoTokenizer } from '@huggingface/transformers'; + * + * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/bert-base-uncased'); + * const { input_ids } = await tokenizer('I love transformers!'); + * // Tensor { + * // data: BigInt64Array(6) [101n, 1045n, 2293n, 19081n, 999n, 102n], + * // dims: [1, 6], + * // type: 'int64', + * // size: 6, + * // } + * ``` + */ +export class AutoTokenizer { + /** + * Instantiate one of the tokenizer classes of the library from a pretrained model. + * + * The tokenizer class to instantiate is selected based on the `tokenizer_class` property of the config object + * (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible) + * + * @param {string} pretrained_model_name_or_path The name or path of the pretrained model. Can be either: + * - A string, the *model id* of a pretrained tokenizer hosted inside a model repo on huggingface.co. + * Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a + * user or organization name, like `dbmdz/bert-base-german-cased`. + * - A path to a *directory* containing tokenizer files, e.g., `./my_model_directory/`. + * @param {import('../../tokenization_utils.js').PretrainedTokenizerOptions} options Additional options for loading the tokenizer. + * + * @returns {Promise} A new instance of the PreTrainedTokenizer class. + */ + static async from_pretrained( + pretrained_model_name_or_path, + { progress_callback = null, config = null, cache_dir = null, local_files_only = false, revision = 'main' } = {}, + ) { + const [tokenizerJSON, tokenizerConfig] = await loadTokenizer(pretrained_model_name_or_path, { + progress_callback, + config, + cache_dir, + local_files_only, + revision, + }); + + // Some tokenizers are saved with the "Fast" suffix, so we remove that if present. + const tokenizerName = tokenizerConfig.tokenizer_class?.replace(/Fast$/, '') ?? 'PreTrainedTokenizer'; + + let cls = AllTokenizers[tokenizerName]; + if (!cls) { + logger.warn(`Unknown tokenizer class "${tokenizerName}", attempting to construct from base class.`); + cls = PreTrainedTokenizer; + } + return new cls(tokenizerJSON, tokenizerConfig); + } +} diff --git a/packages/transformers/src/models/bart/modeling_bart.js b/packages/transformers/src/models/bart/modeling_bart.js new file mode 100644 index 0000000..e5d0343 --- /dev/null +++ b/packages/transformers/src/models/bart/modeling_bart.js @@ -0,0 +1,29 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class BartPretrainedModel extends PreTrainedModel {} + +/** + * The bare BART Model outputting raw hidden-states without any specific head on top. + */ +export class BartModel extends BartPretrainedModel {} + +/** + * The BART Model with a language modeling head. Can be used for summarization. + */ +export class BartForConditionalGeneration extends BartPretrainedModel {} + +/** + * Bart model with a sequence classification/head on top (a linear layer on top of the pooled output) + */ +export class BartForSequenceClassification extends BartPretrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/bart/tokenization_bart.js b/packages/transformers/src/models/bart/tokenization_bart.js new file mode 100644 index 0000000..2178a2d --- /dev/null +++ b/packages/transformers/src/models/bart/tokenization_bart.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class BartTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/beit/image_processing_beit.js b/packages/transformers/src/models/beit/image_processing_beit.js new file mode 100644 index 0000000..dd4604b --- /dev/null +++ b/packages/transformers/src/models/beit/image_processing_beit.js @@ -0,0 +1,3 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class BeitFeatureExtractor extends ImageProcessor {} diff --git a/packages/transformers/src/models/beit/modeling_beit.js b/packages/transformers/src/models/beit/modeling_beit.js new file mode 100644 index 0000000..c4e1393 --- /dev/null +++ b/packages/transformers/src/models/beit/modeling_beit.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class BeitPreTrainedModel extends PreTrainedModel {} +export class BeitModel extends BeitPreTrainedModel {} +export class BeitForImageClassification extends BeitPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/bert/modeling_bert.js b/packages/transformers/src/models/bert/modeling_bert.js new file mode 100644 index 0000000..3179108 --- /dev/null +++ b/packages/transformers/src/models/bert/modeling_bert.js @@ -0,0 +1,70 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { + SequenceClassifierOutput, + MaskedLMOutput, + TokenClassifierOutput, + QuestionAnsweringModelOutput, +} from '../modeling_outputs.js'; + +export class BertPreTrainedModel extends PreTrainedModel {} +export class BertModel extends BertPreTrainedModel {} + +/** + * BertForMaskedLM is a class representing a BERT model for masked language modeling. + */ +export class BertForMaskedLM extends BertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for masked language modeling. + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +/** + * BertForSequenceClassification is a class representing a BERT model for sequence classification. + */ +export class BertForSequenceClassification extends BertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * BertForTokenClassification is a class representing a BERT model for token classification. + */ +export class BertForTokenClassification extends BertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * BertForQuestionAnswering is a class representing a BERT model for question answering. + */ +export class BertForQuestionAnswering extends BertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for question answering. + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/bert/tokenization_bert.js b/packages/transformers/src/models/bert/tokenization_bert.js new file mode 100644 index 0000000..821e58e --- /dev/null +++ b/packages/transformers/src/models/bert/tokenization_bert.js @@ -0,0 +1,5 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class BertTokenizer extends PreTrainedTokenizer { + return_token_type_ids = true; +} diff --git a/packages/transformers/src/models/bit/image_processing_bit.js b/packages/transformers/src/models/bit/image_processing_bit.js new file mode 100644 index 0000000..d9e0ee3 --- /dev/null +++ b/packages/transformers/src/models/bit/image_processing_bit.js @@ -0,0 +1,3 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class BitImageProcessor extends ImageProcessor {} diff --git a/packages/transformers/src/models/blenderbot/modeling_blenderbot.js b/packages/transformers/src/models/blenderbot/modeling_blenderbot.js new file mode 100644 index 0000000..5e37141 --- /dev/null +++ b/packages/transformers/src/models/blenderbot/modeling_blenderbot.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class BlenderbotPreTrainedModel extends PreTrainedModel {} + +/** + * The bare Blenderbot Model outputting raw hidden-states without any specific head on top. + */ +export class BlenderbotModel extends BlenderbotPreTrainedModel {} + +/** + * The Blenderbot Model with a language modeling head. Can be used for summarization. + */ +export class BlenderbotForConditionalGeneration extends BlenderbotPreTrainedModel {} diff --git a/packages/transformers/src/models/blenderbot/tokenization_blenderbot.js b/packages/transformers/src/models/blenderbot/tokenization_blenderbot.js new file mode 100644 index 0000000..77ba25b --- /dev/null +++ b/packages/transformers/src/models/blenderbot/tokenization_blenderbot.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class BlenderbotTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/blenderbot_small/modeling_blenderbot_small.js b/packages/transformers/src/models/blenderbot_small/modeling_blenderbot_small.js new file mode 100644 index 0000000..92515f0 --- /dev/null +++ b/packages/transformers/src/models/blenderbot_small/modeling_blenderbot_small.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class BlenderbotSmallPreTrainedModel extends PreTrainedModel {} + +/** + * The bare BlenderbotSmall Model outputting raw hidden-states without any specific head on top. + */ +export class BlenderbotSmallModel extends BlenderbotSmallPreTrainedModel {} + +/** + * The BlenderbotSmall Model with a language modeling head. Can be used for summarization. + */ +export class BlenderbotSmallForConditionalGeneration extends BlenderbotSmallPreTrainedModel {} diff --git a/packages/transformers/src/models/blenderbot_small/tokenization_blenderbot_small.js b/packages/transformers/src/models/blenderbot_small/tokenization_blenderbot_small.js new file mode 100644 index 0000000..0e770d1 --- /dev/null +++ b/packages/transformers/src/models/blenderbot_small/tokenization_blenderbot_small.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class BlenderbotSmallTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/bloom/modeling_bloom.js b/packages/transformers/src/models/bloom/modeling_bloom.js new file mode 100644 index 0000000..6966ca6 --- /dev/null +++ b/packages/transformers/src/models/bloom/modeling_bloom.js @@ -0,0 +1,16 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +/** + * The Bloom Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). + */ +export class BloomPreTrainedModel extends PreTrainedModel {} + +/** + * The bare Bloom Model transformer outputting raw hidden-states without any specific head on top. + */ +export class BloomModel extends BloomPreTrainedModel {} + +/** + * The Bloom Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). + */ +export class BloomForCausalLM extends BloomPreTrainedModel {} diff --git a/packages/transformers/src/models/bloom/tokenization_bloom.js b/packages/transformers/src/models/bloom/tokenization_bloom.js new file mode 100644 index 0000000..961b222 --- /dev/null +++ b/packages/transformers/src/models/bloom/tokenization_bloom.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class BloomTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/camembert/modeling_camembert.js b/packages/transformers/src/models/camembert/modeling_camembert.js new file mode 100644 index 0000000..bcaa07e --- /dev/null +++ b/packages/transformers/src/models/camembert/modeling_camembert.js @@ -0,0 +1,74 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { + MaskedLMOutput, + QuestionAnsweringModelOutput, + SequenceClassifierOutput, + TokenClassifierOutput, +} from '../modeling_outputs.js'; + +export class CamembertPreTrainedModel extends PreTrainedModel {} + +/** + * The bare CamemBERT Model transformer outputting raw hidden-states without any specific head on top. + */ +export class CamembertModel extends CamembertPreTrainedModel {} + +/** + * CamemBERT Model with a `language modeling` head on top. + */ +export class CamembertForMaskedLM extends CamembertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for masked language modeling. + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +/** + * CamemBERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. + */ +export class CamembertForSequenceClassification extends CamembertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * CamemBERT Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. + */ +export class CamembertForTokenClassification extends CamembertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * CamemBERT Model with a span classification head on top for extractive question-answering tasks + */ +export class CamembertForQuestionAnswering extends CamembertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for question answering. + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/camembert/tokenization_camembert.js b/packages/transformers/src/models/camembert/tokenization_camembert.js new file mode 100644 index 0000000..448fcbd --- /dev/null +++ b/packages/transformers/src/models/camembert/tokenization_camembert.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class CamembertTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/chatterbox/feature_extraction_chatterbox.js b/packages/transformers/src/models/chatterbox/feature_extraction_chatterbox.js new file mode 100644 index 0000000..4b77bc0 --- /dev/null +++ b/packages/transformers/src/models/chatterbox/feature_extraction_chatterbox.js @@ -0,0 +1,22 @@ +import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class ChatterboxFeatureExtractor extends FeatureExtractor { + /** + * Asynchronously extracts input values from a given audio using the provided configuration. + * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array. + * @returns {Promise<{ input_values: Tensor; }>} The extracted input values. + */ + async _call(audio) { + validate_audio_inputs(audio, 'ChatterboxFeatureExtractor'); + + if (audio instanceof Float64Array) { + audio = new Float32Array(audio); + } + + const shape = [1 /* batch_size */, audio.length /* num_samples */]; + return { + input_values: new Tensor('float32', audio, shape), + }; + } +} diff --git a/packages/transformers/src/models/chatterbox/modeling_chatterbox.js b/packages/transformers/src/models/chatterbox/modeling_chatterbox.js new file mode 100644 index 0000000..3f28c39 --- /dev/null +++ b/packages/transformers/src/models/chatterbox/modeling_chatterbox.js @@ -0,0 +1,187 @@ +import { sessionRun } from '../session.js'; +import { PreTrainedModel, decoder_forward, decoder_prepare_inputs_for_generation } from '../modeling_utils.js'; +import { cat, ones, full, Tensor } from '../../utils/tensor.js'; + +const SILENCE_TOKEN = 4299n; +const START_SPEECH_TOKEN = 6561n; + +export class ChatterboxPreTrainedModel extends PreTrainedModel { + forward_params = [ + 'input_ids', + 'inputs_embeds', + 'attention_mask', + 'position_ids', + 'audio_values', + 'exaggeration', + 'audio_features', + 'audio_tokens', + 'speaker_embeddings', + 'speaker_features', + 'past_key_values', + ]; + main_input_name = 'input_ids'; + + _return_dict_in_generate_keys = ['audio_tokens', 'speaker_embeddings', 'speaker_features']; +} +export class ChatterboxModel extends ChatterboxPreTrainedModel { + /** + * @param {Tensor} audio_values + * @returns {Promise<{audio_features: Tensor, audio_tokens: Tensor, speaker_embeddings: Tensor, speaker_features: Tensor}>} + */ + async encode_speech(audio_values) { + return sessionRun(this.sessions['speech_encoder'], { + audio_values, + }); + } + + async forward({ + // Produced by the tokenizer/processor: + input_ids = null, + attention_mask = null, + audio_values = null, + exaggeration = null, + + // Used during generation: + position_ids = null, + inputs_embeds = null, + past_key_values = null, + + // Generic generation parameters + generation_config = null, + logits_processor = null, + num_logits_to_keep = null, + + // Speaker embeddings/features (useful for re-using pre-computed speaker data) + audio_features = null, // float32[batch_size,sequence_length,1024] + audio_tokens = null, // int64[batch_size,audio_sequence_length] + speaker_embeddings = null, // float32[batch_size,192] + speaker_features = null, // float32[batch_size,feature_dim,80] + + // TODO: needed? + ...kwargs + }) { + let speech_encoder_outputs; + if (!inputs_embeds) { + const expected_inputs = this.sessions['embed_tokens'].inputNames; + const embed_model_inputs = { input_ids }; + if (expected_inputs.includes('exaggeration')) { + // Support the following types for exaggeration: + // 1. null/undefined (no exaggeration): use the default of 0.5 + // 2. number: broadcast to (batch_size,) + // 3. number[]: convert to Tensor of shape (batch_size,) + // 4. Tensor of shape (batch_size, 1) + if (!(exaggeration instanceof Tensor)) { + const batch_size = input_ids.dims[0]; + if (exaggeration == null) { + exaggeration = full([batch_size], 0.5); + } else if (typeof exaggeration === 'number') { + exaggeration = full([batch_size], exaggeration); + } else if (Array.isArray(exaggeration)) { + exaggeration = new Tensor('float32', exaggeration, [batch_size]); + } else { + throw new Error('Unsupported type for `exaggeration` input'); + } + } + embed_model_inputs.exaggeration = exaggeration; + } + if (expected_inputs.includes('position_ids')) { + embed_model_inputs.position_ids = position_ids; + } + + ({ inputs_embeds } = await sessionRun(this.sessions['embed_tokens'], embed_model_inputs)); + + if (audio_features && audio_tokens && speaker_embeddings && speaker_features) { + // Use pre-computed speech encoder outputs + speech_encoder_outputs = { audio_features, audio_tokens, speaker_embeddings, speaker_features }; + } + + if (speech_encoder_outputs || audio_values) { + speech_encoder_outputs ??= await this.encode_speech(audio_values); + + // Update LLM inputs + inputs_embeds = cat([speech_encoder_outputs.audio_features, inputs_embeds], 1); + attention_mask = ones([inputs_embeds.dims[0], inputs_embeds.dims[1]]); + } else { + const target_length = inputs_embeds.dims[1]; + if (!past_key_values || target_length !== 1) { + throw new Error('Incorrect state encountered during generation.'); + } + const past_length = past_key_values.get_seq_length(); + attention_mask = ones([inputs_embeds.dims[0], past_length + target_length]); + } + } + + const outputs = await decoder_forward( + this, + { + inputs_embeds, + past_key_values, + attention_mask, + generation_config, + logits_processor, + num_logits_to_keep, + }, + false, + ); + return { + ...outputs, + ...speech_encoder_outputs, + }; + } + + prepare_inputs_for_generation(input_ids, model_inputs, generation_config) { + if (!model_inputs.position_ids && this.sessions['embed_tokens'].inputNames.includes('position_ids')) { + // If position_ids are not provided, we create them on the fly using the position of the START_SPEECH_TOKEN + if (model_inputs.input_ids.dims[1] === 1) { + const position_ids = Array.from( + { + length: input_ids.length, + }, + (_, i) => input_ids[i].length - input_ids[i].findLastIndex((x) => x == START_SPEECH_TOKEN) - 1, + ); + model_inputs.position_ids = new Tensor('int64', position_ids, [input_ids.length, 1]); + } else { + const batched_input_ids = model_inputs.input_ids.tolist(); + const position_ids_list = batched_input_ids.map((ids) => { + let position = 0; + return ids.map((id) => (id >= START_SPEECH_TOKEN ? 0 : position++)); + }); + model_inputs.position_ids = new Tensor('int64', position_ids_list.flat(), model_inputs.input_ids.dims); + } + } + if (model_inputs.input_ids.dims[1] === 1) { + // We are in generation mode and no longer need the audio inputs + delete model_inputs.audio_values; + delete model_inputs.audio_features; + delete model_inputs.audio_tokens; + delete model_inputs.speaker_embeddings; + delete model_inputs.speaker_features; + } + return decoder_prepare_inputs_for_generation(this, input_ids, model_inputs, generation_config); + } + + /** @type {PreTrainedModel['generate']} */ + async generate(params) { + const { sequences, audio_tokens, speaker_embeddings, speaker_features } = /** @type {any} */ ( + await super.generate({ + ...params, + return_dict_in_generate: true, + }) + ); + + const new_tokens = sequences.slice(null, [ + /** @type {Tensor} */ (params.input_ids).dims[1], // Exclude start of speech token + -1, // Exclude end of speech token + ]); + + const silence_tokens = full([new_tokens.dims[0], 3], SILENCE_TOKEN); // Add 3 silence tokens + const speech_tokens = cat([audio_tokens, new_tokens, silence_tokens], 1); + + const { waveform } = await sessionRun(this.sessions['conditional_decoder'], { + speech_tokens, + speaker_features, + speaker_embeddings, + }); + return waveform; + } +} diff --git a/packages/transformers/src/models/chatterbox/processing_chatterbox.js b/packages/transformers/src/models/chatterbox/processing_chatterbox.js new file mode 100644 index 0000000..0ac9f72 --- /dev/null +++ b/packages/transformers/src/models/chatterbox/processing_chatterbox.js @@ -0,0 +1,17 @@ +import { AutoFeatureExtractor } from '../auto/feature_extraction_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { Processor } from '../../processing_utils.js'; + +/** + * Represents a ChatterboxProcessor that extracts features from an audio input. + */ +export class ChatterboxProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static feature_extractor_class = AutoFeatureExtractor; + + async _call(text, audio = null) { + const text_features = this.tokenizer(text); + const audio_features = audio ? await this.feature_extractor(audio) : {}; + return { ...text_features, ...audio_features }; + } +} diff --git a/packages/transformers/src/models/chinese_clip/image_processing_chinese_clip.js b/packages/transformers/src/models/chinese_clip/image_processing_chinese_clip.js new file mode 100644 index 0000000..aca88a8 --- /dev/null +++ b/packages/transformers/src/models/chinese_clip/image_processing_chinese_clip.js @@ -0,0 +1,3 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class ChineseCLIPFeatureExtractor extends ImageProcessor {} diff --git a/packages/transformers/src/models/chinese_clip/modeling_chinese_clip.js b/packages/transformers/src/models/chinese_clip/modeling_chinese_clip.js new file mode 100644 index 0000000..47e5f2d --- /dev/null +++ b/packages/transformers/src/models/chinese_clip/modeling_chinese_clip.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class ChineseCLIPPreTrainedModel extends PreTrainedModel {} + +export class ChineseCLIPModel extends ChineseCLIPPreTrainedModel {} diff --git a/packages/transformers/src/models/chmv2/image_processing_chmv2.js b/packages/transformers/src/models/chmv2/image_processing_chmv2.js new file mode 100644 index 0000000..c147ee0 --- /dev/null +++ b/packages/transformers/src/models/chmv2/image_processing_chmv2.js @@ -0,0 +1,3 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class CHMv2ImageProcessor extends ImageProcessor {} diff --git a/packages/transformers/src/models/chmv2/modeling_chmv2.js b/packages/transformers/src/models/chmv2/modeling_chmv2.js new file mode 100644 index 0000000..380907f --- /dev/null +++ b/packages/transformers/src/models/chmv2/modeling_chmv2.js @@ -0,0 +1,4 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class CHMv2PreTrainedModel extends PreTrainedModel {} +export class CHMv2ForDepthEstimation extends CHMv2PreTrainedModel {} diff --git a/packages/transformers/src/models/clap/feature_extraction_clap.js b/packages/transformers/src/models/clap/feature_extraction_clap.js new file mode 100644 index 0000000..f6175af --- /dev/null +++ b/packages/transformers/src/models/clap/feature_extraction_clap.js @@ -0,0 +1,160 @@ +import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { Tensor } from '../../utils/tensor.js'; +import { mel_filter_bank, spectrogram, window_function } from '../../utils/audio.js'; +import { random } from '../../utils/random.js'; + +export class ClapFeatureExtractor extends FeatureExtractor { + constructor(config) { + super(config); + + this.mel_filters = mel_filter_bank( + this.config.nb_frequency_bins, // num_frequency_bins + this.config.feature_size, // num_mel_filters + this.config.frequency_min, // min_frequency + this.config.frequency_max, // max_frequency + this.config.sampling_rate, // sampling_rate + null, // norm + 'htk', // mel_scale + ); + + this.mel_filters_slaney = mel_filter_bank( + this.config.nb_frequency_bins, // num_frequency_bins + this.config.feature_size, // num_mel_filters + this.config.frequency_min, // min_frequency + this.config.frequency_max, // max_frequency + this.config.sampling_rate, // sampling_rate + 'slaney', // norm + 'slaney', // mel_scale + ); + + this.window = window_function(this.config.fft_window_size, 'hann'); + } + + /** + * Extracts the mel spectrogram and prepares it for the mode based on the `truncation` and `padding` arguments. + * + * Four different path are possible: + * - `truncation="fusion"` and the length of the waveform is greater than the max length: the mel spectrogram + * will be computed on the entire audio. 3 random crops and a dowsampled version of the full mel spectrogram + * are then stacked together. They will later be used for `feature_fusion`. + * - `truncation="rand_trunc"` and the length of the waveform is smaller than the max length: the audio is + * padded based on `padding`. + * - `truncation="fusion"` and the length of the waveform is smaller than the max length: the audio is padded + * based on `padding`, and is repeated `4` times. + * - `truncation="rand_trunc"` and the length of the waveform is greater than the max length: the mel + * spectrogram will be computed on a random crop of the waveform. + * + * @param {Float32Array|Float64Array} waveform The input waveform. + * @param {number} max_length The maximum length of the waveform. + * @param {string} truncation The truncation strategy to use. + * @param {string} padding The padding strategy to use. + * @returns {Promise} An object containing the mel spectrogram data as a Float32Array, its dimensions as an array of numbers, and a boolean indicating whether the waveform was longer than the max length. + * @private + */ + async _get_input_mel(waveform, max_length, truncation, padding) { + /** @type {Tensor} */ + let input_mel; + let longer = false; + const diff = waveform.length - max_length; + if (diff > 0) { + if (truncation === 'rand_trunc') { + longer = true; + const idx = Math.floor(random.random() * (diff + 1)); + waveform = waveform.subarray(idx, idx + max_length); + + input_mel = await this._extract_fbank_features( + waveform, + this.mel_filters_slaney, + this.config.nb_max_samples, + ); + } else { + // TODO implement fusion strategy + throw new Error(`Truncation strategy "${truncation}" not implemented`); + } + } else { + if (diff < 0) { + let padded = new Float64Array(max_length); // already padded with zeros + padded.set(waveform); + + if (padding === 'repeat') { + for (let i = waveform.length; i < max_length; i += waveform.length) { + padded.set(waveform.subarray(0, Math.min(waveform.length, max_length - i)), i); + } + } else if (padding === 'repeatpad') { + for (let i = waveform.length; i < -diff; i += waveform.length) { + padded.set(waveform, i); + } + } + waveform = padded; + } + + if (truncation === 'fusion') { + throw new Error(`Truncation strategy "${truncation}" not implemented`); + } + + input_mel = await this._extract_fbank_features( + waveform, + this.mel_filters_slaney, + this.config.nb_max_samples, + ); + } + + return input_mel.unsqueeze_(0); + } + + /** + * Compute the log-mel spectrogram of the provided `waveform` using the Hann window. + * In CLAP, two different filter banks are used depending on the truncation pattern: + * - `self.mel_filters`: they correspond to the default parameters of `torchaudio` which can be obtained from + * calling `torchaudio.transforms.MelSpectrogram().mel_scale.fb`. These filters are used when `truncation` + * is set to `"fusion"`. + * - `self.mel_filteres_slaney` : they correspond to the default parameters of `librosa` which used + * `librosa.filters.mel` when computing the mel spectrogram. These filters were only used in the original + * implementation when the truncation mode is not `"fusion"`. + * + * @param {Float32Array|Float64Array} waveform The audio waveform to process. + * @param {number[][]} mel_filters The mel filters to use. + * @param {number} [max_length=null] The maximum number of frames to return. + * @returns {Promise} An object containing the log-Mel spectrogram data as a Float32Array and its dimensions as an array of numbers. + */ + async _extract_fbank_features(waveform, mel_filters, max_length = null) { + // NOTE: We don't pad/truncate since that is passed in as `max_num_frames` + return spectrogram( + waveform, + this.window, // window + this.config.fft_window_size, // frame_length + this.config.hop_length, // hop_length + { + power: 2.0, + mel_filters, + log_mel: 'dB', + + // Custom + max_num_frames: max_length, + do_pad: false, + transpose: true, + }, + ); + } + + /** + * Asynchronously extracts features from a given audio using the provided configuration. + * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array. + * @returns {Promise<{ input_features: Tensor }>} A Promise resolving to an object containing the extracted input features as a Tensor. + */ + async _call(audio, { max_length = null } = {}) { + validate_audio_inputs(audio, 'ClapFeatureExtractor'); + + // convert to mel spectrogram, truncate and pad if needed. + const padded_inputs = await this._get_input_mel( + audio, + max_length ?? this.config.nb_max_samples, + this.config.truncation, + this.config.padding, + ); + + return { + input_features: padded_inputs.unsqueeze_(0), + }; + } +} diff --git a/packages/transformers/src/models/clap/modeling_clap.js b/packages/transformers/src/models/clap/modeling_clap.js new file mode 100644 index 0000000..dacbe96 --- /dev/null +++ b/packages/transformers/src/models/clap/modeling_clap.js @@ -0,0 +1,79 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class ClapPreTrainedModel extends PreTrainedModel {} + +export class ClapModel extends ClapPreTrainedModel {} + +/** + * CLAP Text Model with a projection layer on top (a linear layer on top of the pooled output). + * + * **Example:** Compute text embeddings with `ClapTextModelWithProjection`. + * + * ```javascript + * import { AutoTokenizer, ClapTextModelWithProjection } from '@huggingface/transformers'; + * + * // Load tokenizer and text model + * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/clap-htsat-unfused'); + * const text_model = await ClapTextModelWithProjection.from_pretrained('Xenova/clap-htsat-unfused'); + * + * // Run tokenization + * const texts = ['a sound of a cat', 'a sound of a dog']; + * const text_inputs = tokenizer(texts, { padding: true, truncation: true }); + * + * // Compute embeddings + * const { text_embeds } = await text_model(text_inputs); + * // Tensor { + * // dims: [ 2, 512 ], + * // type: 'float32', + * // data: Float32Array(1024) [ ... ], + * // size: 1024 + * // } + * ``` + */ +export class ClapTextModelWithProjection extends ClapPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'text_model', + }); + } +} + +/** + * CLAP Audio Model with a projection layer on top (a linear layer on top of the pooled output). + * + * **Example:** Compute audio embeddings with `ClapAudioModelWithProjection`. + * + * ```javascript + * import { AutoProcessor, ClapAudioModelWithProjection, read_audio } from '@huggingface/transformers'; + * + * // Load processor and audio model + * const processor = await AutoProcessor.from_pretrained('Xenova/clap-htsat-unfused'); + * const audio_model = await ClapAudioModelWithProjection.from_pretrained('Xenova/clap-htsat-unfused'); + * + * // Read audio and run processor + * const audio = await read_audio('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cat_meow.wav'); + * const audio_inputs = await processor(audio); + * + * // Compute embeddings + * const { audio_embeds } = await audio_model(audio_inputs); + * // Tensor { + * // dims: [ 1, 512 ], + * // type: 'float32', + * // data: Float32Array(512) [ ... ], + * // size: 512 + * // } + * ``` + */ +export class ClapAudioModelWithProjection extends ClapPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'audio_model', + }); + } +} diff --git a/packages/transformers/src/models/clip/image_processing_clip.js b/packages/transformers/src/models/clip/image_processing_clip.js new file mode 100644 index 0000000..d536d66 --- /dev/null +++ b/packages/transformers/src/models/clip/image_processing_clip.js @@ -0,0 +1,4 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class CLIPImageProcessor extends ImageProcessor {} +export class CLIPFeatureExtractor extends CLIPImageProcessor {} diff --git a/packages/transformers/src/models/clip/modeling_clip.js b/packages/transformers/src/models/clip/modeling_clip.js new file mode 100644 index 0000000..a4806d9 --- /dev/null +++ b/packages/transformers/src/models/clip/modeling_clip.js @@ -0,0 +1,150 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class CLIPPreTrainedModel extends PreTrainedModel {} + +/** + * CLIP Text and Vision Model with a projection layers on top + * + * **Example:** Perform zero-shot image classification with a `CLIPModel`. + * + * ```javascript + * import { AutoTokenizer, AutoProcessor, CLIPModel, RawImage } from '@huggingface/transformers'; + * + * // Load tokenizer, processor, and model + * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/clip-vit-base-patch16'); + * const processor = await AutoProcessor.from_pretrained('Xenova/clip-vit-base-patch16'); + * const model = await CLIPModel.from_pretrained('Xenova/clip-vit-base-patch16'); + * + * // Run tokenization + * const texts = ['a photo of a car', 'a photo of a football match'] + * const text_inputs = tokenizer(texts, { padding: true, truncation: true }); + * + * // Read image and run processor + * const image = await RawImage.read('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg'); + * const image_inputs = await processor(image); + * + * // Run model with both text and pixel inputs + * const output = await model({ ...text_inputs, ...image_inputs }); + * // { + * // logits_per_image: Tensor { + * // dims: [ 1, 2 ], + * // data: Float32Array(2) [ 18.579734802246094, 24.31830596923828 ], + * // }, + * // logits_per_text: Tensor { + * // dims: [ 2, 1 ], + * // data: Float32Array(2) [ 18.579734802246094, 24.31830596923828 ], + * // }, + * // text_embeds: Tensor { + * // dims: [ 2, 512 ], + * // data: Float32Array(1024) [ ... ], + * // }, + * // image_embeds: Tensor { + * // dims: [ 1, 512 ], + * // data: Float32Array(512) [ ... ], + * // } + * // } + * ``` + */ +export class CLIPModel extends CLIPPreTrainedModel {} + +/** + * The text model from CLIP without any head or projection on top. + */ +export class CLIPTextModel extends CLIPPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'text_model', + }); + } +} + +/** + * CLIP Text Model with a projection layer on top (a linear layer on top of the pooled output) + * + * **Example:** Compute text embeddings with `CLIPTextModelWithProjection`. + * + * ```javascript + * import { AutoTokenizer, CLIPTextModelWithProjection } from '@huggingface/transformers'; + * + * // Load tokenizer and text model + * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/clip-vit-base-patch16'); + * const text_model = await CLIPTextModelWithProjection.from_pretrained('Xenova/clip-vit-base-patch16'); + * + * // Run tokenization + * const texts = ['a photo of a car', 'a photo of a football match']; + * const text_inputs = tokenizer(texts, { padding: true, truncation: true }); + * + * // Compute embeddings + * const { text_embeds } = await text_model(text_inputs); + * // Tensor { + * // dims: [ 2, 512 ], + * // type: 'float32', + * // data: Float32Array(1024) [ ... ], + * // size: 1024 + * // } + * ``` + */ +export class CLIPTextModelWithProjection extends CLIPPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'text_model', + }); + } +} + +/** + * The vision model from CLIP without any head or projection on top. + */ +export class CLIPVisionModel extends CLIPPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'vision_model', + }); + } +} + +/** + * CLIP Vision Model with a projection layer on top (a linear layer on top of the pooled output) + * + * **Example:** Compute vision embeddings with `CLIPVisionModelWithProjection`. + * + * ```javascript + * import { AutoProcessor, CLIPVisionModelWithProjection, RawImage } from '@huggingface/transformers'; + * + * // Load processor and vision model + * const processor = await AutoProcessor.from_pretrained('Xenova/clip-vit-base-patch16'); + * const vision_model = await CLIPVisionModelWithProjection.from_pretrained('Xenova/clip-vit-base-patch16'); + * + * // Read image and run processor + * const image = await RawImage.read('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg'); + * const image_inputs = await processor(image); + * + * // Compute embeddings + * const { image_embeds } = await vision_model(image_inputs); + * // Tensor { + * // dims: [ 1, 512 ], + * // type: 'float32', + * // data: Float32Array(512) [ ... ], + * // size: 512 + * // } + * ``` + */ +export class CLIPVisionModelWithProjection extends CLIPPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'vision_model', + }); + } +} diff --git a/packages/transformers/src/models/clip/tokenization_clip.js b/packages/transformers/src/models/clip/tokenization_clip.js new file mode 100644 index 0000000..c56755c --- /dev/null +++ b/packages/transformers/src/models/clip/tokenization_clip.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class CLIPTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/clipseg/modeling_clipseg.js b/packages/transformers/src/models/clipseg/modeling_clipseg.js new file mode 100644 index 0000000..7993dc0 --- /dev/null +++ b/packages/transformers/src/models/clipseg/modeling_clipseg.js @@ -0,0 +1,53 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class CLIPSegPreTrainedModel extends PreTrainedModel {} + +export class CLIPSegModel extends CLIPSegPreTrainedModel {} + +/** + * CLIPSeg model with a Transformer-based decoder on top for zero-shot and one-shot image segmentation. + * + * **Example:** Perform zero-shot image segmentation with a `CLIPSegForImageSegmentation` model. + * + * ```javascript + * import { AutoTokenizer, AutoProcessor, CLIPSegForImageSegmentation, RawImage } from '@huggingface/transformers'; + * + * // Load tokenizer, processor, and model + * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/clipseg-rd64-refined'); + * const processor = await AutoProcessor.from_pretrained('Xenova/clipseg-rd64-refined'); + * const model = await CLIPSegForImageSegmentation.from_pretrained('Xenova/clipseg-rd64-refined'); + * + * // Run tokenization + * const texts = ['a glass', 'something to fill', 'wood', 'a jar']; + * const text_inputs = tokenizer(texts, { padding: true, truncation: true }); + * + * // Read image and run processor + * const image = await RawImage.read('https://github.com/timojl/clipseg/blob/master/example_image.jpg?raw=true'); + * const image_inputs = await processor(image); + * + * // Run model with both text and pixel inputs + * const { logits } = await model({ ...text_inputs, ...image_inputs }); + * // logits: Tensor { + * // dims: [4, 352, 352], + * // type: 'float32', + * // data: Float32Array(495616) [ ... ], + * // size: 495616 + * // } + * ``` + * + * You can visualize the predictions as follows: + * ```javascript + * const preds = logits + * .unsqueeze_(1) + * .sigmoid_() + * .mul_(255) + * .round_() + * .to('uint8'); + * + * for (let i = 0; i < preds.dims[0]; ++i) { + * const img = RawImage.fromTensor(preds[i]); + * img.save(`prediction_${i}.png`); + * } + * ``` + */ +export class CLIPSegForImageSegmentation extends CLIPSegPreTrainedModel {} diff --git a/packages/transformers/src/models/code_llama/tokenization_code_llama.js b/packages/transformers/src/models/code_llama/tokenization_code_llama.js new file mode 100644 index 0000000..dab78be --- /dev/null +++ b/packages/transformers/src/models/code_llama/tokenization_code_llama.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class CodeLlamaTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/codegen/modeling_codegen.js b/packages/transformers/src/models/codegen/modeling_codegen.js new file mode 100644 index 0000000..040cc2f --- /dev/null +++ b/packages/transformers/src/models/codegen/modeling_codegen.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class CodeGenPreTrainedModel extends PreTrainedModel {} + +/** + * CodeGenModel is a class representing a code generation model without a language model head. + */ +export class CodeGenModel extends CodeGenPreTrainedModel {} + +/** + * CodeGenForCausalLM is a class that represents a code generation model based on the GPT-2 architecture. It extends the `CodeGenPreTrainedModel` class. + */ +export class CodeGenForCausalLM extends CodeGenPreTrainedModel {} diff --git a/packages/transformers/src/models/codegen/tokenization_codegen.js b/packages/transformers/src/models/codegen/tokenization_codegen.js new file mode 100644 index 0000000..c4f232f --- /dev/null +++ b/packages/transformers/src/models/codegen/tokenization_codegen.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class CodeGenTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/cohere/modeling_cohere.js b/packages/transformers/src/models/cohere/modeling_cohere.js new file mode 100644 index 0000000..b276948 --- /dev/null +++ b/packages/transformers/src/models/cohere/modeling_cohere.js @@ -0,0 +1,9 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +/** + * The bare Cohere Model outputting raw hidden-states without any specific head on top. + */ +export class CoherePreTrainedModel extends PreTrainedModel {} +export class CohereModel extends CoherePreTrainedModel {} + +export class CohereForCausalLM extends CoherePreTrainedModel {} diff --git a/packages/transformers/src/models/cohere/tokenization_cohere.js b/packages/transformers/src/models/cohere/tokenization_cohere.js new file mode 100644 index 0000000..35b6d23 --- /dev/null +++ b/packages/transformers/src/models/cohere/tokenization_cohere.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class CohereTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/cohere2/modeling_cohere2.js b/packages/transformers/src/models/cohere2/modeling_cohere2.js new file mode 100644 index 0000000..3cfa725 --- /dev/null +++ b/packages/transformers/src/models/cohere2/modeling_cohere2.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Cohere2PreTrainedModel extends PreTrainedModel {} +export class Cohere2Model extends Cohere2PreTrainedModel {} +export class Cohere2ForCausalLM extends Cohere2PreTrainedModel {} diff --git a/packages/transformers/src/models/cohere_asr/feature_extraction_cohere_asr.js b/packages/transformers/src/models/cohere_asr/feature_extraction_cohere_asr.js new file mode 100644 index 0000000..8dcd1e4 --- /dev/null +++ b/packages/transformers/src/models/cohere_asr/feature_extraction_cohere_asr.js @@ -0,0 +1,117 @@ +import { validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { Random } from '../../utils/random.js'; +import { ParakeetFeatureExtractor } from '../parakeet/feature_extraction_parakeet.js'; + +export class CohereAsrFeatureExtractor extends ParakeetFeatureExtractor { + /** + * Apply deterministic dithering seeded by the waveform length. + * @param {Float64Array} waveform + * @returns {Float64Array} The dithered waveform (mutated in-place). + */ + _apply_dither(waveform) { + const dither = this.config.dither ?? 0; + if (dither <= 0) return waveform; + + const rng = new Random(waveform.length); + for (let i = 0; i < waveform.length; ++i) { + waveform[i] += dither * rng.gauss(); + } + return waveform; + } + + /** + * Split audio into chunks at energy-based boundaries for long audio. + * @param {Float32Array|Float64Array} audio The raw audio waveform. + * @returns {(Float32Array|Float64Array)[]} Array of audio chunks. + */ + split_audio(audio) { + const max_audio_clip_s = this.config.max_audio_clip_s ?? 35.0; + const overlap_chunk_second = this.config.overlap_chunk_second ?? 5.0; + const min_energy_window_samples = this.config.min_energy_window_samples ?? 1600; + const sampling_rate = this.config.sampling_rate; + + const chunk_size = Math.max(1, Math.round(max_audio_clip_s * sampling_rate)); + const boundary_context_size = Math.max(1, Math.round(overlap_chunk_second * sampling_rate)); + + if (audio.length <= chunk_size) { + return [audio]; + } + + const chunks = []; + let idx = 0; + const total_samples = audio.length; + + while (idx < total_samples) { + if (idx + chunk_size >= total_samples) { + chunks.push(audio.slice(idx, total_samples)); + break; + } + + const search_start = Math.max(idx, idx + chunk_size - boundary_context_size); + const search_end = Math.min(idx + chunk_size, total_samples); + + let split_point; + if (search_end <= search_start) { + split_point = idx + chunk_size; + } else { + split_point = this._find_split_point_energy(audio, search_start, search_end, min_energy_window_samples); + } + + split_point = Math.max(idx + 1, Math.min(split_point, total_samples)); + chunks.push(audio.slice(idx, split_point)); + idx = split_point; + } + + return chunks; + } + + /** + * Find the quietest point (minimum energy) within a segment of audio. + * @param {Float32Array|Float64Array} waveform + * @param {number} start_idx + * @param {number} end_idx + * @param {number} window_size + * @returns {number} Index of the quietest point. + */ + _find_split_point_energy(waveform, start_idx, end_idx, window_size) { + const segment_len = end_idx - start_idx; + if (segment_len <= window_size) { + return Math.floor((start_idx + end_idx) / 2); + } + + let min_energy = Infinity; + let quietest_idx = start_idx; + const upper = segment_len - window_size; + + for (let i = 0; i <= upper; i += window_size) { + let energy = 0; + for (let j = 0; j < window_size; ++j) { + const val = waveform[start_idx + i + j]; + energy += val * val; + } + energy = Math.sqrt(energy / window_size); + + if (energy < min_energy) { + min_energy = energy; + quietest_idx = start_idx + i; + } + } + + return quietest_idx; + } + + /** + * Extracts features from a given audio waveform. + * @param {Float32Array|Float64Array} audio The audio data. + * @returns {Promise<{ input_features: import('../../utils/tensor.js').Tensor; attention_mask: import('../../utils/tensor.js').Tensor; }>} + */ + async _call(audio) { + validate_audio_inputs(audio, 'CohereAsrFeatureExtractor'); + + // Clone to Float64Array and apply dithering before feature extraction + const waveform = new Float64Array(audio); + this._apply_dither(waveform); + + return super._call(waveform); + } +} diff --git a/packages/transformers/src/models/cohere_asr/modeling_cohere_asr.js b/packages/transformers/src/models/cohere_asr/modeling_cohere_asr.js new file mode 100644 index 0000000..c25d5fd --- /dev/null +++ b/packages/transformers/src/models/cohere_asr/modeling_cohere_asr.js @@ -0,0 +1,11 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class CohereAsrPreTrainedModel extends PreTrainedModel { + requires_attention_mask = false; + main_input_name = 'input_features'; + forward_params = ['input_features', 'decoder_input_ids', 'decoder_attention_mask', 'past_key_values']; +} + +export class CohereAsrModel extends CohereAsrPreTrainedModel {} + +export class CohereAsrForConditionalGeneration extends CohereAsrPreTrainedModel {} diff --git a/packages/transformers/src/models/cohere_asr/processing_cohere_asr.js b/packages/transformers/src/models/cohere_asr/processing_cohere_asr.js new file mode 100644 index 0000000..7cbcc37 --- /dev/null +++ b/packages/transformers/src/models/cohere_asr/processing_cohere_asr.js @@ -0,0 +1,55 @@ +import { AutoFeatureExtractor } from '../auto/feature_extraction_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { Processor } from '../../processing_utils.js'; + +const NO_SPACE_LANGUAGES = new Set(['ja', 'zh']); + +export class CohereAsrProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static feature_extractor_class = AutoFeatureExtractor; + static uses_processor_config = true; + + /** + * Build the 10-token decoder prompt for the given language. + * @param {string} [language='en'] Language code. + * @returns {number[]} Token IDs for the decoder prompt. + */ + get_decoder_prompt_ids(language = 'en') { + const tokens = [ + '▁', + '<|startofcontext|>', + '<|startoftranscript|>', + '<|emo:undefined|>', + `<|${language}|>`, + `<|${language}|>`, + '<|pnc|>', + '<|noitn|>', + '<|notimestamp|>', + '<|nodiarize|>', + ]; + return this.tokenizer.convert_tokens_to_ids(tokens); + } + + /** + * Join chunk texts back together, using the appropriate separator for the language. + * @param {string[]} texts Decoded texts, one per chunk. + * @param {string} [language='en'] Language code. + * @returns {string} The joined text. + */ + static join_chunks(texts, language = 'en') { + const non_empty = texts.filter((t) => t && t.trim()); + if (non_empty.length === 0) return ''; + const separator = NO_SPACE_LANGUAGES.has(language) ? '' : ' '; + const parts = [non_empty[0].trimEnd(), ...non_empty.slice(1).map((t) => t.trim())]; + return parts.join(separator); + } + + /** + * Calls the feature_extractor function with the given audio input. + * @param {any} audio The audio input to extract features from. + * @returns {Promise} + */ + async _call(audio) { + return await this.feature_extractor(audio); + } +} diff --git a/packages/transformers/src/models/cohere_asr/tokenization_cohere_asr.js b/packages/transformers/src/models/cohere_asr/tokenization_cohere_asr.js new file mode 100644 index 0000000..440b78f --- /dev/null +++ b/packages/transformers/src/models/cohere_asr/tokenization_cohere_asr.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class CohereAsrTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/convbert/modeling_convbert.js b/packages/transformers/src/models/convbert/modeling_convbert.js new file mode 100644 index 0000000..4cd4dbc --- /dev/null +++ b/packages/transformers/src/models/convbert/modeling_convbert.js @@ -0,0 +1,76 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { + MaskedLMOutput, + QuestionAnsweringModelOutput, + SequenceClassifierOutput, + TokenClassifierOutput, +} from '../modeling_outputs.js'; + +export class ConvBertPreTrainedModel extends PreTrainedModel {} + +/** + * The bare ConvBERT Model transformer outputting raw hidden-states without any specific head on top. + */ +export class ConvBertModel extends ConvBertPreTrainedModel {} + +/** + * ConvBERT Model with a language modeling head on top. + */ +export class ConvBertForMaskedLM extends ConvBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for masked language modeling. + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +/** + * ConvBERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) + */ +export class ConvBertForSequenceClassification extends ConvBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * ConvBERT Model with a token classification head on top (a linear layer on top of the hidden-states output) + * e.g. for Named-Entity-Recognition (NER) tasks. + */ +export class ConvBertForTokenClassification extends ConvBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * ConvBERT Model with a span classification head on top for extractive question-answering tasks like SQuAD + * (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`) + */ +export class ConvBertForQuestionAnswering extends ConvBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for question answering. + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/convbert/tokenization_convbert.js b/packages/transformers/src/models/convbert/tokenization_convbert.js new file mode 100644 index 0000000..9d2a516 --- /dev/null +++ b/packages/transformers/src/models/convbert/tokenization_convbert.js @@ -0,0 +1,5 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class ConvBertTokenizer extends PreTrainedTokenizer { + return_token_type_ids = true; +} diff --git a/packages/transformers/src/models/convnext/image_processing_convnext.js b/packages/transformers/src/models/convnext/image_processing_convnext.js new file mode 100644 index 0000000..bcbc78d --- /dev/null +++ b/packages/transformers/src/models/convnext/image_processing_convnext.js @@ -0,0 +1,44 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class ConvNextImageProcessor extends ImageProcessor { + constructor(config) { + super(config); + + /** + * Percentage of the image to crop. Only has an effect if this.size < 384. + */ + // @ts-expect-error TS2339 + this.crop_pct = this.config.crop_pct ?? 224 / 256; + } + + async resize(image) { + const shortest_edge = this.size?.shortest_edge; + if (shortest_edge === undefined) { + throw new Error(`Size dictionary must contain 'shortest_edge' key.`); + } + + if (shortest_edge < 384) { + // maintain same ratio, resizing shortest edge to shortest_edge/crop_pct + const resize_shortest_edge = Math.floor(shortest_edge / this.crop_pct); + + const [newWidth, newHeight] = this.get_resize_output_image_size(image, { + shortest_edge: resize_shortest_edge, + }); + + image = await image.resize(newWidth, newHeight, { + resample: this.resample, + }); + + // then crop to (shortest_edge, shortest_edge) + image = await image.center_crop(shortest_edge, shortest_edge); + } else { + // warping (no cropping) when evaluated at 384 or larger + image = await image.resize(shortest_edge, shortest_edge, { + resample: this.resample, + }); + } + + return image; + } +} +export class ConvNextFeatureExtractor extends ConvNextImageProcessor {} diff --git a/packages/transformers/src/models/convnext/modeling_convnext.js b/packages/transformers/src/models/convnext/modeling_convnext.js new file mode 100644 index 0000000..aefa624 --- /dev/null +++ b/packages/transformers/src/models/convnext/modeling_convnext.js @@ -0,0 +1,21 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class ConvNextPreTrainedModel extends PreTrainedModel {} + +/** + * The bare ConvNext model outputting raw features without any specific head on top. + */ +export class ConvNextModel extends ConvNextPreTrainedModel {} + +/** + * ConvNext Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. + */ +export class ConvNextForImageClassification extends ConvNextPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/convnextv2/modeling_convnextv2.js b/packages/transformers/src/models/convnextv2/modeling_convnextv2.js new file mode 100644 index 0000000..fb46a4f --- /dev/null +++ b/packages/transformers/src/models/convnextv2/modeling_convnextv2.js @@ -0,0 +1,21 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class ConvNextV2PreTrainedModel extends PreTrainedModel {} + +/** + * The bare ConvNextV2 model outputting raw features without any specific head on top. + */ +export class ConvNextV2Model extends ConvNextV2PreTrainedModel {} + +/** + * ConvNextV2 Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. + */ +export class ConvNextV2ForImageClassification extends ConvNextV2PreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/d_fine/modeling_d_fine.js b/packages/transformers/src/models/d_fine/modeling_d_fine.js new file mode 100644 index 0000000..4bf4e62 --- /dev/null +++ b/packages/transformers/src/models/d_fine/modeling_d_fine.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { RTDetrObjectDetectionOutput } from '../rt_detr/modeling_rt_detr.js'; + +export class DFinePreTrainedModel extends PreTrainedModel {} +export class DFineModel extends DFinePreTrainedModel {} +export class DFineForObjectDetection extends DFinePreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new RTDetrObjectDetectionOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/dac/feature_extraction_dac.js b/packages/transformers/src/models/dac/feature_extraction_dac.js new file mode 100644 index 0000000..8932300 --- /dev/null +++ b/packages/transformers/src/models/dac/feature_extraction_dac.js @@ -0,0 +1,3 @@ +import { EncodecFeatureExtractor } from '../encodec/feature_extraction_encodec.js'; + +export class DacFeatureExtractor extends EncodecFeatureExtractor {} diff --git a/packages/transformers/src/models/dac/modeling_dac.js b/packages/transformers/src/models/dac/modeling_dac.js new file mode 100644 index 0000000..c491e26 --- /dev/null +++ b/packages/transformers/src/models/dac/modeling_dac.js @@ -0,0 +1,76 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { sessionRun } from '../session.js'; +import { ModelOutput } from '../modeling_outputs.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class DacEncoderOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.audio_codes Discrete code embeddings, of shape `(batch_size, num_quantizers, codes_length)`. + */ + constructor({ audio_codes }) { + super(); + this.audio_codes = audio_codes; + } +} + +export class DacDecoderOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.audio_values Decoded audio values, of shape `(batch_size, num_channels, sequence_length)`. + */ + constructor({ audio_values }) { + super(); + this.audio_values = audio_values; + } +} + +export class DacPreTrainedModel extends PreTrainedModel { + main_input_name = 'input_values'; + forward_params = ['input_values']; +} + +/** + * The DAC (Descript Audio Codec) model. + */ +export class DacModel extends DacPreTrainedModel { + /** + * Encodes the input audio waveform into discrete codes. + * @param {Object} inputs Model inputs + * @param {Tensor} [inputs.input_values] Float values of the input audio waveform, of shape `(batch_size, channels, sequence_length)`). + * @returns {Promise} The output tensor of shape `(batch_size, num_codebooks, sequence_length)`. + */ + async encode(inputs) { + return new DacEncoderOutput(await sessionRun(this.sessions['encoder_model'], inputs)); + } + + /** + * Decodes the given frames into an output audio waveform. + * @param {DacEncoderOutput} inputs The encoded audio codes. + * @returns {Promise} The output tensor of shape `(batch_size, num_channels, sequence_length)`. + */ + async decode(inputs) { + return new DacDecoderOutput(await sessionRun(this.sessions['decoder_model'], inputs)); + } +} + +export class DacEncoderModel extends DacPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'encoder_model', + }); + } +} +export class DacDecoderModel extends DacPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'decoder_model', + }); + } +} diff --git a/packages/transformers/src/models/deberta/modeling_deberta.js b/packages/transformers/src/models/deberta/modeling_deberta.js new file mode 100644 index 0000000..7d3f2d8 --- /dev/null +++ b/packages/transformers/src/models/deberta/modeling_deberta.js @@ -0,0 +1,75 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { + MaskedLMOutput, + QuestionAnsweringModelOutput, + SequenceClassifierOutput, + TokenClassifierOutput, +} from '../modeling_outputs.js'; + +export class DebertaPreTrainedModel extends PreTrainedModel {} + +/** + * The bare DeBERTa Model transformer outputting raw hidden-states without any specific head on top. + */ +export class DebertaModel extends DebertaPreTrainedModel {} + +/** + * DeBERTa Model with a `language modeling` head on top. + */ +export class DebertaForMaskedLM extends DebertaPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for masked language modeling. + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +/** + * DeBERTa Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) + */ +export class DebertaForSequenceClassification extends DebertaPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * DeBERTa Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. + */ +export class DebertaForTokenClassification extends DebertaPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * DeBERTa Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear + * layers on top of the hidden-states output to compute `span start logits` and `span end logits`). + */ +export class DebertaForQuestionAnswering extends DebertaPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for question answering. + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/deberta/tokenization_deberta.js b/packages/transformers/src/models/deberta/tokenization_deberta.js new file mode 100644 index 0000000..a015035 --- /dev/null +++ b/packages/transformers/src/models/deberta/tokenization_deberta.js @@ -0,0 +1,5 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class DebertaTokenizer extends PreTrainedTokenizer { + return_token_type_ids = true; +} diff --git a/packages/transformers/src/models/deberta_v2/modeling_deberta_v2.js b/packages/transformers/src/models/deberta_v2/modeling_deberta_v2.js new file mode 100644 index 0000000..a797057 --- /dev/null +++ b/packages/transformers/src/models/deberta_v2/modeling_deberta_v2.js @@ -0,0 +1,75 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { + MaskedLMOutput, + QuestionAnsweringModelOutput, + SequenceClassifierOutput, + TokenClassifierOutput, +} from '../modeling_outputs.js'; + +export class DebertaV2PreTrainedModel extends PreTrainedModel {} + +/** + * The bare DeBERTa-V2 Model transformer outputting raw hidden-states without any specific head on top. + */ +export class DebertaV2Model extends DebertaV2PreTrainedModel {} + +/** + * DeBERTa-V2 Model with a `language modeling` head on top. + */ +export class DebertaV2ForMaskedLM extends DebertaV2PreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for masked language modeling. + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +/** + * DeBERTa-V2 Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) + */ +export class DebertaV2ForSequenceClassification extends DebertaV2PreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * DeBERTa-V2 Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. + */ +export class DebertaV2ForTokenClassification extends DebertaV2PreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * DeBERTa-V2 Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear + * layers on top of the hidden-states output to compute `span start logits` and `span end logits`). + */ +export class DebertaV2ForQuestionAnswering extends DebertaV2PreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for question answering. + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/deberta_v2/tokenization_deberta_v2.js b/packages/transformers/src/models/deberta_v2/tokenization_deberta_v2.js new file mode 100644 index 0000000..f23a6c4 --- /dev/null +++ b/packages/transformers/src/models/deberta_v2/tokenization_deberta_v2.js @@ -0,0 +1,5 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class DebertaV2Tokenizer extends PreTrainedTokenizer { + return_token_type_ids = true; +} diff --git a/packages/transformers/src/models/decision_transformer/modeling_decision_transformer.js b/packages/transformers/src/models/decision_transformer/modeling_decision_transformer.js new file mode 100644 index 0000000..8359d5d --- /dev/null +++ b/packages/transformers/src/models/decision_transformer/modeling_decision_transformer.js @@ -0,0 +1,9 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class DecisionTransformerPreTrainedModel extends PreTrainedModel {} + +/** + * The model builds upon the GPT2 architecture to perform autoregressive prediction of actions in an offline RL setting. + * Refer to the paper for more details: https://huggingface.co/papers/2106.01345 + */ +export class DecisionTransformerModel extends DecisionTransformerPreTrainedModel {} diff --git a/packages/transformers/src/models/deepseek_v3/modeling_deepseek_v3.js b/packages/transformers/src/models/deepseek_v3/modeling_deepseek_v3.js new file mode 100644 index 0000000..f4b6bdc --- /dev/null +++ b/packages/transformers/src/models/deepseek_v3/modeling_deepseek_v3.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class DeepseekV3PreTrainedModel extends PreTrainedModel {} +export class DeepseekV3Model extends DeepseekV3PreTrainedModel {} +export class DeepseekV3ForCausalLM extends DeepseekV3PreTrainedModel {} diff --git a/packages/transformers/src/models/deepseek_v4/modeling_deepseek_v4.js b/packages/transformers/src/models/deepseek_v4/modeling_deepseek_v4.js new file mode 100644 index 0000000..2b34ab2 --- /dev/null +++ b/packages/transformers/src/models/deepseek_v4/modeling_deepseek_v4.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class DeepseekV4PreTrainedModel extends PreTrainedModel {} +export class DeepseekV4Model extends DeepseekV4PreTrainedModel {} +export class DeepseekV4ForCausalLM extends DeepseekV4PreTrainedModel {} diff --git a/packages/transformers/src/models/deit/image_processing_deit.js b/packages/transformers/src/models/deit/image_processing_deit.js new file mode 100644 index 0000000..cd39839 --- /dev/null +++ b/packages/transformers/src/models/deit/image_processing_deit.js @@ -0,0 +1,4 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class DeiTImageProcessor extends ImageProcessor {} +export class DeiTFeatureExtractor extends DeiTImageProcessor {} diff --git a/packages/transformers/src/models/deit/modeling_deit.js b/packages/transformers/src/models/deit/modeling_deit.js new file mode 100644 index 0000000..82b62eb --- /dev/null +++ b/packages/transformers/src/models/deit/modeling_deit.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class DeiTPreTrainedModel extends PreTrainedModel {} +export class DeiTModel extends DeiTPreTrainedModel {} +export class DeiTForImageClassification extends DeiTPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/depth_anything/modeling_depth_anything.js b/packages/transformers/src/models/depth_anything/modeling_depth_anything.js new file mode 100644 index 0000000..128dda0 --- /dev/null +++ b/packages/transformers/src/models/depth_anything/modeling_depth_anything.js @@ -0,0 +1,8 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class DepthAnythingPreTrainedModel extends PreTrainedModel {} + +/** + * Depth Anything Model with a depth estimation head on top (consisting of 3 convolutional layers) e.g. for KITTI, NYUv2. + */ +export class DepthAnythingForDepthEstimation extends DepthAnythingPreTrainedModel {} diff --git a/packages/transformers/src/models/depth_pro/modeling_depth_pro.js b/packages/transformers/src/models/depth_pro/modeling_depth_pro.js new file mode 100644 index 0000000..aa85069 --- /dev/null +++ b/packages/transformers/src/models/depth_pro/modeling_depth_pro.js @@ -0,0 +1,4 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class DepthProPreTrainedModel extends PreTrainedModel {} +export class DepthProForDepthEstimation extends DepthProPreTrainedModel {} diff --git a/packages/transformers/src/models/detr/image_processing_detr.js b/packages/transformers/src/models/detr/image_processing_detr.js new file mode 100644 index 0000000..f7bc230 --- /dev/null +++ b/packages/transformers/src/models/detr/image_processing_detr.js @@ -0,0 +1,51 @@ +import { + ImageProcessor, + post_process_object_detection, + post_process_panoptic_segmentation, + post_process_instance_segmentation, +} from '../../image_processors_utils.js'; + +import { full } from '../../utils/tensor.js'; + +/** + * @typedef {Object} DetrFeatureExtractorResultProps + * @property {import('../../utils/tensor.js').Tensor} pixel_mask + * @typedef {import('../../image_processors_utils.js').ImageProcessorResult & DetrFeatureExtractorResultProps} DetrFeatureExtractorResult + */ + +export class DetrImageProcessor extends ImageProcessor { + /** + * Calls the feature extraction process on an array of images, preprocesses + * each image, and concatenates the resulting features into a single Tensor. + * @param {import('../../utils/image.js').RawImage[]} images The image(s) to extract features from. + * @returns {Promise} An object containing the concatenated pixel values of the preprocessed images. + */ + async _call(images) { + const result = await super._call(images); + + // TODO support differently-sized images, for now assume all images are the same size. + // TODO support different mask sizes (not just 64x64) + // Currently, just fill pixel mask with 1s + const maskSize = [result.pixel_values.dims[0], 64, 64]; + const pixel_mask = full(maskSize, 1n); + + return { ...result, pixel_mask }; + } + + /** @type {typeof post_process_object_detection} */ + post_process_object_detection(...args) { + return post_process_object_detection(...args); + } + + /** @type {typeof post_process_panoptic_segmentation} */ + post_process_panoptic_segmentation(...args) { + return post_process_panoptic_segmentation(...args); + } + + /** @type {typeof post_process_instance_segmentation} */ + post_process_instance_segmentation(...args) { + return post_process_instance_segmentation(...args); + } +} + +export class DetrFeatureExtractor extends DetrImageProcessor {} // NOTE: extends DetrImageProcessor diff --git a/packages/transformers/src/models/detr/modeling_detr.js b/packages/transformers/src/models/detr/modeling_detr.js new file mode 100644 index 0000000..23c720d --- /dev/null +++ b/packages/transformers/src/models/detr/modeling_detr.js @@ -0,0 +1,54 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { ModelOutput } from '../modeling_outputs.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class DetrPreTrainedModel extends PreTrainedModel {} +export class DetrModel extends DetrPreTrainedModel {} +export class DetrForObjectDetection extends DetrPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new DetrObjectDetectionOutput(await super._call(model_inputs)); + } +} + +export class DetrForSegmentation extends DetrPreTrainedModel { + /** + * Runs the model with the provided inputs + * @param {Object} model_inputs Model inputs + * @returns {Promise} Object containing segmentation outputs + */ + async _call(model_inputs) { + return new DetrSegmentationOutput(await super._call(model_inputs)); + } +} + +export class DetrObjectDetectionOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.logits Classification logits (including no-object) for all queries. + * @param {Tensor} output.pred_boxes Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). + * These values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding possible padding). + */ + constructor({ logits, pred_boxes }) { + super(); + this.logits = logits; + this.pred_boxes = pred_boxes; + } +} + +export class DetrSegmentationOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.logits The output logits of the model. + * @param {Tensor} output.pred_boxes Predicted boxes. + * @param {Tensor} output.pred_masks Predicted masks. + */ + constructor({ logits, pred_boxes, pred_masks }) { + super(); + this.logits = logits; + this.pred_boxes = pred_boxes; + this.pred_masks = pred_masks; + } +} diff --git a/packages/transformers/src/models/dinov2/modeling_dinov2.js b/packages/transformers/src/models/dinov2/modeling_dinov2.js new file mode 100644 index 0000000..dacce35 --- /dev/null +++ b/packages/transformers/src/models/dinov2/modeling_dinov2.js @@ -0,0 +1,21 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class Dinov2PreTrainedModel extends PreTrainedModel {} + +/** + * The bare DINOv2 Model transformer outputting raw hidden-states without any specific head on top. + */ +export class Dinov2Model extends Dinov2PreTrainedModel {} + +/** + * Dinov2 Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. + */ +export class Dinov2ForImageClassification extends Dinov2PreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/dinov2_with_registers/modeling_dinov2_with_registers.js b/packages/transformers/src/models/dinov2_with_registers/modeling_dinov2_with_registers.js new file mode 100644 index 0000000..81b0e1b --- /dev/null +++ b/packages/transformers/src/models/dinov2_with_registers/modeling_dinov2_with_registers.js @@ -0,0 +1,21 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class Dinov2WithRegistersPreTrainedModel extends PreTrainedModel {} + +/** + * The bare Dinov2WithRegisters Model transformer outputting raw hidden-states without any specific head on top. + */ +export class Dinov2WithRegistersModel extends Dinov2WithRegistersPreTrainedModel {} + +/** + * Dinov2WithRegisters Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. + */ +export class Dinov2WithRegistersForImageClassification extends Dinov2WithRegistersPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/dinov3_convnext/modeling_dinov3_convnext.js b/packages/transformers/src/models/dinov3_convnext/modeling_dinov3_convnext.js new file mode 100644 index 0000000..4408487 --- /dev/null +++ b/packages/transformers/src/models/dinov3_convnext/modeling_dinov3_convnext.js @@ -0,0 +1,4 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class DINOv3ConvNextPreTrainedModel extends PreTrainedModel {} +export class DINOv3ConvNextModel extends DINOv3ConvNextPreTrainedModel {} diff --git a/packages/transformers/src/models/dinov3_vit/image_processing_dinov3_vit.js b/packages/transformers/src/models/dinov3_vit/image_processing_dinov3_vit.js new file mode 100644 index 0000000..bd7edc6 --- /dev/null +++ b/packages/transformers/src/models/dinov3_vit/image_processing_dinov3_vit.js @@ -0,0 +1,3 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class DINOv3ViTImageProcessor extends ImageProcessor {} diff --git a/packages/transformers/src/models/dinov3_vit/modeling_dinov3_vit.js b/packages/transformers/src/models/dinov3_vit/modeling_dinov3_vit.js new file mode 100644 index 0000000..d9a0697 --- /dev/null +++ b/packages/transformers/src/models/dinov3_vit/modeling_dinov3_vit.js @@ -0,0 +1,4 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class DINOv3ViTPreTrainedModel extends PreTrainedModel {} +export class DINOv3ViTModel extends DINOv3ViTPreTrainedModel {} diff --git a/packages/transformers/src/models/distilbert/modeling_distilbert.js b/packages/transformers/src/models/distilbert/modeling_distilbert.js new file mode 100644 index 0000000..e4c57b5 --- /dev/null +++ b/packages/transformers/src/models/distilbert/modeling_distilbert.js @@ -0,0 +1,70 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { + MaskedLMOutput, + QuestionAnsweringModelOutput, + SequenceClassifierOutput, + TokenClassifierOutput, +} from '../modeling_outputs.js'; + +export class DistilBertPreTrainedModel extends PreTrainedModel {} +export class DistilBertModel extends DistilBertPreTrainedModel {} + +/** + * DistilBertForSequenceClassification is a class representing a DistilBERT model for sequence classification. + */ +export class DistilBertForSequenceClassification extends DistilBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * DistilBertForTokenClassification is a class representing a DistilBERT model for token classification. + */ +export class DistilBertForTokenClassification extends DistilBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * DistilBertForQuestionAnswering is a class representing a DistilBERT model for question answering. + */ +export class DistilBertForQuestionAnswering extends DistilBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for question answering. + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} + +/** + * DistilBertForMaskedLM is a class representing a DistilBERT model for masking task. + */ +export class DistilBertForMaskedLM extends DistilBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/distilbert/tokenization_distilbert.js b/packages/transformers/src/models/distilbert/tokenization_distilbert.js new file mode 100644 index 0000000..6df621f --- /dev/null +++ b/packages/transformers/src/models/distilbert/tokenization_distilbert.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class DistilBertTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/donut/image_processing_donut.js b/packages/transformers/src/models/donut/image_processing_donut.js new file mode 100644 index 0000000..79f6602 --- /dev/null +++ b/packages/transformers/src/models/donut/image_processing_donut.js @@ -0,0 +1,29 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class DonutImageProcessor extends ImageProcessor { + pad_image(pixelData, imgDims, padSize, options = {}) { + const [imageHeight, imageWidth, imageChannels] = imgDims; + + let image_mean = this.image_mean; + if (!Array.isArray(this.image_mean)) { + image_mean = new Array(imageChannels).fill(image_mean); + } + + let image_std = this.image_std; + if (!Array.isArray(image_std)) { + image_std = new Array(imageChannels).fill(image_mean); + } + + const constant_values = image_mean.map((x, i) => -x / image_std[i]); + + return super.pad_image(pixelData, imgDims, padSize, { + center: true, + + // Since normalization is done after padding, we need to use certain constant values to ensure the same behaviour is observed. + // For more information, see https://github.com/huggingface/transformers/blob/main/src/transformers/models/donut/image_processing_donut.py#L433-L451 + constant_values, + ...options, + }); + } +} +export class DonutFeatureExtractor extends DonutImageProcessor {} diff --git a/packages/transformers/src/models/donut_swin/modeling_donut_swin.js b/packages/transformers/src/models/donut_swin/modeling_donut_swin.js new file mode 100644 index 0000000..b8cdad0 --- /dev/null +++ b/packages/transformers/src/models/donut_swin/modeling_donut_swin.js @@ -0,0 +1,79 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class DonutSwinPreTrainedModel extends PreTrainedModel {} + +/** + * The bare Donut Swin Model transformer outputting raw hidden-states without any specific head on top. + * + * **Example:** Step-by-step Document Parsing. + * + * ```javascript + * import { AutoProcessor, AutoTokenizer, AutoModelForVision2Seq, RawImage } from '@huggingface/transformers'; + * + * // Choose model to use + * const model_id = 'Xenova/donut-base-finetuned-cord-v2'; + * + * // Prepare image inputs + * const processor = await AutoProcessor.from_pretrained(model_id); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/receipt.png'; + * const image = await RawImage.read(url); + * const image_inputs = await processor(image); + * + * // Prepare decoder inputs + * const tokenizer = await AutoTokenizer.from_pretrained(model_id); + * const task_prompt = ''; + * const decoder_input_ids = tokenizer(task_prompt, { + * add_special_tokens: false, + * }).input_ids; + * + * // Create the model + * const model = await AutoModelForVision2Seq.from_pretrained(model_id); + * + * // Run inference + * const output = await model.generate(image_inputs.pixel_values, { + * decoder_input_ids, + * max_length: model.config.decoder.max_position_embeddings, + * }); + * + * // Decode output + * const decoded = tokenizer.batch_decode(output)[0]; + * // CINNAMON SUGAR 17,000 1 x 17,000 17,000 17,000 20,000 3,000 + * ``` + * + * **Example:** Step-by-step Document Visual Question Answering (DocVQA) + * + * ```javascript + * import { AutoProcessor, AutoTokenizer, AutoModelForVision2Seq, RawImage } from '@huggingface/transformers'; + * + * // Choose model to use + * const model_id = 'Xenova/donut-base-finetuned-docvqa'; + * + * // Prepare image inputs + * const processor = await AutoProcessor.from_pretrained(model_id); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/invoice.png'; + * const image = await RawImage.read(url); + * const image_inputs = await processor(image); + * + * // Prepare decoder inputs + * const tokenizer = await AutoTokenizer.from_pretrained(model_id); + * const question = 'What is the invoice number?'; + * const task_prompt = `${question}`; + * const decoder_input_ids = tokenizer(task_prompt, { + * add_special_tokens: false, + * }).input_ids; + * + * // Create the model + * const model = await AutoModelForVision2Seq.from_pretrained(model_id); + * + * // Run inference + * const output = await model.generate(image_inputs.pixel_values, { + * decoder_input_ids, + * max_length: model.config.decoder.max_position_embeddings, + * }); + * + * // Decode output + * const decoded = tokenizer.batch_decode(output)[0]; + * // What is the invoice number? us-001 + * ``` + */ +export class DonutSwinModel extends DonutSwinPreTrainedModel {} diff --git a/packages/transformers/src/models/dpt/image_processing_dpt.js b/packages/transformers/src/models/dpt/image_processing_dpt.js new file mode 100644 index 0000000..d30c6bf --- /dev/null +++ b/packages/transformers/src/models/dpt/image_processing_dpt.js @@ -0,0 +1,4 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class DPTImageProcessor extends ImageProcessor {} +export class DPTFeatureExtractor extends DPTImageProcessor {} // NOTE: extends DPTImageProcessor diff --git a/packages/transformers/src/models/dpt/modeling_dpt.js b/packages/transformers/src/models/dpt/modeling_dpt.js new file mode 100644 index 0000000..76770fe --- /dev/null +++ b/packages/transformers/src/models/dpt/modeling_dpt.js @@ -0,0 +1,51 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class DPTPreTrainedModel extends PreTrainedModel {} + +/** + * The bare DPT Model transformer outputting raw hidden-states without any specific head on top. + */ +export class DPTModel extends DPTPreTrainedModel {} + +/** + * DPT Model with a depth estimation head on top (consisting of 3 convolutional layers) e.g. for KITTI, NYUv2. + * + * **Example:** Depth estimation w/ `Xenova/dpt-hybrid-midas`. + * ```javascript + * import { DPTForDepthEstimation, AutoProcessor, RawImage, interpolate_4d } from '@huggingface/transformers'; + * + * // Load model and processor + * const model_id = 'Xenova/dpt-hybrid-midas'; + * const model = await DPTForDepthEstimation.from_pretrained(model_id); + * const processor = await AutoProcessor.from_pretrained(model_id); + * + * // Load image from URL + * const url = 'http://images.cocodataset.org/val2017/000000039769.jpg'; + * const image = await RawImage.read(url); + * + * // Prepare image for the model + * const inputs = await processor(image); + * + * // Run model + * const { predicted_depth } = await model(inputs); + * + * // Interpolate to original size + * const prediction = (await interpolate_4d(predicted_depth.unsqueeze(1), { + * size: image.size.reverse(), + * mode: 'bilinear', + * })).squeeze(1); + * + * // Visualize the prediction + * const min = prediction.min().item(); + * const max = prediction.max().item(); + * const formatted = prediction.sub_(min).div_(max - min).mul_(255).to('uint8'); + * const depth = RawImage.fromTensor(formatted); + * // RawImage { + * // data: Uint8Array(307200) [ 85, 85, 84, ... ], + * // width: 640, + * // height: 480, + * // channels: 1 + * // } + * ``` + */ +export class DPTForDepthEstimation extends DPTPreTrainedModel {} diff --git a/packages/transformers/src/models/efficientnet/image_processing_efficientnet.js b/packages/transformers/src/models/efficientnet/image_processing_efficientnet.js new file mode 100644 index 0000000..a85f837 --- /dev/null +++ b/packages/transformers/src/models/efficientnet/image_processing_efficientnet.js @@ -0,0 +1,12 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class EfficientNetImageProcessor extends ImageProcessor { + constructor(config) { + super(config); + // @ts-expect-error TS2339 + this.include_top = this.config.include_top ?? true; + if (this.include_top) { + this.image_std = this.image_std.map((x) => x * x); + } + } +} diff --git a/packages/transformers/src/models/efficientnet/modeling_efficientnet.js b/packages/transformers/src/models/efficientnet/modeling_efficientnet.js new file mode 100644 index 0000000..799dc6f --- /dev/null +++ b/packages/transformers/src/models/efficientnet/modeling_efficientnet.js @@ -0,0 +1,21 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class EfficientNetPreTrainedModel extends PreTrainedModel {} + +/** + * The bare EfficientNet model outputting raw features without any specific head on top. + */ +export class EfficientNetModel extends EfficientNetPreTrainedModel {} + +/** + * EfficientNet Model with an image classification head on top (a linear layer on top of the pooled features). + */ +export class EfficientNetForImageClassification extends EfficientNetPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/electra/modeling_electra.js b/packages/transformers/src/models/electra/modeling_electra.js new file mode 100644 index 0000000..5c7aca9 --- /dev/null +++ b/packages/transformers/src/models/electra/modeling_electra.js @@ -0,0 +1,77 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { + MaskedLMOutput, + QuestionAnsweringModelOutput, + SequenceClassifierOutput, + TokenClassifierOutput, +} from '../modeling_outputs.js'; + +export class ElectraPreTrainedModel extends PreTrainedModel {} + +/** + * The bare Electra Model transformer outputting raw hidden-states without any specific head on top. + * Identical to the BERT model except that it uses an additional linear layer between the embedding + * layer and the encoder if the hidden size and embedding size are different. + */ +export class ElectraModel extends ElectraPreTrainedModel {} +// TODO add ElectraForPreTraining +/** + * Electra model with a language modeling head on top. + */ +export class ElectraForMaskedLM extends ElectraPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for masked language modeling. + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +/** + * ELECTRA Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) + */ +export class ElectraForSequenceClassification extends ElectraPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * Electra model with a token classification head on top. + */ +export class ElectraForTokenClassification extends ElectraPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * LECTRA Model with a span classification head on top for extractive question-answering tasks like SQuAD + * (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). + */ +export class ElectraForQuestionAnswering extends ElectraPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for question answering. + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/electra/tokenization_electra.js b/packages/transformers/src/models/electra/tokenization_electra.js new file mode 100644 index 0000000..845f5c8 --- /dev/null +++ b/packages/transformers/src/models/electra/tokenization_electra.js @@ -0,0 +1,5 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class ElectraTokenizer extends PreTrainedTokenizer { + return_token_type_ids = true; +} diff --git a/packages/transformers/src/models/encodec/feature_extraction_encodec.js b/packages/transformers/src/models/encodec/feature_extraction_encodec.js new file mode 100644 index 0000000..0849831 --- /dev/null +++ b/packages/transformers/src/models/encodec/feature_extraction_encodec.js @@ -0,0 +1,33 @@ +import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class EncodecFeatureExtractor extends FeatureExtractor { + /** + * Asynchronously extracts input values from a given audio using the provided configuration. + * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array. + * @returns {Promise<{ input_values: Tensor; }>} The extracted input values. + */ + async _call(audio) { + validate_audio_inputs(audio, 'EncodecFeatureExtractor'); + + if (audio instanceof Float64Array) { + audio = new Float32Array(audio); + } + + const num_channels = this.config.feature_size; + if (audio.length % num_channels !== 0) { + throw new Error( + `The length of the audio data must be a multiple of the number of channels (${num_channels}).`, + ); + } + + const shape = [ + 1 /* batch_size */, + num_channels /* num_channels */, + audio.length / num_channels /* num_samples */, + ]; + return { + input_values: new Tensor('float32', audio, shape), + }; + } +} diff --git a/packages/transformers/src/models/ernie4_5/modeling_ernie4_5.js b/packages/transformers/src/models/ernie4_5/modeling_ernie4_5.js new file mode 100644 index 0000000..9f3e7f1 --- /dev/null +++ b/packages/transformers/src/models/ernie4_5/modeling_ernie4_5.js @@ -0,0 +1,7 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Ernie4_5PretrainedModel extends PreTrainedModel {} + +export class Ernie4_5Model extends Ernie4_5PretrainedModel {} + +export class Ernie4_5ForCausalLM extends Ernie4_5PretrainedModel {} diff --git a/packages/transformers/src/models/esm/modeling_esm.js b/packages/transformers/src/models/esm/modeling_esm.js new file mode 100644 index 0000000..01dfcc5 --- /dev/null +++ b/packages/transformers/src/models/esm/modeling_esm.js @@ -0,0 +1,55 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { MaskedLMOutput, SequenceClassifierOutput, TokenClassifierOutput } from '../modeling_outputs.js'; + +export class EsmPreTrainedModel extends PreTrainedModel {} + +/** + * The bare ESM Model transformer outputting raw hidden-states without any specific head on top. + */ +export class EsmModel extends EsmPreTrainedModel {} + +/** + * ESM Model with a `language modeling` head on top. + */ +export class EsmForMaskedLM extends EsmPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for masked language modeling. + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +/** + * ESM Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) + */ +export class EsmForSequenceClassification extends EsmPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * ESM Model with a token classification head on top (a linear layer on top of the hidden-states output) + * e.g. for Named-Entity-Recognition (NER) tasks. + */ +export class EsmForTokenClassification extends EsmPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/esm/tokenization_esm.js b/packages/transformers/src/models/esm/tokenization_esm.js new file mode 100644 index 0000000..1fc0072 --- /dev/null +++ b/packages/transformers/src/models/esm/tokenization_esm.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class EsmTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/eurobert/modeling_eurobert.js b/packages/transformers/src/models/eurobert/modeling_eurobert.js new file mode 100644 index 0000000..83da962 --- /dev/null +++ b/packages/transformers/src/models/eurobert/modeling_eurobert.js @@ -0,0 +1,41 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { MaskedLMOutput, SequenceClassifierOutput, TokenClassifierOutput } from '../modeling_outputs.js'; + +export class EuroBertPreTrainedModel extends PreTrainedModel {} +export class EuroBertModel extends EuroBertPreTrainedModel {} + +export class EuroBertForMaskedLM extends EuroBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for masked language modeling. + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +export class EuroBertForSequenceClassification extends EuroBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +export class EuroBertForTokenClassification extends EuroBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/exaone/modeling_exaone.js b/packages/transformers/src/models/exaone/modeling_exaone.js new file mode 100644 index 0000000..3bb46df --- /dev/null +++ b/packages/transformers/src/models/exaone/modeling_exaone.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class ExaonePreTrainedModel extends PreTrainedModel {} +export class ExaoneModel extends ExaonePreTrainedModel {} +export class ExaoneForCausalLM extends ExaonePreTrainedModel {} diff --git a/packages/transformers/src/models/falcon/modeling_falcon.js b/packages/transformers/src/models/falcon/modeling_falcon.js new file mode 100644 index 0000000..d7562ce --- /dev/null +++ b/packages/transformers/src/models/falcon/modeling_falcon.js @@ -0,0 +1,10 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +/** + * The bare Falcon Model outputting raw hidden-states without any specific head on top. + */ +export class FalconPreTrainedModel extends PreTrainedModel {} + +export class FalconModel extends FalconPreTrainedModel {} + +export class FalconForCausalLM extends FalconPreTrainedModel {} diff --git a/packages/transformers/src/models/falcon/tokenization_falcon.js b/packages/transformers/src/models/falcon/tokenization_falcon.js new file mode 100644 index 0000000..1b91f72 --- /dev/null +++ b/packages/transformers/src/models/falcon/tokenization_falcon.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class FalconTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/falcon_h1/modeling_falcon_h1.js b/packages/transformers/src/models/falcon_h1/modeling_falcon_h1.js new file mode 100644 index 0000000..74cb479 --- /dev/null +++ b/packages/transformers/src/models/falcon_h1/modeling_falcon_h1.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class FalconH1PreTrainedModel extends PreTrainedModel {} +export class FalconH1Model extends FalconH1PreTrainedModel {} +export class FalconH1ForCausalLM extends FalconH1PreTrainedModel {} diff --git a/packages/transformers/src/models/fastvit/modeling_fastvit.js b/packages/transformers/src/models/fastvit/modeling_fastvit.js new file mode 100644 index 0000000..60d093c --- /dev/null +++ b/packages/transformers/src/models/fastvit/modeling_fastvit.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class FastViTPreTrainedModel extends PreTrainedModel {} +export class FastViTModel extends FastViTPreTrainedModel {} +export class FastViTForImageClassification extends FastViTPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/feature_extractors.js b/packages/transformers/src/models/feature_extractors.js new file mode 100644 index 0000000..bc1a7b1 --- /dev/null +++ b/packages/transformers/src/models/feature_extractors.js @@ -0,0 +1,21 @@ +export * from './audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.js'; +export * from './encodec/feature_extraction_encodec.js'; +export * from './chatterbox/feature_extraction_chatterbox.js'; +export * from './clap/feature_extraction_clap.js'; +export * from './cohere_asr/feature_extraction_cohere_asr.js'; +export * from './dac/feature_extraction_dac.js'; +export * from './gemma3n/feature_extraction_gemma3n.js'; +export * from './gemma4/feature_extraction_gemma4.js'; +export * from './granite_speech/feature_extraction_granite_speech.js'; +export * from './moonshine/feature_extraction_moonshine.js'; +export * from './parakeet/feature_extraction_parakeet.js'; +export * from './pyannote/feature_extraction_pyannote.js'; +export * from './seamless_m4t/feature_extraction_seamless_m4t.js'; +export * from './snac/feature_extraction_snac.js'; +export * from './speecht5/feature_extraction_speecht5.js'; +export * from './wav2vec2/feature_extraction_wav2vec2.js'; +export * from './wespeaker/feature_extraction_wespeaker.js'; +export * from './voxtral_realtime/feature_extraction_voxtral_realtime.js'; +export * from './whisper/feature_extraction_whisper.js'; + +export { FeatureExtractor } from '../feature_extraction_utils.js'; diff --git a/packages/transformers/src/models/florence2/modeling_florence2.js b/packages/transformers/src/models/florence2/modeling_florence2.js new file mode 100644 index 0000000..0b483e4 --- /dev/null +++ b/packages/transformers/src/models/florence2/modeling_florence2.js @@ -0,0 +1,116 @@ +import { PreTrainedModel, encoder_forward, decoder_forward } from '../modeling_utils.js'; +import { cat, ones } from '../../utils/tensor.js'; + +export class Florence2PreTrainedModel extends PreTrainedModel { + forward_params = [ + // Encoder inputs + 'input_ids', + 'inputs_embeds', + 'attention_mask', + 'pixel_values', + + // Decoder inputs + 'encoder_outputs', + 'decoder_input_ids', + 'decoder_inputs_embeds', + 'decoder_attention_mask', + 'past_key_values', + ]; + main_input_name = 'inputs_embeds'; +} + +export class Florence2ForConditionalGeneration extends Florence2PreTrainedModel { + _merge_input_ids_with_image_features({ inputs_embeds, image_features, input_ids, attention_mask }) { + return { + inputs_embeds: cat( + [ + image_features, // image embeds + inputs_embeds, // task prefix embeds + ], + 1, + ), + attention_mask: cat( + [ + ones(image_features.dims.slice(0, 2)), // image attention mask + attention_mask, // task prefix attention mask + ], + 1, + ), + }; + } + + async _prepare_inputs_embeds({ input_ids, pixel_values, inputs_embeds, attention_mask }) { + if (!input_ids && !pixel_values) { + throw new Error('Either `input_ids` or `pixel_values` should be provided.'); + } + + // 1. Possibly, extract the input embeddings + let text_features, image_features; + if (input_ids) { + text_features = await this.encode_text({ input_ids }); + } + if (pixel_values) { + image_features = await this.encode_image({ pixel_values }); + } + + // 2. Possibly, merge text and images + if (text_features && image_features) { + ({ inputs_embeds, attention_mask } = this._merge_input_ids_with_image_features({ + inputs_embeds: text_features, + image_features, + input_ids, + attention_mask, + })); + } else { + inputs_embeds = text_features || image_features; + } + + return { inputs_embeds, attention_mask }; + } + + async forward({ + input_ids, + pixel_values, + attention_mask, + decoder_input_ids, + decoder_attention_mask, + encoder_outputs, + past_key_values, + + inputs_embeds, + decoder_inputs_embeds, + num_logits_to_keep = null, + }) { + if (!inputs_embeds) { + ({ inputs_embeds, attention_mask } = await this._prepare_inputs_embeds({ + input_ids, + pixel_values, + inputs_embeds, + attention_mask, + })); + } + + if (!encoder_outputs) { + // Must compute encoder outputs + let { last_hidden_state } = await encoder_forward(this, { inputs_embeds, attention_mask }); + encoder_outputs = last_hidden_state; + } + + if (!decoder_inputs_embeds) { + if (!decoder_input_ids) { + throw new Error('Either `decoder_input_ids` or `decoder_inputs_embeds` should be provided.'); + } + decoder_inputs_embeds = await this.encode_text({ input_ids: decoder_input_ids }); + } + + const decoderFeeds = { + inputs_embeds: decoder_inputs_embeds, + attention_mask: decoder_attention_mask, + encoder_attention_mask: attention_mask, + encoder_hidden_states: encoder_outputs, + past_key_values, + num_logits_to_keep, + }; + return await decoder_forward(this, decoderFeeds, true); + } +} diff --git a/packages/transformers/src/models/florence2/processing_florence2.js b/packages/transformers/src/models/florence2/processing_florence2.js new file mode 100644 index 0000000..8574cf3 --- /dev/null +++ b/packages/transformers/src/models/florence2/processing_florence2.js @@ -0,0 +1,134 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; + +export class Florence2Processor extends Processor { + static tokenizer_class = AutoTokenizer; + static image_processor_class = AutoImageProcessor; + + constructor(config, components, chat_template) { + super(config, components, chat_template); + + const { + // @ts-expect-error TS2339 + tasks_answer_post_processing_type, + // @ts-expect-error TS2339 + task_prompts_without_inputs, + // @ts-expect-error TS2339 + task_prompts_with_input, + } = this.image_processor.config; + + /** @type {Map} */ + this.tasks_answer_post_processing_type = new Map(Object.entries(tasks_answer_post_processing_type ?? {})); + + /** @type {Map} */ + this.task_prompts_without_inputs = new Map(Object.entries(task_prompts_without_inputs ?? {})); + + /** @type {Map} */ + this.task_prompts_with_input = new Map(Object.entries(task_prompts_with_input ?? {})); + + this.regexes = { + quad_boxes: + /(.+?)/gm, + bboxes: /([^<]+)?/gm, + }; + this.size_per_bin = 1000; + } + + /** + * Helper function to construct prompts from input texts + * @param {string|string[]} text + * @returns {string[]} + */ + construct_prompts(text) { + if (typeof text === 'string') { + text = [text]; + } + + const prompts = []; + for (const t of text) { + // 1. fixed task prompts without additional inputs + if (this.task_prompts_without_inputs.has(t)) { + prompts.push(this.task_prompts_without_inputs.get(t)); + } + // 2. task prompts with additional inputs + else { + for (const [task, prompt] of this.task_prompts_with_input) { + if (t.includes(task)) { + prompts.push(prompt.replaceAll('{input}', t).replaceAll(task, '')); + break; + } + } + + // 3. default prompt + if (prompts.length !== text.length) { + prompts.push(t); + } + } + } + return prompts; + } + + /** + * Post-process the output of the model to each of the task outputs. + * @param {string} text The text to post-process. + * @param {string} task The task to post-process the text for. + * @param {[number, number]} image_size The size of the image. height x width. + */ + post_process_generation(text, task, image_size) { + const task_answer_post_processing_type = this.tasks_answer_post_processing_type.get(task) ?? 'pure_text'; + + // remove the special tokens + text = text.replaceAll('', '').replaceAll('', ''); + + let final_answer; + switch (task_answer_post_processing_type) { + case 'pure_text': + final_answer = text; + break; + + case 'description_with_bboxes': + case 'bboxes': + case 'phrase_grounding': + case 'ocr': + const key = task_answer_post_processing_type === 'ocr' ? 'quad_boxes' : 'bboxes'; + const matches = text.matchAll(this.regexes[key]); + const labels = []; + const items = []; + for (const [_, label, ...locations] of matches) { + // Push new label, or duplicate the last label + labels.push(label ? label.trim() : (labels.at(-1) ?? '')); + items.push( + locations.map( + (x, i) => + // NOTE: Add 0.5 to use the center position of the bin as the coordinate. + ((Number(x) + 0.5) / this.size_per_bin) * image_size[i % 2], + ), + ); + } + final_answer = { labels, [key]: items }; + break; + + default: + throw new Error(`Task "${task}" (of type "${task_answer_post_processing_type}") not yet implemented.`); + } + + return { [task]: final_answer }; + } + + // NOTE: images and text are switched from the python version + // `images` is required, `text` is optional + async _call(images, text = null, kwargs = {}) { + if (!images && !text) { + throw new Error('Either text or images must be provided'); + } + + const image_inputs = await this.image_processor(images, kwargs); + const text_inputs = text ? this.tokenizer(this.construct_prompts(text), kwargs) : {}; + + return { + ...image_inputs, + ...text_inputs, + }; + } +} diff --git a/packages/transformers/src/models/gemma/modeling_gemma.js b/packages/transformers/src/models/gemma/modeling_gemma.js new file mode 100644 index 0000000..d525f7f --- /dev/null +++ b/packages/transformers/src/models/gemma/modeling_gemma.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +/** + * The bare Gemma Model outputting raw hidden-states without any specific head on top. + */ +export class GemmaPreTrainedModel extends PreTrainedModel {} + +/** + * The bare Gemma Model outputting raw hidden-states without any specific head on top. + */ +export class GemmaModel extends GemmaPreTrainedModel {} + +export class GemmaForCausalLM extends GemmaPreTrainedModel {} diff --git a/packages/transformers/src/models/gemma/tokenization_gemma.js b/packages/transformers/src/models/gemma/tokenization_gemma.js new file mode 100644 index 0000000..4a71158 --- /dev/null +++ b/packages/transformers/src/models/gemma/tokenization_gemma.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class GemmaTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/gemma2/modeling_gemma2.js b/packages/transformers/src/models/gemma2/modeling_gemma2.js new file mode 100644 index 0000000..352eb78 --- /dev/null +++ b/packages/transformers/src/models/gemma2/modeling_gemma2.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +/** + * The bare Gemma2 Model outputting raw hidden-states without any specific head on top. + */ +export class Gemma2PreTrainedModel extends PreTrainedModel {} + +/** + * The bare Gemma2 Model outputting raw hidden-states without any specific head on top. + */ +export class Gemma2Model extends Gemma2PreTrainedModel {} + +export class Gemma2ForCausalLM extends Gemma2PreTrainedModel {} diff --git a/packages/transformers/src/models/gemma3/image_processing_gemma3.js b/packages/transformers/src/models/gemma3/image_processing_gemma3.js new file mode 100644 index 0000000..44b0685 --- /dev/null +++ b/packages/transformers/src/models/gemma3/image_processing_gemma3.js @@ -0,0 +1,3 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class Gemma3ImageProcessor extends ImageProcessor {} diff --git a/packages/transformers/src/models/gemma3/modeling_gemma3.js b/packages/transformers/src/models/gemma3/modeling_gemma3.js new file mode 100644 index 0000000..98db2c5 --- /dev/null +++ b/packages/transformers/src/models/gemma3/modeling_gemma3.js @@ -0,0 +1,16 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { LlavaForConditionalGeneration } from '../llava/modeling_llava.js'; + +/** + * The bare Gemma3 Model outputting raw hidden-states without any specific head on top. + */ +export class Gemma3PreTrainedModel extends PreTrainedModel {} + +/** + * The bare Gemma3 Model outputting raw hidden-states without any specific head on top. + */ +export class Gemma3Model extends Gemma3PreTrainedModel {} + +export class Gemma3ForConditionalGeneration extends LlavaForConditionalGeneration {} + +export class Gemma3ForCausalLM extends Gemma3ForConditionalGeneration {} diff --git a/packages/transformers/src/models/gemma3/processing_gemma3.js b/packages/transformers/src/models/gemma3/processing_gemma3.js new file mode 100644 index 0000000..3ec523e --- /dev/null +++ b/packages/transformers/src/models/gemma3/processing_gemma3.js @@ -0,0 +1,45 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; + +export class Gemma3Processor extends Processor { + static tokenizer_class = AutoTokenizer; + static image_processor_class = AutoImageProcessor; + static uses_processor_config = true; + static uses_chat_template_file = true; + + constructor(config, components, chat_template) { + super(config, components, chat_template); + this.image_seq_length = this.config.image_seq_length; + + const { boi_token, image_token, eoi_token } = this.tokenizer.config; + this.boi_token = boi_token; + this.image_token = image_token; + this.eoi_token = eoi_token; + const image_tokens_expanded = image_token.repeat(this.image_seq_length); + this.full_image_sequence = `\n\n${boi_token}${image_tokens_expanded}${eoi_token}\n\n`; + } + + /** + * @param {string|string[]} text + * @param {import('../../utils/image.js').RawImage|import('../../utils/image.js').RawImage[]} [images] + * @param {Object} [options] + */ + async _call(text, images = null, options = {}) { + if (typeof text === 'string') { + text = [text]; + } + + let image_inputs; + if (images) { + image_inputs = await this.image_processor(images, options); + text = text.map((prompt) => prompt.replaceAll(this.boi_token, this.full_image_sequence)); + } + + const text_inputs = this.tokenizer(text, options); + return { + ...text_inputs, + ...image_inputs, + }; + } +} diff --git a/packages/transformers/src/models/gemma3n/feature_extraction_gemma3n.js b/packages/transformers/src/models/gemma3n/feature_extraction_gemma3n.js new file mode 100644 index 0000000..e090975 --- /dev/null +++ b/packages/transformers/src/models/gemma3n/feature_extraction_gemma3n.js @@ -0,0 +1,89 @@ +import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { full, Tensor } from '../../utils/tensor.js'; +import { mel_filter_bank, spectrogram, window_function } from '../../utils/audio.js'; + +export class Gemma3nAudioFeatureExtractor extends FeatureExtractor { + constructor(config) { + super(config); + + const { fft_length, feature_size, min_frequency, max_frequency, sampling_rate, frame_length } = this.config; + + const mel_filters = mel_filter_bank( + Math.floor(1 + fft_length / 2), // num_frequency_bins + feature_size, // num_mel_filters + min_frequency, // min_frequency + max_frequency, // max_frequency + sampling_rate, // sampling_rate + null, // norm + 'htk', // mel_scale + false, // triangularize_in_mel_space + ); + this.mel_filters = mel_filters; + + this.window = window_function(frame_length, 'hann'); + } + + /** + * Computes the log-Mel spectrogram of the provided audio waveform. + * @param {Float32Array|Float64Array} waveform The audio waveform to process. + * @param {number} max_length The maximum number of frames to return. + * @returns {Promise} An object containing the log-Mel spectrogram data as a Float32Array and its dimensions as an array of numbers. + */ + async _extract_fbank_features(waveform, max_length) { + // NOTE: We don't pad/truncate since that is passed in as `max_num_frames` + return spectrogram( + waveform, + this.window, // window + this.config.frame_length, // frame_length + this.config.hop_length, // hop_length + { + fft_length: this.config.fft_length, + center: false, + onesided: true, + preemphasis: this.config.preemphasis, + preemphasis_htk_flavor: this.config.preemphasis_htk_flavor, + mel_filters: this.mel_filters, + log_mel: 'log', + mel_floor: this.config.mel_floor, + remove_dc_offset: false, + + // Custom + transpose: true, + }, + ); + } + + /** + * Asynchronously extracts features from a given audio using the provided configuration. + * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array. + * @param {Object} options Optional parameters for feature extraction. + * @param {number} [options.max_length=480_000] If provided, defines the maximum length of the audio to allow. + * Audio longer than this will be truncated if `truncation=True`. + * @param {boolean} [options.truncation=true] Whether or not to truncate audio above `max_length`. + * @param {boolean} [options.padding=true] Whether to pad the sequence to a multiple of `pad_to_multiple_of`. + * @param {number} [options.pad_to_multiple_of=128] The number to pad the sequence to a multiple of. + * @returns {Promise<{ input_features: Tensor, input_features_mask: Tensor }>} A Promise resolving to an object containing the extracted input features and attention masks as Tensors. + */ + async _call(audio, { max_length = 480_000, truncation = true, padding = true, pad_to_multiple_of = 128 } = {}) { + validate_audio_inputs(audio, 'Gemma3nAudioFeatureExtractor'); + if (truncation && audio.length > max_length) { + audio = audio.slice(0, max_length); + } + if (padding && audio.length % pad_to_multiple_of !== 0) { + const padding_length = pad_to_multiple_of - (audio.length % pad_to_multiple_of); + const padded_audio = new Float64Array(audio.length + padding_length); + padded_audio.set(audio); + if (this.config.padding_value !== 0) { + padded_audio.fill(this.config.padding_value, audio.length); + } + audio = padded_audio; + } + + const features = await this._extract_fbank_features(audio, this.config.max_length); + const padded_attention_mask = full([1, features.dims[0]], true); + return { + input_features: features.unsqueeze_(0), + input_features_mask: padded_attention_mask, + }; + } +} diff --git a/packages/transformers/src/models/gemma3n/modeling_gemma3n.js b/packages/transformers/src/models/gemma3n/modeling_gemma3n.js new file mode 100644 index 0000000..9a02553 --- /dev/null +++ b/packages/transformers/src/models/gemma3n/modeling_gemma3n.js @@ -0,0 +1,122 @@ +import { + PreTrainedModel, + decoder_forward, + default_merge_input_ids_with_image_features, + default_merge_input_ids_with_audio_features, +} from '../modeling_utils.js'; +import { sessionRun } from '../session.js'; + +export class Gemma3nPreTrainedModel extends PreTrainedModel { + forward_params = [ + 'input_ids', + 'attention_mask', + 'inputs_embeds', + 'per_layer_inputs', + + 'position_ids', + 'pixel_values', + 'input_features', + 'input_features_mask', + 'past_key_values', + ]; +} +export class Gemma3nForConditionalGeneration extends Gemma3nPreTrainedModel { + async forward({ + // Produced by the tokenizer/processor: + input_ids = null, + attention_mask = null, + pixel_values = null, + input_features = null, + input_features_mask = null, + + // Used during generation: + position_ids = null, + inputs_embeds = null, + per_layer_inputs = null, + past_key_values = null, + + // Generic generation parameters + generation_config = null, + logits_processor = null, + num_logits_to_keep = null, + + // TODO: needed? + ...kwargs + }) { + if (!inputs_embeds || !per_layer_inputs) { + // 1. Extract the text embeddings. + ({ inputs_embeds, per_layer_inputs } = await sessionRun(this.sessions['embed_tokens'], { + input_ids, + })); + if (input_ids.dims[1] !== 1) { + if (pixel_values) { + const { image_features } = await this._encode_vision({ pixel_values, ...kwargs }); + ({ inputs_embeds, attention_mask } = this._merge_input_ids_with_image_features({ + image_features, + inputs_embeds, + input_ids, + attention_mask, + })); + } + + if (input_features) { + // Encode the audio + const { audio_features } = await sessionRun(this.sessions['audio_encoder'], { + input_features, + input_features_mask, + }); + ({ inputs_embeds, attention_mask } = this._merge_input_ids_with_audio_features({ + audio_features, + inputs_embeds, + input_ids, + attention_mask, + })); + } + } + } + + const outputs = await decoder_forward( + this, + { + inputs_embeds, + per_layer_inputs, + past_key_values, + attention_mask, + position_ids, + generation_config, + logits_processor, + num_logits_to_keep, + }, + true, + ); + return outputs; + } + + _encode_vision(kwargs) { + return sessionRun(this.sessions['vision_encoder'], { pixel_values: kwargs.pixel_values }); + } + + _merge_input_ids_with_image_features(kwargs) { + const vision_hidden_size = kwargs.image_features.dims.at(-1); + const reshaped_image_hidden_states = kwargs.image_features.view(-1, vision_hidden_size); + return default_merge_input_ids_with_image_features({ + // @ts-ignore + image_token_id: this.config.image_token_id, + ...kwargs, + image_features: reshaped_image_hidden_states, + }); + } + _merge_input_ids_with_audio_features(kwargs) { + const audio_hidden_size = kwargs.audio_features.dims.at(-1); + const reshaped_audio_features = kwargs.audio_features.view(-1, audio_hidden_size); + + return default_merge_input_ids_with_audio_features({ + // @ts-ignore + audio_token_id: this.config.audio_token_id, + ...kwargs, + audio_features: reshaped_audio_features, + }); + } +} + +export class Gemma3nForCausalLM extends Gemma3nForConditionalGeneration {} diff --git a/packages/transformers/src/models/gemma3n/processing_gemma3n.js b/packages/transformers/src/models/gemma3n/processing_gemma3n.js new file mode 100644 index 0000000..6b43e9d --- /dev/null +++ b/packages/transformers/src/models/gemma3n/processing_gemma3n.js @@ -0,0 +1,78 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; +import { AutoFeatureExtractor } from '../auto/feature_extraction_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { RawImage } from '../../utils/image.js'; +import { RawAudio } from '../../utils/audio.js'; + +export class Gemma3nProcessor extends Processor { + static image_processor_class = AutoImageProcessor; + static feature_extractor_class = AutoFeatureExtractor; + static tokenizer_class = AutoTokenizer; + static uses_processor_config = true; + static uses_chat_template_file = true; + + constructor(config, components, chat_template) { + super(config, components, chat_template); + this.audio_seq_length = this.config.audio_seq_length; + this.image_seq_length = this.config.image_seq_length; + + const { + // Audio tokens + audio_token_id, + boa_token, + audio_token, + eoa_token, + + // Image tokens + image_token_id, + boi_token, + image_token, + eoi_token, + } = this.tokenizer.config; + + this.audio_token_id = audio_token_id; + this.boa_token = boa_token; + this.audio_token = audio_token; + const audio_tokens_expanded = audio_token.repeat(this.audio_seq_length); + this.full_audio_sequence = `\n\n${boa_token}${audio_tokens_expanded}${eoa_token}\n\n`; + + this.image_token_id = image_token_id; + this.boi_token = boi_token; + this.image_token = image_token; + const image_tokens_expanded = image_token.repeat(this.image_seq_length); + this.full_image_sequence = `\n\n${boi_token}${image_tokens_expanded}${eoi_token}\n\n`; + } + + /** + * + * @param {string|string[]} text + * @param {RawImage|RawImage[]|RawImage[][]} images + * @param {RawAudio|RawAudio[]|RawAudio[][]} audio + * @returns {Promise} + */ + async _call(text, images = null, audio = null, options = {}) { + if (typeof text === 'string') { + text = [text]; + } + + let audio_inputs; + if (audio) { + audio_inputs = await this.feature_extractor(audio, options); + + text = text.map((prompt) => prompt.replaceAll(this.audio_token, this.full_audio_sequence)); + } + let image_inputs; + if (images) { + image_inputs = await this.image_processor(images, options); + text = text.map((prompt) => prompt.replaceAll(this.image_token, this.full_image_sequence)); + } + + let text_inputs = this.tokenizer(text, options); + return { + ...text_inputs, + ...image_inputs, + ...audio_inputs, + }; + } +} diff --git a/packages/transformers/src/models/gemma4/feature_extraction_gemma4.js b/packages/transformers/src/models/gemma4/feature_extraction_gemma4.js new file mode 100644 index 0000000..0df14b9 --- /dev/null +++ b/packages/transformers/src/models/gemma4/feature_extraction_gemma4.js @@ -0,0 +1,74 @@ +import { validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { Tensor } from '../../utils/tensor.js'; +import { spectrogram } from '../../utils/audio.js'; +import { Gemma3nAudioFeatureExtractor } from '../gemma3n/feature_extraction_gemma3n.js'; + +export class Gemma4AudioFeatureExtractor extends Gemma3nAudioFeatureExtractor { + /** + * @override + * Gemma4 uses semicausal padding, unfold(frame_length+1) framing, and + * additive mel_floor — all controlled via flags on the shared spectrogram(). + */ + async _extract_fbank_features(waveform, max_length) { + const { frame_length, hop_length, fft_length } = this.config; + + // Compute frame count matching Python's unfold(size=frame_length+1, step=hop_length) + const pad_left = Math.floor(frame_length / 2); + const num_frames = Math.floor((waveform.length + pad_left - (frame_length + 1)) / hop_length) + 1; + + return spectrogram(waveform, this.window, frame_length, hop_length, { + fft_length, + center: true, + pad_mode: 'semicausal', + onesided: true, + preemphasis: this.config.preemphasis, + preemphasis_htk_flavor: this.config.preemphasis_htk_flavor, + mel_filters: this.mel_filters, + log_mel: 'log', + mel_floor: this.config.mel_floor, + mel_floor_mode: 'add', + remove_dc_offset: false, + transpose: true, + max_num_frames: num_frames, + }); + } + + /** + * @override + * Wraps the base class result with a frame-aware attention mask + * and zeros out features for invalid (padded) frames. + */ + async _call(audio, options = {}) { + validate_audio_inputs(audio, 'Gemma4AudioFeatureExtractor'); + + const original_length = audio.length; + const result = await super._call(audio, options); + + const { input_features } = result; + const [, num_frames, num_features] = input_features.dims; + + // Build frame-aware mask: a frame is valid only when all its samples are real audio. + const { frame_length, hop_length } = this.config; + const pad_left = Math.floor(frame_length / 2); + const frame_size_for_unfold = frame_length + 1; + + const sample_mask = new Uint8Array(original_length + pad_left + (options.pad_to_multiple_of ?? 128)); + sample_mask.fill(1, pad_left, pad_left + original_length); + + const frame_mask = new Uint8Array(num_frames); + for (let i = 0; i < num_frames; ++i) { + frame_mask[i] = sample_mask[i * hop_length + frame_size_for_unfold - 1] ? 1 : 0; + } + + // Zero out features for invalid frames (matching Python's speech * mask[..., None]) + const feat_data = /** @type {Float32Array} */ (input_features.data); + for (let i = 0; i < num_frames; ++i) { + if (!frame_mask[i]) { + feat_data.fill(0, i * num_features, (i + 1) * num_features); + } + } + + result.input_features_mask = new Tensor('bool', frame_mask, [1, num_frames]); + return result; + } +} diff --git a/packages/transformers/src/models/gemma4/image_processing_gemma4.js b/packages/transformers/src/models/gemma4/image_processing_gemma4.js new file mode 100644 index 0000000..cfbd6ca --- /dev/null +++ b/packages/transformers/src/models/gemma4/image_processing_gemma4.js @@ -0,0 +1,178 @@ +import { Callable } from '../../utils/generic.js'; +import { Tensor, stack } from '../../utils/tensor.js'; +import { RawImage } from '../../utils/image.js'; + +/** + * Compute the target size for aspect-ratio-preserving resize. + * + * @param {number} height Original image height. + * @param {number} width Original image width. + * @param {number} patch_size Size of each patch in pixels. + * @param {number} max_patches Maximum number of patches allowed. + * @param {number} pooling_kernel_size Spatial pooling kernel size. + * @returns {[number, number]} Target [height, width]. + */ +function get_aspect_ratio_preserving_size(height, width, patch_size, max_patches, pooling_kernel_size) { + const target_px = max_patches * patch_size ** 2; + const factor = Math.sqrt(target_px / (height * width)); + const side_mult = pooling_kernel_size * patch_size; + + let target_height = Math.floor((factor * height) / side_mult) * side_mult; + let target_width = Math.floor((factor * width) / side_mult) * side_mult; + + if (target_height === 0 && target_width === 0) { + throw new Error( + `Attempting to resize to a 0 x 0 image. Resized height should be divisible by ` + + `\`pooling_kernel_size * patch_size\`=${side_mult}.`, + ); + } + + const max_side_length = Math.floor(max_patches / pooling_kernel_size ** 2) * side_mult; + if (target_height === 0) { + target_height = side_mult; + target_width = Math.min(Math.floor(width / height) * side_mult, max_side_length); + } else if (target_width === 0) { + target_width = side_mult; + target_height = Math.min(Math.floor(height / width) * side_mult, max_side_length); + } + + return [target_height, target_width]; +} + +/** + * Convert HWC image data to patches and position IDs, then pad. + * + * Patchification produces the same layout as Python's: + * CHW.reshape(C, pH, ps, pW, ps).transpose(1, 3, 2, 4, 0).reshape(pH*pW, ps*ps*C) + * The transpose yields (pH, pW, ps_h, ps_w, C), so within each patch the + * flattened order is (dy, dx, c) — which is exactly HWC order. + * This lets us read directly from HWC source data without a CHW conversion. + * + * @param {Float32Array} hwc_data Image pixel data in HWC layout. + * @param {number} H Image height. + * @param {number} W Image width. + * @param {number} C Number of channels. + * @param {number} patch_size Patch size in pixels. + * @param {number} max_patches Maximum patches (for padding). + * @param {number} pooling_kernel_size Pooling kernel size (for soft token count). + * @returns {{ patches: Tensor, positions: Tensor, num_soft_tokens: number }} + */ +function patchify(hwc_data, H, W, C, patch_size, max_patches, pooling_kernel_size) { + const num_patches_h = Math.floor(H / patch_size); + const num_patches_w = Math.floor(W / patch_size); + const num_patches = num_patches_h * num_patches_w; + const patch_dim = patch_size * patch_size * C; + + // Patchify: iterate (pH, pW, dy, dx, c) — reads HWC in order + const patch_data = new Float32Array(max_patches * patch_dim); // zero-padded + let out = 0; + for (let ph = 0; ph < num_patches_h; ++ph) { + for (let pw = 0; pw < num_patches_w; ++pw) { + for (let dy = 0; dy < patch_size; ++dy) { + const row_offset = (ph * patch_size + dy) * W * C + pw * patch_size * C; + for (let dx = 0; dx < patch_size; ++dx) { + const src = row_offset + dx * C; + for (let c = 0; c < C; ++c) { + patch_data[out++] = hwc_data[src + c]; + } + } + } + } + } + + // Position IDs: meshgrid(arange(W), arange(H), indexing="xy") → [col, row] + const pos_data = new BigInt64Array(max_patches * 2).fill(-1n); // -1 padding + let idx = 0; + for (let row = 0; row < num_patches_h; ++row) { + for (let col = 0; col < num_patches_w; ++col) { + pos_data[idx++] = BigInt(col); + pos_data[idx++] = BigInt(row); + } + } + + return { + patches: new Tensor('float32', patch_data, [max_patches, patch_dim]), + positions: new Tensor('int64', pos_data, [max_patches, 2]), + num_soft_tokens: Math.floor(num_patches / pooling_kernel_size ** 2), + }; +} + +export class Gemma4ImageProcessor extends Callable { + /** @param {Record} config */ + constructor(config) { + super(); + this.config = config; + this.patch_size = config.patch_size ?? 16; + this.max_soft_tokens = config.max_soft_tokens ?? 280; + this.pooling_kernel_size = config.pooling_kernel_size ?? 3; + this.resample = config.resample ?? 3; // bicubic + this.rescale_factor = config.rescale_factor ?? 1 / 255; + this.do_rescale = config.do_rescale ?? true; + this.do_resize = config.do_resize ?? true; + this.do_convert_rgb = config.do_convert_rgb ?? true; + } + + /** + * @param {RawImage|RawImage[]} images + * @returns {Promise<{ pixel_values: Tensor, image_position_ids: Tensor, num_soft_tokens_per_image: number[] }>} + */ + async _call(images) { + if (!Array.isArray(images)) { + images = [images]; + } + + const { patch_size, pooling_kernel_size } = this; + const max_patches = this.max_soft_tokens * pooling_kernel_size ** 2; + + const all_patches = []; + const all_positions = []; + const num_soft_tokens_per_image = []; + + for (let image of images) { + if (this.do_convert_rgb) { + image = image.rgb(); + } + + if (this.do_resize) { + const [target_h, target_w] = get_aspect_ratio_preserving_size( + image.height, + image.width, + patch_size, + max_patches, + pooling_kernel_size, + ); + if (target_h !== image.height || target_w !== image.width) { + image = await image.resize(target_w, target_h, { resample: this.resample }); + } + } + + // Rescale in-place + const pixelData = Float32Array.from(image.data); + if (this.do_rescale) { + for (let i = 0; i < pixelData.length; ++i) { + pixelData[i] *= this.rescale_factor; + } + } + + const { patches, positions, num_soft_tokens } = patchify( + pixelData, + image.height, + image.width, + image.channels, + patch_size, + max_patches, + pooling_kernel_size, + ); + + all_patches.push(patches); + all_positions.push(positions); + num_soft_tokens_per_image.push(num_soft_tokens); + } + + return { + pixel_values: stack(all_patches, 0), + image_position_ids: stack(all_positions, 0), + num_soft_tokens_per_image, + }; + } +} diff --git a/packages/transformers/src/models/gemma4/modeling_gemma4.js b/packages/transformers/src/models/gemma4/modeling_gemma4.js new file mode 100644 index 0000000..be3a84c --- /dev/null +++ b/packages/transformers/src/models/gemma4/modeling_gemma4.js @@ -0,0 +1,27 @@ +import { Gemma3nForConditionalGeneration } from '../gemma3n/modeling_gemma3n.js'; +import { sessionRun } from '../session.js'; + +export class Gemma4ForConditionalGeneration extends Gemma3nForConditionalGeneration { + forward_params = [ + 'input_ids', + 'attention_mask', + 'inputs_embeds', + 'per_layer_inputs', + + 'position_ids', + 'pixel_values', + 'image_position_ids', + 'input_features', + 'input_features_mask', + 'past_key_values', + ]; + + _encode_vision(kwargs) { + return sessionRun(this.sessions['vision_encoder'], { + pixel_values: kwargs.pixel_values, + pixel_position_ids: kwargs.image_position_ids, + }); + } +} + +export class Gemma4ForCausalLM extends Gemma4ForConditionalGeneration {} diff --git a/packages/transformers/src/models/gemma4/processing_gemma4.js b/packages/transformers/src/models/gemma4/processing_gemma4.js new file mode 100644 index 0000000..6b58778 --- /dev/null +++ b/packages/transformers/src/models/gemma4/processing_gemma4.js @@ -0,0 +1,108 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { Gemma4ImageProcessor } from './image_processing_gemma4.js'; +import { Gemma4AudioFeatureExtractor } from './feature_extraction_gemma4.js'; +import { PROCESSOR_NAME, CHAT_TEMPLATE_NAME } from '../../utils/constants.js'; +import { getModelJSON, getModelText } from '../../utils/hub.js'; + +export class Gemma4Processor extends Processor { + static uses_processor_config = true; + static uses_chat_template_file = true; + + constructor(config, components, chat_template) { + super(config, components, chat_template); + + this.audio_ms_per_token = this.config.audio_ms_per_token ?? 40; + this.audio_seq_length = this.config.audio_seq_length ?? 750; + this.image_seq_length = this.config.image_seq_length ?? 280; + + const { audio_token, boa_token, eoa_token, image_token, boi_token, eoi_token } = this.tokenizer.config; + this.audio_token = audio_token; + this.boa_token = boa_token; + this.eoa_token = eoa_token; + this.image_token = image_token; + this.boi_token = boi_token; + this.eoi_token = eoi_token; + } + + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + const [config, tokenizer, chat_template] = await Promise.all([ + getModelJSON(pretrained_model_name_or_path, PROCESSOR_NAME, true, options), + AutoTokenizer.from_pretrained(pretrained_model_name_or_path, options), + getModelText(pretrained_model_name_or_path, CHAT_TEMPLATE_NAME, false, options), + ]); + + const components = { tokenizer }; + if (config.image_processor) { + components.image_processor = new Gemma4ImageProcessor(config.image_processor); + } + if (config.feature_extractor) { + components.feature_extractor = new Gemma4AudioFeatureExtractor(config.feature_extractor); + } + + return new this(config, components, chat_template); + } + + /** + * Compute the number of audio soft tokens for a single waveform. + * Replicates the audio encoder's sequence-length arithmetic: + * mel framing → two SSCP conv layers (kernel=3, stride=2) → cap. + * @param {number} num_samples + * @param {number} sampling_rate + * @returns {number} + */ + _compute_audio_num_tokens(num_samples, sampling_rate) { + const frame_length = Math.round((sampling_rate * 20.0) / 1000.0); + const hop_length = Math.round((sampling_rate * 10.0) / 1000.0); + + const pad_left = Math.floor(frame_length / 2); + let t = Math.floor((num_samples + pad_left - frame_length - 1) / hop_length) + 1; + if (t <= 0) return 0; + + // Two SSCP conv layers: kernel=3, stride=2, pad=1+1 + for (let i = 0; i < 2; ++i) t = Math.floor((t - 1) / 2) + 1; + + return Math.min(t, this.audio_seq_length); + } + + async _call(text, images = null, audio = null, options = {}) { + if (typeof text === 'string') { + text = [text]; + } + + let image_inputs; + if (images) { + image_inputs = await this.image_processor(images, options); + const counts = image_inputs.num_soft_tokens_per_image; + let i = 0; + text = text.map((t) => + t.replaceAll( + this.image_token, + () => `\n\n${this.boi_token}${this.image_token.repeat(counts[i++])}${this.eoi_token}\n\n`, + ), + ); + } + + let audio_inputs; + if (audio) { + const audio_array = Array.isArray(audio) ? audio : [audio]; + audio_inputs = await this.feature_extractor(audio_array[0], options); + + const sampling_rate = this.feature_extractor.config.sampling_rate ?? 16000; + let i = 0; + text = text.map((t) => + t.replaceAll( + this.audio_token, + () => + `\n\n${this.boa_token}${this.audio_token.repeat(this._compute_audio_num_tokens(audio_array[i++].length, sampling_rate))}${this.eoa_token}\n\n`, + ), + ); + } + + return { + ...this.tokenizer(text, options), + ...image_inputs, + ...audio_inputs, + }; + } +} diff --git a/packages/transformers/src/models/glm/modeling_glm.js b/packages/transformers/src/models/glm/modeling_glm.js new file mode 100644 index 0000000..771ddfc --- /dev/null +++ b/packages/transformers/src/models/glm/modeling_glm.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class GlmPreTrainedModel extends PreTrainedModel {} +export class GlmModel extends GlmPreTrainedModel {} +export class GlmForCausalLM extends GlmPreTrainedModel {} diff --git a/packages/transformers/src/models/glm46v/image_processing_glm46v.js b/packages/transformers/src/models/glm46v/image_processing_glm46v.js new file mode 100644 index 0000000..be1c9de --- /dev/null +++ b/packages/transformers/src/models/glm46v/image_processing_glm46v.js @@ -0,0 +1,12 @@ +import { Qwen2VLImageProcessor } from '../qwen2_vl/image_processing_qwen2_vl.js'; +import { smart_resize } from '../../image_processors_utils.js'; + +export class Glm46VImageProcessor extends Qwen2VLImageProcessor { + /** @type {Qwen2VLImageProcessor['get_resize_output_image_size']} */ + get_resize_output_image_size(image, size) { + const factor = this.patch_size * this.merge_size; + // @ts-expect-error ts(2339) + const temporal_factor = this.config.temporal_patch_size ?? 2; + return smart_resize(image.height, image.width, factor, this.min_pixels, this.max_pixels, temporal_factor); + } +} diff --git a/packages/transformers/src/models/glm46v/processing_glm46v.js b/packages/transformers/src/models/glm46v/processing_glm46v.js new file mode 100644 index 0000000..50199ac --- /dev/null +++ b/packages/transformers/src/models/glm46v/processing_glm46v.js @@ -0,0 +1,5 @@ +import { Qwen2VLProcessor } from '../qwen2_vl/processing_qwen2_vl.js'; + +export class Glm46VProcessor extends Qwen2VLProcessor { + static image_token = '<|image|>'; +} diff --git a/packages/transformers/src/models/glm_moe_dsa/modeling_glm_moe_dsa.js b/packages/transformers/src/models/glm_moe_dsa/modeling_glm_moe_dsa.js new file mode 100644 index 0000000..1aa0681 --- /dev/null +++ b/packages/transformers/src/models/glm_moe_dsa/modeling_glm_moe_dsa.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class GlmMoeDsaPreTrainedModel extends PreTrainedModel {} +export class GlmMoeDsaModel extends GlmMoeDsaPreTrainedModel {} +export class GlmMoeDsaForCausalLM extends GlmMoeDsaPreTrainedModel {} diff --git a/packages/transformers/src/models/glm_ocr/modeling_glm_ocr.js b/packages/transformers/src/models/glm_ocr/modeling_glm_ocr.js new file mode 100644 index 0000000..0d1a262 --- /dev/null +++ b/packages/transformers/src/models/glm_ocr/modeling_glm_ocr.js @@ -0,0 +1,78 @@ +import { Qwen2_5_VLForConditionalGeneration } from '../qwen2_5_vl/modeling_qwen2_5_vl.js'; + +export class GlmOcrForConditionalGeneration extends Qwen2_5_VLForConditionalGeneration { + /** + * Compute 3D positional indices for vision tokens. + * Temporal is constant, height is repeat-interleaved, width tiles. + * @param {number} start_position + * @param {number[]} grid_thw [T, H, W] + * @param {number} temp_merge_size + * @param {number} spatial_merge_size + * @returns {number[]} Flat array of length 3 * seq_len: [temporal..., height..., width...] + */ + get_vision_position_ids(start_position, grid_thw, temp_merge_size, spatial_merge_size) { + const llm_grid_t = Math.floor(grid_thw[0] / temp_merge_size); + const llm_grid_h = Math.floor(grid_thw[1] / spatial_merge_size); + const llm_grid_w = Math.floor(grid_thw[2] / spatial_merge_size); + const seq_len = llm_grid_h * llm_grid_w * llm_grid_t; + + const t_pos = Array.from({ length: seq_len }, () => start_position); + const h_pos = Array.from( + { length: seq_len }, + (_, i) => start_position + Math.floor(i / (llm_grid_w * llm_grid_t)), + ); + const w_pos = Array.from({ length: seq_len }, (_, i) => start_position + (i % llm_grid_w)); + + return [...t_pos, ...h_pos, ...w_pos]; + } + + /** + * GlmOcr uses mm_token_type_ids-style grouping (image tokens identified by image_token_id) + * instead of vision_start_token_id scanning used by Qwen2VL. + * After a vision segment, position advances by max(h, w) / spatial_merge_size. + */ + _get_multimodal_rope_positions({ + filtered_ids, + image_grid_thw_list, + video_grid_thw_list, + spatial_merge_size, + state, + }) { + // @ts-ignore + const { image_token_id } = this.config; + + // Build modality groups: 0=text, 1=image (by image_token_id) + const groups = []; + let group_start = 0; + let current_type = filtered_ids[0] == image_token_id ? 1 : 0; + for (let j = 1; j <= filtered_ids.length; ++j) { + const t = j < filtered_ids.length ? (filtered_ids[j] == image_token_id ? 1 : 0) : -1; + if (t !== current_type) { + groups.push([current_type, group_start, j]); + group_start = j; + current_type = t; + } + } + + let current_pos = 0; + /** @type {number[][]} */ + const llm_pos_ids_list = []; + + for (const [modality_type, start_idx, end_idx] of groups) { + if (modality_type === 0) { + const text_len = end_idx - start_idx; + llm_pos_ids_list.push(Array.from({ length: 3 * text_len }, (_, i) => current_pos + (i % text_len))); + current_pos += text_len; + } else { + const grid_thw = image_grid_thw_list[state.image_index++].map(Number); + const temp_merge_size = grid_thw[0]; + llm_pos_ids_list.push( + this.get_vision_position_ids(current_pos, grid_thw, temp_merge_size, spatial_merge_size), + ); + current_pos += Math.max(grid_thw[1], grid_thw[2]) / spatial_merge_size; + } + } + + return llm_pos_ids_list; + } +} diff --git a/packages/transformers/src/models/glpn/image_processing_glpn.js b/packages/transformers/src/models/glpn/image_processing_glpn.js new file mode 100644 index 0000000..7f86a6e --- /dev/null +++ b/packages/transformers/src/models/glpn/image_processing_glpn.js @@ -0,0 +1,3 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class GLPNFeatureExtractor extends ImageProcessor {} diff --git a/packages/transformers/src/models/glpn/modeling_glpn.js b/packages/transformers/src/models/glpn/modeling_glpn.js new file mode 100644 index 0000000..2849bd3 --- /dev/null +++ b/packages/transformers/src/models/glpn/modeling_glpn.js @@ -0,0 +1,47 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class GLPNPreTrainedModel extends PreTrainedModel {} + +/** + * The bare GLPN encoder (Mix-Transformer) outputting raw hidden-states without any specific head on top. + */ +export class GLPNModel extends GLPNPreTrainedModel {} + +/** + * import { GLPNForDepthEstimation, AutoProcessor, RawImage, interpolate_4d } from '@huggingface/transformers'; + * + * // Load model and processor + * const model_id = 'Xenova/glpn-kitti'; + * const model = await GLPNForDepthEstimation.from_pretrained(model_id); + * const processor = await AutoProcessor.from_pretrained(model_id); + * + * // Load image from URL + * const url = 'http://images.cocodataset.org/val2017/000000039769.jpg'; + * const image = await RawImage.read(url); + * + * // Prepare image for the model + * const inputs = await processor(image); + * + * // Run model + * const { predicted_depth } = await model(inputs); + * + * // Interpolate to original size + * const prediction = (await interpolate_4d(predicted_depth.unsqueeze(1), { + * size: image.size.reverse(), + * mode: 'bilinear', + * })).squeeze(1); + * + * // Visualize the prediction + * const min = prediction.min().item(); + * const max = prediction.max().item(); + * const formatted = prediction.sub_(min).div_(max - min).mul_(255).to('uint8'); + * const depth = RawImage.fromTensor(formatted); + * // RawImage { + * // data: Uint8Array(307200) [ 85, 85, 84, ... ], + * // width: 640, + * // height: 480, + * // channels: 1 + * // } + * ``` + */ +export class GLPNForDepthEstimation extends GLPNPreTrainedModel {} diff --git a/packages/transformers/src/models/gpt2/modeling_gpt2.js b/packages/transformers/src/models/gpt2/modeling_gpt2.js new file mode 100644 index 0000000..19e62ae --- /dev/null +++ b/packages/transformers/src/models/gpt2/modeling_gpt2.js @@ -0,0 +1,12 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class GPT2PreTrainedModel extends PreTrainedModel {} + +export class GPT2Model extends GPT2PreTrainedModel {} + +/** + * GPT-2 language model head on top of the GPT-2 base model. This model is suitable for text generation tasks. + */ +export class GPT2LMHeadModel extends GPT2PreTrainedModel {} +// export class GPT2ForSequenceClassification extends GPT2PreTrainedModel { +// TODO diff --git a/packages/transformers/src/models/gpt2/tokenization_gpt2.js b/packages/transformers/src/models/gpt2/tokenization_gpt2.js new file mode 100644 index 0000000..b145b69 --- /dev/null +++ b/packages/transformers/src/models/gpt2/tokenization_gpt2.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class GPT2Tokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/gpt_bigcode/modeling_gpt_bigcode.js b/packages/transformers/src/models/gpt_bigcode/modeling_gpt_bigcode.js new file mode 100644 index 0000000..42f0a3a --- /dev/null +++ b/packages/transformers/src/models/gpt_bigcode/modeling_gpt_bigcode.js @@ -0,0 +1,6 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class GPTBigCodePreTrainedModel extends PreTrainedModel {} +export class GPTBigCodeModel extends GPTBigCodePreTrainedModel {} + +export class GPTBigCodeForCausalLM extends GPTBigCodePreTrainedModel {} diff --git a/packages/transformers/src/models/gpt_neo/modeling_gpt_neo.js b/packages/transformers/src/models/gpt_neo/modeling_gpt_neo.js new file mode 100644 index 0000000..24efb90 --- /dev/null +++ b/packages/transformers/src/models/gpt_neo/modeling_gpt_neo.js @@ -0,0 +1,6 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class GPTNeoPreTrainedModel extends PreTrainedModel {} +export class GPTNeoModel extends GPTNeoPreTrainedModel {} + +export class GPTNeoForCausalLM extends GPTNeoPreTrainedModel {} diff --git a/packages/transformers/src/models/gpt_neox/modeling_gpt_neox.js b/packages/transformers/src/models/gpt_neox/modeling_gpt_neox.js new file mode 100644 index 0000000..df148f5 --- /dev/null +++ b/packages/transformers/src/models/gpt_neox/modeling_gpt_neox.js @@ -0,0 +1,6 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class GPTNeoXPreTrainedModel extends PreTrainedModel {} +export class GPTNeoXModel extends GPTNeoXPreTrainedModel {} + +export class GPTNeoXForCausalLM extends GPTNeoXPreTrainedModel {} diff --git a/packages/transformers/src/models/gpt_neox/tokenization_gpt_neox.js b/packages/transformers/src/models/gpt_neox/tokenization_gpt_neox.js new file mode 100644 index 0000000..84d82ff --- /dev/null +++ b/packages/transformers/src/models/gpt_neox/tokenization_gpt_neox.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class GPTNeoXTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/gpt_oss/modeling_gpt_oss.js b/packages/transformers/src/models/gpt_oss/modeling_gpt_oss.js new file mode 100644 index 0000000..4aea7cf --- /dev/null +++ b/packages/transformers/src/models/gpt_oss/modeling_gpt_oss.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class GptOssPreTrainedModel extends PreTrainedModel {} +export class GptOssModel extends GptOssPreTrainedModel {} +export class GptOssForCausalLM extends GptOssPreTrainedModel {} diff --git a/packages/transformers/src/models/gptj/modeling_gptj.js b/packages/transformers/src/models/gptj/modeling_gptj.js new file mode 100644 index 0000000..c4150cf --- /dev/null +++ b/packages/transformers/src/models/gptj/modeling_gptj.js @@ -0,0 +1,6 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class GPTJPreTrainedModel extends PreTrainedModel {} +export class GPTJModel extends GPTJPreTrainedModel {} + +export class GPTJForCausalLM extends GPTJPreTrainedModel {} diff --git a/packages/transformers/src/models/granite/modeling_granite.js b/packages/transformers/src/models/granite/modeling_granite.js new file mode 100644 index 0000000..8b52d83 --- /dev/null +++ b/packages/transformers/src/models/granite/modeling_granite.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class GranitePreTrainedModel extends PreTrainedModel {} +export class GraniteModel extends GranitePreTrainedModel {} +export class GraniteForCausalLM extends GranitePreTrainedModel {} diff --git a/packages/transformers/src/models/granite_speech/feature_extraction_granite_speech.js b/packages/transformers/src/models/granite_speech/feature_extraction_granite_speech.js new file mode 100644 index 0000000..117dcf0 --- /dev/null +++ b/packages/transformers/src/models/granite_speech/feature_extraction_granite_speech.js @@ -0,0 +1,58 @@ +import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { mel_filter_bank, spectrogram, window_function } from '../../utils/audio.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class GraniteSpeechFeatureExtractor extends FeatureExtractor { + constructor(config) { + super(config); + + const { n_fft, win_length, n_mels, sample_rate } = config.melspec_kwargs; + + // torchaudio uses HTK mel scale with no norm by default + this.mel_filters = mel_filter_bank( + Math.floor(1 + n_fft / 2), // num_frequency_bins = 257 + n_mels, // 80 + 0, // min_frequency + sample_rate / 2, // max_frequency = 8000 + sample_rate, // 16000 + null, // norm (torchaudio default: no norm) + 'htk', // mel_scale (torchaudio default) + ); + + // torchaudio center-pads the window when win_length < n_fft: + // pad_amount = (n_fft - win_length) // 2 on each side + const raw_window = window_function(win_length, 'hann'); + this.window = new Float64Array(n_fft); + const pad = Math.floor((n_fft - win_length) / 2); + this.window.set(raw_window, pad); + } + + /** + * Extract mel spectrogram features from audio, matching the Python GraniteSpeechFeatureExtractor. + * @param {Float32Array|Float64Array} audio The audio waveform. + * @returns {Promise<{input_features: Tensor}>} + */ + async _call(audio) { + validate_audio_inputs(audio, 'GraniteSpeechFeatureExtractor'); + + const { n_fft, hop_length, n_mels } = this.config.melspec_kwargs; + + // Truncate to even number of frames for pair-stacking + const num_frames = Math.floor(audio.length / hop_length) + 1; + const max_num_frames = num_frames - (num_frames % 2); + + const mel = await spectrogram(audio, this.window, n_fft, hop_length, { + power: 2.0, + mel_filters: this.mel_filters, + log_mel: 'log10_max_norm', + transpose: true, // [time, n_mels] + max_num_frames, + do_pad: false, + }); + + // Stack adjacent frame pairs: [time, n_mels] → [1, time/2, 2*n_mels] + const input_features = mel.view(-1, 2 * n_mels).unsqueeze_(0); + + return { input_features }; + } +} diff --git a/packages/transformers/src/models/granite_speech/modeling_granite_speech.js b/packages/transformers/src/models/granite_speech/modeling_granite_speech.js new file mode 100644 index 0000000..5fe9922 --- /dev/null +++ b/packages/transformers/src/models/granite_speech/modeling_granite_speech.js @@ -0,0 +1,5 @@ +import { UltravoxModel } from '../ultravox/modeling_ultravox.js'; + +export class GraniteSpeechForConditionalGeneration extends UltravoxModel { + forward_params = ['input_ids', 'attention_mask', 'input_features', 'past_key_values']; +} diff --git a/packages/transformers/src/models/granite_speech/processing_granite_speech.js b/packages/transformers/src/models/granite_speech/processing_granite_speech.js new file mode 100644 index 0000000..f70b56f --- /dev/null +++ b/packages/transformers/src/models/granite_speech/processing_granite_speech.js @@ -0,0 +1,62 @@ +import { AutoFeatureExtractor } from '../auto/feature_extraction_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { Processor } from '../../processing_utils.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class GraniteSpeechProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static feature_extractor_class = AutoFeatureExtractor; + static uses_processor_config = true; + + /** + * Compute the number of audio tokens for a given raw audio length. + * @param {number} audioLength Raw audio sample count. + * @returns {number} Number of projector output tokens. + */ + _get_num_audio_features(audioLength) { + const { hop_length } = this.feature_extractor.config.melspec_kwargs; + const { projector_window_size, projector_downsample_rate } = this.feature_extractor.config; + const effective_window_size = Math.floor(projector_window_size / projector_downsample_rate); + const mel_length = Math.floor(audioLength / hop_length) + 1; + const encoder_length = Math.floor(mel_length / 2); + const nblocks = Math.ceil(encoder_length / projector_window_size); + return nblocks * effective_window_size; + } + + /** + * @param {string} text The text input to process. + * @param {Float32Array} audio The audio input to process. + */ + async _call(text, audio = null, kwargs = {}) { + if (Array.isArray(text)) { + throw new Error('Batched inputs are not supported yet.'); + } + + let audio_inputs = {}; + if (audio) { + const { input_features } = await this.feature_extractor(audio); + audio_inputs['input_features'] = input_features; + + // Compute audio embed sizes and mask in the processor + const audio_embed_size = this._get_num_audio_features(audio.length); + const mask_data = new Uint8Array(audio_embed_size).fill(1); + audio_inputs['input_features_mask'] = new Tensor('bool', mask_data, [1, audio_embed_size]); + + const audio_token = this.config.audio_token ?? '<|audio|>'; + if (!text.includes(audio_token)) { + throw new Error(`The input text does not contain the audio token ${audio_token}.`); + } + text = text.replaceAll(audio_token, audio_token.repeat(audio_embed_size)); + } + + const text_inputs = this.tokenizer(text, { + add_special_tokens: false, + ...kwargs, + }); + + return { + ...text_inputs, + ...audio_inputs, + }; + } +} diff --git a/packages/transformers/src/models/granitemoehybrid/modeling_granitemoehybrid.js b/packages/transformers/src/models/granitemoehybrid/modeling_granitemoehybrid.js new file mode 100644 index 0000000..0c78663 --- /dev/null +++ b/packages/transformers/src/models/granitemoehybrid/modeling_granitemoehybrid.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class GraniteMoeHybridPreTrainedModel extends PreTrainedModel {} +export class GraniteMoeHybridModel extends GraniteMoeHybridPreTrainedModel {} +export class GraniteMoeHybridForCausalLM extends GraniteMoeHybridPreTrainedModel {} diff --git a/packages/transformers/src/models/grounding_dino/image_processing_grounding_dino.js b/packages/transformers/src/models/grounding_dino/image_processing_grounding_dino.js new file mode 100644 index 0000000..9a408e1 --- /dev/null +++ b/packages/transformers/src/models/grounding_dino/image_processing_grounding_dino.js @@ -0,0 +1,25 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; +import { ones } from '../../utils/tensor.js'; + +/** + * @typedef {Object} GroundingDinoFeatureExtractorResultProps + * @property {import('../../utils/tensor.js').Tensor} pixel_mask + * @typedef {import('../../image_processors_utils.js').ImageProcessorResult & GroundingDinoFeatureExtractorResultProps} GroundingDinoFeatureExtractorResult + */ + +export class GroundingDinoImageProcessor extends ImageProcessor { + /** + * Calls the feature extraction process on an array of images, preprocesses + * each image, and concatenates the resulting features into a single Tensor. + * @param {import('../../utils/image.js').RawImage[]} images The image(s) to extract features from. + * @returns {Promise} An object containing the concatenated pixel values of the preprocessed images. + */ + async _call(images) { + const result = await super._call(images); + + const dims = result.pixel_values.dims; + const pixel_mask = ones([dims[0], dims[2], dims[3]]); + + return { ...result, pixel_mask }; + } +} diff --git a/packages/transformers/src/models/grounding_dino/modeling_grounding_dino.js b/packages/transformers/src/models/grounding_dino/modeling_grounding_dino.js new file mode 100644 index 0000000..662cc7f --- /dev/null +++ b/packages/transformers/src/models/grounding_dino/modeling_grounding_dino.js @@ -0,0 +1,4 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class GroundingDinoPreTrainedModel extends PreTrainedModel {} +export class GroundingDinoForObjectDetection extends GroundingDinoPreTrainedModel {} diff --git a/packages/transformers/src/models/grounding_dino/processing_grounding_dino.js b/packages/transformers/src/models/grounding_dino/processing_grounding_dino.js new file mode 100644 index 0000000..010e0a4 --- /dev/null +++ b/packages/transformers/src/models/grounding_dino/processing_grounding_dino.js @@ -0,0 +1,100 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { center_to_corners_format } from '../../image_processors_utils.js'; + +/** + * Get token ids of phrases from posmaps and input_ids. + * @param {import('../../utils/tensor.js').Tensor} posmaps A boolean tensor of unbatched text-thresholded logits related to the detected bounding boxes of shape `(hidden_size, )`. + * @param {import('../../utils/tensor.js').Tensor} input_ids A tensor of token ids of shape `(sequence_length, )`. + */ +function get_phrases_from_posmap(posmaps, input_ids) { + const left_idx = 0; + const right_idx = posmaps.dims.at(-1) - 1; + + const posmaps_list = posmaps.tolist(); + posmaps_list.fill(false, 0, left_idx + 1); + posmaps_list.fill(false, right_idx); + + const input_ids_list = input_ids.tolist(); + return posmaps_list + .map((val, idx) => (val ? idx : null)) + .filter((idx) => idx !== null) + .map((i) => input_ids_list[i]); +} + +export class GroundingDinoProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static image_processor_class = AutoImageProcessor; + + /** + * @typedef {import('../../utils/image.js').RawImage} RawImage + */ + /** + * + * @param {RawImage|RawImage[]|RawImage[][]} images + * @param {string|string[]} text + * @returns {Promise} + */ + async _call(images, text, options = {}) { + const image_inputs = images ? await this.image_processor(images, options) : {}; + const text_inputs = text ? this.tokenizer(text, options) : {}; + + return { + ...text_inputs, + ...image_inputs, + }; + } + post_process_grounded_object_detection( + outputs, + input_ids, + { box_threshold = 0.25, text_threshold = 0.25, target_sizes = null } = {}, + ) { + const { logits, pred_boxes } = outputs; + const batch_size = logits.dims[0]; + + if (target_sizes !== null && target_sizes.length !== batch_size) { + throw Error('Make sure that you pass in as many target sizes as the batch dimension of the logits'); + } + const num_queries = logits.dims.at(1); + + const probs = logits.sigmoid(); // (batch_size, num_queries, 256) + const scores = probs.max(-1).tolist(); // (batch_size, num_queries) + + // Convert to [x0, y0, x1, y1] format + const boxes = pred_boxes + .tolist() // (batch_size, num_queries, 4) + .map((batch) => batch.map((box) => center_to_corners_format(box))); + + const results = []; + for (let i = 0; i < batch_size; ++i) { + const target_size = target_sizes !== null ? target_sizes[i] : null; + + // Convert from relative [0, 1] to absolute [0, height] coordinates + if (target_size !== null) { + boxes[i] = boxes[i].map((box) => box.map((x, j) => x * target_size[(j + 1) % 2])); + } + + const batch_scores = scores[i]; + const final_scores = []; + const final_phrases = []; + const final_boxes = []; + for (let j = 0; j < num_queries; ++j) { + const score = batch_scores[j]; + if (score <= box_threshold) { + continue; + } + const box = boxes[i][j]; + const prob = probs[i][j]; + + final_scores.push(score); + final_boxes.push(box); + + const phrases = get_phrases_from_posmap(prob.gt(text_threshold), input_ids[i]); + final_phrases.push(phrases); + } + results.push({ scores: final_scores, boxes: final_boxes, labels: this.batch_decode(final_phrases) }); + } + return results; + } +} diff --git a/packages/transformers/src/models/groupvit/modeling_groupvit.js b/packages/transformers/src/models/groupvit/modeling_groupvit.js new file mode 100644 index 0000000..d99b454 --- /dev/null +++ b/packages/transformers/src/models/groupvit/modeling_groupvit.js @@ -0,0 +1,4 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class GroupViTPreTrainedModel extends PreTrainedModel {} +export class GroupViTModel extends GroupViTPreTrainedModel {} diff --git a/packages/transformers/src/models/helium/modeling_helium.js b/packages/transformers/src/models/helium/modeling_helium.js new file mode 100644 index 0000000..87e8e51 --- /dev/null +++ b/packages/transformers/src/models/helium/modeling_helium.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class HeliumPreTrainedModel extends PreTrainedModel {} +export class HeliumModel extends HeliumPreTrainedModel {} +export class HeliumForCausalLM extends HeliumPreTrainedModel {} diff --git a/packages/transformers/src/models/herbert/tokenization_herbert.js b/packages/transformers/src/models/herbert/tokenization_herbert.js new file mode 100644 index 0000000..19a73a4 --- /dev/null +++ b/packages/transformers/src/models/herbert/tokenization_herbert.js @@ -0,0 +1,5 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class HerbertTokenizer extends PreTrainedTokenizer { + return_token_type_ids = true; +} diff --git a/packages/transformers/src/models/hiera/modeling_hiera.js b/packages/transformers/src/models/hiera/modeling_hiera.js new file mode 100644 index 0000000..d0c9f31 --- /dev/null +++ b/packages/transformers/src/models/hiera/modeling_hiera.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class HieraPreTrainedModel extends PreTrainedModel {} +export class HieraModel extends HieraPreTrainedModel {} +export class HieraForImageClassification extends HieraPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/hrm_text/modeling_hrm_text.js b/packages/transformers/src/models/hrm_text/modeling_hrm_text.js new file mode 100644 index 0000000..56dbea7 --- /dev/null +++ b/packages/transformers/src/models/hrm_text/modeling_hrm_text.js @@ -0,0 +1,18 @@ +import { PreTrainedModel, decoder_prepare_inputs_for_generation } from '../modeling_utils.js'; +import { ones_like, zeros_like } from '../../utils/tensor.js'; + +export class HrmTextPreTrainedModel extends PreTrainedModel {} +export class HrmTextModel extends HrmTextPreTrainedModel {} +export class HrmTextForCausalLM extends HrmTextPreTrainedModel { + forward_params = ['input_ids', 'attention_mask', 'token_type_ids', 'past_key_values']; + + prepare_inputs_for_generation(input_ids, model_inputs, generation_config) { + const prepared = decoder_prepare_inputs_for_generation(this, input_ids, model_inputs, generation_config); + + // HRM-Text can be pretrained as a PrefixLM. + // token_type_ids=1 marks tokens inside the bidirectional prefix block, + // and 0 marks autoregressively generated tokens. + prepared.token_type_ids = (prepared.past_key_values ? zeros_like : ones_like)(prepared.input_ids); + return prepared; + } +} diff --git a/packages/transformers/src/models/hubert/modeling_hubert.js b/packages/transformers/src/models/hubert/modeling_hubert.js new file mode 100644 index 0000000..c2284dc --- /dev/null +++ b/packages/transformers/src/models/hubert/modeling_hubert.js @@ -0,0 +1,62 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { CausalLMOutput, SequenceClassifierOutput } from '../modeling_outputs.js'; +import { Wav2Vec2PreTrainedModel } from '../wav2vec2/modeling_wav2vec2.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class HubertPreTrainedModel extends PreTrainedModel {} + +/** + * The bare Hubert Model transformer outputting raw hidden-states without any specific head on top. + * + * **Example:** Load and run a `HubertModel` for feature extraction. + * + * ```javascript + * import { AutoProcessor, AutoModel, read_audio } from '@huggingface/transformers'; + * + * // Read and preprocess audio + * const processor = await AutoProcessor.from_pretrained('Xenova/hubert-base-ls960'); + * const audio = await read_audio('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav', 16000); + * const inputs = await processor(audio); + * + * // Load and run model with inputs + * const model = await AutoModel.from_pretrained('Xenova/hubert-base-ls960'); + * const output = await model(inputs); + * // { + * // last_hidden_state: Tensor { + * // dims: [ 1, 549, 768 ], + * // type: 'float32', + * // data: Float32Array(421632) [0.0682469978928566, 0.08104046434164047, -0.4975186586380005, ...], + * // size: 421632 + * // } + * // } + * ``` + */ +export class HubertModel extends Wav2Vec2PreTrainedModel {} + +/** + * Hubert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). + */ +export class HubertForCTC extends Wav2Vec2PreTrainedModel { + /** + * @param {Object} model_inputs + * @param {Tensor} model_inputs.input_values Float values of input raw speech waveform. + * @param {Tensor} model_inputs.attention_mask Mask to avoid performing convolution and attention on padding token indices. Mask values selected in [0, 1] + */ + async _call(model_inputs) { + return new CausalLMOutput(await super._call(model_inputs)); + } +} + +/** + * Hubert Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. + */ +export class HubertForSequenceClassification extends Wav2Vec2PreTrainedModel { + /** + * Calls the model on new inputs. + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.js b/packages/transformers/src/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.js new file mode 100644 index 0000000..9086290 --- /dev/null +++ b/packages/transformers/src/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class HunYuanDenseV1PreTrainedModel extends PreTrainedModel {} +export class HunYuanDenseV1Model extends HunYuanDenseV1PreTrainedModel {} +export class HunYuanDenseV1ForCausalLM extends HunYuanDenseV1PreTrainedModel {} diff --git a/packages/transformers/src/models/idefics3/image_processing_idefics3.js b/packages/transformers/src/models/idefics3/image_processing_idefics3.js new file mode 100644 index 0000000..4697526 --- /dev/null +++ b/packages/transformers/src/models/idefics3/image_processing_idefics3.js @@ -0,0 +1,224 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; +import { cat, full, interpolate_4d, slice, stack } from '../../utils/tensor.js'; + +export class Idefics3ImageProcessor extends ImageProcessor { + constructor(config) { + super(config); + + this.do_image_splitting = config.do_image_splitting ?? true; + this.max_image_size = config.max_image_size; + } + + /** + * @typedef {import('../../utils/image.js').RawImage} RawImage + * @typedef {import('../../utils/tensor.js').Tensor} Tensor + */ + + /** + * Calculate size to resize images to, to be multiples of `vision_encoder_max_size` while preserving the aspect ratio. + * @param {Tensor} pixel_values Tensor of the image to resize. + * @param {number} vision_encoder_max_size Maximum size of the output image. If the image is larger than this size, + * it will be split into patches of this size, and the original image will be concatenated with the patches, resized to max_size. + */ + get_resize_for_vision_encoder(pixel_values, vision_encoder_max_size) { + let [height, width] = pixel_values.dims.slice(-2); + + const aspect_ratio = width / height; + if (width >= height) { + width = Math.ceil(width / vision_encoder_max_size) * vision_encoder_max_size; + height = Math.floor(width / aspect_ratio); + height = Math.ceil(height / vision_encoder_max_size) * vision_encoder_max_size; + } else { + height = Math.ceil(height / vision_encoder_max_size) * vision_encoder_max_size; + width = Math.floor(height * aspect_ratio); + width = Math.ceil(width / vision_encoder_max_size) * vision_encoder_max_size; + } + return { height, width }; + } + + /** @param {RawImage|RawImage[]|RawImage[][]} images */ + async _call(images, { do_image_splitting = null, return_row_col_info = false } = {}) { + /** @type {RawImage[][]} */ + let batched_2d_images; + if (!Array.isArray(images)) { + batched_2d_images = [[images]]; + } else { + if (images.length === 0 || !images[0]) { + throw new Error('No images provided.'); + } + if (!Array.isArray(images[0])) { + batched_2d_images = [/** @type {RawImage[]} */ (images)]; + } else { + batched_2d_images = /** @type {RawImage[][]} */ (images); + } + } + + // List of tensors, each with shape [patches, channels, height, width] + let all_pixel_values = []; + let images_list_rows = []; + let images_list_cols = []; + + const original_sizes = []; + const reshaped_input_sizes = []; + for (const image_batch of batched_2d_images) { + let images_list = await Promise.all(image_batch.map((x) => this.preprocess(x))); + + // Original sizes of images + original_sizes.push(...images_list.map((x) => x.original_size)); + + // Reshaped sizes of images, before padding or cropping + reshaped_input_sizes.push(...images_list.map((x) => x.reshaped_input_size)); + + // Convert images to 4D tensors for easier processing + images_list.forEach((x) => x.pixel_values.unsqueeze_(0)); + + const { longest_edge } = this.max_image_size; + + /** @type {Tensor[]} */ + let images_tensor; + if (do_image_splitting ?? this.do_image_splitting) { + let image_rows = new Array(images_list.length); + let image_cols = new Array(images_list.length); + + // We first resize both height and width of each image to the nearest max_image_size multiple, disregarding the aspect ratio + images_tensor = await Promise.all( + images_list.map(async (x, i) => { + const new_size = this.get_resize_for_vision_encoder(x.pixel_values, longest_edge); + + const resized = await interpolate_4d(x.pixel_values, { + size: [new_size.height, new_size.width], + }); + + const { frames, num_splits_h, num_splits_w } = await this.split_image( + resized, + this.max_image_size, + ); + image_rows[i] = num_splits_h; + image_cols[i] = num_splits_w; + return cat(frames, 0); + }), + ); + + images_list_rows.push(image_rows); + images_list_cols.push(image_cols); + } else { + /** @type {[number, number]} */ + const size = [longest_edge, longest_edge]; + images_tensor = await Promise.all(images_list.map((x) => interpolate_4d(x.pixel_values, { size }))); + + images_list_rows.push(new Array(images_list.length).fill(0)); + images_list_cols.push(new Array(images_list.length).fill(0)); + } + + all_pixel_values.push(cat(images_tensor, 0)); + } + + const batch_size = all_pixel_values.length; + const [n, c, h, w] = all_pixel_values[0].dims; + + // Stack pixel values + let pixel_values; + let pixel_attention_mask; + if (batch_size === 1) { + pixel_values = all_pixel_values[0].unsqueeze_(0); + pixel_attention_mask = full([batch_size, n, h, w], true); + } else { + // Add padding (if necessary) to images with less patches than the maximum number of patches + const max_num_patches = Math.max(...all_pixel_values.map((x) => x.dims.at(0))); + + pixel_attention_mask = full([batch_size, max_num_patches, h, w], true); + const pixel_attention_mask_data = pixel_attention_mask.data; + const pixel_attention_mask_stride = max_num_patches * h * w; + for (let i = 0; i < batch_size; ++i) { + const num_patches = all_pixel_values[i].dims[0]; + if (num_patches < max_num_patches) { + all_pixel_values[i] = cat( + [all_pixel_values[i], full([max_num_patches - num_patches, c, h, w], 0)], + 0, + ); + + const start_offset = i * pixel_attention_mask_stride + num_patches * h * w; + const end_offset = (i + 1) * pixel_attention_mask_stride; + + // @ts-ignore + pixel_attention_mask_data.fill(false, start_offset, end_offset); + } + } + pixel_values = stack(all_pixel_values, 0); + } + + return { + pixel_values, + pixel_attention_mask, + + original_sizes, + reshaped_input_sizes, + ...(return_row_col_info ? { rows: images_list_rows, cols: images_list_cols } : {}), + }; + } + + async split_image(pixel_values, { longest_edge }) { + const max_height = longest_edge; + const max_width = longest_edge; + + const frames = []; + + const [height, width] = pixel_values.dims.slice(-2); + + let num_splits_h = 0, + num_splits_w = 0; + + if (height > max_height || width > max_width) { + // Calculate the number of splits + num_splits_h = Math.ceil(height / max_height); + num_splits_w = Math.ceil(width / max_width); + + // Calculate the optimal width and height for the sub-images + const optimal_height = Math.ceil(height / num_splits_h); + const optimal_width = Math.ceil(width / num_splits_w); + + // Iterate through each row and column + for (let r = 0; r < num_splits_h; ++r) { + for (let c = 0; c < num_splits_w; ++c) { + let start_x, start_y, end_x, end_y; + if (r === num_splits_h - 1) { + // At bottom + start_y = height - optimal_height; + end_y = height; + } else { + start_y = r * optimal_height; + end_y = (r + 1) * optimal_height; + } + if (c === num_splits_w - 1) { + // At right + start_x = width - optimal_width; + end_x = width; + } else { + start_x = c * optimal_width; + end_x = (c + 1) * optimal_width; + } + + const starts = [start_y, start_x]; + const ends = [end_y, end_x]; + + const patch = await slice(pixel_values, starts, ends, [2, 3]); + frames.push(patch); + } + } + + // Resize the global image to match max dimensions for memory efficiency + const global_image_height = max_height; + const global_image_width = max_width; + + if (height !== global_image_height || width !== global_image_width) { + pixel_values = await interpolate_4d(pixel_values, { + size: [global_image_height, global_image_width], + }); + } + } + + frames.push(pixel_values); + + return { frames, num_splits_h, num_splits_w }; + } +} diff --git a/packages/transformers/src/models/idefics3/modeling_idefics3.js b/packages/transformers/src/models/idefics3/modeling_idefics3.js new file mode 100644 index 0000000..7c6de5e --- /dev/null +++ b/packages/transformers/src/models/idefics3/modeling_idefics3.js @@ -0,0 +1,15 @@ +import { LlavaForConditionalGeneration } from '../llava/modeling_llava.js'; + +/** + * The Idefics3 model which consists of a vision backbone and a language model. + */ +export class Idefics3ForConditionalGeneration extends LlavaForConditionalGeneration { + forward_params = [ + 'input_ids', + 'attention_mask', + 'pixel_values', + 'pixel_attention_mask', + 'position_ids', + 'past_key_values', + ]; +} diff --git a/packages/transformers/src/models/idefics3/processing_idefics3.js b/packages/transformers/src/models/idefics3/processing_idefics3.js new file mode 100644 index 0000000..021656c --- /dev/null +++ b/packages/transformers/src/models/idefics3/processing_idefics3.js @@ -0,0 +1,144 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { RawImage } from '../../utils/image.js'; +import { count } from '../../utils/core.js'; + +/** + * Prompt with expanded image tokens for when the image is split into patches. + * @private + */ +function _prompt_split_image( + image_seq_len, + image_rows, + image_cols, + fake_token_around_image, + image_token, + global_img_token, +) { + let text_split_images = ''; + for (let n_h = 0; n_h < image_rows; ++n_h) { + for (let n_w = 0; n_w < image_cols; ++n_w) { + text_split_images += + fake_token_around_image + `` + image_token.repeat(image_seq_len); + } + text_split_images += '\n'; + } + + text_split_images += + `\n${fake_token_around_image}` + + `${global_img_token}` + + image_token.repeat(image_seq_len) + + `${fake_token_around_image}`; + return text_split_images; +} + +/** + * Prompt with expanded image tokens for a single image. + * @private + */ +function _prompt_single_image(image_seq_len, fake_token_around_image, image_token, global_img_token) { + return ( + `${fake_token_around_image}` + + `${global_img_token}` + + image_token.repeat(image_seq_len) + + `${fake_token_around_image}` + ); +} + +function get_image_prompt_string( + image_rows, + image_cols, + image_seq_len, + fake_token_around_image, + image_token, + global_img_token, +) { + if (image_rows === 0 && image_cols === 0) { + return _prompt_single_image(image_seq_len, fake_token_around_image, image_token, global_img_token); + } + return _prompt_split_image( + image_seq_len, + image_rows, + image_cols, + fake_token_around_image, + image_token, + global_img_token, + ); +} + +export class Idefics3Processor extends Processor { + static image_processor_class = AutoImageProcessor; + static tokenizer_class = AutoTokenizer; + static uses_processor_config = true; + + fake_image_token = ''; + image_token = ''; + global_img_token = ''; + + /** + * + * @param {string|string[]} text + * @param {RawImage|RawImage[]|RawImage[][]} images + * @returns {Promise} + */ + async _call(text, images = null, options = {}) { + options.return_row_col_info ??= true; + + let image_inputs; + + if (images) { + image_inputs = await this.image_processor(images, options); + } + + // NOTE: We assume text is present + if (!Array.isArray(text)) { + text = [text]; + } + + const image_rows = image_inputs.rows ?? [new Array(text.length).fill(0)]; + const image_cols = image_inputs.cols ?? [new Array(text.length).fill(0)]; + + const image_seq_len = this.config.image_seq_len; + const n_images_in_text = []; + const prompt_strings = []; + for (let i = 0; i < text.length; ++i) { + const sample = text[i]; + const sample_rows = image_rows[i]; + const sample_cols = image_cols[i]; + + n_images_in_text.push(count(sample, this.image_token)); + + // Replace the image token with fake tokens around the expanded image token sequence of length `image_seq_len` + const image_prompt_strings = sample_rows.map((n_rows, j) => + get_image_prompt_string( + n_rows, + sample_cols[j], + image_seq_len, + this.fake_image_token, + this.image_token, + this.global_img_token, + ), + ); + + const split_sample = sample.split(this.image_token); + if (split_sample.length === 0) { + throw new Error('The image token should be present in the text.'); + } + + // Place in the image prompt strings where the image tokens are + let new_sample = split_sample[0]; + for (let j = 0; j < image_prompt_strings.length; ++j) { + new_sample += image_prompt_strings[j] + split_sample[j + 1]; + } + prompt_strings.push(new_sample); + } + + const text_inputs = this.tokenizer(prompt_strings); + + return { + ...text_inputs, + ...image_inputs, + }; + } +} diff --git a/packages/transformers/src/models/ijepa/modeling_ijepa.js b/packages/transformers/src/models/ijepa/modeling_ijepa.js new file mode 100644 index 0000000..29abcaa --- /dev/null +++ b/packages/transformers/src/models/ijepa/modeling_ijepa.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class IJepaPreTrainedModel extends PreTrainedModel {} +export class IJepaModel extends IJepaPreTrainedModel {} +export class IJepaForImageClassification extends IJepaPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/image_processors.js b/packages/transformers/src/models/image_processors.js new file mode 100644 index 0000000..d7cfef3 --- /dev/null +++ b/packages/transformers/src/models/image_processors.js @@ -0,0 +1,52 @@ +export * from './beit/image_processing_beit.js'; +export * from './bit/image_processing_bit.js'; +export * from './chinese_clip/image_processing_chinese_clip.js'; +export * from './chmv2/image_processing_chmv2.js'; +export * from './clip/image_processing_clip.js'; +export * from './convnext/image_processing_convnext.js'; +export * from './deit/image_processing_deit.js'; +export * from './detr/image_processing_detr.js'; +export * from './dinov3_vit/image_processing_dinov3_vit.js'; +export * from './donut/image_processing_donut.js'; +export * from './dpt/image_processing_dpt.js'; +export * from './efficientnet/image_processing_efficientnet.js'; +export * from './gemma3/image_processing_gemma3.js'; +export * from './gemma4/image_processing_gemma4.js'; +export * from './glm46v/image_processing_glm46v.js'; +export * from './glpn/image_processing_glpn.js'; +export * from './grounding_dino/image_processing_grounding_dino.js'; +export * from './idefics3/image_processing_idefics3.js'; +export * from './janus/image_processing_janus.js'; +export * from './jina_clip/image_processing_jina_clip.js'; +export * from './lfm2_vl/image_processing_lfm2_vl.js'; +export * from './llava_onevision/image_processing_llava_onevision.js'; +export * from './mask2former/image_processing_mask2former.js'; +export * from './maskformer/image_processing_maskformer.js'; +export * from './mobilenet_v1/image_processing_mobilenet_v1.js'; +export * from './mobilenet_v2/image_processing_mobilenet_v2.js'; +export * from './mobilenet_v3/image_processing_mobilenet_v3.js'; +export * from './mobilenet_v4/image_processing_mobilenet_v4.js'; +export * from './mobilevit/image_processing_mobilevit.js'; +export * from './nougat/image_processing_nougat.js'; +export * from './owlv2/image_processing_owlv2.js'; +export * from './owlvit/image_processing_owlvit.js'; +export * from './phi3_v/image_processing_phi3_v.js'; +export * from './pixtral/image_processing_pixtral.js'; +export * from './pvt/image_processing_pvt.js'; +export * from './qwen2_vl/image_processing_qwen2_vl.js'; +export * from './rt_detr/image_processing_rt_detr.js'; +export * from './sam/image_processing_sam.js'; +export * from './sam2/image_processing_sam2.js'; +export * from './sam3/image_processing_sam3.js'; +export * from './sapiens/image_processing_sapiens.js'; +export * from './segformer/image_processing_segformer.js'; +export * from './siglip/image_processing_siglip.js'; +export * from './smolvlm/image_processing_smolvlm.js'; +export * from './swin2sr/image_processing_swin2sr.js'; +export * from './vit/image_processing_vit.js'; +export * from './vitmatte/image_processing_vitmatte.js'; +export * from './vitpose/image_processing_vitpose.js'; +export * from './yolos/image_processing_yolos.js'; + +// For legacy support, ImageFeatureExtractor is an alias for ImageProcessor +export { ImageProcessor, ImageProcessor as ImageFeatureExtractor } from '../image_processors_utils.js'; diff --git a/packages/transformers/src/models/jais/modeling_jais.js b/packages/transformers/src/models/jais/modeling_jais.js new file mode 100644 index 0000000..bc6f79c --- /dev/null +++ b/packages/transformers/src/models/jais/modeling_jais.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class JAISPreTrainedModel extends PreTrainedModel {} + +/** + * The bare JAIS Model transformer outputting raw hidden-states without any specific head on top. + */ +export class JAISModel extends JAISPreTrainedModel {} + +/** + * The JAIS Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). + */ +export class JAISLMHeadModel extends JAISPreTrainedModel {} diff --git a/packages/transformers/src/models/janus/image_processing_janus.js b/packages/transformers/src/models/janus/image_processing_janus.js new file mode 100644 index 0000000..50b5221 --- /dev/null +++ b/packages/transformers/src/models/janus/image_processing_janus.js @@ -0,0 +1,24 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class VLMImageProcessor extends ImageProcessor { + constructor(config) { + super({ + do_pad: true, + pad_size: { + width: config.image_size, + height: config.image_size, + }, + ...config, + }); + // @ts-expect-error TS2339 + this.constant_values = this.config.background_color.map((x) => x * this.rescale_factor); + } + + pad_image(pixelData, imgDims, padSize, options) { + return super.pad_image(pixelData, imgDims, padSize, { + constant_values: this.constant_values, + center: true, + ...options, + }); + } +} diff --git a/packages/transformers/src/models/janus/processing_janus.js b/packages/transformers/src/models/janus/processing_janus.js new file mode 100644 index 0000000..c149f0c --- /dev/null +++ b/packages/transformers/src/models/janus/processing_janus.js @@ -0,0 +1,120 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { mergeArrays } from '../../utils/core.js'; +import { Tensor } from '../../utils/tensor.js'; +import { RawImage } from '../../utils/image.js'; + +export class VLChatProcessor extends Processor { + static image_processor_class = AutoImageProcessor; + static tokenizer_class = AutoTokenizer; + static uses_processor_config = true; + + constructor(config, components, chat_template) { + super(config, components, chat_template); + + this.image_tag = this.config.image_tag; + this.image_start_tag = this.config.image_start_tag; + this.image_end_tag = this.config.image_end_tag; + this.num_image_tokens = this.config.num_image_tokens; + } + + /** + * @typedef {Object} MultimodalMessageProperties Additional properties for multimodal messages. + * @property {(RawImage | string | URL)[]} [images] The images in the message. + * @typedef {(import('../../tokenization_utils.js').Message & MultimodalMessageProperties)[]} MultimodalConversation The conversation possibly containing multimodal inputs. + */ + + /** + * @typedef {Object} VLCChatProcessorResult The processed input. + * @property {Tensor} input_ids The input IDs. + * @property {Tensor} attention_mask The attention mask. + * @property {Tensor} images_seq_mask The image sequence mask. + * @property {Tensor} images_emb_mask The image embedding mask. + */ + + /** + * @param {MultimodalConversation} conversation The chat messages to process. + * @param {Object} options Additional options for processing. + * @param {RawImage|RawImage[]} [options.images] The images to process, if not set in the conversation. + * @param {string} [options.chat_template="default"] The chat template to use. + * @returns {Promise} The processed input. + */ + async _call(conversation, { images = null, chat_template = 'default' } = {}) { + if (!images) { + images = await Promise.all( + conversation + .filter((msg) => msg.images) + .flatMap((msg) => msg.images) + .map((img) => RawImage.read(img)), + ); + } else if (!Array.isArray(images)) { + images = [images]; + } + + const tokenizer = this.tokenizer; + const result = tokenizer.apply_chat_template(conversation, { + tokenize: false, + add_generation_prompt: true, + chat_template, + }); + + const encode = (text) => tokenizer.encode(text, { add_special_tokens: false }); + const parts = /** @type {string} */ (result).split(this.image_tag); + const num_images = parts.length - 1; + if (images.length !== num_images) { + throw new Error( + `Number of images provided (${images.length}) does not match number of "${this.image_tag}" image tags (${num_images})`, + ); + } + + const [image_placeholder_tag_id, image_start_tag_id, image_end_tag_id] = tokenizer.convert_tokens_to_ids([ + this.image_tag, + this.image_start_tag, + this.image_end_tag, + ]); + + let input_ids = encode(parts[0]); + let images_seq_mask = new Array(input_ids.length).fill(false); + for (let i = 1; i < parts.length; ++i) { + const placeholder_image_tokens = new Array(this.num_image_tokens).fill(image_placeholder_tag_id); + const tokens = encode(parts[i]); + input_ids = mergeArrays( + input_ids, + [image_start_tag_id], + placeholder_image_tokens, + [image_end_tag_id], + tokens, + ); + const image_mask = new Array(this.num_image_tokens).fill(true); + images_seq_mask = mergeArrays( + images_seq_mask, + [false], + image_mask, + [false], + new Array(tokens.length).fill(false), + ); + } + + const dims = [1, input_ids.length]; + const final = { + input_ids: new Tensor('int64', input_ids, dims), + attention_mask: new Tensor('int64', new Array(input_ids.length).fill(1), dims), + images_seq_mask: new Tensor('bool', images_seq_mask, dims), + images_emb_mask: new Tensor('bool', new Array(num_images * this.num_image_tokens).fill(true), [ + 1, + num_images, + this.num_image_tokens, + ]), + }; + + if (images && images.length > 0) { + const image_inputs = await this.image_processor(images); + // Set the batch_size dimension to 1 + image_inputs.pixel_values.unsqueeze_(0); + return { ...final, ...image_inputs }; + } + + return final; + } +} diff --git a/packages/transformers/src/models/jina_clip/image_processing_jina_clip.js b/packages/transformers/src/models/jina_clip/image_processing_jina_clip.js new file mode 100644 index 0000000..ada0361 --- /dev/null +++ b/packages/transformers/src/models/jina_clip/image_processing_jina_clip.js @@ -0,0 +1,25 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class JinaCLIPImageProcessor extends ImageProcessor { + constructor(config) { + // JinaCLIPImageProcessor uses a custom preprocessor_config.json, so we configure it here + const { resize_mode, fill_color, interpolation, size, ...other } = config; + + const new_size = + resize_mode === 'squash' + ? { width: size, height: size } + : resize_mode === 'shortest' + ? { shortest_edge: size } + : { longest_edge: size }; + + const resample = interpolation === 'bicubic' ? 3 : 2; + super({ + ...other, + size: new_size, + resample, + do_center_crop: true, + crop_size: size, + do_normalize: true, + }); + } +} diff --git a/packages/transformers/src/models/jina_clip/modeling_jina_clip.js b/packages/transformers/src/models/jina_clip/modeling_jina_clip.js new file mode 100644 index 0000000..c7cd66a --- /dev/null +++ b/packages/transformers/src/models/jina_clip/modeling_jina_clip.js @@ -0,0 +1,65 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { ones, full } from '../../utils/tensor.js'; + +export class JinaCLIPPreTrainedModel extends PreTrainedModel {} + +export class JinaCLIPModel extends JinaCLIPPreTrainedModel { + async forward(model_inputs) { + const missing_text_inputs = !model_inputs.input_ids; + const missing_image_inputs = !model_inputs.pixel_values; + + if (missing_text_inputs && missing_image_inputs) { + throw new Error('Either `input_ids` or `pixel_values` should be provided.'); + } + + // If either `input_ids` or `pixel_values` aren't passed, we need to create dummy input since the model requires a value to be specified. + if (missing_text_inputs) { + // NOTE: We cannot pass zero-dimension tensor as input for input_ids. + // Fortunately, the majority of time is spent in the vision encoder, so this shouldn't significantly impact performance. + model_inputs.input_ids = ones([model_inputs.pixel_values.dims[0], 1]); + } + + if (missing_image_inputs) { + // NOTE: Since we create a zero-sized tensor, this does not increase computation time. + // @ts-ignore + const { image_size } = this.config.vision_config; + model_inputs.pixel_values = full([0, 3, image_size, image_size], 0.0); // (pass zero-dimension tensor) + } + + const { text_embeddings, image_embeddings, l2norm_text_embeddings, l2norm_image_embeddings } = + await super.forward(model_inputs); + + const result = {}; + if (!missing_text_inputs) { + result.text_embeddings = text_embeddings; + result.l2norm_text_embeddings = l2norm_text_embeddings; + } + if (!missing_image_inputs) { + result.image_embeddings = image_embeddings; + result.l2norm_image_embeddings = l2norm_image_embeddings; + } + return result; + } +} + +export class JinaCLIPTextModel extends JinaCLIPPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'text_model', + }); + } +} + +export class JinaCLIPVisionModel extends JinaCLIPPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'vision_model', + }); + } +} diff --git a/packages/transformers/src/models/jina_clip/processing_jina_clip.js b/packages/transformers/src/models/jina_clip/processing_jina_clip.js new file mode 100644 index 0000000..3531353 --- /dev/null +++ b/packages/transformers/src/models/jina_clip/processing_jina_clip.js @@ -0,0 +1,22 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; + +export class JinaCLIPProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static image_processor_class = AutoImageProcessor; + + async _call(text = null, images = null, kwargs = {}) { + if (!text && !images) { + throw new Error('Either text or images must be provided'); + } + + const text_inputs = text ? this.tokenizer(text, kwargs) : {}; + const image_inputs = images ? await this.image_processor(images, kwargs) : {}; + + return { + ...text_inputs, + ...image_inputs, + }; + } +} diff --git a/packages/transformers/src/models/lfm2/modeling_lfm2.js b/packages/transformers/src/models/lfm2/modeling_lfm2.js new file mode 100644 index 0000000..a611067 --- /dev/null +++ b/packages/transformers/src/models/lfm2/modeling_lfm2.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Lfm2PreTrainedModel extends PreTrainedModel {} +export class Lfm2Model extends Lfm2PreTrainedModel {} +export class Lfm2ForCausalLM extends Lfm2PreTrainedModel {} diff --git a/packages/transformers/src/models/lfm2_moe/modeling_lfm2_moe.js b/packages/transformers/src/models/lfm2_moe/modeling_lfm2_moe.js new file mode 100644 index 0000000..acfc373 --- /dev/null +++ b/packages/transformers/src/models/lfm2_moe/modeling_lfm2_moe.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Lfm2MoePreTrainedModel extends PreTrainedModel {} +export class Lfm2MoeModel extends Lfm2MoePreTrainedModel {} +export class Lfm2MoeForCausalLM extends Lfm2MoePreTrainedModel {} diff --git a/packages/transformers/src/models/lfm2_vl/image_processing_lfm2_vl.js b/packages/transformers/src/models/lfm2_vl/image_processing_lfm2_vl.js new file mode 100644 index 0000000..4066bbc --- /dev/null +++ b/packages/transformers/src/models/lfm2_vl/image_processing_lfm2_vl.js @@ -0,0 +1,305 @@ +import { ImageProcessor, smart_resize } from '../../image_processors_utils.js'; +import { Tensor, cat, interpolate_4d, stack } from '../../utils/tensor.js'; + +/** + * @typedef {import('../../utils/image.js').RawImage} RawImage + */ + +/** + * Returns the closest integer to `number` that is divisible by `factor`. + * @param {number} number + * @param {number} factor + * @returns {number} + */ +function round_by_factor(number, factor) { + return Math.round(number / factor) * factor; +} + +/** + * Find the closest aspect ratio from target_ratios to match the input aspect ratio. + * @param {number} aspect_ratio + * @param {number[][]} target_ratios + * @param {number} width + * @param {number} height + * @param {number} image_size + * @returns {number[]} + */ +function find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size) { + let best_ratio_diff = Infinity; + let best_ratio = [1, 1]; + const area = width * height; + for (const ratio of target_ratios) { + const ratio_diff = Math.abs(aspect_ratio - ratio[0] / ratio[1]); + if (ratio_diff < best_ratio_diff) { + best_ratio_diff = ratio_diff; + best_ratio = ratio; + } else if (ratio_diff === best_ratio_diff && area > 0.5 * image_size * image_size * ratio[0] * ratio[1]) { + best_ratio = ratio; + } + } + return best_ratio; +} + +/** + * Compute all valid (width, height) tile ratios for the given range. + * @param {number} min_tiles + * @param {number} max_tiles + * @returns {number[][]} + */ +function get_target_ratios(min_tiles, max_tiles) { + /** @type {number[][]} */ + const ratios = []; + const seen = new Set(); + for (let n = min_tiles; n <= max_tiles; ++n) { + for (let w = 1; w <= n; ++w) { + for (let h = 1; h <= n; ++h) { + const product = w * h; + if (product >= min_tiles && product <= max_tiles) { + const key = (w << 16) | h; + if (!seen.has(key)) { + seen.add(key); + ratios.push([w, h]); + } + } + } + } + } + return ratios.sort((a, b) => a[0] * a[1] - b[0] * b[1]); +} + +/** + * Convert image tensor to flattened patches. + * + * Equivalent to PyTorch: `images.reshape(B, C, ph, ps, pw, ps).permute(0, 2, 4, 3, 5, 1).reshape(B, ph*pw, -1)` + * @param {Tensor} images Shape: [batch, channels, height, width] + * @param {number} patch_size + * @returns {Tensor} Shape: [batch, num_patches, patch_size * patch_size * channels] + */ +function convert_image_to_patches(images, patch_size) { + const [B, C, H, W] = images.dims; + const ph = Math.floor(H / patch_size), + pw = Math.floor(W / patch_size); + const patch_dim = patch_size * patch_size * C; + const data = /** @type {Float32Array} */ (images.data); + const result = new Float32Array(B * ph * pw * patch_dim); + const ch_stride = H * W; + + for (let b = 0; b < B; ++b) { + const b_src = b * C * ch_stride; + const b_dst = b * ph * pw * patch_dim; + for (let py = 0; py < ph; ++py) { + for (let px = 0; px < pw; ++px) { + let off = b_dst + (py * pw + px) * patch_dim; + for (let dy = 0; dy < patch_size; ++dy) { + const row = (py * patch_size + dy) * W + px * patch_size; + for (let dx = 0; dx < patch_size; ++dx) { + const pixel = row + dx; + for (let c = 0; c < C; ++c) { + result[off++] = data[b_src + c * ch_stride + pixel]; + } + } + } + } + } + } + + return new Tensor('float32', result, [B, ph * pw, patch_dim]); +} + +/** + * Pad patches along the patch dimension to `target_length`. + * @param {Tensor} patches Shape: [1, current_length, patch_dim] + * @param {number} target_length + * @returns {{ padded: Tensor, mask: Tensor }} + */ +function pad_along_first_dim(patches, target_length) { + const [, len, dim] = patches.dims; + const mask_data = new BigInt64Array(target_length); + mask_data.fill(1n, 0, len); + + let padded = patches; + if (len < target_length) { + const padded_data = new Float32Array(target_length * dim); + padded_data.set(/** @type {Float32Array} */ (patches.data)); + padded = new Tensor('float32', padded_data, [1, target_length, dim]); + } + + return { padded, mask: new Tensor('int64', mask_data, [target_length]) }; +} + +export class Lfm2VlImageProcessor extends ImageProcessor { + constructor(/** @type {Record} */ config) { + super(config); + this.downsample_factor = config.downsample_factor ?? 2; + this.do_image_splitting = config.do_image_splitting ?? true; + this.min_tiles = config.min_tiles ?? 2; + this.max_tiles = config.max_tiles ?? 10; + this.use_thumbnail = config.use_thumbnail ?? true; + this.min_image_tokens = config.min_image_tokens ?? 64; + this.max_image_tokens = config.max_image_tokens ?? 256; + this.encoder_patch_size = config.encoder_patch_size ?? config.patch_size ?? 16; + this.tile_size = config.tile_size ?? 512; + this.max_pixels_tolerance = config.max_pixels_tolerance ?? 2.0; + this.return_row_col_info = config.return_row_col_info ?? false; + + const max_thumbnail_patches = this.max_image_tokens * this.downsample_factor ** 2; + const tile_size_patches = this.do_image_splitting ? (this.tile_size / this.encoder_patch_size) ** 2 : 0; + this.max_num_patches = Math.max(max_thumbnail_patches, tile_size_patches); + } + + /** + * Check if the image is too large to be processed as a single tile. + * @param {number} height + * @param {number} width + * @returns {boolean} + */ + _is_image_too_large(height, width) { + const total_factor = this.encoder_patch_size * this.downsample_factor; + const h_bar = Math.max(this.encoder_patch_size, round_by_factor(height, total_factor)); + const w_bar = Math.max(this.encoder_patch_size, round_by_factor(width, total_factor)); + return ( + h_bar * w_bar > + this.max_image_tokens * (this.encoder_patch_size * this.downsample_factor) ** 2 * this.max_pixels_tolerance + ); + } + + /** + * Get the grid layout for tiling a large image. + * @param {number} height + * @param {number} width + * @returns {{ grid_width: number, grid_height: number, target_width: number, target_height: number }} + */ + _get_grid_layout(height, width) { + const target_ratios = get_target_ratios(this.min_tiles, this.max_tiles); + const [grid_width, grid_height] = find_closest_aspect_ratio( + width / height, + target_ratios, + width, + height, + this.tile_size, + ); + return { + grid_width, + grid_height, + target_width: this.tile_size * grid_width, + target_height: this.tile_size * grid_height, + }; + } + + /** @param {RawImage|RawImage[]|RawImage[][]} images */ + // @ts-expect-error + async _call(images, { return_row_col_info = null } = {}) { + /** @type {RawImage[][]} */ + let batched_images; + if (!Array.isArray(images)) { + batched_images = [[images]]; + } else if (!Array.isArray(images[0])) { + batched_images = [/** @type {RawImage[]} */ (images)]; + } else { + batched_images = /** @type {RawImage[][]} */ (images); + } + + /** @type {Tensor[]} */ + const all_pixel_values = []; + /** @type {Tensor[]} */ + const all_pixel_masks = []; + /** @type {number[][]} */ + const all_spatial_shapes = []; + /** @type {number[]} */ + const all_rows = []; + /** @type {number[]} */ + const all_cols = []; + /** @type {number[][]} */ + const all_image_sizes = []; + + for (const image_batch of batched_images) { + const preprocessed = await Promise.all(image_batch.map((x) => this.preprocess(x, { do_pad: false }))); + + for (const { pixel_values } of preprocessed) { + const [, height, width] = pixel_values.dims; + const img = pixel_values.unsqueeze_(0); + + const total_factor = this.encoder_patch_size * this.downsample_factor; + const f2 = total_factor ** 2; + const [new_width, new_height] = smart_resize( + Math.max(total_factor, height), + Math.max(total_factor, width), + total_factor, + this.min_image_tokens * f2, + this.max_image_tokens * f2, + ).map((x) => Math.max(total_factor, x)); + + /** @type {Tensor[]} */ + let tiles; + let num_rows = 1, + num_cols = 1; + + const is_large = this._is_image_too_large(height, width); + const do_splitting = this.do_image_splitting && !(this.min_tiles === 1 && this.max_tiles === 1); + + if (is_large && do_splitting) { + const { grid_width, grid_height, target_width, target_height } = this._get_grid_layout( + height, + width, + ); + num_rows = grid_height; + num_cols = grid_width; + + const resized = await interpolate_4d(img, { + size: [target_height, target_width], + }); + + tiles = []; + for (let r = 0; r < grid_height; ++r) { + for (let c = 0; c < grid_width; ++c) { + const y = r * this.tile_size; + const x = c * this.tile_size; + tiles.push(resized.slice(null, null, [y, y + this.tile_size], [x, x + this.tile_size])); + } + } + + if (this.use_thumbnail && grid_width * grid_height !== 1) { + tiles.push(await interpolate_4d(img, { size: [new_height, new_width] })); + } + } else { + tiles = [await interpolate_4d(img, { size: [new_height, new_width] })]; + } + + for (const tile of tiles) { + const [, , th, tw] = tile.dims; + const patches = convert_image_to_patches(tile, this.encoder_patch_size); + const { padded, mask } = pad_along_first_dim(patches, this.max_num_patches); + + all_pixel_values.push(padded); + all_pixel_masks.push(mask); + all_spatial_shapes.push([ + Math.floor(th / this.encoder_patch_size), + Math.floor(tw / this.encoder_patch_size), + ]); + } + + all_rows.push(num_rows); + all_cols.push(num_cols); + all_image_sizes.push([new_height, new_width]); + } + } + + /** @type {Record} */ + const result = { + pixel_values: cat(all_pixel_values, 0), + pixel_attention_mask: stack(all_pixel_masks, 0), + spatial_shapes: new Tensor('int64', BigInt64Array.from(all_spatial_shapes.flat(), BigInt), [ + all_spatial_shapes.length, + 2, + ]), + }; + + if (return_row_col_info ?? this.return_row_col_info) { + result.image_rows = all_rows; + result.image_cols = all_cols; + result.image_sizes = all_image_sizes; + } + + return result; + } +} diff --git a/packages/transformers/src/models/lfm2_vl/modeling_lfm2_vl.js b/packages/transformers/src/models/lfm2_vl/modeling_lfm2_vl.js new file mode 100644 index 0000000..9d7cba4 --- /dev/null +++ b/packages/transformers/src/models/lfm2_vl/modeling_lfm2_vl.js @@ -0,0 +1,13 @@ +import { LlavaForConditionalGeneration } from '../llava/modeling_llava.js'; + +export class Lfm2VlForConditionalGeneration extends LlavaForConditionalGeneration { + forward_params = [ + 'input_ids', + 'attention_mask', + 'pixel_values', + 'pixel_attention_mask', + 'spatial_shapes', + 'position_ids', + 'past_key_values', + ]; +} diff --git a/packages/transformers/src/models/lfm2_vl/processing_lfm2_vl.js b/packages/transformers/src/models/lfm2_vl/processing_lfm2_vl.js new file mode 100644 index 0000000..ec2755a --- /dev/null +++ b/packages/transformers/src/models/lfm2_vl/processing_lfm2_vl.js @@ -0,0 +1,77 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; + +/** + * @typedef {import('../../utils/image.js').RawImage} RawImage + */ + +export class Lfm2VlProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static image_processor_class = AutoImageProcessor; + + /** + * @param {RawImage|RawImage[]} images + * @param {string|string[]|null} [text] + * @param {Record} [kwargs] + */ + async _call(images, text = null, kwargs = {}) { + const { image_rows, image_cols, image_sizes, ...image_inputs } = await this.image_processor(images, { + ...kwargs, + return_row_col_info: true, + }); + + if (text) { + const image_token = this.config.image_token ?? ''; + const { + tile_size = 512, + downsample_factor = 2, + encoder_patch_size = 16, + use_thumbnail = true, + } = /** @type {Record} */ (this.image_processor.config); + + const ds = (/** @type {number} */ s) => Math.ceil(Math.floor(s / encoder_patch_size) / downsample_factor); + const tokens_per_tile = ds(tile_size) ** 2; + const image_start = this.config.image_start_token ?? '<|image_start|>'; + const image_end = this.config.image_end_token ?? '<|image_end|>'; + const thumbnail_token = this.config.image_thumbnail ?? '<|img_thumbnail|>'; + + if (!Array.isArray(text)) text = [text]; + + let image_idx = 0; + text = text.map((sample) => { + const parts = sample.split(image_token); + return ( + parts[0] + + parts + .slice(1) + .map((part) => { + const idx = image_idx++; + const [h, w] = image_sizes[idx]; + const rows = image_rows[idx], + cols = image_cols[idx]; + const tokens_for_image = ds(h) * ds(w); + + let expanded = image_start; + if (rows > 1 || cols > 1) { + const tile_str = image_token.repeat(tokens_per_tile); + for (let r = 0; r < rows; ++r) + for (let c = 0; c < cols; ++c) + expanded += `<|img_row_${r + 1}_col_${c + 1}|>` + tile_str; + if (use_thumbnail) expanded += thumbnail_token + image_token.repeat(tokens_for_image); + } else { + expanded += image_token.repeat(tokens_for_image); + } + return expanded + image_end + part; + }) + .join('') + ); + }); + } + + return { + ...image_inputs, + ...(text ? this.tokenizer(text, kwargs) : {}), + }; + } +} diff --git a/packages/transformers/src/models/lighton_ocr/modeling_lighton_ocr.js b/packages/transformers/src/models/lighton_ocr/modeling_lighton_ocr.js new file mode 100644 index 0000000..8e8f74a --- /dev/null +++ b/packages/transformers/src/models/lighton_ocr/modeling_lighton_ocr.js @@ -0,0 +1,3 @@ +import { LlavaForConditionalGeneration } from '../llava/modeling_llava.js'; + +export class LightOnOcrForConditionalGeneration extends LlavaForConditionalGeneration {} diff --git a/packages/transformers/src/models/llama/modeling_llama.js b/packages/transformers/src/models/llama/modeling_llama.js new file mode 100644 index 0000000..9e1e0ae --- /dev/null +++ b/packages/transformers/src/models/llama/modeling_llama.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +/** + * The bare LLama Model outputting raw hidden-states without any specific head on top. + */ +export class LlamaPreTrainedModel extends PreTrainedModel {} + +/** + * The bare LLaMA Model outputting raw hidden-states without any specific head on top. + */ +export class LlamaModel extends LlamaPreTrainedModel {} + +export class LlamaForCausalLM extends LlamaPreTrainedModel {} diff --git a/packages/transformers/src/models/llama/tokenization_llama.js b/packages/transformers/src/models/llama/tokenization_llama.js new file mode 100644 index 0000000..e26c028 --- /dev/null +++ b/packages/transformers/src/models/llama/tokenization_llama.js @@ -0,0 +1,5 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class LlamaTokenizer extends PreTrainedTokenizer { + padding_side = 'left'; +} diff --git a/packages/transformers/src/models/llama4/modeling_llama4.js b/packages/transformers/src/models/llama4/modeling_llama4.js new file mode 100644 index 0000000..6e46900 --- /dev/null +++ b/packages/transformers/src/models/llama4/modeling_llama4.js @@ -0,0 +1,4 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Llama4PreTrainedModel extends PreTrainedModel {} +export class Llama4ForCausalLM extends Llama4PreTrainedModel {} diff --git a/packages/transformers/src/models/llava/modeling_llava.js b/packages/transformers/src/models/llava/modeling_llava.js new file mode 100644 index 0000000..3b8fcd7 --- /dev/null +++ b/packages/transformers/src/models/llava/modeling_llava.js @@ -0,0 +1,26 @@ +import { PreTrainedModel, default_merge_input_ids_with_image_features } from '../modeling_utils.js'; + +export class LlavaPreTrainedModel extends PreTrainedModel { + forward_params = ['input_ids', 'attention_mask', 'pixel_values', 'position_ids', 'past_key_values']; +} + +/** + * The LLAVA model which consists of a vision backbone and a language model. + */ +export class LlavaForConditionalGeneration extends LlavaPreTrainedModel { + _merge_input_ids_with_image_features(kwargs) { + const vision_hidden_size = kwargs.image_features.dims.at(-1); + const reshaped_image_hidden_states = kwargs.image_features.view(-1, vision_hidden_size); + + return default_merge_input_ids_with_image_features({ + // @ts-ignore + image_token_id: this.config.image_token_index ?? this.config.image_token_id, + ...kwargs, + image_features: reshaped_image_hidden_states, + }); + } +} + +export class Moondream1ForConditionalGeneration extends LlavaForConditionalGeneration {} // NOTE: extends LlavaForConditionalGeneration + +export class LlavaQwen2ForCausalLM extends LlavaForConditionalGeneration {} // NOTE: extends LlavaForConditionalGeneration diff --git a/packages/transformers/src/models/llava/processing_llava.js b/packages/transformers/src/models/llava/processing_llava.js new file mode 100644 index 0000000..5e4394b --- /dev/null +++ b/packages/transformers/src/models/llava/processing_llava.js @@ -0,0 +1,41 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; + +export class LlavaProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static image_processor_class = AutoImageProcessor; + static uses_processor_config = true; + + /** + * @typedef {import('../../utils/image.js').RawImage} RawImage + */ + + // `images` is required, `text` is optional + async _call(/** @type {RawImage|RawImage[]} */ images, text = null, kwargs = {}) { + const image_inputs = await this.image_processor(images, kwargs); + + if (text) { + const [height, width] = image_inputs.pixel_values.dims.slice(-2); + + const { image_token, patch_size, num_additional_image_tokens } = this.config; + const num_image_tokens = + Math.floor(height / patch_size) * Math.floor(width / patch_size) + num_additional_image_tokens; + + text = structuredClone(text); // Avoid modifying the original text input + if (!Array.isArray(text)) { + text = [text]; + } + for (let i = 0; i < text.length; ++i) { + text[i] = text[i].replace(image_token, image_token.repeat(num_image_tokens)); + } + } + + const text_inputs = text ? this.tokenizer(text, kwargs) : {}; + + return { + ...image_inputs, + ...text_inputs, + }; + } +} diff --git a/packages/transformers/src/models/llava_onevision/image_processing_llava_onevision.js b/packages/transformers/src/models/llava_onevision/image_processing_llava_onevision.js new file mode 100644 index 0000000..e813f52 --- /dev/null +++ b/packages/transformers/src/models/llava_onevision/image_processing_llava_onevision.js @@ -0,0 +1,3 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class LlavaOnevisionImageProcessor extends ImageProcessor {} diff --git a/packages/transformers/src/models/llava_onevision/modeling_llava_onevision.js b/packages/transformers/src/models/llava_onevision/modeling_llava_onevision.js new file mode 100644 index 0000000..8542d31 --- /dev/null +++ b/packages/transformers/src/models/llava_onevision/modeling_llava_onevision.js @@ -0,0 +1 @@ +export { LlavaForConditionalGeneration as LlavaOnevisionForConditionalGeneration } from '../llava/modeling_llava.js'; diff --git a/packages/transformers/src/models/longt5/modeling_longt5.js b/packages/transformers/src/models/longt5/modeling_longt5.js new file mode 100644 index 0000000..6bae05a --- /dev/null +++ b/packages/transformers/src/models/longt5/modeling_longt5.js @@ -0,0 +1,15 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +/** + * An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. + */ +export class LongT5PreTrainedModel extends PreTrainedModel {} + +/** + * The bare LONGT5 Model transformer outputting raw hidden-states without any specific head on top. + */ +export class LongT5Model extends LongT5PreTrainedModel {} + +/** + * LONGT5 Model with a `language modeling` head on top. + */ +export class LongT5ForConditionalGeneration extends LongT5PreTrainedModel {} diff --git a/packages/transformers/src/models/m2m_100/modeling_m2m_100.js b/packages/transformers/src/models/m2m_100/modeling_m2m_100.js new file mode 100644 index 0000000..d6814c1 --- /dev/null +++ b/packages/transformers/src/models/m2m_100/modeling_m2m_100.js @@ -0,0 +1,7 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class M2M100PreTrainedModel extends PreTrainedModel {} + +export class M2M100Model extends M2M100PreTrainedModel {} + +export class M2M100ForConditionalGeneration extends M2M100PreTrainedModel {} diff --git a/packages/transformers/src/models/m2m_100/tokenization_m2m_100.js b/packages/transformers/src/models/m2m_100/tokenization_m2m_100.js new file mode 100644 index 0000000..db17795 --- /dev/null +++ b/packages/transformers/src/models/m2m_100/tokenization_m2m_100.js @@ -0,0 +1,34 @@ +import { PreTrainedTokenizer, _build_translation_inputs } from '../../tokenization_utils.js'; + +/** + * The M2M100Tokenizer class is used to tokenize text for M2M100 ("Many-to-Many") models. + * + * M2M100 is a multilingual encoder-decoder (seq-to-seq) model trained for Many-to-Many + * multilingual translation. It was introduced in this [paper](https://huggingface.co/papers/2010.11125) + * and first released in [this](https://github.com/pytorch/fairseq/tree/master/examples/m2m_100) repository. + * + * For a list of supported languages (along with their language codes), + * @see {@link https://huggingface.co/facebook/m2m100_418M#languages-covered} + */ +export class M2M100Tokenizer extends PreTrainedTokenizer { + constructor(tokenizerJSON, tokenizerConfig) { + super(tokenizerJSON, tokenizerConfig); + + this.languageRegex = /^__[a-z]{2,3}__$/; + this.language_codes = this.all_special_tokens + .filter((x) => this.languageRegex.test(x)) + .map((x) => x.slice(2, -2)); + this.lang_to_token = (x) => `__${x}__`; + } + + /** + * Helper function to build translation inputs for an `M2M100Tokenizer`. + * @param {string|string[]} raw_inputs The text to tokenize. + * @param {Object} tokenizer_options Options to be sent to the tokenizer + * @param {Object} generate_kwargs Generation options. + * @returns {Object} Object to be passed to the model. + */ + _build_translation_inputs(raw_inputs, tokenizer_options, generate_kwargs) { + return _build_translation_inputs(this, raw_inputs, tokenizer_options, generate_kwargs); + } +} diff --git a/packages/transformers/src/models/marian/modeling_marian.js b/packages/transformers/src/models/marian/modeling_marian.js new file mode 100644 index 0000000..28888b9 --- /dev/null +++ b/packages/transformers/src/models/marian/modeling_marian.js @@ -0,0 +1,7 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class MarianPreTrainedModel extends PreTrainedModel {} + +export class MarianModel extends MarianPreTrainedModel {} + +export class MarianMTModel extends MarianPreTrainedModel {} diff --git a/packages/transformers/src/models/marian/tokenization_marian.js b/packages/transformers/src/models/marian/tokenization_marian.js new file mode 100644 index 0000000..cc78793 --- /dev/null +++ b/packages/transformers/src/models/marian/tokenization_marian.js @@ -0,0 +1,56 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; +import { mergeArrays } from '../../utils/core.js'; +import { logger } from '../../utils/logger.js'; + +/** + * @todo This model is not yet supported by Hugging Face's "fast" tokenizers library (https://github.com/huggingface/tokenizers). + * Therefore, this implementation (which is based on fast tokenizers) may produce slightly inaccurate results. + */ +export class MarianTokenizer extends PreTrainedTokenizer { + /** + * Create a new MarianTokenizer instance. + * @param {Object} tokenizerJSON The JSON of the tokenizer. + * @param {Object} tokenizerConfig The config of the tokenizer. + */ + constructor(tokenizerJSON, tokenizerConfig) { + super(tokenizerJSON, tokenizerConfig); + + this.languageRegex = /^(>>\w+<<)\s*/g; + + this.supported_language_codes = Array.from(this.get_vocab().keys()).filter((x) => this.languageRegex.test(x)); + + logger.warn( + 'WARNING: `MarianTokenizer` is not yet supported by Hugging Face\'s "fast" tokenizers library. Therefore, you may experience slightly inaccurate results.', + ); + } + + /** + * Encodes a single text. Overriding this method is necessary since the language codes + * must be removed before encoding with sentencepiece model. + * @see https://github.com/huggingface/transformers/blob/12d51db243a00726a548a43cc333390ebae731e3/src/transformers/models/marian/tokenization_marian.py#L204-L213 + * + * @param {string|null} text The text to encode. + * @returns {string[]|null} The encoded tokens. + */ + _encode_text(text) { + if (text === null) return null; + + // Check if text starts with language code: + const [matchInfo, ...remainder] = text.trim().split(this.languageRegex); + + if (remainder.length === 0) { + // No language code, encode normally + return super._encode_text(matchInfo); + } else if (remainder.length === 2) { + // Text starts with language code, so we do not encode it with sentencepiece. + const [language, text] = remainder; + + if (!this.supported_language_codes.includes(language)) { + logger.warn( + `Unsupported language code "${language}" detected, which may lead to unexpected behavior. Should be one of: ${JSON.stringify(this.supported_language_codes)}`, + ); + } + return mergeArrays([language], super._encode_text(text)); + } + } +} diff --git a/packages/transformers/src/models/mask2former/image_processing_mask2former.js b/packages/transformers/src/models/mask2former/image_processing_mask2former.js new file mode 100644 index 0000000..f8def92 --- /dev/null +++ b/packages/transformers/src/models/mask2former/image_processing_mask2former.js @@ -0,0 +1,4 @@ +import { MaskFormerImageProcessor } from '../maskformer/image_processing_maskformer.js'; + +// NOTE: extends MaskFormerImageProcessor +export class Mask2FormerImageProcessor extends MaskFormerImageProcessor {} diff --git a/packages/transformers/src/models/maskformer/image_processing_maskformer.js b/packages/transformers/src/models/maskformer/image_processing_maskformer.js new file mode 100644 index 0000000..fc48944 --- /dev/null +++ b/packages/transformers/src/models/maskformer/image_processing_maskformer.js @@ -0,0 +1,17 @@ +import { + ImageProcessor, + post_process_panoptic_segmentation, + post_process_instance_segmentation, +} from '../../image_processors_utils.js'; + +export class MaskFormerImageProcessor extends ImageProcessor { + /** @type {typeof post_process_panoptic_segmentation} */ + post_process_panoptic_segmentation(...args) { + return post_process_panoptic_segmentation(...args); + } + /** @type {typeof post_process_instance_segmentation} */ + post_process_instance_segmentation(...args) { + return post_process_instance_segmentation(...args); + } +} +export class MaskFormerFeatureExtractor extends MaskFormerImageProcessor {} diff --git a/packages/transformers/src/models/maskformer/modeling_maskformer.js b/packages/transformers/src/models/maskformer/modeling_maskformer.js new file mode 100644 index 0000000..85903e8 --- /dev/null +++ b/packages/transformers/src/models/maskformer/modeling_maskformer.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class MaskFormerPreTrainedModel extends PreTrainedModel {} +export class MaskFormerModel extends MaskFormerPreTrainedModel {} +export class MaskFormerForInstanceSegmentation extends MaskFormerPreTrainedModel {} diff --git a/packages/transformers/src/models/mbart/modeling_mbart.js b/packages/transformers/src/models/mbart/modeling_mbart.js new file mode 100644 index 0000000..5b0e19c --- /dev/null +++ b/packages/transformers/src/models/mbart/modeling_mbart.js @@ -0,0 +1,31 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class MBartPreTrainedModel extends PreTrainedModel {} + +/** + * The bare MBART Model outputting raw hidden-states without any specific head on top. + */ +export class MBartModel extends MBartPreTrainedModel {} + +/** + * The MBART Model with a language modeling head. Can be used for summarization, after fine-tuning the pretrained models. + */ +export class MBartForConditionalGeneration extends MBartPreTrainedModel {} + +/** + * MBart model with a sequence classification/head on top (a linear layer on top of the pooled output). + */ +export class MBartForSequenceClassification extends MBartPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +export class MBartForCausalLM extends MBartPreTrainedModel {} diff --git a/packages/transformers/src/models/mbart/tokenization_mbart.js b/packages/transformers/src/models/mbart/tokenization_mbart.js new file mode 100644 index 0000000..87032f6 --- /dev/null +++ b/packages/transformers/src/models/mbart/tokenization_mbart.js @@ -0,0 +1,22 @@ +import { PreTrainedTokenizer, _build_translation_inputs } from '../../tokenization_utils.js'; + +export class MBartTokenizer extends PreTrainedTokenizer { + constructor(tokenizerJSON, tokenizerConfig) { + super(tokenizerJSON, tokenizerConfig); + + this.languageRegex = /^[a-z]{2}_[A-Z]{2}$/; + this.language_codes = this.all_special_tokens.filter((x) => this.languageRegex.test(x)).map((x) => x); + this.lang_to_token = (x) => x; // Identity function + } + + /** + * Helper function to build translation inputs for an `MBartTokenizer`. + * @param {string|string[]} raw_inputs The text to tokenize. + * @param {Object} tokenizer_options Options to be sent to the tokenizer + * @param {Object} generate_kwargs Generation options. + * @returns {Object} Object to be passed to the model. + */ + _build_translation_inputs(raw_inputs, tokenizer_options, generate_kwargs) { + return _build_translation_inputs(this, raw_inputs, tokenizer_options, generate_kwargs); + } +} diff --git a/packages/transformers/src/models/mbart50/tokenization_mbart50.js b/packages/transformers/src/models/mbart50/tokenization_mbart50.js new file mode 100644 index 0000000..c872411 --- /dev/null +++ b/packages/transformers/src/models/mbart50/tokenization_mbart50.js @@ -0,0 +1,3 @@ +import { MBartTokenizer } from '../mbart/tokenization_mbart.js'; + +export class MBart50Tokenizer extends MBartTokenizer {} // NOTE: extends MBartTokenizer diff --git a/packages/transformers/src/models/metric3d/modeling_metric3d.js b/packages/transformers/src/models/metric3d/modeling_metric3d.js new file mode 100644 index 0000000..ea98741 --- /dev/null +++ b/packages/transformers/src/models/metric3d/modeling_metric3d.js @@ -0,0 +1,4 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Metric3DPreTrainedModel extends PreTrainedModel {} +export class Metric3DForDepthEstimation extends Metric3DPreTrainedModel {} diff --git a/packages/transformers/src/models/metric3dv2/modeling_metric3dv2.js b/packages/transformers/src/models/metric3dv2/modeling_metric3dv2.js new file mode 100644 index 0000000..f1859d1 --- /dev/null +++ b/packages/transformers/src/models/metric3dv2/modeling_metric3dv2.js @@ -0,0 +1,4 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Metric3Dv2PreTrainedModel extends PreTrainedModel {} +export class Metric3Dv2ForDepthEstimation extends Metric3Dv2PreTrainedModel {} diff --git a/packages/transformers/src/models/mgp_str/modeling_mgp_str.js b/packages/transformers/src/models/mgp_str/modeling_mgp_str.js new file mode 100644 index 0000000..b46c573 --- /dev/null +++ b/packages/transformers/src/models/mgp_str/modeling_mgp_str.js @@ -0,0 +1,30 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { ModelOutput } from '../modeling_outputs.js'; + +export class MgpstrModelOutput extends ModelOutput { + constructor({ char_logits, bpe_logits, wp_logits }) { + super(); + this.char_logits = char_logits; + this.bpe_logits = bpe_logits; + this.wp_logits = wp_logits; + } + + get logits() { + return [this.char_logits, this.bpe_logits, this.wp_logits]; + } +} + +export class MgpstrPreTrainedModel extends PreTrainedModel {} + +/** + * MGP-STR Model transformer with three classification heads on top + * (three A^3 modules and three linear layer on top of the transformer encoder output) for scene text recognition (STR). + */ +export class MgpstrForSceneTextRecognition extends MgpstrPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new MgpstrModelOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/mgp_str/processing_mgp_str.js b/packages/transformers/src/models/mgp_str/processing_mgp_str.js new file mode 100644 index 0000000..a7f14b9 --- /dev/null +++ b/packages/transformers/src/models/mgp_str/processing_mgp_str.js @@ -0,0 +1,169 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { max, softmax } from '../../utils/maths.js'; + +const DECODE_TYPE_MAPPING = { + char: ['char_decode', 1], + bpe: ['bpe_decode', 2], + wp: ['wp_decode', 102], +}; +export class MgpstrProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static image_processor_class = AutoImageProcessor; + + /** + * @returns {import('./tokenization_mgp_str.js').MgpstrTokenizer} The character tokenizer. + */ + get char_tokenizer() { + return this.components.char_tokenizer; + } + + /** + * @returns {import('../gpt2/tokenization_gpt2.js').GPT2Tokenizer} The BPE tokenizer. + */ + get bpe_tokenizer() { + return this.components.bpe_tokenizer; + } + + /** + * @returns {import('../bert/tokenization_bert.js').BertTokenizer} The WordPiece tokenizer. + */ + get wp_tokenizer() { + return this.components.wp_tokenizer; + } + + /** + * Helper function to decode the model prediction logits. + * @param {import('../../utils/tensor.js').Tensor} pred_logits Model prediction logits. + * @param {string} format Type of model prediction. Must be one of ['char', 'bpe', 'wp']. + * @returns {[string[], number[]]} The decoded sentences and their confidence scores. + */ + _decode_helper(pred_logits, format) { + if (!DECODE_TYPE_MAPPING.hasOwnProperty(format)) { + throw new Error(`Format ${format} is not supported.`); + } + + const [decoder_name, eos_token] = DECODE_TYPE_MAPPING[format]; + const decoder = this[decoder_name].bind(this); + + const [batch_size, batch_max_length] = pred_logits.dims; + const conf_scores = []; + const all_ids = []; + + /** @type {number[][][]} */ + const pred_logits_list = pred_logits.tolist(); + for (let i = 0; i < batch_size; ++i) { + const logits = pred_logits_list[i]; + const ids = []; + const scores = []; + + // Start and index=1 to skip the first token + for (let j = 1; j < batch_max_length; ++j) { + // NOTE: == to match bigint and number + const [max_prob, max_prob_index] = max(softmax(logits[j])); + scores.push(max_prob); + if (max_prob_index == eos_token) { + break; + } + ids.push(max_prob_index); + } + + const confidence_score = scores.length > 0 ? scores.reduce((a, b) => a * b, 1) : 0; + + all_ids.push(ids); + conf_scores.push(confidence_score); + } + + const decoded = decoder(all_ids); + return [decoded, conf_scores]; + } + + /** + * Convert a list of lists of char token ids into a list of strings by calling char tokenizer. + * @param {number[][]} sequences List of tokenized input ids. + * @returns {string[]} The list of char decoded sentences. + */ + char_decode(sequences) { + return this.char_tokenizer.batch_decode(sequences).map((str) => str.replaceAll(' ', '')); + } + + /** + * Convert a list of lists of BPE token ids into a list of strings by calling BPE tokenizer. + * @param {number[][]} sequences List of tokenized input ids. + * @returns {string[]} The list of BPE decoded sentences. + */ + bpe_decode(sequences) { + return this.bpe_tokenizer.batch_decode(sequences); + } + + /** + * Convert a list of lists of word piece token ids into a list of strings by calling word piece tokenizer. + * @param {number[][]} sequences List of tokenized input ids. + * @returns {string[]} The list of wp decoded sentences. + */ + wp_decode(sequences) { + return this.wp_tokenizer.batch_decode(sequences).map((str) => str.replaceAll(' ', '')); + } + + /** + * Convert a list of lists of token ids into a list of strings by calling decode. + * @param {[import('../../utils/tensor.js').Tensor, import('../../utils/tensor.js').Tensor, import('../../utils/tensor.js').Tensor]} sequences List of tokenized input ids. + * @returns {{generated_text: string[], scores: number[], char_preds: string[], bpe_preds: string[], wp_preds: string[]}} + * Dictionary of all the outputs of the decoded results. + * - generated_text: The final results after fusion of char, bpe, and wp. + * - scores: The final scores after fusion of char, bpe, and wp. + * - char_preds: The list of character decoded sentences. + * - bpe_preds: The list of BPE decoded sentences. + * - wp_preds: The list of wp decoded sentences. + */ + // @ts-expect-error The type of this method is not compatible with the one in the base class. + batch_decode([char_logits, bpe_logits, wp_logits]) { + const [char_preds, char_scores] = this._decode_helper(char_logits, 'char'); + const [bpe_preds, bpe_scores] = this._decode_helper(bpe_logits, 'bpe'); + const [wp_preds, wp_scores] = this._decode_helper(wp_logits, 'wp'); + + const generated_text = []; + const scores = []; + for (let i = 0; i < char_preds.length; ++i) { + const [max_score, max_score_index] = max([char_scores[i], bpe_scores[i], wp_scores[i]]); + generated_text.push([char_preds[i], bpe_preds[i], wp_preds[i]][max_score_index]); + scores.push(max_score); + } + + return { + generated_text, + scores, + char_preds, + bpe_preds, + wp_preds, + }; + } + /** @type {typeof Processor.from_pretrained} */ + static async from_pretrained(...args) { + const base = await super.from_pretrained(...args); + + // Load Transformers.js-compatible versions of the BPE and WordPiece tokenizers + const bpe_tokenizer = await AutoTokenizer.from_pretrained('Xenova/gpt2'); // openai-community/gpt2 + const wp_tokenizer = await AutoTokenizer.from_pretrained('Xenova/bert-base-uncased'); // google-bert/bert-base-uncased + + // Update components + base.components = { + image_processor: base.image_processor, + char_tokenizer: base.tokenizer, + bpe_tokenizer: bpe_tokenizer, + wp_tokenizer: wp_tokenizer, + }; + return base; + } + + async _call(images, text = null) { + const result = await this.image_processor(images); + + if (text) { + result.labels = this.tokenizer(text).input_ids; + } + + return result; + } +} diff --git a/packages/transformers/src/models/mgp_str/tokenization_mgp_str.js b/packages/transformers/src/models/mgp_str/tokenization_mgp_str.js new file mode 100644 index 0000000..df0f90b --- /dev/null +++ b/packages/transformers/src/models/mgp_str/tokenization_mgp_str.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class MgpstrTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/mimi/modeling_mimi.js b/packages/transformers/src/models/mimi/modeling_mimi.js new file mode 100644 index 0000000..b7a83b1 --- /dev/null +++ b/packages/transformers/src/models/mimi/modeling_mimi.js @@ -0,0 +1,76 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { sessionRun } from '../session.js'; +import { ModelOutput } from '../modeling_outputs.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class MimiEncoderOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.audio_codes Discrete code embeddings, of shape `(batch_size, num_quantizers, codes_length)`. + */ + constructor({ audio_codes }) { + super(); + this.audio_codes = audio_codes; + } +} + +export class MimiDecoderOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.audio_values Decoded audio values, of shape `(batch_size, num_channels, sequence_length)`. + */ + constructor({ audio_values }) { + super(); + this.audio_values = audio_values; + } +} + +export class MimiPreTrainedModel extends PreTrainedModel { + main_input_name = 'input_values'; + forward_params = ['input_values']; +} + +/** + * The Mimi neural audio codec model. + */ +export class MimiModel extends MimiPreTrainedModel { + /** + * Encodes the input audio waveform into discrete codes. + * @param {Object} inputs Model inputs + * @param {Tensor} [inputs.input_values] Float values of the input audio waveform, of shape `(batch_size, channels, sequence_length)`). + * @returns {Promise} The output tensor of shape `(batch_size, num_codebooks, sequence_length)`. + */ + async encode(inputs) { + return new MimiEncoderOutput(await sessionRun(this.sessions['encoder_model'], inputs)); + } + + /** + * Decodes the given frames into an output audio waveform. + * @param {MimiEncoderOutput} inputs The encoded audio codes. + * @returns {Promise} The output tensor of shape `(batch_size, num_channels, sequence_length)`. + */ + async decode(inputs) { + return new MimiDecoderOutput(await sessionRun(this.sessions['decoder_model'], inputs)); + } +} + +export class MimiEncoderModel extends MimiPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'encoder_model', + }); + } +} +export class MimiDecoderModel extends MimiPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'decoder_model', + }); + } +} diff --git a/packages/transformers/src/models/ministral/modeling_ministral.js b/packages/transformers/src/models/ministral/modeling_ministral.js new file mode 100644 index 0000000..c31cdc5 --- /dev/null +++ b/packages/transformers/src/models/ministral/modeling_ministral.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class MinistralPreTrainedModel extends PreTrainedModel {} +export class MinistralModel extends MinistralPreTrainedModel {} +export class MinistralForCausalLM extends MinistralPreTrainedModel {} diff --git a/packages/transformers/src/models/ministral3/modeling_ministral3.js b/packages/transformers/src/models/ministral3/modeling_ministral3.js new file mode 100644 index 0000000..1d9c05e --- /dev/null +++ b/packages/transformers/src/models/ministral3/modeling_ministral3.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Ministral3PreTrainedModel extends PreTrainedModel {} +export class Ministral3Model extends Ministral3PreTrainedModel {} +export class Ministral3ForCausalLM extends Ministral3PreTrainedModel {} diff --git a/packages/transformers/src/models/mistral/modeling_mistral.js b/packages/transformers/src/models/mistral/modeling_mistral.js new file mode 100644 index 0000000..24d7233 --- /dev/null +++ b/packages/transformers/src/models/mistral/modeling_mistral.js @@ -0,0 +1,10 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +/** + * The bare Mistral Model outputting raw hidden-states without any specific head on top. + */ +export class MistralPreTrainedModel extends PreTrainedModel {} + +export class MistralModel extends MistralPreTrainedModel {} + +export class MistralForCausalLM extends MistralPreTrainedModel {} diff --git a/packages/transformers/src/models/mistral3/modeling_mistral3.js b/packages/transformers/src/models/mistral3/modeling_mistral3.js new file mode 100644 index 0000000..45d0aee --- /dev/null +++ b/packages/transformers/src/models/mistral3/modeling_mistral3.js @@ -0,0 +1,3 @@ +import { LlavaForConditionalGeneration } from '../llava/modeling_llava.js'; + +export class Mistral3ForConditionalGeneration extends LlavaForConditionalGeneration {} diff --git a/packages/transformers/src/models/mistral4/modeling_mistral4.js b/packages/transformers/src/models/mistral4/modeling_mistral4.js new file mode 100644 index 0000000..6eabfa3 --- /dev/null +++ b/packages/transformers/src/models/mistral4/modeling_mistral4.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Mistral4PreTrainedModel extends PreTrainedModel {} +export class Mistral4Model extends Mistral4PreTrainedModel {} +export class Mistral4ForCausalLM extends Mistral4PreTrainedModel {} diff --git a/packages/transformers/src/models/mobilebert/modeling_mobilebert.js b/packages/transformers/src/models/mobilebert/modeling_mobilebert.js new file mode 100644 index 0000000..006cb3b --- /dev/null +++ b/packages/transformers/src/models/mobilebert/modeling_mobilebert.js @@ -0,0 +1,50 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { MaskedLMOutput, QuestionAnsweringModelOutput, SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class MobileBertPreTrainedModel extends PreTrainedModel {} +export class MobileBertModel extends MobileBertPreTrainedModel {} + +/** + * MobileBertForMaskedLM is a class representing a MobileBERT model for masking task. + */ +export class MobileBertForMaskedLM extends MobileBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +/** + * MobileBert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) + */ +export class MobileBertForSequenceClassification extends MobileBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * MobileBert Model with a span classification head on top for extractive question-answering tasks + */ +export class MobileBertForQuestionAnswering extends MobileBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/mobilebert/tokenization_mobilebert.js b/packages/transformers/src/models/mobilebert/tokenization_mobilebert.js new file mode 100644 index 0000000..d47723f --- /dev/null +++ b/packages/transformers/src/models/mobilebert/tokenization_mobilebert.js @@ -0,0 +1,5 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class MobileBertTokenizer extends PreTrainedTokenizer { + return_token_type_ids = true; +} diff --git a/packages/transformers/src/models/mobilellm/modeling_mobilellm.js b/packages/transformers/src/models/mobilellm/modeling_mobilellm.js new file mode 100644 index 0000000..220e46c --- /dev/null +++ b/packages/transformers/src/models/mobilellm/modeling_mobilellm.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class MobileLLMPreTrainedModel extends PreTrainedModel {} +export class MobileLLMModel extends MobileLLMPreTrainedModel {} +export class MobileLLMForCausalLM extends MobileLLMPreTrainedModel {} diff --git a/packages/transformers/src/models/mobilenet_v1/image_processing_mobilenet_v1.js b/packages/transformers/src/models/mobilenet_v1/image_processing_mobilenet_v1.js new file mode 100644 index 0000000..4d7ad7b --- /dev/null +++ b/packages/transformers/src/models/mobilenet_v1/image_processing_mobilenet_v1.js @@ -0,0 +1,4 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class MobileNetV1ImageProcessor extends ImageProcessor {} +export class MobileNetV1FeatureExtractor extends MobileNetV1ImageProcessor {} diff --git a/packages/transformers/src/models/mobilenet_v1/modeling_mobilenet_v1.js b/packages/transformers/src/models/mobilenet_v1/modeling_mobilenet_v1.js new file mode 100644 index 0000000..b1783c4 --- /dev/null +++ b/packages/transformers/src/models/mobilenet_v1/modeling_mobilenet_v1.js @@ -0,0 +1,24 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class MobileNetV1PreTrainedModel extends PreTrainedModel {} + +/** + * The bare MobileNetV1 model outputting raw hidden-states without any specific head on top. + */ +export class MobileNetV1Model extends MobileNetV1PreTrainedModel {} + +/** + * MobileNetV1 model with an image classification head on top (a linear layer on top of the pooled features), + * e.g. for ImageNet. + */ +export class MobileNetV1ForImageClassification extends MobileNetV1PreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +export class MobileNetV1ForSemanticSegmentation extends MobileNetV1PreTrainedModel {} diff --git a/packages/transformers/src/models/mobilenet_v2/image_processing_mobilenet_v2.js b/packages/transformers/src/models/mobilenet_v2/image_processing_mobilenet_v2.js new file mode 100644 index 0000000..68442b0 --- /dev/null +++ b/packages/transformers/src/models/mobilenet_v2/image_processing_mobilenet_v2.js @@ -0,0 +1,4 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class MobileNetV2ImageProcessor extends ImageProcessor {} +export class MobileNetV2FeatureExtractor extends MobileNetV2ImageProcessor {} diff --git a/packages/transformers/src/models/mobilenet_v2/modeling_mobilenet_v2.js b/packages/transformers/src/models/mobilenet_v2/modeling_mobilenet_v2.js new file mode 100644 index 0000000..127baa6 --- /dev/null +++ b/packages/transformers/src/models/mobilenet_v2/modeling_mobilenet_v2.js @@ -0,0 +1,23 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class MobileNetV2PreTrainedModel extends PreTrainedModel {} + +/** + * The bare MobileNetV2 model outputting raw hidden-states without any specific head on top. + */ +export class MobileNetV2Model extends MobileNetV2PreTrainedModel {} + +/** + * MobileNetV2 model with an image classification head on top (a linear layer on top of the pooled features), + * e.g. for ImageNet. + */ +export class MobileNetV2ForImageClassification extends MobileNetV2PreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} +export class MobileNetV2ForSemanticSegmentation extends MobileNetV2PreTrainedModel {} diff --git a/packages/transformers/src/models/mobilenet_v3/image_processing_mobilenet_v3.js b/packages/transformers/src/models/mobilenet_v3/image_processing_mobilenet_v3.js new file mode 100644 index 0000000..1d76efa --- /dev/null +++ b/packages/transformers/src/models/mobilenet_v3/image_processing_mobilenet_v3.js @@ -0,0 +1,4 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class MobileNetV3ImageProcessor extends ImageProcessor {} +export class MobileNetV3FeatureExtractor extends MobileNetV3ImageProcessor {} diff --git a/packages/transformers/src/models/mobilenet_v3/modeling_mobilenet_v3.js b/packages/transformers/src/models/mobilenet_v3/modeling_mobilenet_v3.js new file mode 100644 index 0000000..b7200c6 --- /dev/null +++ b/packages/transformers/src/models/mobilenet_v3/modeling_mobilenet_v3.js @@ -0,0 +1,23 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class MobileNetV3PreTrainedModel extends PreTrainedModel {} + +/** + * The bare MobileNetV3 model outputting raw hidden-states without any specific head on top. + */ +export class MobileNetV3Model extends MobileNetV3PreTrainedModel {} + +/** + * MobileNetV3 model with an image classification head on top (a linear layer on top of the pooled features), + * e.g. for ImageNet. + */ +export class MobileNetV3ForImageClassification extends MobileNetV3PreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} +export class MobileNetV3ForSemanticSegmentation extends MobileNetV3PreTrainedModel {} diff --git a/packages/transformers/src/models/mobilenet_v4/image_processing_mobilenet_v4.js b/packages/transformers/src/models/mobilenet_v4/image_processing_mobilenet_v4.js new file mode 100644 index 0000000..999c9a8 --- /dev/null +++ b/packages/transformers/src/models/mobilenet_v4/image_processing_mobilenet_v4.js @@ -0,0 +1,4 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class MobileNetV4ImageProcessor extends ImageProcessor {} +export class MobileNetV4FeatureExtractor extends MobileNetV4ImageProcessor {} diff --git a/packages/transformers/src/models/mobilenet_v4/modeling_mobilenet_v4.js b/packages/transformers/src/models/mobilenet_v4/modeling_mobilenet_v4.js new file mode 100644 index 0000000..d6e92ff --- /dev/null +++ b/packages/transformers/src/models/mobilenet_v4/modeling_mobilenet_v4.js @@ -0,0 +1,23 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class MobileNetV4PreTrainedModel extends PreTrainedModel {} + +/** + * The bare MobileNetV4 model outputting raw hidden-states without any specific head on top. + */ +export class MobileNetV4Model extends MobileNetV4PreTrainedModel {} + +/** + * MobileNetV4 model with an image classification head on top (a linear layer on top of the pooled features), + * e.g. for ImageNet. + */ +export class MobileNetV4ForImageClassification extends MobileNetV4PreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} +export class MobileNetV4ForSemanticSegmentation extends MobileNetV4PreTrainedModel {} diff --git a/packages/transformers/src/models/mobilevit/image_processing_mobilevit.js b/packages/transformers/src/models/mobilevit/image_processing_mobilevit.js new file mode 100644 index 0000000..bbd0fd3 --- /dev/null +++ b/packages/transformers/src/models/mobilevit/image_processing_mobilevit.js @@ -0,0 +1,4 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class MobileViTImageProcessor extends ImageProcessor {} +export class MobileViTFeatureExtractor extends MobileViTImageProcessor {} diff --git a/packages/transformers/src/models/mobilevit/modeling_mobilevit.js b/packages/transformers/src/models/mobilevit/modeling_mobilevit.js new file mode 100644 index 0000000..09292c5 --- /dev/null +++ b/packages/transformers/src/models/mobilevit/modeling_mobilevit.js @@ -0,0 +1,14 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class MobileViTPreTrainedModel extends PreTrainedModel {} +export class MobileViTModel extends MobileViTPreTrainedModel {} +export class MobileViTForImageClassification extends MobileViTPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} +// TODO: MobileViTForSemanticSegmentation diff --git a/packages/transformers/src/models/mobilevitv2/modeling_mobilevitv2.js b/packages/transformers/src/models/mobilevitv2/modeling_mobilevitv2.js new file mode 100644 index 0000000..8eb4f2f --- /dev/null +++ b/packages/transformers/src/models/mobilevitv2/modeling_mobilevitv2.js @@ -0,0 +1,14 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class MobileViTV2PreTrainedModel extends PreTrainedModel {} +export class MobileViTV2Model extends MobileViTV2PreTrainedModel {} +export class MobileViTV2ForImageClassification extends MobileViTV2PreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} +// TODO: MobileViTV2ForSemanticSegmentation diff --git a/packages/transformers/src/models/modeling_outputs.js b/packages/transformers/src/models/modeling_outputs.js new file mode 100644 index 0000000..73fce1f --- /dev/null +++ b/packages/transformers/src/models/modeling_outputs.js @@ -0,0 +1,149 @@ +/** + * @typedef {import('../utils/tensor.js').Tensor} Tensor + */ + +export class ModelOutput {} + +/** + * Base class for model's outputs, with potential hidden states and attentions. + */ +export class BaseModelOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.last_hidden_state Sequence of hidden-states at the output of the last layer of the model. + * @param {Tensor} [output.hidden_states] Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + * @param {Tensor} [output.attentions] Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + */ + constructor({ last_hidden_state, hidden_states = null, attentions = null }) { + super(); + this.last_hidden_state = last_hidden_state; + this.hidden_states = hidden_states; + this.attentions = attentions; + } +} + +/** + * Base class for outputs of sentence classification models. + */ +export class SequenceClassifierOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.logits classification (or regression if config.num_labels==1) scores (before SoftMax). + * @param {Record} [output.attentions] Object of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. + * Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + */ + constructor({ logits, ...attentions }) { + super(); + this.logits = logits; + const attentions_list = Object.values(attentions); + if (attentions_list.length > 0) { + // Only set attentions if they are not empty + this.attentions = attentions_list; + } + } +} + +/** + * Base class for outputs of token classification models. + */ +export class TokenClassifierOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.logits Classification scores (before SoftMax). + */ + constructor({ logits }) { + super(); + this.logits = logits; + } +} + +/** + * Base class for masked language models outputs. + */ +export class MaskedLMOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.logits Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + */ + constructor({ logits }) { + super(); + this.logits = logits; + } +} + +/** + * Base class for outputs of question answering models. + */ +export class QuestionAnsweringModelOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.start_logits Span-start scores (before SoftMax). + * @param {Tensor} output.end_logits Span-end scores (before SoftMax). + */ + constructor({ start_logits, end_logits }) { + super(); + this.start_logits = start_logits; + this.end_logits = end_logits; + } +} + +/** + * Base class for causal language model (or autoregressive) outputs. + */ +export class CausalLMOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.logits Prediction scores of the language modeling head (scores for each vocabulary token before softmax). + */ + constructor({ logits }) { + super(); + this.logits = logits; + } +} + +/** + * Base class for causal language model (or autoregressive) outputs. + */ +export class CausalLMOutputWithPast extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.logits Prediction scores of the language modeling head (scores for each vocabulary token before softmax). + * @param {Tensor} output.past_key_values Contains pre-computed hidden-states (key and values in the self-attention blocks) + * that can be used (see `past_key_values` input) to speed up sequential decoding. + */ + constructor({ logits, past_key_values }) { + super(); + this.logits = logits; + this.past_key_values = past_key_values; + } +} + +export class Seq2SeqLMOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.logits The output logits of the model. + * @param {Tensor} output.past_key_values An tensor of key/value pairs that represent the previous state of the model. + * @param {Tensor} output.encoder_outputs The output of the encoder in a sequence-to-sequence model. + * @param {Tensor} [output.decoder_attentions] Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the self-attention heads. + * @param {Tensor} [output.cross_attentions] Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the weighted average in the cross-attention heads. + */ + constructor({ logits, past_key_values, encoder_outputs, decoder_attentions = null, cross_attentions = null }) { + super(); + this.logits = logits; + this.past_key_values = past_key_values; + this.encoder_outputs = encoder_outputs; + this.decoder_attentions = decoder_attentions; + this.cross_attentions = cross_attentions; + } +} + +export class ImageMattingOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.alphas Estimated alpha values, of shape `(batch_size, num_channels, height, width)`. + */ + constructor({ alphas }) { + super(); + this.alphas = alphas; + } +} diff --git a/packages/transformers/src/models/modeling_utils.js b/packages/transformers/src/models/modeling_utils.js new file mode 100644 index 0000000..ec9f174 --- /dev/null +++ b/packages/transformers/src/models/modeling_utils.js @@ -0,0 +1,1726 @@ +import { Callable } from '../utils/generic.js'; +import { constructSessions, sessionRun } from './session.js'; +import { AutoConfig, getCacheNames } from '../configs.js'; +import { Tensor, full_like, cat, zeros_like, ones_like, ones } from '../utils/tensor.js'; +import { DataTypeMap } from '../utils/dtypes.js'; + +// These will be populated by registry.js +export let MODEL_MAPPING_NAMES = null; + +/** + * Register task mappings (called by registry.js after defining full mappings) + * @param {Object} mappings - Object with mapping names as keys + */ +export function registerTaskMappings(mappings) { + MODEL_MAPPING_NAMES = mappings; +} +import { GITHUB_ISSUE_URL } from '../utils/constants.js'; +import { getModelJSON } from '../utils/hub.js'; +import { Seq2SeqLMOutput } from './modeling_outputs.js'; +import { + LogitsProcessorList, + ForcedBOSTokenLogitsProcessor, + ForcedEOSTokenLogitsProcessor, + SuppressTokensLogitsProcessor, + SuppressTokensAtBeginLogitsProcessor, + NoRepeatNGramLogitsProcessor, + RepetitionPenaltyLogitsProcessor, + NoBadWordsLogitsProcessor, + MinLengthLogitsProcessor, + MinNewTokensLengthLogitsProcessor, + TemperatureLogitsWarper, + ClassifierFreeGuidanceLogitsProcessor, +} from '../generation/logits_process.js'; +import { GenerationConfig } from '../generation/configuration_utils.js'; +import { EosTokenCriteria, MaxLengthCriteria, StoppingCriteriaList } from '../generation/stopping_criteria.js'; +import { LogitsSampler } from '../generation/logits_sampler.js'; +import { DefaultProgressCallback, pick } from '../utils/core.js'; +import { ModelOutput } from './modeling_outputs.js'; +import { logger } from '../utils/logger.js'; +import { DynamicCache } from '../cache_utils.js'; +import { get_model_files } from '../utils/model_registry/get_model_files.js'; +import { get_file_metadata } from '../utils/model_registry/get_file_metadata.js'; +import { MODEL_SESSION_CONFIG, MODEL_TYPES } from './session_config.js'; + +/** + * Converts an array or Tensor of integers to an int64 Tensor. + * @param {any[]|Tensor} items The input integers to be converted. + * @returns {Tensor} The int64 Tensor with the converted values. + * @throws {Error} If the input array is empty or the input is a batched Tensor and not all sequences have the same length. + * @private + */ +function toI64Tensor(items) { + if (items instanceof Tensor) { + return items; + } + // items is an array + if (items.length === 0) { + throw Error('items must be non-empty'); + } + + if (Array.isArray(items[0])) { + // batched + if (items.some((x) => x.length !== items[0].length)) { + throw Error( + "Unable to create tensor, you should probably activate truncation and/or padding with 'padding=True' and/or 'truncation=True' to have batched tensors with the same length.", + ); + } + + return new Tensor('int64', BigInt64Array.from(items.flat().map((x) => BigInt(x))), [ + items.length, + items[0].length, + ]); + } else { + //flat + return new Tensor('int64', BigInt64Array.from(items.map((x) => BigInt(x))), [1, items.length]); + } +} + +/** + * Creates a boolean tensor with a single value. + * @param {boolean} value The value of the tensor. + * @returns {Tensor} The boolean tensor. + * @private + */ +export function boolTensor(value) { + return new Tensor('bool', [value], [1]); +} + +export { getSessionsConfig, getTextOnlySessions, MODEL_TYPES } from './session_config.js'; + +/** + * Runtime-only model type configuration (forward functions, generation flags). + * Session/file configuration lives in `MODEL_SESSION_CONFIG` (session_config.js) + * and is merged in at lookup time by `resolveTypeConfig` to avoid duplication. + */ +const MODEL_RUNTIME_CONFIG = { + [MODEL_TYPES.DecoderOnly]: { + can_generate: true, + forward: decoder_forward, + prepare_inputs: decoder_prepare_inputs_for_generation, + }, + [MODEL_TYPES.DecoderOnlyWithoutHead]: { + can_generate: false, + forward: decoder_forward, + prepare_inputs: decoder_prepare_inputs_for_generation, + }, + [MODEL_TYPES.Seq2Seq]: { + can_generate: true, + forward: seq2seq_forward, + prepare_inputs: encoder_decoder_prepare_inputs_for_generation, + }, + [MODEL_TYPES.Vision2Seq]: { + can_generate: true, + forward: seq2seq_forward, + prepare_inputs: encoder_decoder_prepare_inputs_for_generation, + }, + [MODEL_TYPES.Musicgen]: { + can_generate: true, + forward: seq2seq_forward, + }, + [MODEL_TYPES.EncoderDecoder]: { + can_generate: false, + forward: seq2seq_forward, + }, + [MODEL_TYPES.ImageTextToText]: { + can_generate: true, + forward: image_text_to_text_forward, + prepare_inputs: multimodal_text_to_text_prepare_inputs_for_generation, + }, + [MODEL_TYPES.AudioTextToText]: { + can_generate: true, + forward: audio_text_to_text_forward, + prepare_inputs: multimodal_text_to_text_prepare_inputs_for_generation, + }, + [MODEL_TYPES.ImageAudioTextToText]: { + can_generate: true, + prepare_inputs: multimodal_text_to_text_prepare_inputs_for_generation, + }, + [MODEL_TYPES.Phi3V]: { + can_generate: true, + prepare_inputs: multimodal_text_to_text_prepare_inputs_for_generation, + }, + [MODEL_TYPES.MultiModality]: { + can_generate: true, + }, + [MODEL_TYPES.AutoEncoder]: { + can_generate: false, + forward: auto_encoder_forward, + }, + [MODEL_TYPES.Chatterbox]: { + can_generate: true, + forward: encoder_forward, + }, + [MODEL_TYPES.VoxtralRealtime]: { + can_generate: true, + prepare_inputs: decoder_prepare_inputs_for_generation, + }, + default: { + can_generate: false, + forward: encoder_forward, + }, +}; + +/** + * Resolves the model type config for a given class name and config. + * @param {string} modelName The name of the class being used to load. + * @param {Object} config The model config. + * @returns {{ typeConfig: Object, textOnly: boolean, modelType: number|undefined }} + */ +function resolveTypeConfig(modelName, config) { + let modelType = MODEL_TYPE_MAPPING.get(modelName); + let textOnly = false; + + // Detect cross-architecture loading: e.g., ForCausalLM class loading a ForConditionalGeneration model. + // In this case, use the native architecture's type config (for forward/sessions) in text-only mode. + const nativeArch = config?.architectures?.[0]; + if ( + nativeArch && + nativeArch !== modelName && + modelName?.endsWith('ForCausalLM') && + nativeArch.endsWith('ForConditionalGeneration') + ) { + const nativeType = MODEL_TYPE_MAPPING.get(nativeArch); + if (nativeType !== undefined) { + modelType = nativeType; + textOnly = true; + } + } + + const runtimeConfig = MODEL_RUNTIME_CONFIG[modelType] ?? MODEL_RUNTIME_CONFIG.default; + const sessionConfig = MODEL_SESSION_CONFIG[modelType] ?? MODEL_SESSION_CONFIG.default; + return { typeConfig: { ...runtimeConfig, ...sessionConfig }, textOnly, modelType }; +} + +export const MODEL_TYPE_MAPPING = new Map(); +export const MODEL_NAME_TO_CLASS_MAPPING = new Map(); +export const MODEL_CLASS_TO_NAME_MAPPING = new Map(); + +/** + * A base class for pre-trained models that provides the model configuration and an ONNX session. + */ +export class PreTrainedModel extends Callable { + main_input_name = 'input_ids'; + forward_params = ['input_ids', 'attention_mask']; + + _return_dict_in_generate_keys = null; + + /** + * Creates a new instance of the `PreTrainedModel` class. + * @param {import('../configs.js').PretrainedConfig} config The model configuration. + * @param {Record} sessions The inference sessions for the model. + * @param {Record} configs Additional configuration files (e.g., generation_config.json). + */ + constructor(config, sessions, configs) { + super(); + + this.config = config; + this.sessions = sessions; + this.configs = configs; + + const modelName = MODEL_CLASS_TO_NAME_MAPPING.get(this.constructor); + const { typeConfig } = resolveTypeConfig(modelName, config); + + this.can_generate = typeConfig.can_generate; + this._forward = typeConfig.forward; + this._prepare_inputs_for_generation = typeConfig.prepare_inputs; + + if (this.can_generate) { + this.forward_params.push('past_key_values'); + } + + /** @type {import('../configs.js').TransformersJSConfig} */ + this.custom_config = this.config['transformers.js_config'] ?? {}; + } + + /** + * Disposes of all the ONNX sessions that were created during inference. + * @returns {Promise} An array of promises, one for each ONNX session that is being disposed. + * @todo Use https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/FinalizationRegistry + */ + async dispose() { + const promises = []; + for (const session of Object.values(this.sessions)) { + promises.push(session.release?.()); + } + return await Promise.all(promises); + } + + /** + * Instantiate one of the model classes of the library from a pretrained model. + * + * The model class to instantiate is selected based on the `model_type` property of the config object + * (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible) + * + * @param {string} pretrained_model_name_or_path The name or path of the pretrained model. Can be either: + * - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co. + * Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a + * user or organization name, like `dbmdz/bert-base-german-cased`. + * - A path to a *directory* containing model weights, e.g., `./my_model_directory/`. + * @param {import('../utils/hub.js').PretrainedModelOptions} options Additional options for loading the model. + * + * @returns {Promise} A new instance of the `PreTrainedModel` class. + */ + static async from_pretrained( + pretrained_model_name_or_path, + { + progress_callback = null, + config = null, + cache_dir = null, + local_files_only = false, + revision = 'main', + model_file_name = null, + subfolder = 'onnx', + device = null, + dtype = null, + use_external_data_format = null, + session_options = {}, + } = {}, + ) { + const options = { + progress_callback, + config, + cache_dir, + local_files_only, + revision, + model_file_name, + subfolder, + device, + dtype, + use_external_data_format, + session_options, + }; + + const modelName = MODEL_CLASS_TO_NAME_MAPPING.get(this); + + config = options.config = await AutoConfig.from_pretrained(pretrained_model_name_or_path, options); + + const { typeConfig, textOnly, modelType } = resolveTypeConfig(modelName, config); + + if (modelType === undefined) { + const type = modelName ?? config?.model_type; + if (type !== 'custom') { + logger.warn( + `Model type for '${type}' not found, assuming encoder-only architecture. Please report this at ${GITHUB_ISSUE_URL}.`, + ); + } + } + + // If a progress callback is provided AND it hasn't already been wrapped + // by pipeline() (which does its own aggregation), gather file metadata + // upfront so we can emit `progress_total` events. This lets consumers + // render a single overall progress bar when calling from_pretrained() directly. + if (progress_callback && !(progress_callback instanceof DefaultProgressCallback)) { + /** @type {import('../utils/core.js').FilesLoadingMap} */ + const files_loading = {}; + + try { + const expected_files = await get_model_files(pretrained_model_name_or_path, { + config, + dtype, + device, + model_file_name, + }); + + const metadata = await Promise.all( + expected_files.map((file) => get_file_metadata(pretrained_model_name_or_path, file, options)), + ); + metadata.forEach((m, i) => { + if (m.exists) { + // config.json is fetched by AutoConfig.from_pretrained() above + const isAlreadyLoaded = expected_files[i] === 'config.json'; + files_loading[expected_files[i]] = { + loaded: isAlreadyLoaded ? (m.size ?? 0) : 0, + total: m.size ?? 0, + }; + } + }); + } catch (e) { + // If we fail to get metadata, we can still proceed without total progress. + // This may happen with local-only models or custom cache setups. + logger.warn(`Unable to fetch model file metadata for total progress tracking: ${e}`); + } + + if (Object.keys(files_loading).length > 0) { + options.progress_callback = new DefaultProgressCallback(progress_callback, files_loading); + } + } + + const sessions = typeConfig.sessions(config, options, textOnly); + const promises = [ + constructSessions(pretrained_model_name_or_path, sessions, options, typeConfig.cache_sessions), + ]; + if (typeConfig.optional_configs) { + promises.push(get_optional_configs(pretrained_model_name_or_path, typeConfig.optional_configs, options)); + } + const info = await Promise.all(promises); + + // @ts-ignore + return new this(config, ...info); + } + + /** + * Runs the model with the provided inputs + * @param {Object} model_inputs Object containing input tensors + * @returns {Promise} Object containing output tensors + */ + async _call(model_inputs) { + return await this.forward(model_inputs); + } + + /** + * Forward method for a pretrained model. If not overridden by a subclass, the correct forward method + * will be chosen based on the model type. + * @param {Object} model_inputs The input data to the model in the format specified in the ONNX model. + * @returns {Promise} The output data from the model in the format specified in the ONNX model. + * @throws {Error} This method must be implemented in subclasses. + */ + async forward(model_inputs) { + return await this._forward(this, model_inputs); + } + + /** + * Get the model's generation config, if it exists. + * @returns {GenerationConfig|null} The model's generation config if it exists, otherwise `null`. + */ + get generation_config() { + return this.configs?.generation_config ?? null; + } + + /** + * @param {GenerationConfig} generation_config + * @param {number} input_ids_seq_length The starting sequence length for the input ids. + * @returns {LogitsProcessorList} + * @private + */ + _get_logits_processor( + generation_config, + input_ids_seq_length, + // encoder_input_ids, TODO + // prefix_allowed_tokens_fn, TODO + logits_processor = null, + ) { + const processors = new LogitsProcessorList(); + + // if (generation_config.diversity_penalty !== null && generation_config.diversity_penalty > 0.0) { + // processors.push(new HammingDiversityLogitsProcessor( + // generation_config.diversity_penalty, + // generation_config.num_beams, + // generation_config.num_beam_groups + // )); + // } + + // if (generation_config.encoder_repetition_penalty !== null && generation_config.encoder_repetition_penalty !== 1.0) { + // processors.push(new EncoderRepetitionPenaltyLogitsProcessor( + // generation_config.encoder_repetition_penalty, + // encoder_input_ids + // )); + // } + + if (generation_config.repetition_penalty !== null && generation_config.repetition_penalty !== 1.0) { + processors.push(new RepetitionPenaltyLogitsProcessor(generation_config.repetition_penalty)); + } + + if (generation_config.no_repeat_ngram_size !== null && generation_config.no_repeat_ngram_size > 0) { + processors.push(new NoRepeatNGramLogitsProcessor(generation_config.no_repeat_ngram_size)); + } + + // if (generation_config.encoder_no_repeat_ngram_size !== null && generation_config.encoder_no_repeat_ngram_size > 0) { + // if (this.config.is_encoder_decoder) { + // processors.push(new EncoderNoRepeatNGramLogitsProcessor( + // generation_config.encoder_no_repeat_ngram_size, + // encoder_input_ids + // )); + // } else { + // throw new Error("It's impossible to use `encoder_no_repeat_ngram_size` with decoder-only architecture"); + // } + // } + + if (generation_config.bad_words_ids !== null) { + processors.push( + new NoBadWordsLogitsProcessor(generation_config.bad_words_ids, generation_config.eos_token_id), + ); + } + + if ( + generation_config.min_length !== null && + generation_config.eos_token_id !== null && + generation_config.min_length > 0 + ) { + processors.push(new MinLengthLogitsProcessor(generation_config.min_length, generation_config.eos_token_id)); + } + + if ( + generation_config.min_new_tokens !== null && + generation_config.eos_token_id !== null && + generation_config.min_new_tokens > 0 + ) { + processors.push( + new MinNewTokensLengthLogitsProcessor( + input_ids_seq_length, + generation_config.min_new_tokens, + generation_config.eos_token_id, + ), + ); + } + + // if (prefix_allowed_tokens_fn !== null) { + // processors.push(new PrefixConstrainedLogitsProcessor( + // prefix_allowed_tokens_fn, + // generation_config.num_beams / generation_config.num_beam_groups + // )); + // } + + if (generation_config.forced_bos_token_id !== null) { + processors.push(new ForcedBOSTokenLogitsProcessor(generation_config.forced_bos_token_id)); + } + + if (generation_config.forced_eos_token_id !== null) { + processors.push( + new ForcedEOSTokenLogitsProcessor(generation_config.max_length, generation_config.forced_eos_token_id), + ); + } + + // if (generation_config.remove_invalid_values === true) { + // processors.push(new InfNanRemoveLogitsProcessor()); + // } + + // if (generation_config.exponential_decay_length_penalty !== null) { + // processors.push(new ExponentialDecayLengthPenalty( + // generation_config.exponential_decay_length_penalty, + // generation_config.eos_token_id, + // input_ids_seq_length + // )); + // } + + if (generation_config.suppress_tokens !== null) { + processors.push(new SuppressTokensLogitsProcessor(generation_config.suppress_tokens)); + } + + if (generation_config.begin_suppress_tokens !== null) { + const begin_index = + input_ids_seq_length > 1 || generation_config.forced_bos_token_id === null + ? input_ids_seq_length + : input_ids_seq_length + 1; + + processors.push( + new SuppressTokensAtBeginLogitsProcessor(generation_config.begin_suppress_tokens, begin_index), + ); + } + + // DEPRECATED: https://github.com/huggingface/transformers/pull/29485 + // if (generation_config.forced_decoder_ids !== null) { + // processors.push(new ForceTokensLogitsProcessor(generation_config.forced_decoder_ids)); + // } + + // 8. prepare batched CFG externally + if (generation_config.guidance_scale !== null && generation_config.guidance_scale > 1) { + processors.push(new ClassifierFreeGuidanceLogitsProcessor(generation_config.guidance_scale)); + } + + if (generation_config.temperature === 0 && generation_config.do_sample) { + logger.warn( + '`do_sample` changed to false because `temperature: 0` implies greedy sampling (always selecting the most likely token), which is incompatible with `do_sample: true`.', + ); + generation_config.do_sample = false; + } + + if (generation_config.do_sample) { + if (generation_config.temperature !== null && generation_config.temperature !== 1.0) { + processors.push(new TemperatureLogitsWarper(generation_config.temperature)); + } + // TODO: Add TopPLogitsWarper and TopKLogitsWarper + // if (generation_config.top_k !== null && generation_config.top_k !== 0) { + // processors.push(new TopKLogitsWarper(generation_config.top_k)); + // } + // if (generation_config.top_p !== null && generation_config.top_p < 1.0) { + // processors.push(new TopPLogitsWarper(generation_config.top_p)); + // } + } + + if (logits_processor !== null) { + processors.extend(logits_processor); + } + + // `LogitNormalization` should always be the last logit processor, when present + // if (generation_config.renormalize_logits === true) { + // processors.push(new LogitNormalization()); + // } + + return processors; + } + + /** + * This function merges multiple generation configs together to form a final generation config to be used by the model for text generation. + * It first creates an empty `GenerationConfig` object, then it applies the model's own `generation_config` property to it. Finally, if a `generation_config` object was passed in the arguments, it overwrites the corresponding properties in the final config with those of the passed config object. + * @param {GenerationConfig|null} generation_config A `GenerationConfig` object containing generation parameters. + * @param {Object} kwargs Additional generation parameters to be used in place of those in the `generation_config` object. + * @returns {GenerationConfig} The final generation config object to be used by the model for text generation. + */ + _prepare_generation_config(generation_config, kwargs, cls = GenerationConfig) { + // Create empty generation config (contains defaults) + // We pass `this.config` so that if `eos_token_id` or `bos_token_id` exist in the model's config, we will use them + const config = { ...this.config }; + for (const key of ['decoder', 'generator', 'text_config']) { + // Special case: some models have generation attributes set in the decoder. + // Use them if still unset in the generation config. + if (key in config) { + Object.assign(config, config[key]); + } + } + + const gen_config = new cls(config); + + // Apply model's generation config, if it exists + Object.assign(gen_config, this.generation_config ?? {}); + + // Next, use any generation config specified by the user + // when calling `generate` + if (generation_config) { + Object.assign(gen_config, generation_config); + } + + // Finally, if any kwargs were passed, use them to overwrite + if (kwargs) { + Object.assign(gen_config, pick(kwargs, Object.getOwnPropertyNames(gen_config))); + } + + return gen_config; + } + + /** + * + * @param {GenerationConfig} generation_config + * @param {import('../generation/stopping_criteria.js').StoppingCriteria|import('../generation/stopping_criteria.js').StoppingCriteria[]|StoppingCriteriaList} [stopping_criteria=null] + */ + _get_stopping_criteria(generation_config, stopping_criteria = null) { + const criteria = new StoppingCriteriaList(); + + if (generation_config.max_length !== null) { + criteria.push( + new MaxLengthCriteria(generation_config.max_length, this.config.max_position_embeddings ?? null), + ); + } + // if (generation_config.max_time !== null) { + // criteria.push(new MaxTimeCriteria(generation_config.max_time)); + // } + if (generation_config.eos_token_id !== null) { + criteria.push(new EosTokenCriteria(generation_config.eos_token_id)); + } + + if (stopping_criteria) { + criteria.extend(stopping_criteria); + } + return criteria; + } + + /** + * Confirms that the model class is compatible with generation. + * If not, raises an exception that points to the right class to use. + */ + _validate_model_class() { + if (!this.can_generate) { + const generate_compatible_mappings = [ + MODEL_MAPPING_NAMES.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES, + // MODEL_MAPPING_NAMES.MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING, // TODO + MODEL_MAPPING_NAMES.MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES, + MODEL_MAPPING_NAMES.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES, + MODEL_MAPPING_NAMES.MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES, + ].filter(Boolean); // Filter out null mappings (in case registry hasn't loaded yet) + + const modelName = MODEL_CLASS_TO_NAME_MAPPING.get(this.constructor); + + const generate_compatible_classes = new Set(); + const modelType = this.config.model_type; + for (const model_mapping of generate_compatible_mappings) { + const supported_models = model_mapping?.get(modelType); + if (supported_models) { + generate_compatible_classes.add(supported_models); + } + } + + let errorMessage = `The current model class (${modelName}) is not compatible with \`.generate()\`, as it doesn't have a language model head.`; + if (generate_compatible_classes.size > 0) { + errorMessage += ` Please use the following class instead: ${[...generate_compatible_classes].join(', ')}`; + } + throw Error(errorMessage); + } + } + + prepare_inputs_for_generation(...args) { + if (!this._prepare_inputs_for_generation) { + throw new Error('prepare_inputs_for_generation is not implemented for this model.'); + } + return this._prepare_inputs_for_generation(this, ...args); + } + + /** + * + * @param {Object} inputs + * @param {bigint[][]} inputs.generated_input_ids + * @param {Object} inputs.outputs + * @param {Object} inputs.model_inputs + * @param {boolean} inputs.is_encoder_decoder + * @returns {Object} The updated model inputs for the next generation iteration. + */ + _update_model_kwargs_for_generation({ generated_input_ids, outputs, model_inputs, is_encoder_decoder }) { + // update past_key_values + model_inputs['past_key_values'] = getPastKeyValues(outputs, model_inputs.past_key_values); + + // update inputs for next run + model_inputs['input_ids'] = new Tensor('int64', generated_input_ids.flat(), [generated_input_ids.length, 1]); + + if (!is_encoder_decoder) { + // update attention mask + model_inputs.attention_mask = cat( + [model_inputs.attention_mask, ones([model_inputs.attention_mask.dims[0], 1])], + 1, + ); + } else if ('decoder_attention_mask' in model_inputs) { + model_inputs.decoder_attention_mask = cat( + [model_inputs.decoder_attention_mask, ones([model_inputs.decoder_attention_mask.dims[0], 1])], + 1, + ); + } + + // force recreate position_ids in next iteration + model_inputs['position_ids'] = null; + + return model_inputs; + } + + /** + * This function extracts the model-specific `inputs` for generation. + * @param {Object} params + * @param {Tensor} [params.inputs=null] + * @param {number} [params.bos_token_id=null] + * @param {Record} [params.model_kwargs] + * @returns {{inputs_tensor: Tensor, model_inputs: Record & {past_key_values?: DynamicCache}, model_input_name: string}} The model-specific inputs for generation. + */ + _prepare_model_inputs({ inputs, bos_token_id, model_kwargs }) { + const model_inputs = pick(model_kwargs, this.forward_params); + const input_name = this.main_input_name; + if (input_name in model_inputs) { + if (inputs) { + throw new Error( + '`inputs`: {inputs}` were passed alongside {input_name} which is not allowed. ' + + 'Make sure to either pass {inputs} or {input_name}=...', + ); + } + } else { + model_inputs[input_name] = inputs; + } + + const inputs_tensor = model_inputs[input_name]; + + return { inputs_tensor, model_inputs, model_input_name: input_name }; + } + + async _prepare_encoder_decoder_kwargs_for_generation({ + inputs_tensor, + model_inputs, + model_input_name, + generation_config, + }) { + if ( + this.sessions['model'].inputNames.includes('inputs_embeds') && + !model_inputs.inputs_embeds && + '_prepare_inputs_embeds' in this + ) { + // Encoder expects `inputs_embeds` instead of `input_ids` + const { input_ids, pixel_values, attention_mask, ...kwargs } = model_inputs; + // @ts-ignore + const prepared_inputs = await this._prepare_inputs_embeds(model_inputs); + model_inputs = { + ...kwargs, + ...pick(prepared_inputs, ['inputs_embeds', 'attention_mask']), + }; + } + let { last_hidden_state } = await encoder_forward(this, model_inputs); + + // for classifier free guidance we need to add a 'null' input to our encoder hidden states + if (generation_config.guidance_scale !== null && generation_config.guidance_scale > 1) { + last_hidden_state = cat([last_hidden_state, full_like(last_hidden_state, 0.0)], 0); + + if ('attention_mask' in model_inputs) { + model_inputs['attention_mask'] = cat( + [model_inputs['attention_mask'], zeros_like(model_inputs['attention_mask'])], + 0, + ); + } + } else if (model_inputs.decoder_input_ids) { + // Ensure that the encoder outputs have the same batch size as the decoder inputs, + // allowing for more efficient batched generation for single inputs + const decoder_input_ids_batch_size = toI64Tensor(model_inputs.decoder_input_ids).dims[0]; + if (decoder_input_ids_batch_size !== last_hidden_state.dims[0]) { + if (last_hidden_state.dims[0] !== 1) { + throw new Error( + `The encoder outputs have a different batch size (${last_hidden_state.dims[0]}) than the decoder inputs (${decoder_input_ids_batch_size}).`, + ); + } + last_hidden_state = cat( + Array.from({ length: decoder_input_ids_batch_size }, () => last_hidden_state), + 0, + ); + } + } + model_inputs['encoder_outputs'] = last_hidden_state; + + return model_inputs; + } + + /** + * Prepares `decoder_input_ids` for generation with encoder-decoder models + * @param {*} param0 + */ + _prepare_decoder_input_ids_for_generation({ + batch_size, + model_input_name, + model_kwargs, + decoder_start_token_id, + bos_token_id, + generation_config, + }) { + let { decoder_input_ids, ...model_inputs } = model_kwargs; + + // Prepare input ids if the user has not defined `decoder_input_ids` manually. + if (!(decoder_input_ids instanceof Tensor)) { + if (!decoder_input_ids) { + decoder_start_token_id ??= bos_token_id; + + if (this.config.model_type === 'musicgen') { + // Custom logic (TODO: move to Musicgen class) + decoder_input_ids = Array.from( + { + // @ts-expect-error TS2339 + length: batch_size * this.config.decoder.num_codebooks, + }, + () => [decoder_start_token_id], + ); + } else if (Array.isArray(decoder_start_token_id)) { + if (decoder_start_token_id.length !== batch_size) { + throw new Error( + `\`decoder_start_token_id\` expcted to have length ${batch_size} but got ${decoder_start_token_id.length}`, + ); + } + decoder_input_ids = decoder_start_token_id; + } else { + decoder_input_ids = Array.from( + { + length: batch_size, + }, + () => [decoder_start_token_id], + ); + } + } else if (!Array.isArray(decoder_input_ids[0])) { + // Correct batch size + decoder_input_ids = Array.from( + { + length: batch_size, + }, + () => decoder_input_ids, + ); + } + decoder_input_ids = toI64Tensor(decoder_input_ids); + } + + model_inputs['decoder_attention_mask'] = ones_like(decoder_input_ids); + + return { input_ids: decoder_input_ids, model_inputs }; + } + + /** + * Generates sequences of token ids for models with a language modeling head. + * @param {import('../generation/parameters.js').GenerationFunctionParameters} options + * @returns {Promise} The output of the model, which can contain the generated token ids, attentions, and scores. + */ + async generate({ + inputs = null, + generation_config = null, + logits_processor = null, + stopping_criteria = null, + streamer = null, + + // inputs_attention_mask = null, + ...kwargs + }) { + this._validate_model_class(); + + // Update generation config with defaults and kwargs + generation_config = this._prepare_generation_config(generation_config, kwargs); + + // 3. Define model inputs + let { inputs_tensor, model_inputs, model_input_name } = this._prepare_model_inputs({ + inputs, + model_kwargs: /** @type {Record} */ (kwargs), + }); + + const is_encoder_decoder = this.config.is_encoder_decoder; + + // 4. Define other model kwargs + if (!is_encoder_decoder) { + // decoder-only models should use left-padding for generation + } else if (!('encoder_outputs' in model_inputs)) { + // if model is encoder decoder encoder_outputs are created + // and added to `model_kwargs` + model_inputs = await this._prepare_encoder_decoder_kwargs_for_generation({ + inputs_tensor, + model_inputs, + model_input_name, + generation_config, + }); + } + + // 5. Prepare `input_ids` which will be used for auto-regressive generation + // TODO: Update to align with HF transformers' implementation + let input_ids; + if (is_encoder_decoder) { + // Generating from the encoder outputs + ({ input_ids, model_inputs } = this._prepare_decoder_input_ids_for_generation({ + batch_size: model_inputs[model_input_name].dims.at(0), + model_input_name, + model_kwargs: model_inputs, + decoder_start_token_id: generation_config.decoder_start_token_id, + bos_token_id: generation_config.bos_token_id, + generation_config, + })); + } else { + input_ids = model_inputs[model_input_name]; + } + + // 6. Prepare `max_length` depending on other stopping criteria. + let input_ids_length = input_ids.dims.at(-1); + + if (generation_config.max_new_tokens !== null) { + generation_config.max_length = input_ids_length + generation_config.max_new_tokens; + } + + // input_ids_length = model_inputs[model_input_name].dims.at(1); + // // inputs instanceof Tensor ? : inputs.length; + + // // decoder-only + // if (input_ids_length === 0) { + // throw Error("Must supply a non-empty array of input token ids.") + // } + + // let decoder_input_ids = + // generation_config.decoder_input_ids + // ?? generation_config.decoder_start_token_id + // ?? generation_config.bos_token_id + // ?? generation_config.eos_token_id; + + // Update logits processor + // 8. prepare distribution pre_processing samplers + const prepared_logits_processor = this._get_logits_processor( + generation_config, + input_ids_length, + logits_processor, + ); + + // 9. prepare stopping criteria + const prepared_stopping_criteria = this._get_stopping_criteria(generation_config, stopping_criteria); + + // /** @type {number[]} */ + // let eos_token_ids = generation_config.eos_token_id; + // if (eos_token_ids !== null && !Array.isArray(eos_token_ids)) { + // eos_token_ids = [eos_token_ids]; + // } + + const numInputs = model_inputs[model_input_name].dims.at(0); + + // TODO: + // done is a list of booleans to keep track of which inputs are done + // const done = new Array(numInputs).fill(false); + // For efficiency purposes, we remove completed rows from model_inputs + // when the beam is complete, and we keep track of the row index + // const rowIndexToBatchIndex = new Map(); + + const sampler = LogitsSampler.getSampler(generation_config); + + // TODO make > numInputs + const scores = new Array(numInputs).fill(0); + /** @type {bigint[][]} */ + const all_input_ids = input_ids.tolist(); + if (streamer) { + streamer.put(all_input_ids); + } + // const all_generated_input_ids = Array.from({ length: numInputs }, () => []); + + // NOTE: For now, we don't support spawning new beams + // TODO: when we do, we simply copy past key values and accumulate into single large tensor + + //////////////////////////////////////////////////// + // Generic search which handles 4 generation modes: + // - GenerationMode.GREEDY_SEARCH + // - GenerationMode.SAMPLE + // - GenerationMode.BEAM_SEARCH + // - GenerationMode.BEAM_SAMPLE + //////////////////////////////////////////////////// + let outputs; + let attentions = {}; + let return_dict_items = {}; + while (true) { + // prepare model inputs + model_inputs = this.prepare_inputs_for_generation(all_input_ids, model_inputs, generation_config); + outputs = await this.forward(model_inputs); + + if (generation_config.return_dict_in_generate) { + if (generation_config.output_attentions) { + // Get attentions if they are present + const token_attentions = getAttentions(outputs); + for (const key in token_attentions) { + if (!(key in attentions)) { + attentions[key] = []; + } + attentions[key].push(token_attentions[key]); + } + } else if (this._return_dict_in_generate_keys) { + Object.assign(return_dict_items, pick(outputs, this._return_dict_in_generate_keys)); + } + } + + // Logits are of the form [batch_size, out_seq_length, vocab_size] + // In most cases, this will be [batch_size, 1, vocab_size] + // So, we select the last token's logits: + // (equivalent to `logits = outputs.logits[:, -1, :]`) + // The `.to('float32')` is necessary for models with float16 logits, + // and is a no-op for float32 logits. + // TODO: Support float16 sampling in the sampler directly + const logits = outputs.logits.slice(null, -1, null).to('float32'); + + const next_tokens_scores = prepared_logits_processor(all_input_ids, logits); + + /** @type {[bigint][]} */ + const generated_input_ids = []; + // const new_kv_cache = [];// NOTE: Only used for beam search when concatenating new kv + // Loop over each batch + for (let batch_idx = 0; batch_idx < next_tokens_scores.dims.at(0); ++batch_idx) { + const logs = next_tokens_scores[batch_idx]; + + const sampledTokens = await sampler(logs); + for (const [newTokenId, logProb] of sampledTokens) { + const bigint = BigInt(newTokenId); + // TODO: If branching, use previous beam as a starting point + // update generated ids, model inputs, and length for next step + scores[batch_idx] += logProb; + all_input_ids[batch_idx].push(bigint); + generated_input_ids.push([bigint]); + + // TODO: Support beam search + break; + } + } + if (streamer) { + streamer.put(generated_input_ids); + } + + const stop = prepared_stopping_criteria(all_input_ids); + if (stop.every((x) => x)) { + break; + } + + model_inputs = this._update_model_kwargs_for_generation({ + generated_input_ids, + outputs, + model_inputs, + is_encoder_decoder, + }); + } + + if (streamer) { + streamer.end(); + } + + // TODO: ensure all_input_ids is padded correctly... + const sequences = new Tensor('int64', all_input_ids.flat(), [all_input_ids.length, all_input_ids[0].length]); + + // Update past key values from the final forward pass + const past_key_values = getPastKeyValues(outputs, model_inputs.past_key_values); + + // Dispose output tensors not held by the cache + const cachedTensors = new Set(Object.values(past_key_values)); + for (const tensor of Object.values(outputs)) { + if (tensor.location === 'gpu-buffer' && !cachedTensors.has(tensor)) { + tensor.dispose(); + } + } + + // Dispose cache tensors if no one needs them + const keepCacheAlive = 'past_key_values' in kwargs || generation_config.return_dict_in_generate; + if (!keepCacheAlive) { + await past_key_values.dispose(); + } + + if (generation_config.return_dict_in_generate) { + return { + sequences, + past_key_values, + ...attentions, + ...return_dict_items, + // TODO: + // scores, + // logits, + }; + } + return sequences; + } + + /** + * Helper function to select valid inputs and run through the appropriate encoder (vision, text, audio) based on the input type. + * @param {string} sessionName + * @param {Record} inputs + * @param {string} outputName + * @private + */ + async _encode_input(sessionName, inputs, outputName) { + if (!Object.hasOwn(this.sessions, sessionName)) { + throw new Error(`Model does not have a ${sessionName} session.`); + } + const session = this.sessions[sessionName]; + const output = await sessionRun(session, pick(inputs, session.inputNames)); + return output[outputName]; + } + + async encode_image(inputs) { + return this._encode_input('vision_encoder', inputs, 'image_features'); + } + + async encode_text(inputs) { + return this._encode_input('embed_tokens', inputs, 'inputs_embeds'); + } + + async encode_audio(inputs) { + return this._encode_input('audio_encoder', inputs, 'audio_features'); + } +} + +/** + * Perform forward pass on the seq2seq model (both encoder and decoder). + * @param {Object} self The seq2seq model object. + * @param {Object} model_inputs The input object for the model containing encoder and decoder inputs. + * @returns {Promise} Promise that resolves with the output of the seq2seq model. + * @private + */ +export async function seq2seq_forward(self, model_inputs) { + let { encoder_outputs, input_ids, decoder_input_ids, decoder_attention_mask, ...other_decoder_inputs } = + model_inputs; + // Encode if needed + if (!encoder_outputs) { + const encoder_inputs = pick(model_inputs, self.sessions['model'].inputNames); + // Encoder outputs are not given, so we must compute them. + encoder_outputs = (await encoder_forward(self, encoder_inputs)).last_hidden_state; + } + + other_decoder_inputs.input_ids = decoder_input_ids; + other_decoder_inputs.encoder_hidden_states = encoder_outputs; + + if (self.sessions['decoder_model_merged'].inputNames.includes('encoder_attention_mask')) { + other_decoder_inputs.encoder_attention_mask = model_inputs.attention_mask; + } + + // Pass decoder_attention_mask as attention_mask to the decoder session + if (decoder_attention_mask && !other_decoder_inputs.attention_mask) { + other_decoder_inputs.attention_mask = decoder_attention_mask; + } + + return await decoder_forward(self, other_decoder_inputs, true); +} + +/** + * Forward pass of an encoder model. + * @param {Object} self The encoder model. + * @param {Object} model_inputs The input data to be used for the forward pass. + * @returns {Promise} The model's outputs. + * @private + */ +export async function encoder_forward(self, model_inputs) { + const session = self.sessions['model']; + const encoderFeeds = pick(model_inputs, session.inputNames); + + if (session.inputNames.includes('inputs_embeds') && !encoderFeeds.inputs_embeds) { + if (!model_inputs.input_ids) { + throw new Error('Both `input_ids` and `inputs_embeds` are missing in the model inputs.'); + } + encoderFeeds.inputs_embeds = await self.encode_text({ input_ids: model_inputs.input_ids }); + } + if (session.inputNames.includes('token_type_ids') && !encoderFeeds.token_type_ids) { + if (!encoderFeeds.input_ids) { + throw new Error('Both `input_ids` and `token_type_ids` are missing in the model inputs.'); + } + // Assign default `token_type_ids` (all zeroes) to the `encoderFeeds` if the model expects it, + // but they weren't created by the tokenizer. + encoderFeeds.token_type_ids = zeros_like(encoderFeeds.input_ids); + } + if (session.inputNames.includes('pixel_mask') && !encoderFeeds.pixel_mask) { + if (!encoderFeeds.pixel_values) { + throw new Error('Both `pixel_values` and `pixel_mask` are missing in the model inputs.'); + } + // Assign default `pixel_mask` (all ones) to the `encoderFeeds` if the model expects it, + // but they weren't created by the processor. + const dims = encoderFeeds.pixel_values.dims; + encoderFeeds.pixel_mask = ones([dims[0], dims[2], dims[3]]); + } + + return await sessionRun(session, encoderFeeds); +} + +export async function auto_encoder_forward(self, model_inputs) { + const encoded = await self.encode(model_inputs); + const decoded = await self.decode(encoded); + return decoded; +} + +/** + * Returns a DynamicCache containing past key values from the given decoder results object. + * Always updates in-place when pastKeyValues is provided; creates a new DynamicCache otherwise. + * + * @param {Object} decoderResults The decoder results object. + * @param {DynamicCache} pastKeyValues The previous past key values. + * @returns {DynamicCache} The updated past key values cache. + */ +export function getPastKeyValues(decoderResults, pastKeyValues) { + /** @type {Record} */ + const pkvs = Object.create(null); + + for (const name in decoderResults) { + if (name.startsWith('present')) { + const newName = name + // Hybrid cache architecture + .replace('present_ssm', 'past_ssm') // Mamba + .replace('present_conv', 'past_conv') // LFM2 + .replace('present_recurrent', 'past_recurrent') // Qwen3.5 + .replace('present_compressor', 'past_compressor') // Deepseek V4 + .replace('present_indexer', 'past_indexer') // Deepseek V4 + + // Standard cache architecture + .replace('present', 'past_key_values'); + const is_encoder_pkv = name.includes('encoder'); + if (is_encoder_pkv && pastKeyValues) { + // Optimization introduced by optimum to reuse past key values. + // So, we just replace the constant outputs (`decoderResults[name]`) with the previous past key values. + // https://github.com/huggingface/optimum/blob/0bf2c05fb7e1182b52d21b703cfc95fd9e4ea3dc/optimum/onnxruntime/base.py#L677-L704 + pkvs[newName] = pastKeyValues[newName]; + } else { + pkvs[newName] = decoderResults[name]; + } + } + } + + if (pastKeyValues) { + pastKeyValues.update(pkvs); + return pastKeyValues; + } + return new DynamicCache(pkvs); +} + +/** + * Returns an object containing attentions from the given model output object. + * + * @param {Object} model_output The output of the model. + * @returns {{cross_attentions?: Tensor[]}} An object containing attentions. + */ +export function getAttentions(model_output) { + const attentions = {}; + + for (const attnName of ['cross_attentions', 'encoder_attentions', 'decoder_attentions']) { + for (const name in model_output) { + if (name.startsWith(attnName)) { + if (!(attnName in attentions)) { + attentions[attnName] = []; + } + attentions[attnName].push(model_output[name]); + } + } + } + return attentions; +} + +/** + * Resolve symbolic dims from ONNX inputMetadata for empty-cache initialization. + * Each symbolic dim name is looked up in `symbols`; numeric dims pass through. + * Any unresolved symbolic dim defaults to 0. + * @param {ReadonlyArray} metadataShape + * @param {Record} symbols + * @returns {number[]} + */ +export function resolveCacheShape(metadataShape, symbols) { + return metadataShape.map((d) => { + if (typeof d === 'number') return d; + return symbols[d] ?? 0; + }); +} + +/** + * Adds past key values to the decoder feeds object. If pastKeyValues is null, + * creates a new DynamicCache with zero-filled tensors for each cache entry. + * + * @param {PreTrainedModel} self The model instance. + * @param {Record} decoderFeeds The decoder feeds object to add past key values to. + * @param {DynamicCache|null} pastKeyValues The cache containing past key values. + * @returns {DynamicCache} The past key values cache (existing or newly created). + */ +export function addPastKeyValues(self, decoderFeeds, pastKeyValues) { + if (pastKeyValues && Object.keys(pastKeyValues).length > 0) { + Object.assign(decoderFeeds, pastKeyValues); + return pastKeyValues; + } + + const session = self.sessions['decoder_model_merged'] ?? self.sessions['model']; + const batch_size = (decoderFeeds[self.main_input_name] ?? decoderFeeds.attention_mask)?.dims?.[0] ?? 1; + + const names = getCacheNames(self.config); + const num_heads = self.config?.normalized_config?.num_heads; + /** @type {Record} */ + const symbols = { batch_size }; + if (typeof num_heads === 'number') { + symbols['batch_size x num_heads'] = batch_size * num_heads; + } + /** @type {Record} */ + const entries = Object.create(null); + for (const meta of session.inputMetadata) { + if (!names.has(meta.name)) continue; + const shape = resolveCacheShape(meta.shape, symbols); + const size = shape.reduce((a, b) => a * b, 1); + const cls = DataTypeMap[meta.type]; + const t = new Tensor(meta.type, new cls(size), shape); + decoderFeeds[meta.name] = t; + entries[meta.name] = t; + } + if (pastKeyValues) { + // Populate the (empty) user-provided cache in-place + pastKeyValues.update(entries); + return pastKeyValues; + } + return new DynamicCache(entries); +} + +/** + * Sets `num_logits_to_keep` on `model_inputs` if the decoder session declares it as an input + * and it has not already been set. + * + * `num_logits_to_keep` specifies how many trailing prompt logits the model computes: + * - `0n` (or unset) computes logits for the entire sequence — used for prefill/scoring. + * - `1n` computes only the last token's logits — used during autoregressive generation, + * since only the last prompt token's logits are needed to sample the next token. For long + * sequences, computing all logits uses a lot of memory, so `1n` significantly reduces the + * memory footprint. + * - Any other positive integer keeps the last `num_logits_to_keep` logits. + * + * @param {PreTrainedModel} self The model instance. + * @param {Record} model_inputs The model inputs to mutate. + * @param {bigint} value The value to set (typically `1n` for generation, `0n` as a fallback). + * @private + */ +export function setNumLogitsToKeep(self, model_inputs, value) { + if (model_inputs.num_logits_to_keep) return; + const session = self.sessions['decoder_model_merged'] ?? self.sessions['model']; + if (session?.inputNames.includes('num_logits_to_keep')) { + model_inputs.num_logits_to_keep = new Tensor('int64', [value], []); + } +} + +/** + * Forward pass of a decoder model. + * @param {Object} self The decoder model. + * @param {Object} model_inputs The input data to be used for the forward pass. + * @returns {Promise} The logits and past key values. + * @private + */ +export async function decoder_forward(self, model_inputs, is_encoder_decoder = false) { + const session = self.sessions[is_encoder_decoder ? 'decoder_model_merged' : 'model']; + + const { past_key_values, ...new_model_inputs } = model_inputs; + + if (session.inputNames.includes('use_cache_branch')) { + new_model_inputs.use_cache_branch = boolTensor( + past_key_values != null && Object.keys(past_key_values).length > 0, + ); + } + if ( + session.inputNames.includes('position_ids') && + new_model_inputs.attention_mask && + !new_model_inputs.position_ids + ) { + // NOTE: Handle a special case for paligemma/gemma3 models, where positions are 1-indexed + const start_index = ['paligemma', 'gemma3_text', 'gemma3'].includes(self.config.model_type) ? 1 : 0; + new_model_inputs.position_ids = create_position_ids(new_model_inputs, past_key_values, start_index); + } + + // Fallback for non-generation forward calls (e.g. prefill scoring): compute all logits. + setNumLogitsToKeep(self, new_model_inputs, 0n); + + // Unpack the `past_key_values` object into model inputs + addPastKeyValues(self, new_model_inputs, past_key_values); + + // Select only the inputs that are needed for the current session + const fixed = pick(new_model_inputs, session.inputNames); + return await sessionRun(session, fixed); +} + +/** + * Abstract forward pass function for image-text-to-text or audio-text-to-text models. + * @param {Object} self The model object. + * @param {Object} params Additional parameters. + * @param {Function} [params.encode_function] The function to encode the modality values. + * @param {Function} [params.merge_function] The function to merge the modality features with the input embeddings. + * @param {string[]} [params.modality_input_names] The modality input name. + * @param {string} [params.modality_output_name] The modality output name. + * @param {Tensor} [params.input_ids=null] + * @param {Tensor} [params.attention_mask=null] + * @param {Tensor} [params.position_ids=null] + * @param {Tensor} [params.inputs_embeds=null] + * @param {DynamicCache} [params.past_key_values=null] + * @param {Object} [params.generation_config=null] + * @param {Object} [params.logits_processor=null] + * @param {Tensor} [params.num_logits_to_keep=null] + * @returns {Promise} The model's output tensor + * @private + */ +export async function generic_text_to_text_forward( + self, + { + // Generic parameters: + encode_function, + merge_function, + modality_input_names, + modality_output_name, + + // Produced by the tokenizer/processor: + input_ids = null, + attention_mask = null, + + // Used during generation: + position_ids = null, + inputs_embeds = null, + past_key_values = null, + + // Generic generation parameters + generation_config = null, + logits_processor = null, + num_logits_to_keep = null, + + // Additional parameters + ...kwargs + }, +) { + if (!inputs_embeds) { + // 1. Extract the text embeddings. + inputs_embeds = await self.encode_text({ input_ids, ...kwargs }); + + // 2. Possibly, merge text and modality values + const modality_values = pick(kwargs, modality_input_names); + if (Object.keys(modality_values).length > 0) { + if (input_ids.dims[1] !== 1) { + const modality_features = await encode_function({ + // Pass the modality values under its expected key. + // The caller knows whether this is audio or image. + ...modality_values, + ...kwargs, + }); + ({ inputs_embeds, attention_mask } = merge_function({ + [modality_output_name]: modality_features, + inputs_embeds, + input_ids, + attention_mask, + })); + } else if (past_key_values && input_ids.dims[1] === 1) { + // This branch handles the cache case. + const target_length = input_ids.dims[1]; // always 1 + const past_length = past_key_values.get_seq_length(); + + attention_mask = cat( + [ + ones([input_ids.dims[0], past_length]), + attention_mask.slice(null, [attention_mask.dims[1] - target_length, attention_mask.dims[1]]), + ], + 1, + ); + } + } + } + + if (!position_ids) { + if ( + // Handle special case for qwen vl models + [ + 'qwen2_vl', + 'qwen2_vl_text', + 'qwen2_5_vl', + 'qwen2_5_vl_text', + 'qwen3_vl', + 'qwen3_vl_text', + 'qwen3_vl_moe', + 'qwen3_vl_moe_text', + 'qwen3_5', + 'qwen3_5_text', + 'qwen3_5_moe', + 'qwen3_5_moe_text', + 'glm_ocr', + 'glm_ocr_text', + ].includes(self.config.model_type) + ) { + // @ts-ignore + const { image_grid_thw, video_grid_thw } = kwargs; + [position_ids] = self.get_rope_index(input_ids, image_grid_thw, video_grid_thw, attention_mask); + } + } + + // 3. Call the decoder forward using the updated inputs. + const outputs = await decoder_forward( + self, + { + inputs_embeds, + past_key_values, + attention_mask, + position_ids, + generation_config, + logits_processor, + num_logits_to_keep, + }, + true, + ); + return outputs; +} + +/** + * Forward pass of an audio-text-to-text model. + * @param {Object} self The audio-text-to-text model. + * @param {Object} params The inputs for the audio-text-to-text forward pass. + * @returns {Promise} The model's output tensor. + * @private + */ +export async function audio_text_to_text_forward(self, params) { + return await generic_text_to_text_forward(self, { + ...params, + modality_input_names: ['audio_values', 'input_features'], + modality_output_name: 'audio_features', + encode_function: self.encode_audio.bind(self), + merge_function: self._merge_input_ids_with_audio_features.bind(self), + }); +} + +/** + * Forward pass of an image-text-to-text model. + * @param {Object} self The image-text-to-text model. + * @param {Object} params The inputs for the image-text-to-text forward pass. + * @returns {Promise} The model's output tensor. + * @private + */ +export async function image_text_to_text_forward(self, params) { + return await generic_text_to_text_forward(self, { + ...params, + modality_input_names: ['pixel_values'], + modality_output_name: 'image_features', + encode_function: self.encode_image.bind(self), + merge_function: self._merge_input_ids_with_image_features.bind(self), + }); +} + +/** + * Helper function to perform the following: + * ```python + * x = attention_mask.long().cumsum(-1) - 1 + * x.masked_fill_(attention_mask == 0, 1) + * ``` + * @param {Tensor} attention_mask + * @returns {{data: BigInt64Array, dims: number[]}} + */ +export function cumsum_masked_fill(attention_mask, start_index = 0) { + const [bz, seq_len] = attention_mask.dims; + const attn_mask_data = attention_mask.data; + + const data = new BigInt64Array(attn_mask_data.length); + for (let i = 0; i < bz; ++i) { + const start = i * seq_len; + let sum = BigInt(start_index); + for (let j = 0; j < seq_len; ++j) { + const index = start + j; + if (attn_mask_data[index] === 0n) { + data[index] = BigInt(1); + } else { + // === 1n + data[index] = sum; + sum += attn_mask_data[index]; + } + } + } + return { data, dims: attention_mask.dims }; +} + +/** + * If the model supports providing position_ids, we create position_ids on the fly for batch generation, + * by computing the cumulative sum of the attention mask along the sequence length dimension. + * + * Equivalent to: + * ```python + * position_ids = attention_mask.long().cumsum(-1) - 1 + * position_ids.masked_fill_(attention_mask == 0, 1) + * if past_key_values: + * position_ids = position_ids[:, -input_ids.shape[1] :] + * ``` + */ +export function create_position_ids(model_inputs, past_key_values = null, start_index = 0) { + const { input_ids, inputs_embeds, attention_mask } = model_inputs; + + const { data, dims } = cumsum_masked_fill(attention_mask, start_index); + let position_ids = new Tensor('int64', data, dims); + if (past_key_values) { + const offset = -(input_ids ?? inputs_embeds).dims.at(1); + position_ids = position_ids.slice(null, [offset, null]); + } + return position_ids; +} + +export function decoder_prepare_inputs_for_generation(self, input_ids, model_inputs, generation_config) { + const past_length = model_inputs.past_key_values ? model_inputs.past_key_values.get_seq_length() : 0; + + setNumLogitsToKeep(self, model_inputs, 1n); + + if (!model_inputs.attention_mask) { + // If the attention mask is not provided, we attempt to infer based on provided inputs + let dims; + for (const key of ['input_ids', 'inputs_embeds', 'position_ids']) { + if (model_inputs[key]) { + dims = model_inputs[key].dims; + break; + } + } + if (!dims) { + throw new Error('attention_mask is not provided, and unable to infer its shape from model inputs.'); + } + model_inputs.attention_mask = ones([dims[0], past_length + dims[1]]); + } + + if (model_inputs.past_key_values) { + const { input_ids, attention_mask } = model_inputs; + + // Keep only the unprocessed tokens: + // 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where + // some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as + // input) + if (attention_mask && attention_mask.dims[1] > input_ids.dims[1]) { + // NOTE: not needed since we only pass the generated tokens to the next forward pass + // const offset = -(attention_mask.dims[1] - past_length); + // model_inputs.input_ids = input_ids.slice(null, [offset, null]); + } + // 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. + // We can discard input_ids based on the past_length. + else if (past_length < input_ids.dims[1]) { + // NOTE: Required for phi models. + // See https://github.com/huggingface/transformers/issues/30809#issuecomment-2111918479 for more information. + model_inputs.input_ids = input_ids.slice(null, [past_length, null]); + } + // 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens. + else { + } + } + + return model_inputs; +} + +export function encoder_decoder_prepare_inputs_for_generation(self, input_ids, model_inputs, generation_config) { + if (model_inputs.past_key_values) { + input_ids = input_ids.map((x) => [x.at(-1)]); + } + + setNumLogitsToKeep(self, model_inputs, 1n); + + return { + ...model_inputs, + decoder_input_ids: toI64Tensor(input_ids), + }; +} + +export function multimodal_text_to_text_prepare_inputs_for_generation(self, ...args) { + if (self.config.is_encoder_decoder) { + return encoder_decoder_prepare_inputs_for_generation(self, ...args); + } else { + return decoder_prepare_inputs_for_generation(self, ...args); + } +} + +export function default_merge_input_ids_with_features({ + modality_token_id, + inputs_embeds, + modality_features, + input_ids, + attention_mask, +}) { + const token_positions = input_ids.tolist().map((ids) => + ids.reduce((acc, x, idx) => { + if (x == modality_token_id) acc.push(idx); + return acc; + }, []), + ); + const n_tokens = token_positions.reduce((acc, x) => acc + x.length, 0); + const n_features = modality_features.dims[0]; + if (n_tokens !== n_features) { + throw new Error(`Number of tokens and features do not match: tokens: ${n_tokens}, features ${n_features}`); + } + + // Currently, we require modality features to be in float32 for correct scatter behavior. + // TODO: In future, detect dtype of embeds and cast modality features to the same dtype before scattering. + // modality_features = modality_features.to('float32'); + + // Equivalent to performing a masked_scatter + let img = 0; + for (let i = 0; i < token_positions.length; ++i) { + const tokens = token_positions[i]; + const embeds = inputs_embeds[i]; + for (let j = 0; j < tokens.length; ++j) { + embeds[tokens[j]].data.set(modality_features[img++].data); + } + } + return { inputs_embeds, attention_mask }; +} + +export function default_merge_input_ids_with_image_features({ + image_token_id, + inputs_embeds, + image_features, + input_ids, + attention_mask, +}) { + return default_merge_input_ids_with_features({ + modality_token_id: image_token_id, + inputs_embeds, + modality_features: image_features, + input_ids, + attention_mask, + }); +} + +export function default_merge_input_ids_with_audio_features({ + audio_token_id, + inputs_embeds, + audio_features, + input_ids, + attention_mask, +}) { + return default_merge_input_ids_with_features({ + modality_token_id: audio_token_id, + inputs_embeds, + modality_features: audio_features, + input_ids, + attention_mask, + }); +} + +/** + * Helper function to load multiple optional configuration files + * @param {string} pretrained_model_name_or_path The path to the directory containing the config file. + * @param {Record} names The names of the config files to load. + * @param {import('../utils/hub.js').PretrainedModelOptions} options Additional options for loading the configs. + * @returns {Promise>} A Promise that resolves to a dictionary of configuration objects. + * @private + */ +export async function get_optional_configs(pretrained_model_name_or_path, names, options) { + return Object.fromEntries( + await Promise.all( + Object.keys(names).map(async (name) => { + const config = await getModelJSON(pretrained_model_name_or_path, names[name], false, options); + return [name, config]; + }), + ), + ); +} diff --git a/packages/transformers/src/models/models.js b/packages/transformers/src/models/models.js new file mode 100644 index 0000000..099042c --- /dev/null +++ b/packages/transformers/src/models/models.js @@ -0,0 +1,216 @@ +export * from './albert/modeling_albert.js'; +export * from './apertus/modeling_apertus.js'; +export * from './afmoe/modeling_afmoe.js'; +export * from './arcee/modeling_arcee.js'; +export * from './audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.js'; +export * from './bart/modeling_bart.js'; +export * from './beit/modeling_beit.js'; +export * from './bert/modeling_bert.js'; +export * from './blenderbot/modeling_blenderbot.js'; +export * from './blenderbot_small/modeling_blenderbot_small.js'; +export * from './bloom/modeling_bloom.js'; +export * from './camembert/modeling_camembert.js'; +export * from './chatterbox/modeling_chatterbox.js'; +export * from './chinese_clip/modeling_chinese_clip.js'; +export * from './chmv2/modeling_chmv2.js'; +export * from './clap/modeling_clap.js'; +export * from './clip/modeling_clip.js'; +export * from './clipseg/modeling_clipseg.js'; +export * from './codegen/modeling_codegen.js'; +export * from './cohere/modeling_cohere.js'; +export * from './cohere2/modeling_cohere2.js'; +export * from './cohere_asr/modeling_cohere_asr.js'; +export * from './convbert/modeling_convbert.js'; +export * from './convnext/modeling_convnext.js'; +export * from './convnextv2/modeling_convnextv2.js'; +export * from './d_fine/modeling_d_fine.js'; +export * from './dac/modeling_dac.js'; +export * from './deberta/modeling_deberta.js'; +export * from './deepseek_v3/modeling_deepseek_v3.js'; +export * from './deepseek_v4/modeling_deepseek_v4.js'; +export * from './deberta_v2/modeling_deberta_v2.js'; +export * from './decision_transformer/modeling_decision_transformer.js'; +export * from './deit/modeling_deit.js'; +export * from './depth_anything/modeling_depth_anything.js'; +export * from './depth_pro/modeling_depth_pro.js'; +export * from './detr/modeling_detr.js'; +export * from './dinov2/modeling_dinov2.js'; +export * from './dinov2_with_registers/modeling_dinov2_with_registers.js'; +export * from './dinov3_convnext/modeling_dinov3_convnext.js'; +export * from './dinov3_vit/modeling_dinov3_vit.js'; +export * from './distilbert/modeling_distilbert.js'; +export * from './donut_swin/modeling_donut_swin.js'; +export * from './dpt/modeling_dpt.js'; +export * from './efficientnet/modeling_efficientnet.js'; +export * from './electra/modeling_electra.js'; +export * from './ernie4_5/modeling_ernie4_5.js'; +export * from './esm/modeling_esm.js'; +export * from './eurobert/modeling_eurobert.js'; +export * from './exaone/modeling_exaone.js'; +export * from './falcon/modeling_falcon.js'; +export * from './falcon_h1/modeling_falcon_h1.js'; +export * from './fastvit/modeling_fastvit.js'; +export * from './florence2/modeling_florence2.js'; +export * from './gemma/modeling_gemma.js'; +export * from './gemma2/modeling_gemma2.js'; +export * from './gemma3/modeling_gemma3.js'; +export * from './gemma3n/modeling_gemma3n.js'; +export * from './gemma4/modeling_gemma4.js'; +export * from './glm/modeling_glm.js'; +export * from './glm_moe_dsa/modeling_glm_moe_dsa.js'; +export * from './glm_ocr/modeling_glm_ocr.js'; +export * from './glpn/modeling_glpn.js'; +export * from './gpt_bigcode/modeling_gpt_bigcode.js'; +export * from './gpt_neo/modeling_gpt_neo.js'; +export * from './gpt_neox/modeling_gpt_neox.js'; +export * from './gpt_oss/modeling_gpt_oss.js'; +export * from './gpt2/modeling_gpt2.js'; +export * from './gptj/modeling_gptj.js'; +export * from './granite/modeling_granite.js'; +export * from './granitemoehybrid/modeling_granitemoehybrid.js'; +export * from './granite_speech/modeling_granite_speech.js'; +export * from './grounding_dino/modeling_grounding_dino.js'; +export * from './groupvit/modeling_groupvit.js'; +export * from './helium/modeling_helium.js'; +export * from './hiera/modeling_hiera.js'; +export * from './hrm_text/modeling_hrm_text.js'; +export * from './hubert/modeling_hubert.js'; +export * from './hunyuan_v1_dense/modeling_hunyuan_v1_dense.js'; +export * from './idefics3/modeling_idefics3.js'; +export * from './ijepa/modeling_ijepa.js'; +export * from './jais/modeling_jais.js'; +export * from './jina_clip/modeling_jina_clip.js'; +export * from './lfm2/modeling_lfm2.js'; +export * from './lighton_ocr/modeling_lighton_ocr.js'; +export * from './lfm2_moe/modeling_lfm2_moe.js'; +export * from './lfm2_vl/modeling_lfm2_vl.js'; +export * from './llama/modeling_llama.js'; +export * from './llama4/modeling_llama4.js'; +export * from './llava/modeling_llava.js'; +export * from './llava_onevision/modeling_llava_onevision.js'; +export * from './longt5/modeling_longt5.js'; +export * from './m2m_100/modeling_m2m_100.js'; +export * from './marian/modeling_marian.js'; +export * from './maskformer/modeling_maskformer.js'; +export * from './mbart/modeling_mbart.js'; +export * from './metric3d/modeling_metric3d.js'; +export * from './metric3dv2/modeling_metric3dv2.js'; +export * from './mgp_str/modeling_mgp_str.js'; +export * from './mimi/modeling_mimi.js'; +export * from './ministral/modeling_ministral.js'; +export * from './ministral3/modeling_ministral3.js'; +export * from './mistral/modeling_mistral.js'; +export * from './mistral3/modeling_mistral3.js'; +export * from './mistral4/modeling_mistral4.js'; +export * from './mobilebert/modeling_mobilebert.js'; +export * from './mobilellm/modeling_mobilellm.js'; +export * from './mobilenet_v1/modeling_mobilenet_v1.js'; +export * from './mobilenet_v2/modeling_mobilenet_v2.js'; +export * from './mobilenet_v3/modeling_mobilenet_v3.js'; +export * from './mobilenet_v4/modeling_mobilenet_v4.js'; +export * from './mobilevit/modeling_mobilevit.js'; +export * from './mobilevitv2/modeling_mobilevitv2.js'; +export * from './modernbert/modeling_modernbert.js'; +export * from './modernbert_decoder/modeling_modernbert_decoder.js'; +export * from './moonshine/modeling_moonshine.js'; +export * from './mpnet/modeling_mpnet.js'; +export * from './mpt/modeling_mpt.js'; +export * from './mt5/modeling_mt5.js'; +export * from './multi_modality/modeling_multi_modality.js'; +export * from './musicgen/modeling_musicgen.js'; +export * from './nanochat/modeling_nanochat.js'; +export * from './nemotron_h/modeling_nemotron_h.js'; +export * from './neobert/modeling_neobert.js'; +export * from './nomic_bert/modeling_nomic_bert.js'; +export * from './olmo/modeling_olmo.js'; +export * from './olmo2/modeling_olmo2.js'; +export * from './olmo3/modeling_olmo3.js'; +export * from './olmo_hybrid/modeling_olmo_hybrid.js'; +export * from './openai_privacy_filter/modeling_openai_privacy_filter.js'; +export * from './openelm/modeling_openelm.js'; +export * from './opt/modeling_opt.js'; +export * from './owlv2/modeling_owlv2.js'; +export * from './owlvit/modeling_owlvit.js'; +export * from './paligemma/modeling_paligemma.js'; +export * from './parakeet/modeling_parakeet.js'; +export * from './patchtsmixer/modeling_patchtsmixer.js'; +export * from './patchtst/modeling_patchtst.js'; +export * from './phi/modeling_phi.js'; +export * from './phi3/modeling_phi3.js'; +export * from './phi3_v/modeling_phi3_v.js'; +export * from './pvt/modeling_pvt.js'; +export * from './pyannote/modeling_pyannote.js'; +export * from './qwen2/modeling_qwen2.js'; +export * from './qwen2_moe/modeling_qwen2_moe.js'; +export * from './qwen2_vl/modeling_qwen2_vl.js'; +export * from './qwen2_5_vl/modeling_qwen2_5_vl.js'; +export * from './qwen3/modeling_qwen3.js'; +export * from './qwen3_moe/modeling_qwen3_moe.js'; +export * from './qwen3_next/modeling_qwen3_next.js'; +export * from './qwen3_vl/modeling_qwen3_vl.js'; +export * from './qwen3_vl_moe/modeling_qwen3_vl_moe.js'; +export * from './qwen3_5/modeling_qwen3_5.js'; +export * from './qwen3_5_moe/modeling_qwen3_5_moe.js'; +export * from './resnet/modeling_resnet.js'; +export * from './rf_detr/modeling_rf_detr.js'; +export * from './roberta/modeling_roberta.js'; +export * from './roformer/modeling_roformer.js'; +export * from './rt_detr/modeling_rt_detr.js'; +export * from './rt_detr_v2/modeling_rt_detr_v2.js'; +export * from './sam/modeling_sam.js'; +export * from './sam2/modeling_sam2.js'; +export * from './sapiens/modeling_sapiens.js'; +export * from './segformer/modeling_segformer.js'; +export * from './siglip/modeling_siglip.js'; +export * from './smollm3/modeling_smollm3.js'; +export * from './smolvlm/modeling_smolvlm.js'; +export * from './snac/modeling_snac.js'; +export * from './solar_open/modeling_solar_open.js'; +export * from './speecht5/modeling_speecht5.js'; +export * from './squeezebert/modeling_squeezebert.js'; +export * from './stablelm/modeling_stablelm.js'; +export * from './starcoder2/modeling_starcoder2.js'; +export * from './style_text_to_speech_2/modeling_style_text_to_speech_2.js'; +export * from './supertonic/modeling_supertonic.js'; +export * from './swin/modeling_swin.js'; +export * from './swin2sr/modeling_swin2sr.js'; +export * from './t5/modeling_t5.js'; +export * from './table_transformer/modeling_table_transformer.js'; +export * from './trocr/modeling_trocr.js'; +export * from './ultravox/modeling_ultravox.js'; +export * from './unispeech/modeling_unispeech.js'; +export * from './unispeech_sat/modeling_unispeech_sat.js'; +export * from './vaultgemma/modeling_vaultgemma.js'; +export * from './vision_encoder_decoder/modeling_vision_encoder_decoder.js'; +export * from './vit/modeling_vit.js'; +export * from './vit_mae/modeling_vit_mae.js'; +export * from './vit_msn/modeling_vit_msn.js'; +export * from './vitmatte/modeling_vitmatte.js'; +export * from './vitpose/modeling_vitpose.js'; +export * from './vits/modeling_vits.js'; +export * from './voxtral/modeling_voxtral.js'; +export * from './voxtral_realtime/modeling_voxtral_realtime.js'; +export * from './wav2vec2/modeling_wav2vec2.js'; +export * from './wav2vec2_bert/modeling_wav2vec2_bert.js'; +export * from './wavlm/modeling_wavlm.js'; +export * from './wespeaker_resnet/modeling_wespeaker_resnet.js'; +export * from './whisper/modeling_whisper.js'; +export * from './xlm/modeling_xlm.js'; +export * from './xlm_roberta/modeling_xlm_roberta.js'; +export * from './yolos/modeling_yolos.js'; +export * from './youtu/modeling_youtu.js'; +export * from './zaya/modeling_zaya.js'; + +export { + BaseModelOutput, + CausalLMOutput, + CausalLMOutputWithPast, + ImageMattingOutput, + MaskedLMOutput, + ModelOutput, + QuestionAnsweringModelOutput, + Seq2SeqLMOutput, + SequenceClassifierOutput, + TokenClassifierOutput, +} from './modeling_outputs.js'; +export { PreTrainedModel } from './modeling_utils.js'; diff --git a/packages/transformers/src/models/modernbert/modeling_modernbert.js b/packages/transformers/src/models/modernbert/modeling_modernbert.js new file mode 100644 index 0000000..080fb3b --- /dev/null +++ b/packages/transformers/src/models/modernbert/modeling_modernbert.js @@ -0,0 +1,41 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { MaskedLMOutput, SequenceClassifierOutput, TokenClassifierOutput } from '../modeling_outputs.js'; + +export class ModernBertPreTrainedModel extends PreTrainedModel {} +export class ModernBertModel extends ModernBertPreTrainedModel {} + +export class ModernBertForMaskedLM extends ModernBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for masked language modeling. + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +export class ModernBertForSequenceClassification extends ModernBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +export class ModernBertForTokenClassification extends ModernBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/modernbert_decoder/modeling_modernbert_decoder.js b/packages/transformers/src/models/modernbert_decoder/modeling_modernbert_decoder.js new file mode 100644 index 0000000..954ad32 --- /dev/null +++ b/packages/transformers/src/models/modernbert_decoder/modeling_modernbert_decoder.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class ModernBertDecoderPreTrainedModel extends PreTrainedModel {} +export class ModernBertDecoderModel extends ModernBertDecoderPreTrainedModel {} +export class ModernBertDecoderForCausalLM extends ModernBertDecoderPreTrainedModel {} diff --git a/packages/transformers/src/models/moonshine/feature_extraction_moonshine.js b/packages/transformers/src/models/moonshine/feature_extraction_moonshine.js new file mode 100644 index 0000000..0749a92 --- /dev/null +++ b/packages/transformers/src/models/moonshine/feature_extraction_moonshine.js @@ -0,0 +1,22 @@ +import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class MoonshineFeatureExtractor extends FeatureExtractor { + /** + * Asynchronously extracts input values from a given audio using the provided configuration. + * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array. + * @returns {Promise<{ input_values: Tensor; }>} The extracted input values. + */ + async _call(audio) { + validate_audio_inputs(audio, 'MoonshineFeatureExtractor'); + + if (audio instanceof Float64Array) { + audio = new Float32Array(audio); + } + + const shape = [1 /* batch_size */, audio.length /* num_samples */]; + return { + input_values: new Tensor('float32', audio, shape), + }; + } +} diff --git a/packages/transformers/src/models/moonshine/modeling_moonshine.js b/packages/transformers/src/models/moonshine/modeling_moonshine.js new file mode 100644 index 0000000..88fc067 --- /dev/null +++ b/packages/transformers/src/models/moonshine/modeling_moonshine.js @@ -0,0 +1,14 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class MoonshinePreTrainedModel extends PreTrainedModel { + requires_attention_mask = false; + main_input_name = 'input_values'; + forward_params = ['input_values', 'decoder_input_ids', 'past_key_values']; +} + +/** + * MoonshineModel class for training Moonshine models without a language model head. + */ +export class MoonshineModel extends MoonshinePreTrainedModel {} + +export class MoonshineForConditionalGeneration extends MoonshinePreTrainedModel {} diff --git a/packages/transformers/src/models/moonshine/processing_moonshine.js b/packages/transformers/src/models/moonshine/processing_moonshine.js new file mode 100644 index 0000000..282d04e --- /dev/null +++ b/packages/transformers/src/models/moonshine/processing_moonshine.js @@ -0,0 +1,20 @@ +import { AutoFeatureExtractor } from '../auto/feature_extraction_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { Processor } from '../../processing_utils.js'; + +/** + * Represents a MoonshineProcessor that extracts features from an audio input. + */ +export class MoonshineProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static feature_extractor_class = AutoFeatureExtractor; + + /** + * Calls the feature_extractor function with the given audio input. + * @param {any} audio The audio input to extract features from. + * @returns {Promise} A Promise that resolves with the extracted features. + */ + async _call(audio) { + return await this.feature_extractor(audio); + } +} diff --git a/packages/transformers/src/models/mpnet/modeling_mpnet.js b/packages/transformers/src/models/mpnet/modeling_mpnet.js new file mode 100644 index 0000000..d377871 --- /dev/null +++ b/packages/transformers/src/models/mpnet/modeling_mpnet.js @@ -0,0 +1,74 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { + MaskedLMOutput, + QuestionAnsweringModelOutput, + SequenceClassifierOutput, + TokenClassifierOutput, +} from '../modeling_outputs.js'; + +export class MPNetPreTrainedModel extends PreTrainedModel {} + +/** + * The bare MPNet Model transformer outputting raw hidden-states without any specific head on top. + */ +export class MPNetModel extends MPNetPreTrainedModel {} + +/** + * MPNetForMaskedLM is a class representing a MPNet model for masked language modeling. + */ +export class MPNetForMaskedLM extends MPNetPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for masked language modeling. + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +/** + * MPNetForSequenceClassification is a class representing a MPNet model for sequence classification. + */ +export class MPNetForSequenceClassification extends MPNetPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * MPNetForTokenClassification is a class representing a MPNet model for token classification. + */ +export class MPNetForTokenClassification extends MPNetPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * MPNetForQuestionAnswering is a class representing a MPNet model for question answering. + */ +export class MPNetForQuestionAnswering extends MPNetPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for question answering. + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/mpnet/tokenization_mpnet.js b/packages/transformers/src/models/mpnet/tokenization_mpnet.js new file mode 100644 index 0000000..d5274b8 --- /dev/null +++ b/packages/transformers/src/models/mpnet/tokenization_mpnet.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class MPNetTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/mpt/modeling_mpt.js b/packages/transformers/src/models/mpt/modeling_mpt.js new file mode 100644 index 0000000..157ca4f --- /dev/null +++ b/packages/transformers/src/models/mpt/modeling_mpt.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class MptPreTrainedModel extends PreTrainedModel {} + +/** + * The bare Mpt Model transformer outputting raw hidden-states without any specific head on top. + */ +export class MptModel extends MptPreTrainedModel {} + +/** + * The MPT Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). + */ +export class MptForCausalLM extends MptPreTrainedModel {} diff --git a/packages/transformers/src/models/mt5/modeling_mt5.js b/packages/transformers/src/models/mt5/modeling_mt5.js new file mode 100644 index 0000000..4420cf1 --- /dev/null +++ b/packages/transformers/src/models/mt5/modeling_mt5.js @@ -0,0 +1,10 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class MT5PreTrainedModel extends PreTrainedModel {} + +export class MT5Model extends MT5PreTrainedModel {} + +/** + * A class representing a conditional sequence-to-sequence model based on the MT5 architecture. + */ +export class MT5ForConditionalGeneration extends MT5PreTrainedModel {} diff --git a/packages/transformers/src/models/multi_modality/modeling_multi_modality.js b/packages/transformers/src/models/multi_modality/modeling_multi_modality.js new file mode 100644 index 0000000..b0969dd --- /dev/null +++ b/packages/transformers/src/models/multi_modality/modeling_multi_modality.js @@ -0,0 +1,157 @@ +import { PreTrainedModel, decoder_forward, setNumLogitsToKeep } from '../modeling_utils.js'; +import { sessionRun } from '../session.js'; +import { pick } from '../../utils/core.js'; +import { RawImage } from '../../utils/image.js'; +import { Tensor, cat, full, full_like } from '../../utils/tensor.js'; + +export class MultiModalityPreTrainedModel extends PreTrainedModel {} +export class MultiModalityCausalLM extends MultiModalityPreTrainedModel { + forward_params = [ + // prepare_inputs_embeds + 'input_ids', + 'pixel_values', + 'images_seq_mask', + 'images_emb_mask', + + // language_model + 'attention_mask', + 'position_ids', + 'past_key_values', + ]; + + /** + * @param {ConstructorParameters} args + */ + constructor(...args) { + super(...args); + + // State-based approach to switch out which heads to use during generation + this._generation_mode = 'text'; + } + + async forward(model_inputs) { + const mode = this._generation_mode ?? 'text'; + + // TODO support re-using PKVs for input_ids.dims[1] !== 1 + // if (model_inputs.past_key_values) { + // // && model_inputs.input_ids.dims[1] === 1 + // } + + let output_1; + if (mode === 'text' || !model_inputs.past_key_values) { + const session = this.sessions['prepare_inputs_embeds']; + const prep_inputs = pick(model_inputs, session.inputNames); + output_1 = await sessionRun(session, prep_inputs); + } else { + const session = this.sessions['gen_img_embeds']; + const prep_inputs = pick( + { + image_ids: model_inputs.input_ids, + }, + session.inputNames, + ); + output_1 = await sessionRun(session, prep_inputs); + } + + const input_2 = { ...model_inputs, ...output_1 }; + const output_2 = await decoder_forward(this, input_2); + + const head = this.sessions[mode === 'text' ? 'lm_head' : 'gen_head']; + if (!head) { + throw new Error(`Unable to find "${head}" generation head`); + } + + const output_3 = await sessionRun(head, pick(output_2, head.inputNames)); + + return { + ...output_1, + ...output_2, + ...output_3, + }; + } + + prepare_inputs_for_generation(input_ids, model_inputs, generation_config) { + const has_past_key_values = !!model_inputs.past_key_values; + + setNumLogitsToKeep(this, model_inputs, 1n); + + if (generation_config.guidance_scale !== null && generation_config.guidance_scale > 1) { + if (has_past_key_values) { + model_inputs.input_ids = cat([model_inputs.input_ids, model_inputs.input_ids], 0); + // NOTE: attention_mask handled in generation + } else { + model_inputs.input_ids = cat( + [model_inputs.input_ids, full_like(model_inputs.input_ids, BigInt(generation_config.pad_token_id))], + 0, + ); + model_inputs.attention_mask = cat( + [model_inputs.attention_mask, full_like(model_inputs.attention_mask, 0n)], + 0, + ); + } + } + + if (has_past_key_values || !model_inputs.pixel_values) { + model_inputs.pixel_values = full([0, 0, 3, 384, 384], 1.0); + } + + if (has_past_key_values) { + const num_img_tokens = 0; + const num_text_tokens = 1; + const has_image = num_img_tokens > 0 ? 1 : 0; + + const batch_size = 1; + model_inputs.images_seq_mask = new Tensor( + 'bool', + new Array(num_img_tokens + num_text_tokens).fill(true).fill(false, 0, num_text_tokens), + [batch_size, num_img_tokens + num_text_tokens], + ); + model_inputs.images_emb_mask = new Tensor('bool', new Array(num_img_tokens).fill(!!has_image), [ + batch_size, + 1, + num_img_tokens, + ]); + } + return model_inputs; + } + + /** + * @param {import('../../generation/parameters.js').GenerationFunctionParameters} options + */ + async generate(options) { + this._generation_mode = 'text'; + return super.generate(options); + } + + /** + * @param {import('../../generation/parameters.js').GenerationFunctionParameters} options + */ + async generate_images(options) { + this._generation_mode = 'image'; + + const start_num_tokens = /** @type {Tensor} */ (options.inputs ?? options[this.main_input_name]).dims[1]; + const all_tokens = await super.generate(options); + + const generated_tokens = /** @type {Tensor} */ (all_tokens).slice(null, [start_num_tokens, null]); + + const image_decode = this.sessions['image_decode']; + const { decoded_image } = await sessionRun(image_decode, { + generated_tokens, + }); + + // Equivalent to `np.clip((dec + 1) / 2 * 255, 0, 255)` + const clamped = decoded_image + .add_(1) + .mul_(255 / 2) + .clamp_(0, 255) + .to('uint8'); + + // Return as a list of images + const images = []; + for (const tensor of clamped) { + const img = RawImage.fromTensor(tensor); + images.push(img); + } + return images; + } +} diff --git a/packages/transformers/src/models/musicgen/modeling_musicgen.js b/packages/transformers/src/models/musicgen/modeling_musicgen.js new file mode 100644 index 0000000..333baa0 --- /dev/null +++ b/packages/transformers/src/models/musicgen/modeling_musicgen.js @@ -0,0 +1,139 @@ +import { PreTrainedModel, encoder_decoder_prepare_inputs_for_generation } from '../modeling_utils.js'; +import { sessionRun } from '../session.js'; +import { Tensor } from '../../utils/tensor.js'; +import { ModelOutput } from '../modeling_outputs.js'; + +export class MusicgenPreTrainedModel extends PreTrainedModel {} + +/** + * The bare Musicgen decoder model outputting raw hidden-states without any specific head on top. + */ +export class MusicgenModel extends MusicgenPreTrainedModel {} + +/** + * The MusicGen decoder model with a language modelling head on top. + */ +export class MusicgenForCausalLM extends MusicgenPreTrainedModel {} + +/** + * The composite MusicGen model with a text encoder, audio encoder and Musicgen decoder, + * for music generation tasks with one or both of text and audio prompts. + * + * **Example:** Generate music from text with `Xenova/musicgen-small`. + * ```javascript + * import { AutoTokenizer, MusicgenForConditionalGeneration, RawAudio } from '@huggingface/transformers'; + * + * // Load tokenizer and model + * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/musicgen-small'); + * const model = await MusicgenForConditionalGeneration.from_pretrained( + * 'Xenova/musicgen-small', { dtype: 'fp32' } + * ); + * + * // Prepare text input + * const prompt = '80s pop track with bassy drums and synth'; + * const inputs = tokenizer(prompt); + * + * // Generate audio + * const audio_values = await model.generate({ + * ...inputs, + * max_new_tokens: 512, + * do_sample: true, + * guidance_scale: 3, + * }); + * + * // (Optional) Save the output to a WAV file + * const audio = new RawAudio( + * audio_values.data, + * model.config.audio_encoder.sampling_rate, + * ); + * audio.save('musicgen_out.wav'); + * ``` + */ +export class MusicgenForConditionalGeneration extends PreTrainedModel { + // NOTE: not MusicgenPreTrainedModel + forward_params = [ + 'input_ids', + 'attention_mask', + 'encoder_outputs', + 'decoder_input_ids', + 'decoder_attention_mask', + 'past_key_values', + ]; + + /** + * Apply the pattern mask to the final ids, + * then revert the pattern delay mask by filtering the pad token id in a single step. + * @param {Tensor} outputs The output tensor from the model. + * @returns {Tensor} The filtered output tensor. + */ + _apply_and_filter_by_delay_pattern_mask(outputs) { + const [bs_x_codebooks, seqLength] = outputs.dims; + // @ts-expect-error TS2339 + const num_codebooks = this.config.decoder.num_codebooks; + const upperBound = seqLength - num_codebooks; + + let newDataSize = 0; + for (let i = 0; i < outputs.size; ++i) { + // @ts-expect-error TS2339 + if (outputs.data[i] == this.config.decoder.pad_token_id) { + continue; + } + + const row = i % seqLength; + const col = Math.floor(i / seqLength) % num_codebooks; + + const diff = row - col; + if (diff > 0 && diff <= upperBound) { + outputs.data[newDataSize++] = outputs.data[i]; + } + } + + const batch_size = Math.floor(bs_x_codebooks / num_codebooks); + const inferred = newDataSize / (batch_size * num_codebooks); + // TODO: assert `inferred` is an integer + return new Tensor(outputs.type, outputs.data.slice(0, newDataSize), [batch_size, num_codebooks, inferred]); + } + + prepare_inputs_for_generation(input_ids, model_inputs, generation_config) { + // @ts-expect-error TS2339 + const pad_token_id = BigInt(this.config.decoder.pad_token_id); + + // apply the delay pattern mask + let clonedInputIds = structuredClone(input_ids); + for (let i = 0; i < clonedInputIds.length; ++i) { + for (let j = 0; j < clonedInputIds[i].length; ++j) { + // @ts-expect-error TS2339 + if (i % this.config.decoder.num_codebooks >= j) { + clonedInputIds[i][j] = pad_token_id; + } + } + } + // for classifier free guidance we need to replicate the decoder args across the batch dim + // (we'll split these before sampling) + if (generation_config.guidance_scale !== null && generation_config.guidance_scale > 1) { + // [batch, seqLength] -> [2 * batch, seqLength] + clonedInputIds = clonedInputIds.concat(clonedInputIds); + } + + return encoder_decoder_prepare_inputs_for_generation(this, clonedInputIds, model_inputs, generation_config); + } + + /** + * Generates sequences of token ids for models with a language modeling head. + * @param {import('../../generation/parameters.js').GenerationFunctionParameters} options + * @returns {Promise} The output of the model, which can contain the generated token ids, attentions, and scores. + */ + async generate(options) { + const output_ids = await super.generate(options); + + // apply the pattern mask to the final ids + // tensor: int64[1,batch_size,4,chunk_length] + const audio_codes = this._apply_and_filter_by_delay_pattern_mask(/** @type {Tensor} */ (output_ids)).unsqueeze_( + 0, + ); // append the frame dimension back to the audio codes + + const { audio_values } = await sessionRun(this.sessions['encodec_decode'], { audio_codes }); + + return audio_values; + } +} diff --git a/packages/transformers/src/models/nanochat/modeling_nanochat.js b/packages/transformers/src/models/nanochat/modeling_nanochat.js new file mode 100644 index 0000000..3ad89a4 --- /dev/null +++ b/packages/transformers/src/models/nanochat/modeling_nanochat.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class NanoChatPreTrainedModel extends PreTrainedModel {} +export class NanoChatModel extends NanoChatPreTrainedModel {} +export class NanoChatForCausalLM extends NanoChatPreTrainedModel {} diff --git a/packages/transformers/src/models/nemotron_h/modeling_nemotron_h.js b/packages/transformers/src/models/nemotron_h/modeling_nemotron_h.js new file mode 100644 index 0000000..088cc0a --- /dev/null +++ b/packages/transformers/src/models/nemotron_h/modeling_nemotron_h.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class NemotronHPreTrainedModel extends PreTrainedModel {} +export class NemotronHModel extends NemotronHPreTrainedModel {} +export class NemotronHForCausalLM extends NemotronHPreTrainedModel {} diff --git a/packages/transformers/src/models/neobert/modeling_neobert.js b/packages/transformers/src/models/neobert/modeling_neobert.js new file mode 100644 index 0000000..7d58e02 --- /dev/null +++ b/packages/transformers/src/models/neobert/modeling_neobert.js @@ -0,0 +1,58 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { + MaskedLMOutput, + SequenceClassifierOutput, + TokenClassifierOutput, + QuestionAnsweringModelOutput, +} from '../modeling_outputs.js'; + +export class NeoBertPreTrainedModel extends PreTrainedModel {} +export class NeoBertModel extends NeoBertPreTrainedModel {} + +export class NeoBertForMaskedLM extends NeoBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for masked language modeling. + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +export class NeoBertForSequenceClassification extends NeoBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +export class NeoBertForTokenClassification extends NeoBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} + +export class NeoBertForQuestionAnswering extends NeoBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for question answering. + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/nllb/tokenization_nllb.js b/packages/transformers/src/models/nllb/tokenization_nllb.js new file mode 100644 index 0000000..37b831d --- /dev/null +++ b/packages/transformers/src/models/nllb/tokenization_nllb.js @@ -0,0 +1,35 @@ +import { PreTrainedTokenizer, _build_translation_inputs } from '../../tokenization_utils.js'; + +/** + * The NllbTokenizer class is used to tokenize text for NLLB ("No Language Left Behind") models. + * + * No Language Left Behind (NLLB) is a first-of-its-kind, AI breakthrough project + * that open-sources models capable of delivering high-quality translations directly + * between any pair of 200+ languages — including low-resource languages like Asturian, + * Luganda, Urdu and more. It aims to help people communicate with anyone, anywhere, + * regardless of their language preferences. For more information, check out their + * [paper](https://huggingface.co/papers/2207.04672). + * + * For a list of supported languages (along with their language codes), + * @see {@link https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200} + */ +export class NllbTokenizer extends PreTrainedTokenizer { + constructor(tokenizerJSON, tokenizerConfig) { + super(tokenizerJSON, tokenizerConfig); + + this.languageRegex = /^[a-z]{3}_[A-Z][a-z]{3}$/; + this.language_codes = this.all_special_tokens.filter((x) => this.languageRegex.test(x)); + this.lang_to_token = (x) => x; // Identity function + } + + /** + * Helper function to build translation inputs for an `NllbTokenizer`. + * @param {string|string[]} raw_inputs The text to tokenize. + * @param {Object} tokenizer_options Options to be sent to the tokenizer + * @param {Object} generate_kwargs Generation options. + * @returns {Object} Object to be passed to the model. + */ + _build_translation_inputs(raw_inputs, tokenizer_options, generate_kwargs) { + return _build_translation_inputs(this, raw_inputs, tokenizer_options, generate_kwargs); + } +} diff --git a/packages/transformers/src/models/nomic_bert/modeling_nomic_bert.js b/packages/transformers/src/models/nomic_bert/modeling_nomic_bert.js new file mode 100644 index 0000000..7e50438 --- /dev/null +++ b/packages/transformers/src/models/nomic_bert/modeling_nomic_bert.js @@ -0,0 +1,4 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class NomicBertPreTrainedModel extends PreTrainedModel {} +export class NomicBertModel extends NomicBertPreTrainedModel {} diff --git a/packages/transformers/src/models/nougat/image_processing_nougat.js b/packages/transformers/src/models/nougat/image_processing_nougat.js new file mode 100644 index 0000000..2b20785 --- /dev/null +++ b/packages/transformers/src/models/nougat/image_processing_nougat.js @@ -0,0 +1,4 @@ +import { DonutImageProcessor } from '../donut/image_processing_donut.js'; + +// NOTE: extends DonutImageProcessor +export class NougatImageProcessor extends DonutImageProcessor {} diff --git a/packages/transformers/src/models/nougat/tokenization_nougat.js b/packages/transformers/src/models/nougat/tokenization_nougat.js new file mode 100644 index 0000000..68fa3d4 --- /dev/null +++ b/packages/transformers/src/models/nougat/tokenization_nougat.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class NougatTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/olmo/modeling_olmo.js b/packages/transformers/src/models/olmo/modeling_olmo.js new file mode 100644 index 0000000..e3ceac7 --- /dev/null +++ b/packages/transformers/src/models/olmo/modeling_olmo.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class OlmoPreTrainedModel extends PreTrainedModel {} +export class OlmoModel extends OlmoPreTrainedModel {} +export class OlmoForCausalLM extends OlmoPreTrainedModel {} diff --git a/packages/transformers/src/models/olmo2/modeling_olmo2.js b/packages/transformers/src/models/olmo2/modeling_olmo2.js new file mode 100644 index 0000000..205f0ab --- /dev/null +++ b/packages/transformers/src/models/olmo2/modeling_olmo2.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Olmo2PreTrainedModel extends PreTrainedModel {} +export class Olmo2Model extends Olmo2PreTrainedModel {} +export class Olmo2ForCausalLM extends Olmo2PreTrainedModel {} diff --git a/packages/transformers/src/models/olmo3/modeling_olmo3.js b/packages/transformers/src/models/olmo3/modeling_olmo3.js new file mode 100644 index 0000000..ed34f3a --- /dev/null +++ b/packages/transformers/src/models/olmo3/modeling_olmo3.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Olmo3PreTrainedModel extends PreTrainedModel {} +export class Olmo3Model extends Olmo3PreTrainedModel {} +export class Olmo3ForCausalLM extends Olmo3PreTrainedModel {} diff --git a/packages/transformers/src/models/olmo_hybrid/modeling_olmo_hybrid.js b/packages/transformers/src/models/olmo_hybrid/modeling_olmo_hybrid.js new file mode 100644 index 0000000..5d9961f --- /dev/null +++ b/packages/transformers/src/models/olmo_hybrid/modeling_olmo_hybrid.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class OlmoHybridPreTrainedModel extends PreTrainedModel {} +export class OlmoHybridModel extends OlmoHybridPreTrainedModel {} +export class OlmoHybridForCausalLM extends OlmoHybridPreTrainedModel {} diff --git a/packages/transformers/src/models/openai_privacy_filter/modeling_openai_privacy_filter.js b/packages/transformers/src/models/openai_privacy_filter/modeling_openai_privacy_filter.js new file mode 100644 index 0000000..4c17d73 --- /dev/null +++ b/packages/transformers/src/models/openai_privacy_filter/modeling_openai_privacy_filter.js @@ -0,0 +1,17 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class OpenAIPrivacyFilterPreTrainedModel extends PreTrainedModel {} +export class OpenAIPrivacyFilterModel extends OpenAIPrivacyFilterPreTrainedModel {} + +export class OpenAIPrivacyFilterForTokenClassification extends OpenAIPrivacyFilterPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/openelm/modeling_openelm.js b/packages/transformers/src/models/openelm/modeling_openelm.js new file mode 100644 index 0000000..0765116 --- /dev/null +++ b/packages/transformers/src/models/openelm/modeling_openelm.js @@ -0,0 +1,6 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class OpenELMPreTrainedModel extends PreTrainedModel {} +export class OpenELMModel extends OpenELMPreTrainedModel {} + +export class OpenELMForCausalLM extends OpenELMPreTrainedModel {} diff --git a/packages/transformers/src/models/opt/modeling_opt.js b/packages/transformers/src/models/opt/modeling_opt.js new file mode 100644 index 0000000..b7278e2 --- /dev/null +++ b/packages/transformers/src/models/opt/modeling_opt.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class OPTPreTrainedModel extends PreTrainedModel {} + +/** + * The bare OPT Model outputting raw hidden-states without any specific head on top. + */ +export class OPTModel extends OPTPreTrainedModel {} + +/** + * The OPT Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). + */ +export class OPTForCausalLM extends OPTPreTrainedModel {} diff --git a/packages/transformers/src/models/owlv2/image_processing_owlv2.js b/packages/transformers/src/models/owlv2/image_processing_owlv2.js new file mode 100644 index 0000000..9523cc1 --- /dev/null +++ b/packages/transformers/src/models/owlv2/image_processing_owlv2.js @@ -0,0 +1,4 @@ +import { OwlViTImageProcessor } from '../owlvit/image_processing_owlvit.js'; + +// NOTE: extends OwlViTImageProcessor +export class Owlv2ImageProcessor extends OwlViTImageProcessor {} diff --git a/packages/transformers/src/models/owlv2/modeling_owlv2.js b/packages/transformers/src/models/owlv2/modeling_owlv2.js new file mode 100644 index 0000000..5239835 --- /dev/null +++ b/packages/transformers/src/models/owlv2/modeling_owlv2.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Owlv2PreTrainedModel extends PreTrainedModel {} +export class Owlv2Model extends Owlv2PreTrainedModel {} +export class Owlv2ForObjectDetection extends Owlv2PreTrainedModel {} diff --git a/packages/transformers/src/models/owlvit/image_processing_owlvit.js b/packages/transformers/src/models/owlvit/image_processing_owlvit.js new file mode 100644 index 0000000..0a12636 --- /dev/null +++ b/packages/transformers/src/models/owlvit/image_processing_owlvit.js @@ -0,0 +1,9 @@ +import { ImageProcessor, post_process_object_detection } from '../../image_processors_utils.js'; + +export class OwlViTImageProcessor extends ImageProcessor { + /** @type {typeof post_process_object_detection} */ + post_process_object_detection(...args) { + return post_process_object_detection(...args); + } +} +export class OwlViTFeatureExtractor extends OwlViTImageProcessor {} diff --git a/packages/transformers/src/models/owlvit/modeling_owlvit.js b/packages/transformers/src/models/owlvit/modeling_owlvit.js new file mode 100644 index 0000000..d025358 --- /dev/null +++ b/packages/transformers/src/models/owlvit/modeling_owlvit.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class OwlViTPreTrainedModel extends PreTrainedModel {} +export class OwlViTModel extends OwlViTPreTrainedModel {} +export class OwlViTForObjectDetection extends OwlViTPreTrainedModel {} diff --git a/packages/transformers/src/models/owlvit/processing_owlvit.js b/packages/transformers/src/models/owlvit/processing_owlvit.js new file mode 100644 index 0000000..7e2fa88 --- /dev/null +++ b/packages/transformers/src/models/owlvit/processing_owlvit.js @@ -0,0 +1,7 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +export class OwlViTProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static image_processor_class = AutoImageProcessor; +} diff --git a/packages/transformers/src/models/paligemma/modeling_paligemma.js b/packages/transformers/src/models/paligemma/modeling_paligemma.js new file mode 100644 index 0000000..fff4de6 --- /dev/null +++ b/packages/transformers/src/models/paligemma/modeling_paligemma.js @@ -0,0 +1,3 @@ +import { LlavaForConditionalGeneration } from '../llava/modeling_llava.js'; + +export class PaliGemmaForConditionalGeneration extends LlavaForConditionalGeneration {} diff --git a/packages/transformers/src/models/paligemma/processing_paligemma.js b/packages/transformers/src/models/paligemma/processing_paligemma.js new file mode 100644 index 0000000..bf85ab3 --- /dev/null +++ b/packages/transformers/src/models/paligemma/processing_paligemma.js @@ -0,0 +1,70 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { logger } from '../../utils/logger.js'; + +const IMAGE_TOKEN = ''; + +function build_string_from_input(prompt, bos_token, image_seq_len, image_token, num_images) { + return `${image_token.repeat(image_seq_len * num_images)}${bos_token}${prompt}\n`; +} + +export class PaliGemmaProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static image_processor_class = AutoImageProcessor; + static uses_processor_config = false; + + /** + * @typedef {import('../../utils/image.js').RawImage} RawImage + */ + + // `images` is required, `text` is optional + async _call(/** @type {RawImage|RawImage[]} */ images, text = null, kwargs = {}) { + if (!text) { + logger.warn( + 'You are using PaliGemma without a text prefix. It will perform as a picture-captioning model.', + ); + text = ''; + } + + if (!Array.isArray(images)) { + images = [images]; + } + + if (!Array.isArray(text)) { + text = [text]; + } + + const bos_token = this.tokenizer.bos_token; + // @ts-expect-error TS2339 + const image_seq_length = this.image_processor.config.image_seq_length; + let input_strings; + if (text.some((t) => t.includes(IMAGE_TOKEN))) { + input_strings = text.map((sample) => { + const expanded_sample = sample.replaceAll(IMAGE_TOKEN, IMAGE_TOKEN.repeat(image_seq_length)); + const bos_rfind_index = expanded_sample.lastIndexOf(IMAGE_TOKEN); + const bos_index = bos_rfind_index === -1 ? 0 : bos_rfind_index + IMAGE_TOKEN.length; + return expanded_sample.slice(0, bos_index) + bos_token + expanded_sample.slice(bos_index) + '\n'; + }); + } else { + logger.warn( + 'You are passing both `text` and `images` to `PaliGemmaProcessor`. The processor expects special ' + + 'image tokens in the text, as many tokens as there are images per each text. It is recommended to ' + + 'add `` tokens in the very beginning of your text. For this call, we will infer how many images ' + + 'each text has and add special tokens.', + ); + + input_strings = text.map((sample) => + build_string_from_input(sample, bos_token, image_seq_length, IMAGE_TOKEN, images.length), + ); + } + + const text_inputs = this.tokenizer(input_strings, kwargs); + const image_inputs = await this.image_processor(images, kwargs); + + return { + ...image_inputs, + ...text_inputs, + }; + } +} diff --git a/packages/transformers/src/models/parakeet/feature_extraction_parakeet.js b/packages/transformers/src/models/parakeet/feature_extraction_parakeet.js new file mode 100644 index 0000000..2b971ed --- /dev/null +++ b/packages/transformers/src/models/parakeet/feature_extraction_parakeet.js @@ -0,0 +1,120 @@ +import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { Tensor } from '../../utils/tensor.js'; +import { mel_filter_bank, spectrogram, window_function } from '../../utils/audio.js'; + +const EPSILON = 1e-5; + +export class ParakeetFeatureExtractor extends FeatureExtractor { + constructor(config) { + super(config); + + // Prefer given `mel_filters` from preprocessor_config.json, or calculate them if they don't exist. + this.config.mel_filters ??= mel_filter_bank( + Math.floor(1 + this.config.n_fft / 2), // num_frequency_bins + this.config.feature_size, // num_mel_filters + 0.0, // min_frequency + this.config.sampling_rate / 2, // max_frequency + this.config.sampling_rate, // sampling_rate + 'slaney', // norm + 'slaney', // mel_scale + ); + + const window = window_function(this.config.win_length, 'hann', { + periodic: false, + }); + + this.window = new Float64Array(this.config.n_fft); + const offset = Math.floor((this.config.n_fft - this.config.win_length) / 2); + this.window.set(window, offset); + } + + /** + * Computes the log-Mel spectrogram of the provided audio waveform. + * @param {Float32Array|Float64Array} waveform The audio waveform to process. + * @returns {Promise} An object containing the log-Mel spectrogram data as a Float32Array and its dimensions as an array of numbers. + */ + async _extract_fbank_features(waveform) { + // Parakeet uses a custom preemphasis strategy: Apply preemphasis to entire waveform at once + const preemphasis = this.config.preemphasis; + waveform = new Float64Array(waveform); // Clone to avoid destructive changes + for (let j = waveform.length - 1; j >= 1; --j) { + waveform[j] -= preemphasis * waveform[j - 1]; + } + + const features = await spectrogram( + waveform, + this.window, // window + this.window.length, // frame_length + this.config.hop_length, // hop_length + { + fft_length: this.config.n_fft, + power: 2.0, + mel_filters: this.config.mel_filters, + log_mel: 'log', + mel_floor: -Infinity, + pad_mode: 'constant', + center: true, + + // Custom + transpose: true, + mel_offset: 2 ** -24, + }, + ); + + return features; + } + + /** + * Asynchronously extracts features from a given audio using the provided configuration. + * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array. + * @returns {Promise<{ input_features: Tensor; attention_mask: Tensor; }>} A Promise resolving to an object containing the extracted input features as a Tensor. + */ + async _call(audio) { + validate_audio_inputs(audio, 'ParakeetFeatureExtractor'); + + const features = await this._extract_fbank_features(audio); + + const features_length = Math.floor( + (audio.length + Math.floor(this.config.n_fft / 2) * 2 - this.config.n_fft) / this.config.hop_length, + ); + + const features_data = /** @type {Float32Array} */ (features.data); + features_data.fill(0, features_length * features.dims[1]); + + // normalize mel features, ignoring padding + const [num_frames, num_features] = features.dims; + const sum = new Float64Array(num_features); + const sum_sq = new Float64Array(num_features); + + for (let i = 0; i < features_length; ++i) { + const offset = i * num_features; + for (let j = 0; j < num_features; ++j) { + const val = features_data[offset + j]; + sum[j] += val; + sum_sq[j] += val * val; + } + } + + // Calculate mean and standard deviation, then normalize + const divisor = features_length > 1 ? features_length - 1 : 1; + for (let j = 0; j < num_features; ++j) { + const mean = sum[j] / features_length; + const variance = (sum_sq[j] - features_length * mean * mean) / divisor; + const std = Math.sqrt(variance) + EPSILON; + const inv_std = 1 / std; + + for (let i = 0; i < features_length; ++i) { + const index = i * num_features + j; + features_data[index] = (features_data[index] - mean) * inv_std; + } + } + + const mask_data = new BigInt64Array(num_frames); + mask_data.fill(1n, 0, features_length); + + return { + input_features: features.unsqueeze_(0), + attention_mask: new Tensor('int64', mask_data, [1, num_frames]), + }; + } +} diff --git a/packages/transformers/src/models/parakeet/modeling_parakeet.js b/packages/transformers/src/models/parakeet/modeling_parakeet.js new file mode 100644 index 0000000..784846e --- /dev/null +++ b/packages/transformers/src/models/parakeet/modeling_parakeet.js @@ -0,0 +1,15 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { CausalLMOutput } from '../modeling_outputs.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class ParakeetPreTrainedModel extends PreTrainedModel {} +export class ParakeetForCTC extends ParakeetPreTrainedModel { + /** + * @param {Object} model_inputs + * @param {Tensor} model_inputs.input_values Float values of input raw speech waveform. + * @param {Tensor} model_inputs.attention_mask Mask to avoid performing convolution and attention on padding token indices. Mask values selected in [0, 1] + */ + async _call(model_inputs) { + return new CausalLMOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/patchtsmixer/modeling_patchtsmixer.js b/packages/transformers/src/models/patchtsmixer/modeling_patchtsmixer.js new file mode 100644 index 0000000..307e5b8 --- /dev/null +++ b/packages/transformers/src/models/patchtsmixer/modeling_patchtsmixer.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class PatchTSMixerPreTrainedModel extends PreTrainedModel {} + +/** + * The bare PatchTSMixer Model outputting raw hidden-states without any specific head. + */ +export class PatchTSMixerModel extends PatchTSMixerPreTrainedModel {} + +/** + * The PatchTSMixer for prediction model. + */ +export class PatchTSMixerForPrediction extends PatchTSMixerPreTrainedModel {} diff --git a/packages/transformers/src/models/patchtst/modeling_patchtst.js b/packages/transformers/src/models/patchtst/modeling_patchtst.js new file mode 100644 index 0000000..361f7a7 --- /dev/null +++ b/packages/transformers/src/models/patchtst/modeling_patchtst.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class PatchTSTPreTrainedModel extends PreTrainedModel {} + +/** + * The bare PatchTST Model outputting raw hidden-states without any specific head. + */ +export class PatchTSTModel extends PatchTSTPreTrainedModel {} + +/** + * The PatchTST for prediction model. + */ +export class PatchTSTForPrediction extends PatchTSTPreTrainedModel {} diff --git a/packages/transformers/src/models/phi/modeling_phi.js b/packages/transformers/src/models/phi/modeling_phi.js new file mode 100644 index 0000000..9624613 --- /dev/null +++ b/packages/transformers/src/models/phi/modeling_phi.js @@ -0,0 +1,10 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class PhiPreTrainedModel extends PreTrainedModel {} + +/** + * The bare Phi Model outputting raw hidden-states without any specific head on top. + */ +export class PhiModel extends PhiPreTrainedModel {} + +export class PhiForCausalLM extends PhiPreTrainedModel {} diff --git a/packages/transformers/src/models/phi3/modeling_phi3.js b/packages/transformers/src/models/phi3/modeling_phi3.js new file mode 100644 index 0000000..70213d2 --- /dev/null +++ b/packages/transformers/src/models/phi3/modeling_phi3.js @@ -0,0 +1,10 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Phi3PreTrainedModel extends PreTrainedModel {} + +/** + * The bare Phi3 Model outputting raw hidden-states without any specific head on top. + */ +export class Phi3Model extends Phi3PreTrainedModel {} + +export class Phi3ForCausalLM extends Phi3PreTrainedModel {} diff --git a/packages/transformers/src/models/phi3_v/image_processing_phi3_v.js b/packages/transformers/src/models/phi3_v/image_processing_phi3_v.js new file mode 100644 index 0000000..f8b5732 --- /dev/null +++ b/packages/transformers/src/models/phi3_v/image_processing_phi3_v.js @@ -0,0 +1,165 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; +import { cat, interpolate_4d, slice, stack, Tensor } from '../../utils/tensor.js'; + +const IMAGE_SIZE = 336; +const SLICE_AXES = [2, 3]; // axes to slice on +const { ceil, floor, sqrt } = Math; + +export class Phi3VImageProcessor extends ImageProcessor { + constructor(config) { + super({ + ...config, + do_normalize: true, + do_pad: true, + pad_size: 'custom', + do_convert_rgb: true, + do_resize: true, // Smart resizing "hd_transform" + }); + + this._num_crops = config.num_crops; + } + calc_num_image_tokens_from_image_size(width, height) { + // @ts-expect-error + const { num_img_tokens } = this.config; + return floor( + (floor(height / IMAGE_SIZE) * floor(width / IMAGE_SIZE) + 1) * num_img_tokens + + 1 + + (floor(height / IMAGE_SIZE) + 1) * sqrt(num_img_tokens), + ); + } + + /** @type {ImageProcessor['get_resize_output_image_size']} */ + get_resize_output_image_size(image, size) { + const hd_num = this._num_crops; + const [width, height] = image.size; + + let ratio = width / height; + let scale = 1; + + // Calculate the scaling factor + while (scale * Math.ceil(scale / ratio) <= hd_num) { + scale += 1; + } + scale -= 1; + + // Compute the new dimensions + const new_w = Math.floor(scale * 336); + const new_h = Math.floor(new_w / ratio); + + return [new_w, new_h]; + } + + /** @type {ImageProcessor['pad_image']} */ + pad_image(pixelData, imgDims, padSize, options = {}) { + // Phi3V uses a custom padding strategy: + // - Pad to a multiple of 336 + // - Pad with white pixels + const [imageHeight, imageWidth] = imgDims; + const height = IMAGE_SIZE * ceil(imageHeight / IMAGE_SIZE); + const width = IMAGE_SIZE * ceil(imageWidth / IMAGE_SIZE); + + // NOTE: Since padding is done after normalization, we need to fill with the normalized values + const constant_values = [1, 1, 1].map((x, i) => (x - this.image_mean[i]) / this.image_std[i]); + return super.pad_image( + pixelData, + imgDims, + { width, height }, + { + center: true, + constant_values, + ...options, + }, + ); + } + + async _call(images, { num_crops = null } = {}) { + // @ts-expect-error + this._num_crops = num_crops ??= this.config.num_crops; + if (num_crops < 4 || sqrt(num_crops) % 1 !== 0) { + throw new Error('num_crops must be a square number >= 4'); + } + + if (!Array.isArray(images)) { + images = [images]; + } + + const num_images = images.length; + const imageData = await Promise.all(images.map((x) => this.preprocess(x))); + + const original_sizes = imageData.map((x) => x.original_size); + const reshaped_input_sizes = imageData.map((x) => x.reshaped_input_size); + + // Process each image in batch + const all_pixel_values = []; + for (const { pixel_values } of imageData) { + pixel_values.unsqueeze_(0); // Easier processing as 4D tensor + + const [height, width] = pixel_values.dims.slice(-2); + + // Global image (Tensor of shape [num_channels, height, width]) + const batch_pixel_values = await interpolate_4d(pixel_values, { + size: [IMAGE_SIZE, IMAGE_SIZE], + mode: 'bicubic', + }); + + if (num_crops > 0) { + const patches = []; + const sqrt_patches = sqrt(num_crops); + const patch_width = floor(width / sqrt_patches); + const patch_height = floor(height / sqrt_patches); + for (let y = 0; y < sqrt_patches; ++y) { + for (let x = 0; x < sqrt_patches; ++x) { + let start_x, start_y, end_x, end_y; + if (y === sqrt_patches - 1) { + // At bottom + start_y = height - patch_height; + end_y = height; + } else { + start_y = y * patch_height; + end_y = (y + 1) * patch_height; + } + if (x === sqrt_patches - 1) { + // At right + start_x = width - patch_width; + end_x = width; + } else { + start_x = x * patch_width; + end_x = (x + 1) * patch_width; + } + + const starts = [start_y, start_x]; + const ends = [end_y, end_x]; + const patch = await slice(pixel_values, starts, ends, SLICE_AXES); + patches.push(patch); + } + } + + const resized_tensors = await interpolate_4d(cat(patches, 0), { + size: [IMAGE_SIZE, IMAGE_SIZE], + mode: 'bicubic', + }); // [num_crops, 3, 336, 336] + + // Concatenate the global image with the patches + all_pixel_values.push(cat([batch_pixel_values, resized_tensors], 0)); + } else { + // Only use the global image + // NOTE: Not currently supported in modelling code + all_pixel_values.push(batch_pixel_values); + } + } + + // [num_images, 1 + num_crops, num_channels=3, height, width] + const pixel_values = stack(all_pixel_values, 0); + + // Calculate padded image sizes + const sizes = reshaped_input_sizes.map((x) => x.map((y) => IMAGE_SIZE * ceil(y / IMAGE_SIZE))); + + const image_sizes = new Tensor('int64', sizes.flat(), [num_images, 2]); + + const num_img_tokens = sizes.map(([height, width]) => + this.calc_num_image_tokens_from_image_size(width, height), + ); + + return { pixel_values, original_sizes, reshaped_input_sizes, image_sizes, num_img_tokens }; + } +} diff --git a/packages/transformers/src/models/phi3_v/modeling_phi3_v.js b/packages/transformers/src/models/phi3_v/modeling_phi3_v.js new file mode 100644 index 0000000..1637918 --- /dev/null +++ b/packages/transformers/src/models/phi3_v/modeling_phi3_v.js @@ -0,0 +1,75 @@ +import { PreTrainedModel, decoder_forward } from '../modeling_utils.js'; +import { sessionRun } from '../session.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class Phi3VPreTrainedModel extends PreTrainedModel { + forward_params = [ + 'input_ids', + 'inputs_embeds', + 'attention_mask', + 'position_ids', + 'pixel_values', + 'image_sizes', + 'past_key_values', + ]; +} +export class Phi3VForCausalLM extends Phi3VPreTrainedModel { + async forward({ + // Produced by the tokenizer/processor: + input_ids = null, + attention_mask = null, + pixel_values = null, + image_sizes = null, + + // Used during generation: + position_ids = null, + inputs_embeds = null, + past_key_values = null, + + // Generic generation parameters + generation_config = null, + logits_processor = null, + num_logits_to_keep = null, + + // TODO: needed? + ...kwargs + }) { + if (!inputs_embeds) { + let image_features; + if (pixel_values && input_ids.dims[1] !== 1) { + if (!image_sizes) { + throw new Error('`image_sizes` must be provided when `pixel_values` is provided.'); + } + + // Encode the image + ({ image_features } = await sessionRun(this.sessions['vision_encoder'], { + pixel_values, + image_sizes, + })); + } else { + const hidden_size = this.config.normalized_config.hidden_size; + image_features = new Tensor('float32', [], [0, hidden_size]); + } + + ({ inputs_embeds } = await sessionRun(this.sessions['prepare_inputs_embeds'], { + input_ids, + image_features, + })); + } + + const outputs = await decoder_forward( + this, + { + inputs_embeds, + past_key_values, + attention_mask, + position_ids, + generation_config, + logits_processor, + num_logits_to_keep, + }, + false, + ); + return outputs; + } +} diff --git a/packages/transformers/src/models/phi3_v/processing_phi3_v.js b/packages/transformers/src/models/phi3_v/processing_phi3_v.js new file mode 100644 index 0000000..eb0df7d --- /dev/null +++ b/packages/transformers/src/models/phi3_v/processing_phi3_v.js @@ -0,0 +1,50 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { RawImage } from '../../utils/image.js'; + +const IMAGE_TOKEN = '<|image|>'; +const IMAGE_TOKEN_PATTERN = /<\|image_\d+\|>/g; + +export class Phi3VProcessor extends Processor { + static image_processor_class = AutoImageProcessor; + static tokenizer_class = AutoTokenizer; + + /** + * + * @param {string|string[]} text + * @param {RawImage|RawImage[]} images + * @param { { padding?: boolean, truncation?: boolean, num_crops?: number } | undefined } options + * @returns {Promise} + */ + async _call(text, images = null, { padding = true, truncation = true, num_crops = null } = {}) { + if (!Array.isArray(text)) { + text = [text]; + } + + let text_inputs, image_inputs; + if (images) { + image_inputs = await this.image_processor(images, { num_crops }); + const { num_img_tokens } = image_inputs; + + // The original implementation adds a bos_token before the image tokens + // TODO: Check if this affects performance, since it looks like a bug in the original implementation + const prompt_chunks = text.map((t, i) => + t.split(IMAGE_TOKEN_PATTERN).join(IMAGE_TOKEN.repeat(num_img_tokens[i])), + ); + + text_inputs = this.tokenizer(prompt_chunks, { padding, truncation }); + + // The model expects image tokens to be negative, so we negate the image token ids + const image_token_id = this.tokenizer._tokenizer.token_to_id(IMAGE_TOKEN); + text_inputs.input_ids.map_((id) => (id == image_token_id ? -id : id)); + } else { + text_inputs = this.tokenizer(text); + } + + return { + ...text_inputs, + ...image_inputs, + }; + } +} diff --git a/packages/transformers/src/models/pixtral/image_processing_pixtral.js b/packages/transformers/src/models/pixtral/image_processing_pixtral.js new file mode 100644 index 0000000..d18d5ec --- /dev/null +++ b/packages/transformers/src/models/pixtral/image_processing_pixtral.js @@ -0,0 +1,35 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class PixtralImageProcessor extends ImageProcessor { + /** @type {ImageProcessor['get_resize_output_image_size']} */ + get_resize_output_image_size(image, size) { + const { longest_edge } = size; + if (longest_edge === undefined) { + throw new Error("size must contain 'longest_edge'"); + } + + const [srcWidth, srcHeight] = image.size; + + const ratio = Math.max(srcWidth, srcHeight) / longest_edge; + + let newWidth = srcWidth; + let newHeight = srcHeight; + if (ratio > 1) { + newWidth = Math.floor(srcWidth / ratio); + newHeight = Math.floor(srcHeight / ratio); + } + + // @ts-expect-error TS2339 + const { patch_size, spatial_merge_size } = this.config; + if (!spatial_merge_size) { + throw new Error("config must contain 'spatial_merge_size'"); + } + const real_patch_size = patch_size * spatial_merge_size; + + // Calculate number of tokens + const num_width_tokens = Math.floor((newWidth - 1) / real_patch_size) + 1; + const num_height_tokens = Math.floor((newHeight - 1) / real_patch_size) + 1; + + return [num_width_tokens * real_patch_size, num_height_tokens * real_patch_size]; + } +} diff --git a/packages/transformers/src/models/pixtral/processing_pixtral.js b/packages/transformers/src/models/pixtral/processing_pixtral.js new file mode 100644 index 0000000..6ec75bd --- /dev/null +++ b/packages/transformers/src/models/pixtral/processing_pixtral.js @@ -0,0 +1,46 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; + +export class PixtralProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static image_processor_class = AutoImageProcessor; + static uses_processor_config = true; + + /** + * @typedef {import('../../utils/image.js').RawImage} RawImage + */ + + // `images` is required, `text` is optional + async _call(/** @type {RawImage|RawImage[]} */ images, text = null, kwargs = {}) { + const image_inputs = await this.image_processor(images, kwargs); + + if (text) { + const [height, width] = image_inputs.pixel_values.dims.slice(-2); + + const { image_token, image_break_token, image_end_token, patch_size, spatial_merge_size } = this.config; + const real_patch_size = patch_size * spatial_merge_size; + const num_height_tokens = Math.floor(height / real_patch_size); + const num_width_tokens = Math.floor(width / real_patch_size); + + text = structuredClone(text); // Avoid modifying the original text input + if (!Array.isArray(text)) { + text = [text]; + } + for (let i = 0; i < text.length; ++i) { + const width_tokens = image_token.repeat(num_width_tokens); + const row = width_tokens + image_break_token; + const finalRow = width_tokens + image_end_token; + const full = row.repeat(num_height_tokens - 1) + finalRow; + text[i] = text[i].replace(image_token, full); + } + } + + const text_inputs = text ? this.tokenizer(text, kwargs) : {}; + + return { + ...image_inputs, + ...text_inputs, + }; + } +} diff --git a/packages/transformers/src/models/processors.js b/packages/transformers/src/models/processors.js new file mode 100644 index 0000000..79ae8da --- /dev/null +++ b/packages/transformers/src/models/processors.js @@ -0,0 +1,36 @@ +export * from './chatterbox/processing_chatterbox.js'; +export * from './cohere_asr/processing_cohere_asr.js'; +export * from './florence2/processing_florence2.js'; +export * from './gemma3/processing_gemma3.js'; +export * from './gemma3n/processing_gemma3n.js'; +export * from './gemma4/processing_gemma4.js'; +export * from './glm46v/processing_glm46v.js'; +export * from './granite_speech/processing_granite_speech.js'; +export * from './grounding_dino/processing_grounding_dino.js'; +export * from './idefics3/processing_idefics3.js'; +export * from './janus/processing_janus.js'; +export * from './jina_clip/processing_jina_clip.js'; +export * from './lfm2_vl/processing_lfm2_vl.js'; +export * from './llava/processing_llava.js'; +export * from './mgp_str/processing_mgp_str.js'; +export * from './moonshine/processing_moonshine.js'; +export * from './owlvit/processing_owlvit.js'; +export * from './paligemma/processing_paligemma.js'; +export * from './phi3_v/processing_phi3_v.js'; +export * from './pixtral/processing_pixtral.js'; +export * from './pyannote/processing_pyannote.js'; +export * from './qwen2_vl/processing_qwen2_vl.js'; +export * from './qwen2_5_vl/processing_qwen2_5_vl.js'; +export * from './qwen3_vl/processing_qwen3_vl.js'; +export * from './sam/processing_sam.js'; +export * from './sam2/processing_sam2.js'; +export * from './smolvlm/processing_smolvlm.js'; +export * from './speecht5/processing_speecht5.js'; +export * from './ultravox/processing_ultravox.js'; +export * from './voxtral/processing_voxtral.js'; +export * from './voxtral_realtime/processing_voxtral_realtime.js'; +export * from './wav2vec2/processing_wav2vec2.js'; +export * from './wav2vec2_with_lm/processing_wav2vec2_with_lm.js'; +export * from './whisper/processing_whisper.js'; + +export { Processor } from '../processing_utils.js'; diff --git a/packages/transformers/src/models/pvt/image_processing_pvt.js b/packages/transformers/src/models/pvt/image_processing_pvt.js new file mode 100644 index 0000000..8167055 --- /dev/null +++ b/packages/transformers/src/models/pvt/image_processing_pvt.js @@ -0,0 +1,3 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class PvtImageProcessor extends ImageProcessor {} diff --git a/packages/transformers/src/models/pvt/modeling_pvt.js b/packages/transformers/src/models/pvt/modeling_pvt.js new file mode 100644 index 0000000..0450de8 --- /dev/null +++ b/packages/transformers/src/models/pvt/modeling_pvt.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class PvtPreTrainedModel extends PreTrainedModel {} +export class PvtModel extends PvtPreTrainedModel {} +export class PvtForImageClassification extends PvtPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/pyannote/feature_extraction_pyannote.js b/packages/transformers/src/models/pyannote/feature_extraction_pyannote.js new file mode 100644 index 0000000..df419cb --- /dev/null +++ b/packages/transformers/src/models/pyannote/feature_extraction_pyannote.js @@ -0,0 +1,79 @@ +import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { Tensor } from '../../utils/tensor.js'; +import { max, softmax } from '../../utils/maths.js'; + +export class PyAnnoteFeatureExtractor extends FeatureExtractor { + /** + * Asynchronously extracts features from a given audio using the provided configuration. + * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array. + * @returns {Promise<{ input_values: Tensor; }>} The extracted input features. + */ + async _call(audio) { + validate_audio_inputs(audio, 'PyAnnoteFeatureExtractor'); + + if (audio instanceof Float64Array) { + audio = new Float32Array(audio); + } + + const shape = [1 /* batch_size */, 1 /* num_channels */, audio.length /* num_samples */]; + return { + input_values: new Tensor('float32', audio, shape), + }; + } + + /** + * NOTE: Can return fractional values. `Math.ceil` will ensure correct value. + * @param {number} samples The number of frames in the audio. + * @returns {number} The number of frames in the audio. + */ + samples_to_frames(samples) { + return (samples - this.config.offset) / this.config.step; + } + + /** + * Post-processes the speaker diarization logits output by the model. + * @param {import('../../utils/tensor.js').Tensor} logits The speaker diarization logits output by the model. + * @param {number} num_samples Number of samples in the input audio. + * @returns {Array>} The post-processed speaker diarization results. + */ + post_process_speaker_diarization(logits, num_samples) { + const ratio = num_samples / this.samples_to_frames(num_samples) / this.config.sampling_rate; + + const results = []; + for (const scores of logits.tolist()) { + const accumulated_segments = []; + + let current_speaker = -1; + for (let i = 0; i < scores.length; ++i) { + /** @type {number[]} */ + const probabilities = softmax(scores[i]); + const [score, id] = max(probabilities); + const [start, end] = [i, i + 1]; + + if (id !== current_speaker) { + // Speaker has changed + current_speaker = id; + accumulated_segments.push({ id, start, end, score }); + } else { + // Continue the current segment + accumulated_segments.at(-1).end = end; + accumulated_segments.at(-1).score += score; + } + } + + results.push( + accumulated_segments.map( + // Convert frame-space to time-space + // and compute the confidence + ({ id, start, end, score }) => ({ + id, + start: start * ratio, + end: end * ratio, + confidence: score / (end - start), + }), + ), + ); + } + return results; + } +} diff --git a/packages/transformers/src/models/pyannote/modeling_pyannote.js b/packages/transformers/src/models/pyannote/modeling_pyannote.js new file mode 100644 index 0000000..fa5ecae --- /dev/null +++ b/packages/transformers/src/models/pyannote/modeling_pyannote.js @@ -0,0 +1,77 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { TokenClassifierOutput } from '../modeling_outputs.js'; + +export class PyAnnotePreTrainedModel extends PreTrainedModel {} + +/** + * The bare PyAnnote Model transformer outputting raw hidden-states without any specific head on top. + */ +export class PyAnnoteModel extends PyAnnotePreTrainedModel {} + +/** + * PyAnnote Model with a frame classification head on top for tasks like Speaker Diarization. + * + * **Example:** Load and run a `PyAnnoteForAudioFrameClassification` for speaker diarization. + * + * ```javascript + * import { AutoProcessor, AutoModelForAudioFrameClassification, read_audio } from '@huggingface/transformers'; + * + * // Load model and processor + * const model_id = 'onnx-community/pyannote-segmentation-3.0'; + * const model = await AutoModelForAudioFrameClassification.from_pretrained(model_id); + * const processor = await AutoProcessor.from_pretrained(model_id); + * + * // Read and preprocess audio + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/mlk.wav'; + * const audio = await read_audio(url, processor.feature_extractor.config.sampling_rate); + * const inputs = await processor(audio); + * + * // Run model with inputs + * const { logits } = await model(inputs); + * // { + * // logits: Tensor { + * // dims: [ 1, 767, 7 ], // [batch_size, num_frames, num_classes] + * // type: 'float32', + * // data: Float32Array(5369) [ ... ], + * // size: 5369 + * // } + * // } + * + * const result = processor.post_process_speaker_diarization(logits, audio.length); + * // [ + * // [ + * // { id: 0, start: 0, end: 1.0512535626298245, confidence: 0.8220156481664611 }, + * // { id: 2, start: 1.0512535626298245, end: 2.3398869619825127, confidence: 0.9008811707860472 }, + * // ... + * // ] + * // ] + * + * // Display result + * console.table(result[0], ['start', 'end', 'id', 'confidence']); + * // ┌─────────┬────────────────────┬────────────────────┬────┬─────────────────────┐ + * // │ (index) │ start │ end │ id │ confidence │ + * // ├─────────┼────────────────────┼────────────────────┼────┼─────────────────────┤ + * // │ 0 │ 0 │ 1.0512535626298245 │ 0 │ 0.8220156481664611 │ + * // │ 1 │ 1.0512535626298245 │ 2.3398869619825127 │ 2 │ 0.9008811707860472 │ + * // │ 2 │ 2.3398869619825127 │ 3.5946089560890773 │ 0 │ 0.7521651315796233 │ + * // │ 3 │ 3.5946089560890773 │ 4.578039708226655 │ 2 │ 0.8491978128022479 │ + * // │ 4 │ 4.578039708226655 │ 4.594995410849717 │ 0 │ 0.2935352600416393 │ + * // │ 5 │ 4.594995410849717 │ 6.121008646925269 │ 3 │ 0.6788051309866024 │ + * // │ 6 │ 6.121008646925269 │ 6.256654267909762 │ 0 │ 0.37125512393851134 │ + * // │ 7 │ 6.256654267909762 │ 8.630452635138397 │ 2 │ 0.7467035186353542 │ + * // │ 8 │ 8.630452635138397 │ 10.088643060721703 │ 0 │ 0.7689364814666032 │ + * // │ 9 │ 10.088643060721703 │ 12.58113134631177 │ 2 │ 0.9123324509131324 │ + * // │ 10 │ 12.58113134631177 │ 13.005023911888312 │ 0 │ 0.4828358177572041 │ + * // └─────────┴────────────────────┴────────────────────┴────┴─────────────────────┘ + * ``` + */ +export class PyAnnoteForAudioFrameClassification extends PyAnnotePreTrainedModel { + /** + * Calls the model on new inputs. + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/pyannote/processing_pyannote.js b/packages/transformers/src/models/pyannote/processing_pyannote.js new file mode 100644 index 0000000..910c701 --- /dev/null +++ b/packages/transformers/src/models/pyannote/processing_pyannote.js @@ -0,0 +1,26 @@ +import { Processor } from '../../processing_utils.js'; +import { PyAnnoteFeatureExtractor } from './feature_extraction_pyannote.js'; + +export class PyAnnoteProcessor extends Processor { + static feature_extractor_class = PyAnnoteFeatureExtractor; + + /** + * Calls the feature_extractor function with the given audio input. + * @param {any} audio The audio input to extract features from. + * @returns {Promise} A Promise that resolves with the extracted features. + */ + async _call(audio) { + return await this.feature_extractor(audio); + } + + /** @type {PyAnnoteFeatureExtractor['post_process_speaker_diarization']} */ + post_process_speaker_diarization(...args) { + return /** @type {PyAnnoteFeatureExtractor} */ (this.feature_extractor).post_process_speaker_diarization( + ...args, + ); + } + + get sampling_rate() { + return this.feature_extractor.config.sampling_rate; + } +} diff --git a/packages/transformers/src/models/qwen2/modeling_qwen2.js b/packages/transformers/src/models/qwen2/modeling_qwen2.js new file mode 100644 index 0000000..fbe875e --- /dev/null +++ b/packages/transformers/src/models/qwen2/modeling_qwen2.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +/** + * The bare Qwen2 Model outputting raw hidden-states without any specific head on top. + */ +export class Qwen2PreTrainedModel extends PreTrainedModel {} + +/** + * The bare Qwen2 Model outputting raw hidden-states without any specific head on top. + */ +export class Qwen2Model extends Qwen2PreTrainedModel {} + +export class Qwen2ForCausalLM extends Qwen2PreTrainedModel {} diff --git a/packages/transformers/src/models/qwen2/tokenization_qwen2.js b/packages/transformers/src/models/qwen2/tokenization_qwen2.js new file mode 100644 index 0000000..eac3d33 --- /dev/null +++ b/packages/transformers/src/models/qwen2/tokenization_qwen2.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class Qwen2Tokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/qwen2_5_vl/modeling_qwen2_5_vl.js b/packages/transformers/src/models/qwen2_5_vl/modeling_qwen2_5_vl.js new file mode 100644 index 0000000..bbc8c96 --- /dev/null +++ b/packages/transformers/src/models/qwen2_5_vl/modeling_qwen2_5_vl.js @@ -0,0 +1,9 @@ +import { Qwen2VLForConditionalGeneration, Qwen2VLForCausalLM } from '../qwen2_vl/modeling_qwen2_vl.js'; + +export class Qwen2_5_VLForConditionalGeneration extends Qwen2VLForConditionalGeneration { + image_grid_thw_name = 'image_grid_thw'; +} + +export class Qwen2_5_VLForCausalLM extends Qwen2VLForCausalLM { + image_grid_thw_name = 'image_grid_thw'; +} diff --git a/packages/transformers/src/models/qwen2_5_vl/processing_qwen2_5_vl.js b/packages/transformers/src/models/qwen2_5_vl/processing_qwen2_5_vl.js new file mode 100644 index 0000000..ba0639e --- /dev/null +++ b/packages/transformers/src/models/qwen2_5_vl/processing_qwen2_5_vl.js @@ -0,0 +1,3 @@ +import { Qwen2VLProcessor } from '../qwen2_vl/processing_qwen2_vl.js'; + +export class Qwen2_5_VLProcessor extends Qwen2VLProcessor {} diff --git a/packages/transformers/src/models/qwen2_moe/modeling_qwen2_moe.js b/packages/transformers/src/models/qwen2_moe/modeling_qwen2_moe.js new file mode 100644 index 0000000..efeb931 --- /dev/null +++ b/packages/transformers/src/models/qwen2_moe/modeling_qwen2_moe.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Qwen2MoePreTrainedModel extends PreTrainedModel {} +export class Qwen2MoeModel extends Qwen2MoePreTrainedModel {} +export class Qwen2MoeForCausalLM extends Qwen2MoePreTrainedModel {} diff --git a/packages/transformers/src/models/qwen2_vl/image_processing_qwen2_vl.js b/packages/transformers/src/models/qwen2_vl/image_processing_qwen2_vl.js new file mode 100644 index 0000000..63ac1cd --- /dev/null +++ b/packages/transformers/src/models/qwen2_vl/image_processing_qwen2_vl.js @@ -0,0 +1,63 @@ +import { ImageProcessor, smart_resize } from '../../image_processors_utils.js'; +import { cat, Tensor } from '../../utils/tensor.js'; + +export class Qwen2VLImageProcessor extends ImageProcessor { + constructor(config) { + super(config); + this.min_pixels = config.min_pixels ?? config.size?.shortest_edge; + this.max_pixels = config.max_pixels ?? config.size?.longest_edge; + this.patch_size = config.patch_size; + this.merge_size = config.merge_size; + } + + /** @type {ImageProcessor['get_resize_output_image_size']} */ + get_resize_output_image_size(image, size) { + const factor = this.patch_size * this.merge_size; + return smart_resize(image.height, image.width, factor, this.min_pixels, this.max_pixels); + } + + async _call(images, ...args) { + const { pixel_values, original_sizes, reshaped_input_sizes } = await super._call(images, ...args); + + let patches = pixel_values; + + // @ts-ignore + const { temporal_patch_size, merge_size, patch_size } = this.config; + if (patches.dims[0] === 1) { + // Equivalent to np.tile(patches, (self.temporal_patch_size, 1, 1, 1)) + patches = cat( + Array.from({ length: temporal_patch_size }, () => patches), + 0, + ); + } + + const grid_t = patches.dims[0] / temporal_patch_size; + const channel = patches.dims[1]; + const grid_h = Math.floor(patches.dims[2] / patch_size); + const grid_w = Math.floor(patches.dims[3] / patch_size); + + const flatten_patches = patches + .view( + grid_t, + temporal_patch_size, + channel, + Math.floor(grid_h / merge_size), + merge_size, + patch_size, + Math.floor(grid_w / merge_size), + merge_size, + patch_size, + ) + .permute(0, 3, 6, 4, 7, 2, 1, 5, 8) + .view(grid_t * grid_h * grid_w, channel * temporal_patch_size * patch_size * patch_size); + + const image_grid_thw = new Tensor('int64', [grid_t, grid_h, grid_w], [1, 3]); + + return { + pixel_values: flatten_patches, + image_grid_thw, + original_sizes, + reshaped_input_sizes, + }; + } +} diff --git a/packages/transformers/src/models/qwen2_vl/modeling_qwen2_vl.js b/packages/transformers/src/models/qwen2_vl/modeling_qwen2_vl.js new file mode 100644 index 0000000..ea9f415 --- /dev/null +++ b/packages/transformers/src/models/qwen2_vl/modeling_qwen2_vl.js @@ -0,0 +1,354 @@ +import { + PreTrainedModel, + cumsum_masked_fill, + default_merge_input_ids_with_image_features, + setNumLogitsToKeep, +} from '../modeling_utils.js'; +import { sessionRun } from '../session.js'; +import { stack, Tensor, ones_like, zeros } from '../../utils/tensor.js'; +import { max } from '../../utils/maths.js'; + +export class Qwen2VLPreTrainedModel extends PreTrainedModel { + forward_params = [ + // Text inputs + 'input_ids', + 'attention_mask', + 'position_ids', + 'past_key_values', + + // Vision inputs + 'pixel_values', + 'image_grid_thw', + ]; +} +export class Qwen2VLForConditionalGeneration extends Qwen2VLPreTrainedModel { + // NOTE: This is used as the base class for all Qwen VL models and their CausalLM variants. + // CausalLM variants (e.g., Qwen2VLForCausalLM) extend this class but load only + // embed_tokens + decoder_model_merged (no vision_encoder) via MultimodalLanguageModelOnly type. + image_grid_thw_name = 'grid_thw'; + + /** + * Compute text-only 3D rope position IDs (all 3 dims get the same 1D positions). + * @param {Tensor} input_ids + * @param {Tensor} attention_mask + * @returns {[Tensor, Tensor]} [position_ids, mrope_position_deltas] + */ + _get_text_only_rope_index(input_ids, attention_mask) { + if (attention_mask) { + const { data, dims } = cumsum_masked_fill(attention_mask); + + const position_ids = BigInt64Array.from({ length: 3 * data.length }, (_, i) => data[i % data.length]); + /** @type {bigint[]} */ + const mrope_position_deltas = Array.from( + { length: dims[0] }, + (_, i) => max(data.subarray(dims[1] * i, dims[1] * (i + 1)))[0] + 1n + BigInt(dims[1]), + ); + + return [ + new Tensor('int64', position_ids, [3, ...dims]), + new Tensor('int64', mrope_position_deltas, [mrope_position_deltas.length, 1]), + ]; + } else { + const [batch_size, seq_length] = input_ids.dims; + const position_ids = BigInt64Array.from({ length: 3 * batch_size * seq_length }, (_, i) => + BigInt(Math.floor((i % seq_length) / batch_size)), + ); + + return [new Tensor('int64', position_ids, [3, ...input_ids.dims]), zeros([batch_size, 1])]; + } + } + + /** + * Reorder per-segment position ID lists from [seg1[t,h,w], seg2[t,h,w], ...] into + * global [all_t, all_h, all_w] order, then write back into the position_ids array + * respecting attention mask. + * @param {number[][]} llm_pos_ids_list List of per-segment position arrays, each of length 3*seg_len + * @param {number[]} attn_mask Attention mask for this batch element + * @param {number[][][]} position_ids_list [3][batch][seq] output array to write into + * @param {number} batch_idx Current batch index + * @returns {number[]} Flat reordered positions of length total_len + */ + _reorder_and_write_positions(llm_pos_ids_list, attn_mask, position_ids_list, batch_idx) { + const total_len = llm_pos_ids_list.reduce((acc, x) => acc + x.length, 0); + const llm_positions = new Array(total_len); + let index = 0; + for (let x = 0; x < 3; ++x) { + for (const val of llm_pos_ids_list) { + const seg_len = val.length / 3; + for (let z = x * seg_len; z < (x + 1) * seg_len; ++z) { + llm_positions[index++] = val[z]; + } + } + } + + let count = 0; + for (let y = 0; y < attn_mask.length; ++y) { + if (attn_mask[y] == 1) { + for (let x = 0; x < 3; ++x) { + position_ids_list[x][batch_idx][y] = llm_positions[(x * total_len) / 3 + count]; + } + ++count; + } + } + + return llm_positions; + } + + /** + * Build per-batch position ID segments for multimodal rope. + * Override this in subclasses to change how vision/text segments are identified and positioned. + * @param {object} params + * @param {any[]} params.filtered_ids - attention-masked token IDs for this batch element + * @param {any[][]} params.image_grid_thw_list - all image grid dimensions + * @param {any[][]} params.video_grid_thw_list - all video grid dimensions + * @param {number} params.spatial_merge_size + * @param {{image_index: number, video_index: number}} params.state - mutable counters shared across batches + * @returns {number[][]} llm_pos_ids_list - segments of [t..., h..., w...] positions + */ + _get_multimodal_rope_positions({ + filtered_ids, + image_grid_thw_list, + video_grid_thw_list, + spatial_merge_size, + state, + }) { + // @ts-ignore + const { image_token_id, video_token_id, vision_start_token_id } = this.config; + + const ids = filtered_ids; + const vision_start_indices = ids.reduce((acc, x, idx) => { + if (x == vision_start_token_id) acc.push(idx); + return acc; + }, []); + + const vision_tokens = vision_start_indices.map((x) => ids[x + 1]); + const image_nums = vision_tokens.filter((x) => x == image_token_id).length; + const video_nums = vision_tokens.filter((x) => x == video_token_id).length; + + /** @type {number[][]} */ + const llm_pos_ids_list = []; + let st = 0; + let remain_images = image_nums; + let remain_videos = video_nums; + for (let j = 0; j < vision_tokens.length; ++j) { + const next_image_token = ids.findIndex((x, i) => i > st && x == image_token_id); + const next_video_token = ids.findIndex((x, i) => i > st && x == video_token_id); + + const ed_image = remain_images > 0 && next_image_token !== -1 ? next_image_token : ids.length + 1; + const ed_video = remain_videos > 0 && next_video_token !== -1 ? next_video_token : ids.length + 1; + + let ed; + let t, h, w; + if (ed_image < ed_video) { + [t, h, w] = image_grid_thw_list[state.image_index]; + ++state.image_index; + --remain_images; + ed = ed_image; + } else { + [t, h, w] = video_grid_thw_list[state.video_index]; + ++state.video_index; + --remain_videos; + ed = ed_video; + } + + const [llm_grid_t, llm_grid_h, llm_grid_w] = [ + Number(t), + Math.floor(Number(h) / spatial_merge_size), + Math.floor(Number(w) / spatial_merge_size), + ]; + const text_len = ed - st; + const st_idx = llm_pos_ids_list.length > 0 ? max(llm_pos_ids_list.at(-1))[0] + 1 : 0; + + llm_pos_ids_list.push(Array.from({ length: 3 * text_len }, (_, i) => st_idx + (i % text_len))); + + const offset = text_len + st_idx; + const grid_size = llm_grid_t * llm_grid_h * llm_grid_w; + const t_index = Array.from( + { length: grid_size }, + (_, i) => offset + Math.floor(i / (llm_grid_h * llm_grid_w)), + ); + const h_index = Array.from( + { length: grid_size }, + (_, i) => offset + (Math.floor(i / llm_grid_w) % llm_grid_h), + ); + const w_index = Array.from({ length: grid_size }, (_, i) => offset + (i % llm_grid_w)); + + llm_pos_ids_list.push([t_index, h_index, w_index].flat()); + + st = ed + grid_size; + } + + if (st < ids.length) { + const st_idx = llm_pos_ids_list.length > 0 ? max(llm_pos_ids_list.at(-1))[0] + 1 : 0; + const text_len = ids.length - st; + + llm_pos_ids_list.push(Array.from({ length: 3 * text_len }, (_, i) => st_idx + (i % text_len))); + } + + return llm_pos_ids_list; + } + + /** + * Calculate the 3D rope index based on image and video's temporal, height and width in LLM. + * + * Explanation: + * Each embedding sequence contains vision embedding and text embedding or just contains text embedding. + * + * For pure text embedding sequence, the rotary position embedding has no difference with mordern LLMs. + * Examples: + * input_ids: [T T T T T], here T is for text. + * temporal position_ids: [0, 1, 2, 3, 4] + * height position_ids: [0, 1, 2, 3, 4] + * width position_ids: [0, 1, 2, 3, 4] + * + * For vision and text embedding sequence, we calculate 3D rotary position embedding for vision part + * and 1D rotary position embeddin for text part. + * Examples: + * Assume we have a video input with 3 temporal patches, 2 height patches and 2 width patches. + * input_ids: [V V V V V V V V V V V V T T T T T], here V is for vision. + * vision temporal position_ids: [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2] + * vision height position_ids: [0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1] + * vision width position_ids: [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1] + * text temporal position_ids: [3, 4, 5, 6, 7] + * text height position_ids: [3, 4, 5, 6, 7] + * text width position_ids: [3, 4, 5, 6, 7] + * Here we calculate the text start position_ids as the max vision position_ids plus 1. + * + * @param {Tensor} input_ids Indices of input sequence tokens in the vocabulary. Tensor of shape `(batch_size, sequence_length)`. + * @param {Tensor} image_grid_thw (Optional) The temporal, height and width of feature shape of each image in LLM. Tensor of shape `(num_images, 3)`. + * @param {Tensor} video_grid_thw (Optional) The temporal, height and width of feature shape of each video in LLM. Tensor of shape `(num_videos, 3)`. + * @param {Tensor} attention_mask (Optional) Mask to avoid performing attention on padding token indices. Tensor of shape `(batch_size, sequence_length)`. + * @returns {[Tensor, Tensor]} [position_ids, mrope_position_deltas] + */ + get_rope_index(input_ids, image_grid_thw, video_grid_thw, attention_mask) { + // @ts-ignore + const { vision_config } = this.config; + const spatial_merge_size = vision_config.spatial_merge_size ?? 2; + + if (image_grid_thw || video_grid_thw) { + const total_input_ids = input_ids.tolist(); + if (!attention_mask) { + attention_mask = ones_like(input_ids); + } + + const attention_mask_list = attention_mask.tolist(); + const position_ids_list = Array.from({ length: 3 }, () => + Array.from({ length: input_ids.dims[0] }, () => Array.from({ length: input_ids.dims[1] }, () => 0)), + ); + + const image_grid_thw_list = image_grid_thw ? image_grid_thw.tolist() : []; + const video_grid_thw_list = video_grid_thw ? video_grid_thw.tolist() : []; + const state = { image_index: 0, video_index: 0 }; + + const mrope_position_deltas = []; + for (let i = 0; i < total_input_ids.length; ++i) { + const filtered_ids = total_input_ids[i].filter((_, j) => attention_mask_list[i][j] == 1); + + const llm_pos_ids_list = this._get_multimodal_rope_positions({ + filtered_ids, + image_grid_thw_list, + video_grid_thw_list, + spatial_merge_size, + state, + }); + + const llm_positions = this._reorder_and_write_positions( + llm_pos_ids_list, + attention_mask_list[i], + position_ids_list, + i, + ); + + mrope_position_deltas.push(max(llm_positions)[0] + 1 - total_input_ids[i].length); + } + + return [ + new Tensor('int64', position_ids_list.flat(Infinity), [3, input_ids.dims[0], input_ids.dims[1]]), + new Tensor('int64', mrope_position_deltas, [mrope_position_deltas.length, 1]), + ]; + } else { + return this._get_text_only_rope_index(input_ids, attention_mask); + } + } + + async encode_image({ pixel_values, image_grid_thw }) { + const features = ( + await sessionRun(this.sessions['vision_encoder'], { + pixel_values, + [this.image_grid_thw_name]: image_grid_thw, + }) + ).image_features; + return features; + } + + _merge_input_ids_with_image_features(kwargs) { + return default_merge_input_ids_with_image_features({ + // @ts-ignore + image_token_id: this.config.image_token_id, + ...kwargs, + }); + } + + prepare_inputs_for_generation(input_ids, model_inputs, generation_config) { + setNumLogitsToKeep(this, model_inputs, 1n); + + // Overwritten -- in specific circumstances we don't want to forward image inputs to the model + if (!model_inputs.attention_mask || model_inputs.position_ids) { + return model_inputs; + } + + const session = this.sessions['decoder_model_merged'] ?? this.sessions['model']; + if (!session.inputNames.includes('position_ids')) { + return model_inputs; + } + + // Calculate position_ids and rope_deltas + if (!model_inputs.past_key_values) { + [model_inputs.position_ids, model_inputs.rope_deltas] = this.get_rope_index( + model_inputs.input_ids, + model_inputs.image_grid_thw, + model_inputs.video_grid_thw, + model_inputs.attention_mask, + ); + } else { + model_inputs.pixel_values = null; + // model_inputs.pixel_values_videos = null; + + const past_length = model_inputs.past_key_values.get_seq_length(); + + if (past_length < model_inputs.input_ids.dims[1]) { + // Externally provided `past_key_values` with full input_ids: + // Compute full position_ids, then slice to only the new (unprocessed) tokens. + const [full_position_ids, rope_deltas] = this.get_rope_index( + model_inputs.input_ids, + model_inputs.image_grid_thw, + model_inputs.video_grid_thw, + model_inputs.attention_mask, + ); + model_inputs.rope_deltas = rope_deltas; + model_inputs.position_ids = full_position_ids.slice(null, null, [past_length, null]); + model_inputs.input_ids = model_inputs.input_ids.slice(null, [past_length, null]); + } else { + // Auto-regressive case: single new token. + // `rope_deltas` may be absent when generation starts from externally provided `past_key_values`. + // In that case, recompute from current inputs instead of relying on persisted model state. + if (!model_inputs.rope_deltas) { + [, model_inputs.rope_deltas] = this.get_rope_index( + model_inputs.input_ids, + model_inputs.image_grid_thw, + model_inputs.video_grid_thw, + model_inputs.attention_mask, + ); + } + + const delta = BigInt(past_length); + const rope_deltas_list = model_inputs.rope_deltas.map((x) => delta + x); + model_inputs.position_ids = stack([rope_deltas_list, rope_deltas_list, rope_deltas_list], 0); + } + } + + return model_inputs; + } +} + +export class Qwen2VLForCausalLM extends Qwen2VLForConditionalGeneration {} diff --git a/packages/transformers/src/models/qwen2_vl/processing_qwen2_vl.js b/packages/transformers/src/models/qwen2_vl/processing_qwen2_vl.js new file mode 100644 index 0000000..896aa98 --- /dev/null +++ b/packages/transformers/src/models/qwen2_vl/processing_qwen2_vl.js @@ -0,0 +1,53 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { RawImage } from '../../utils/image.js'; + +export class Qwen2VLProcessor extends Processor { + static image_processor_class = AutoImageProcessor; + static tokenizer_class = AutoTokenizer; + static image_token = '<|image_pad|>'; + + /** + * + * @param {string|string[]} text + * @param {RawImage|RawImage[]} images + * @param {...any} args + * @returns {Promise} + */ + async _call(text, images = null, ...args) { + if (!Array.isArray(text)) { + text = [text]; + } + + let image_inputs, image_grid_thw; + + if (images) { + image_inputs = await this.image_processor(images); + image_grid_thw = image_inputs.image_grid_thw; + } + + if (image_grid_thw) { + // @ts-expect-error TS2551 + let merge_length = this.image_processor.config.merge_size ** 2; + let index = 0; + + const image_token = /** @type {typeof Qwen2VLProcessor} */ (this.constructor).image_token; + const image_grid_thw_list = image_grid_thw.tolist(); + text = text.map((t) => { + while (t.includes(image_token)) { + const prod = Number(image_grid_thw_list[index++].reduce((a, b) => a * b, 1n)); + t = t.replace(image_token, '<|placeholder|>'.repeat(Math.floor(prod / merge_length))); + } + return t.replaceAll('<|placeholder|>', image_token); + }); + } + + const text_inputs = this.tokenizer(text); + + return { + ...text_inputs, + ...image_inputs, + }; + } +} diff --git a/packages/transformers/src/models/qwen3/modeling_qwen3.js b/packages/transformers/src/models/qwen3/modeling_qwen3.js new file mode 100644 index 0000000..fbf88bf --- /dev/null +++ b/packages/transformers/src/models/qwen3/modeling_qwen3.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +/** + * The bare Qwen3 Model outputting raw hidden-states without any specific head on top. + */ +export class Qwen3PreTrainedModel extends PreTrainedModel {} + +/** + * The bare Qwen3 Model outputting raw hidden-states without any specific head on top. + */ +export class Qwen3Model extends Qwen3PreTrainedModel {} + +export class Qwen3ForCausalLM extends Qwen3PreTrainedModel {} diff --git a/packages/transformers/src/models/qwen3_5/modeling_qwen3_5.js b/packages/transformers/src/models/qwen3_5/modeling_qwen3_5.js new file mode 100644 index 0000000..c819b89 --- /dev/null +++ b/packages/transformers/src/models/qwen3_5/modeling_qwen3_5.js @@ -0,0 +1,4 @@ +import { Qwen3VLForConditionalGeneration } from '../qwen3_vl/modeling_qwen3_vl.js'; + +export class Qwen3_5ForConditionalGeneration extends Qwen3VLForConditionalGeneration {} +export class Qwen3_5ForCausalLM extends Qwen3_5ForConditionalGeneration {} diff --git a/packages/transformers/src/models/qwen3_5_moe/modeling_qwen3_5_moe.js b/packages/transformers/src/models/qwen3_5_moe/modeling_qwen3_5_moe.js new file mode 100644 index 0000000..8cb24ab --- /dev/null +++ b/packages/transformers/src/models/qwen3_5_moe/modeling_qwen3_5_moe.js @@ -0,0 +1,4 @@ +import { Qwen3_5ForConditionalGeneration, Qwen3_5ForCausalLM } from '../qwen3_5/modeling_qwen3_5.js'; + +export class Qwen3_5MoeForConditionalGeneration extends Qwen3_5ForConditionalGeneration {} +export class Qwen3_5MoeForCausalLM extends Qwen3_5ForCausalLM {} diff --git a/packages/transformers/src/models/qwen3_moe/modeling_qwen3_moe.js b/packages/transformers/src/models/qwen3_moe/modeling_qwen3_moe.js new file mode 100644 index 0000000..a1f9143 --- /dev/null +++ b/packages/transformers/src/models/qwen3_moe/modeling_qwen3_moe.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Qwen3MoePreTrainedModel extends PreTrainedModel {} +export class Qwen3MoeModel extends Qwen3MoePreTrainedModel {} +export class Qwen3MoeForCausalLM extends Qwen3MoePreTrainedModel {} diff --git a/packages/transformers/src/models/qwen3_next/modeling_qwen3_next.js b/packages/transformers/src/models/qwen3_next/modeling_qwen3_next.js new file mode 100644 index 0000000..f232efa --- /dev/null +++ b/packages/transformers/src/models/qwen3_next/modeling_qwen3_next.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Qwen3NextPreTrainedModel extends PreTrainedModel {} +export class Qwen3NextModel extends Qwen3NextPreTrainedModel {} +export class Qwen3NextForCausalLM extends Qwen3NextPreTrainedModel {} diff --git a/packages/transformers/src/models/qwen3_vl/modeling_qwen3_vl.js b/packages/transformers/src/models/qwen3_vl/modeling_qwen3_vl.js new file mode 100644 index 0000000..b1f77a3 --- /dev/null +++ b/packages/transformers/src/models/qwen3_vl/modeling_qwen3_vl.js @@ -0,0 +1,4 @@ +import { Qwen2_5_VLForConditionalGeneration, Qwen2_5_VLForCausalLM } from '../qwen2_5_vl/modeling_qwen2_5_vl.js'; + +export class Qwen3VLForConditionalGeneration extends Qwen2_5_VLForConditionalGeneration {} +export class Qwen3VLForCausalLM extends Qwen2_5_VLForCausalLM {} diff --git a/packages/transformers/src/models/qwen3_vl/processing_qwen3_vl.js b/packages/transformers/src/models/qwen3_vl/processing_qwen3_vl.js new file mode 100644 index 0000000..d50f462 --- /dev/null +++ b/packages/transformers/src/models/qwen3_vl/processing_qwen3_vl.js @@ -0,0 +1,3 @@ +import { Qwen2_5_VLProcessor } from '../qwen2_5_vl/processing_qwen2_5_vl.js'; + +export class Qwen3VLProcessor extends Qwen2_5_VLProcessor {} diff --git a/packages/transformers/src/models/qwen3_vl_moe/modeling_qwen3_vl_moe.js b/packages/transformers/src/models/qwen3_vl_moe/modeling_qwen3_vl_moe.js new file mode 100644 index 0000000..4c58e73 --- /dev/null +++ b/packages/transformers/src/models/qwen3_vl_moe/modeling_qwen3_vl_moe.js @@ -0,0 +1,4 @@ +import { Qwen3VLForConditionalGeneration, Qwen3VLForCausalLM } from '../qwen3_vl/modeling_qwen3_vl.js'; + +export class Qwen3VLMoeForConditionalGeneration extends Qwen3VLForConditionalGeneration {} +export class Qwen3VLMoeForCausalLM extends Qwen3VLForCausalLM {} diff --git a/packages/transformers/src/models/registry.js b/packages/transformers/src/models/registry.js new file mode 100644 index 0000000..95f9060 --- /dev/null +++ b/packages/transformers/src/models/registry.js @@ -0,0 +1,701 @@ +import { + MODEL_TYPES, + MODEL_TYPE_MAPPING, + MODEL_NAME_TO_CLASS_MAPPING, + MODEL_CLASS_TO_NAME_MAPPING, + PreTrainedModel, + registerTaskMappings, +} from './modeling_utils.js'; + +import * as ALL_MODEL_FILES from './models.js'; + +const MODEL_MAPPING_NAMES_ENCODER_ONLY = new Map([ + ['bert', 'BertModel'], + ['eurobert', 'EuroBertModel'], + ['neobert', 'NeoBertModel'], + ['modernbert', 'ModernBertModel'], + ['nomic_bert', 'NomicBertModel'], + ['roformer', 'RoFormerModel'], + ['electra', 'ElectraModel'], + ['esm', 'EsmModel'], + ['convbert', 'ConvBertModel'], + ['camembert', 'CamembertModel'], + ['deberta', 'DebertaModel'], + ['deberta-v2', 'DebertaV2Model'], + ['mpnet', 'MPNetModel'], + ['albert', 'AlbertModel'], + ['distilbert', 'DistilBertModel'], + ['roberta', 'RobertaModel'], + ['xlm', 'XLMModel'], + ['xlm-roberta', 'XLMRobertaModel'], + ['clap', 'ClapModel'], + ['clip', 'CLIPModel'], + ['clipseg', 'CLIPSegModel'], + ['chinese_clip', 'ChineseCLIPModel'], + ['siglip', 'SiglipModel'], + ['jina_clip', 'JinaCLIPModel'], + ['mobilebert', 'MobileBertModel'], + ['squeezebert', 'SqueezeBertModel'], + ['wav2vec2', 'Wav2Vec2Model'], + ['wav2vec2-bert', 'Wav2Vec2BertModel'], + ['unispeech', 'UniSpeechModel'], + ['unispeech-sat', 'UniSpeechSatModel'], + ['hubert', 'HubertModel'], + ['wavlm', 'WavLMModel'], + ['audio-spectrogram-transformer', 'ASTModel'], + ['vits', 'VitsModel'], + ['pyannote', 'PyAnnoteModel'], + ['wespeaker-resnet', 'WeSpeakerResNetModel'], + + ['detr', 'DetrModel'], + ['rt_detr', 'RTDetrModel'], + ['rt_detr_v2', 'RTDetrV2Model'], + ['rf_detr', 'RFDetrModel'], + ['d_fine', 'DFineModel'], + ['table-transformer', 'TableTransformerModel'], + ['vit', 'ViTModel'], + ['ijepa', 'IJepaModel'], + ['pvt', 'PvtModel'], + ['vit_msn', 'ViTMSNModel'], + ['vit_mae', 'ViTMAEModel'], + ['groupvit', 'GroupViTModel'], + ['fastvit', 'FastViTModel'], + ['mobilevit', 'MobileViTModel'], + ['mobilevitv2', 'MobileViTV2Model'], + ['owlvit', 'OwlViTModel'], + ['owlv2', 'Owlv2Model'], + ['beit', 'BeitModel'], + ['deit', 'DeiTModel'], + ['hiera', 'HieraModel'], + ['convnext', 'ConvNextModel'], + ['convnextv2', 'ConvNextV2Model'], + ['dinov2', 'Dinov2Model'], + ['dinov2_with_registers', 'Dinov2WithRegistersModel'], + ['dinov3_vit', 'DINOv3ViTModel'], + ['dinov3_convnext', 'DINOv3ConvNextModel'], + ['resnet', 'ResNetModel'], + ['swin', 'SwinModel'], + ['swin2sr', 'Swin2SRModel'], + ['donut-swin', 'DonutSwinModel'], + ['yolos', 'YolosModel'], + ['dpt', 'DPTModel'], + ['glpn', 'GLPNModel'], + + ['hifigan', 'SpeechT5HifiGan'], + ['efficientnet', 'EfficientNetModel'], + + ['decision_transformer', 'DecisionTransformerModel'], + ['patchtst', 'PatchTSTModel'], + ['patchtsmixer', 'PatchTSMixerModel'], + + ['mobilenet_v1', 'MobileNetV1Model'], + ['mobilenet_v2', 'MobileNetV2Model'], + ['mobilenet_v3', 'MobileNetV3Model'], + ['mobilenet_v4', 'MobileNetV4Model'], + + ['maskformer', 'MaskFormerModel'], + ['mgp-str', 'MgpstrForSceneTextRecognition'], + + ['style_text_to_speech_2', 'StyleTextToSpeech2Model'], + ['openai_privacy_filter', 'OpenAIPrivacyFilterModel'], +]); + +const MODEL_MAPPING_NAMES_ENCODER_DECODER = new Map([ + ['t5', 'T5Model'], + ['longt5', 'LongT5Model'], + ['mt5', 'MT5Model'], + ['bart', 'BartModel'], + ['mbart', 'MBartModel'], + ['marian', 'MarianModel'], + ['whisper', 'WhisperModel'], + ['cohere_asr', 'CohereAsrModel'], + ['m2m_100', 'M2M100Model'], + ['blenderbot', 'BlenderbotModel'], + ['blenderbot-small', 'BlenderbotSmallModel'], +]); + +const MODEL_MAPPING_NAMES_AUTO_ENCODER = new Map([ + ['mimi', 'MimiModel'], + ['dac', 'DacModel'], + ['snac', 'SnacModel'], +]); + +const MODEL_MAPPING_NAMES_DECODER_ONLY = new Map([ + ['bloom', 'BloomModel'], + ['jais', 'JAISModel'], + ['gpt2', 'GPT2Model'], + ['gpt_oss', 'GptOssModel'], + ['gptj', 'GPTJModel'], + ['gpt_bigcode', 'GPTBigCodeModel'], + ['gpt_neo', 'GPTNeoModel'], + ['gpt_neox', 'GPTNeoXModel'], + ['codegen', 'CodeGenModel'], + ['llama', 'LlamaModel'], + ['apertus', 'ApertusModel'], + ['nanochat', 'NanoChatModel'], + ['arcee', 'ArceeModel'], + ['afmoe', 'AfmoeModel'], + ['lfm2', 'Lfm2Model'], + ['lfm2_moe', 'Lfm2MoeModel'], + ['smollm3', 'SmolLM3Model'], + ['exaone', 'ExaoneModel'], + ['olmo', 'OlmoModel'], + ['olmo2', 'Olmo2Model'], + ['olmo3', 'Olmo3Model'], + ['olmo_hybrid', 'OlmoHybridModel'], + ['mobilellm', 'MobileLLMModel'], + ['granite', 'GraniteModel'], + ['granitemoehybrid', 'GraniteMoeHybridModel'], + ['cohere', 'CohereModel'], + ['cohere2', 'Cohere2Model'], + ['gemma', 'GemmaModel'], + ['gemma2', 'Gemma2Model'], + ['vaultgemma', 'VaultGemmaModel'], + ['gemma3_text', 'Gemma3Model'], + ['helium', 'HeliumModel'], + ['glm', 'GlmModel'], + ['glm_moe_dsa', 'GlmMoeDsaModel'], + ['openelm', 'OpenELMModel'], + ['qwen2', 'Qwen2Model'], + ['qwen2_moe', 'Qwen2MoeModel'], + ['qwen3', 'Qwen3Model'], + ['qwen3_moe', 'Qwen3MoeModel'], + ['qwen3_next', 'Qwen3NextModel'], + ['phi', 'PhiModel'], + ['phi3', 'Phi3Model'], + ['mpt', 'MptModel'], + ['opt', 'OPTModel'], + ['mistral', 'MistralModel'], + ['mistral4', 'Mistral4Model'], + ['ministral', 'MinistralModel'], + ['ministral3', 'Ministral3Model'], + ['ernie4_5', 'Ernie4_5ForCausalLM'], + ['starcoder2', 'Starcoder2Model'], + ['deepseek_v3', 'DeepseekV3Model'], + ['deepseek_v4', 'DeepseekV4Model'], + ['falcon', 'FalconModel'], + ['falcon_h1', 'FalconH1Model'], + ['nemotron_h', 'NemotronHModel'], + ['solar_open', 'SolarOpenModel'], + ['stablelm', 'StableLmModel'], + ['modernbert-decoder', 'ModernBertDecoderModel'], + ['hunyuan_v1_dense', 'HunYuanDenseV1Model'], + ['youtu', 'YoutuModel'], + ['zaya', 'ZayaModel'], + ['hrm_text', 'HrmTextModel'], +]); + +export const MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES = new Map([ + ['speecht5', 'SpeechT5ForSpeechToText'], + ['whisper', 'WhisperForConditionalGeneration'], + ['lite-whisper', 'LiteWhisperForConditionalGeneration'], + ['moonshine', 'MoonshineForConditionalGeneration'], + ['cohere_asr', 'CohereAsrForConditionalGeneration'], +]); + +const MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES = new Map([['speecht5', 'SpeechT5ForTextToSpeech']]); + +const MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES = new Map([ + ['vits', 'VitsModel'], + ['musicgen', 'MusicgenForConditionalGeneration'], + ['supertonic', 'SupertonicForConditionalGeneration'], +]); + +const MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES = new Map([ + ['bert', 'BertForSequenceClassification'], + ['eurobert', 'EuroBertForSequenceClassification'], + ['neobert', 'NeoBertForSequenceClassification'], + ['modernbert', 'ModernBertForSequenceClassification'], + ['roformer', 'RoFormerForSequenceClassification'], + ['electra', 'ElectraForSequenceClassification'], + ['esm', 'EsmForSequenceClassification'], + ['convbert', 'ConvBertForSequenceClassification'], + ['camembert', 'CamembertForSequenceClassification'], + ['deberta', 'DebertaForSequenceClassification'], + ['deberta-v2', 'DebertaV2ForSequenceClassification'], + ['mpnet', 'MPNetForSequenceClassification'], + ['albert', 'AlbertForSequenceClassification'], + ['distilbert', 'DistilBertForSequenceClassification'], + ['roberta', 'RobertaForSequenceClassification'], + ['xlm', 'XLMForSequenceClassification'], + ['xlm-roberta', 'XLMRobertaForSequenceClassification'], + ['bart', 'BartForSequenceClassification'], + ['mbart', 'MBartForSequenceClassification'], + ['mobilebert', 'MobileBertForSequenceClassification'], + ['squeezebert', 'SqueezeBertForSequenceClassification'], +]); + +const MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES = new Map([ + ['bert', 'BertForTokenClassification'], + ['eurobert', 'EuroBertForTokenClassification'], + ['neobert', 'NeoBertForTokenClassification'], + ['modernbert', 'ModernBertForTokenClassification'], + ['roformer', 'RoFormerForTokenClassification'], + ['electra', 'ElectraForTokenClassification'], + ['esm', 'EsmForTokenClassification'], + ['convbert', 'ConvBertForTokenClassification'], + ['camembert', 'CamembertForTokenClassification'], + ['deberta', 'DebertaForTokenClassification'], + ['deberta-v2', 'DebertaV2ForTokenClassification'], + ['mpnet', 'MPNetForTokenClassification'], + ['distilbert', 'DistilBertForTokenClassification'], + ['roberta', 'RobertaForTokenClassification'], + ['xlm', 'XLMForTokenClassification'], + ['xlm-roberta', 'XLMRobertaForTokenClassification'], + ['openai_privacy_filter', 'OpenAIPrivacyFilterForTokenClassification'], +]); + +export const MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES = new Map([ + ['t5', 'T5ForConditionalGeneration'], + ['longt5', 'LongT5ForConditionalGeneration'], + ['mt5', 'MT5ForConditionalGeneration'], + ['bart', 'BartForConditionalGeneration'], + ['mbart', 'MBartForConditionalGeneration'], + ['marian', 'MarianMTModel'], + ['m2m_100', 'M2M100ForConditionalGeneration'], + ['blenderbot', 'BlenderbotForConditionalGeneration'], + ['blenderbot-small', 'BlenderbotSmallForConditionalGeneration'], +]); + +export const MODEL_FOR_CAUSAL_LM_MAPPING_NAMES = new Map([ + ['bloom', 'BloomForCausalLM'], + ['gpt2', 'GPT2LMHeadModel'], + ['gpt_oss', 'GptOssForCausalLM'], + ['jais', 'JAISLMHeadModel'], + ['gptj', 'GPTJForCausalLM'], + ['gpt_bigcode', 'GPTBigCodeForCausalLM'], + ['gpt_neo', 'GPTNeoForCausalLM'], + ['gpt_neox', 'GPTNeoXForCausalLM'], + ['codegen', 'CodeGenForCausalLM'], + ['llama', 'LlamaForCausalLM'], + ['nanochat', 'NanoChatForCausalLM'], + ['apertus', 'ApertusForCausalLM'], + ['llama4_text', 'Llama4ForCausalLM'], + ['arcee', 'ArceeForCausalLM'], + ['afmoe', 'AfmoeForCausalLM'], + ['lfm2', 'Lfm2ForCausalLM'], + ['lfm2_moe', 'Lfm2MoeForCausalLM'], + ['smollm3', 'SmolLM3ForCausalLM'], + ['exaone', 'ExaoneForCausalLM'], + ['olmo', 'OlmoForCausalLM'], + ['olmo2', 'Olmo2ForCausalLM'], + ['olmo3', 'Olmo3ForCausalLM'], + ['olmo_hybrid', 'OlmoHybridForCausalLM'], + ['mobilellm', 'MobileLLMForCausalLM'], + ['granite', 'GraniteForCausalLM'], + ['granitemoehybrid', 'GraniteMoeHybridForCausalLM'], + ['cohere', 'CohereForCausalLM'], + ['cohere2', 'Cohere2ForCausalLM'], + ['gemma', 'GemmaForCausalLM'], + ['gemma2', 'Gemma2ForCausalLM'], + ['vaultgemma', 'VaultGemmaForCausalLM'], + ['gemma3_text', 'Gemma3ForCausalLM'], + ['gemma3', 'Gemma3ForCausalLM'], + ['helium', 'HeliumForCausalLM'], + ['glm', 'GlmForCausalLM'], + ['glm_moe_dsa', 'GlmMoeDsaForCausalLM'], + ['openelm', 'OpenELMForCausalLM'], + ['qwen2', 'Qwen2ForCausalLM'], + ['qwen2_moe', 'Qwen2MoeForCausalLM'], + ['qwen3', 'Qwen3ForCausalLM'], + ['qwen3_moe', 'Qwen3MoeForCausalLM'], + ['qwen3_next', 'Qwen3NextForCausalLM'], + ['qwen2_vl', 'Qwen2VLForCausalLM'], + ['qwen2_5_vl', 'Qwen2_5_VLForCausalLM'], + ['qwen3_vl', 'Qwen3VLForCausalLM'], + ['qwen3_vl_moe', 'Qwen3VLMoeForCausalLM'], + ['qwen3_5', 'Qwen3_5ForCausalLM'], + ['qwen3_5_text', 'Qwen3_5ForCausalLM'], + ['qwen3_5_moe', 'Qwen3_5MoeForCausalLM'], + ['gemma3n', 'Gemma3nForCausalLM'], + ['gemma4', 'Gemma4ForCausalLM'], + ['phi', 'PhiForCausalLM'], + ['phi3', 'Phi3ForCausalLM'], + ['mpt', 'MptForCausalLM'], + ['opt', 'OPTForCausalLM'], + ['mbart', 'MBartForCausalLM'], + ['mistral', 'MistralForCausalLM'], + ['mistral4', 'Mistral4ForCausalLM'], + ['ministral', 'MinistralForCausalLM'], + ['ministral3', 'Ministral3ForCausalLM'], + ['ernie4_5', 'Ernie4_5ForCausalLM'], + ['starcoder2', 'Starcoder2ForCausalLM'], + ['deepseek_v3', 'DeepseekV3ForCausalLM'], + ['deepseek_v4', 'DeepseekV4ForCausalLM'], + ['falcon', 'FalconForCausalLM'], + ['falcon_h1', 'FalconH1ForCausalLM'], + ['nemotron_h', 'NemotronHForCausalLM'], + ['trocr', 'TrOCRForCausalLM'], + ['solar_open', 'SolarOpenForCausalLM'], + ['stablelm', 'StableLmForCausalLM'], + ['modernbert-decoder', 'ModernBertDecoderForCausalLM'], + ['hunyuan_v1_dense', 'HunYuanDenseV1ForCausalLM'], + ['youtu', 'YoutuForCausalLM'], + ['zaya', 'ZayaForCausalLM'], + ['hrm_text', 'HrmTextForCausalLM'], + + // Also image-text-to-text + ['phi3_v', 'Phi3VForCausalLM'], +]); + +const MODEL_FOR_MULTIMODALITY_MAPPING_NAMES = new Map([['multi_modality', 'MultiModalityCausalLM']]); + +const MODEL_FOR_MASKED_LM_MAPPING_NAMES = new Map([ + ['bert', 'BertForMaskedLM'], + ['eurobert', 'EuroBertForMaskedLM'], + ['neobert', 'NeoBertForMaskedLM'], + ['modernbert', 'ModernBertForMaskedLM'], + ['roformer', 'RoFormerForMaskedLM'], + ['electra', 'ElectraForMaskedLM'], + ['esm', 'EsmForMaskedLM'], + ['convbert', 'ConvBertForMaskedLM'], + ['camembert', 'CamembertForMaskedLM'], + ['deberta', 'DebertaForMaskedLM'], + ['deberta-v2', 'DebertaV2ForMaskedLM'], + ['mpnet', 'MPNetForMaskedLM'], + ['albert', 'AlbertForMaskedLM'], + ['distilbert', 'DistilBertForMaskedLM'], + ['roberta', 'RobertaForMaskedLM'], + ['xlm', 'XLMWithLMHeadModel'], + ['xlm-roberta', 'XLMRobertaForMaskedLM'], + ['mobilebert', 'MobileBertForMaskedLM'], + ['squeezebert', 'SqueezeBertForMaskedLM'], +]); + +const MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES = new Map([ + ['bert', 'BertForQuestionAnswering'], + ['neobert', 'NeoBertForQuestionAnswering'], + ['roformer', 'RoFormerForQuestionAnswering'], + ['electra', 'ElectraForQuestionAnswering'], + ['convbert', 'ConvBertForQuestionAnswering'], + ['camembert', 'CamembertForQuestionAnswering'], + ['deberta', 'DebertaForQuestionAnswering'], + ['deberta-v2', 'DebertaV2ForQuestionAnswering'], + ['mpnet', 'MPNetForQuestionAnswering'], + ['albert', 'AlbertForQuestionAnswering'], + ['distilbert', 'DistilBertForQuestionAnswering'], + ['roberta', 'RobertaForQuestionAnswering'], + ['xlm', 'XLMForQuestionAnswering'], + ['xlm-roberta', 'XLMRobertaForQuestionAnswering'], + ['mobilebert', 'MobileBertForQuestionAnswering'], + ['squeezebert', 'SqueezeBertForQuestionAnswering'], +]); + +export const MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES = new Map([ + ['vision-encoder-decoder', 'VisionEncoderDecoderModel'], + ['idefics3', 'Idefics3ForConditionalGeneration'], + ['smolvlm', 'SmolVLMForConditionalGeneration'], +]); + +const MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES = new Map([ + ['llava', 'LlavaForConditionalGeneration'], + ['llava_onevision', 'LlavaOnevisionForConditionalGeneration'], + ['moondream1', 'Moondream1ForConditionalGeneration'], + ['florence2', 'Florence2ForConditionalGeneration'], + ['qwen2_vl', 'Qwen2VLForConditionalGeneration'], + ['qwen2_5_vl', 'Qwen2_5_VLForConditionalGeneration'], + ['qwen3_vl', 'Qwen3VLForConditionalGeneration'], + ['qwen3_vl_moe', 'Qwen3VLMoeForConditionalGeneration'], + ['qwen3_5', 'Qwen3_5ForConditionalGeneration'], + ['qwen3_5_moe', 'Qwen3_5MoeForConditionalGeneration'], + ['lfm2_vl', 'Lfm2VlForConditionalGeneration'], + ['idefics3', 'Idefics3ForConditionalGeneration'], + ['smolvlm', 'SmolVLMForConditionalGeneration'], + ['paligemma', 'PaliGemmaForConditionalGeneration'], + ['llava_qwen2', 'LlavaQwen2ForCausalLM'], + ['gemma3', 'Gemma3ForConditionalGeneration'], + ['gemma3n', 'Gemma3nForConditionalGeneration'], + ['gemma4', 'Gemma4ForConditionalGeneration'], + ['mistral3', 'Mistral3ForConditionalGeneration'], + ['lighton_ocr', 'LightOnOcrForConditionalGeneration'], + ['glm_ocr', 'GlmOcrForConditionalGeneration'], +]); + +const MODEL_FOR_AUDIO_TEXT_TO_TEXT_MAPPING_NAMES = new Map([ + ['granite_speech', 'GraniteSpeechForConditionalGeneration'], + ['ultravox', 'UltravoxModel'], + ['voxtral', 'VoxtralForConditionalGeneration'], + ['voxtral_realtime', 'VoxtralRealtimeForConditionalGeneration'], +]); + +const MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES = new Map([ + ['vision-encoder-decoder', 'VisionEncoderDecoderModel'], +]); + +const MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES = new Map([ + ['vit', 'ViTForImageClassification'], + ['ijepa', 'IJepaForImageClassification'], + ['pvt', 'PvtForImageClassification'], + ['vit_msn', 'ViTMSNForImageClassification'], + ['fastvit', 'FastViTForImageClassification'], + ['mobilevit', 'MobileViTForImageClassification'], + ['mobilevitv2', 'MobileViTV2ForImageClassification'], + ['beit', 'BeitForImageClassification'], + ['deit', 'DeiTForImageClassification'], + ['hiera', 'HieraForImageClassification'], + ['convnext', 'ConvNextForImageClassification'], + ['convnextv2', 'ConvNextV2ForImageClassification'], + ['dinov2', 'Dinov2ForImageClassification'], + ['dinov2_with_registers', 'Dinov2WithRegistersForImageClassification'], + ['resnet', 'ResNetForImageClassification'], + ['swin', 'SwinForImageClassification'], + ['segformer', 'SegformerForImageClassification'], + ['efficientnet', 'EfficientNetForImageClassification'], + ['mobilenet_v1', 'MobileNetV1ForImageClassification'], + ['mobilenet_v2', 'MobileNetV2ForImageClassification'], + ['mobilenet_v3', 'MobileNetV3ForImageClassification'], + ['mobilenet_v4', 'MobileNetV4ForImageClassification'], +]); + +const MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES = new Map([ + ['detr', 'DetrForObjectDetection'], + ['rt_detr', 'RTDetrForObjectDetection'], + ['rt_detr_v2', 'RTDetrV2ForObjectDetection'], + ['rf_detr', 'RFDetrForObjectDetection'], + ['d_fine', 'DFineForObjectDetection'], + ['table-transformer', 'TableTransformerForObjectDetection'], + ['yolos', 'YolosForObjectDetection'], +]); + +const MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES = new Map([ + ['owlvit', 'OwlViTForObjectDetection'], + ['owlv2', 'Owlv2ForObjectDetection'], + ['grounding-dino', 'GroundingDinoForObjectDetection'], +]); + +const MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES = new Map([ + // TODO: Do not add new models here + ['detr', 'DetrForSegmentation'], + ['clipseg', 'CLIPSegForImageSegmentation'], +]); + +const MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES = new Map([ + ['segformer', 'SegformerForSemanticSegmentation'], + ['sapiens', 'SapiensForSemanticSegmentation'], + + ['swin', 'SwinForSemanticSegmentation'], + ['mobilenet_v1', 'MobileNetV1ForSemanticSegmentation'], + ['mobilenet_v2', 'MobileNetV2ForSemanticSegmentation'], + ['mobilenet_v3', 'MobileNetV3ForSemanticSegmentation'], + ['mobilenet_v4', 'MobileNetV4ForSemanticSegmentation'], +]); + +const MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING_NAMES = new Map([ + ['detr', 'DetrForSegmentation'], + ['maskformer', 'MaskFormerForInstanceSegmentation'], +]); + +const MODEL_FOR_MASK_GENERATION_MAPPING_NAMES = new Map([ + ['sam', 'SamModel'], + ['sam2', 'Sam2Model'], + ['edgetam', 'EdgeTamModel'], + ['sam3_tracker', 'Sam3TrackerModel'], +]); + +const MODEL_FOR_CTC_MAPPING_NAMES = new Map([ + ['wav2vec2', 'Wav2Vec2ForCTC'], + ['wav2vec2-bert', 'Wav2Vec2BertForCTC'], + ['unispeech', 'UniSpeechForCTC'], + ['unispeech-sat', 'UniSpeechSatForCTC'], + ['wavlm', 'WavLMForCTC'], + ['hubert', 'HubertForCTC'], + ['parakeet_ctc', 'ParakeetForCTC'], +]); + +const MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES = new Map([ + ['wav2vec2', 'Wav2Vec2ForSequenceClassification'], + ['wav2vec2-bert', 'Wav2Vec2BertForSequenceClassification'], + ['unispeech', 'UniSpeechForSequenceClassification'], + ['unispeech-sat', 'UniSpeechSatForSequenceClassification'], + ['wavlm', 'WavLMForSequenceClassification'], + ['hubert', 'HubertForSequenceClassification'], + ['audio-spectrogram-transformer', 'ASTForAudioClassification'], +]); + +const MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES = new Map([['wavlm', 'WavLMForXVector']]); + +const MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES = new Map([ + ['unispeech-sat', 'UniSpeechSatForAudioFrameClassification'], + ['wavlm', 'WavLMForAudioFrameClassification'], + ['wav2vec2', 'Wav2Vec2ForAudioFrameClassification'], + ['pyannote', 'PyAnnoteForAudioFrameClassification'], +]); + +const MODEL_FOR_IMAGE_MATTING_MAPPING_NAMES = new Map([['vitmatte', 'VitMatteForImageMatting']]); + +const MODEL_FOR_TIME_SERIES_PREDICTION_MAPPING_NAMES = new Map([ + ['patchtst', 'PatchTSTForPrediction'], + ['patchtsmixer', 'PatchTSMixerForPrediction'], +]); + +const MODEL_FOR_IMAGE_TO_IMAGE_MAPPING_NAMES = new Map([['swin2sr', 'Swin2SRForImageSuperResolution']]); + +const MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES = new Map([ + ['chmv2', 'CHMv2ForDepthEstimation'], + ['dpt', 'DPTForDepthEstimation'], + ['depth_anything', 'DepthAnythingForDepthEstimation'], + ['glpn', 'GLPNForDepthEstimation'], + ['sapiens', 'SapiensForDepthEstimation'], + ['depth_pro', 'DepthProForDepthEstimation'], + ['metric3d', 'Metric3DForDepthEstimation'], + ['metric3dv2', 'Metric3Dv2ForDepthEstimation'], +]); + +const MODEL_FOR_NORMAL_ESTIMATION_MAPPING_NAMES = new Map([['sapiens', 'SapiensForNormalEstimation']]); + +const MODEL_FOR_POSE_ESTIMATION_MAPPING_NAMES = new Map([['vitpose', 'VitPoseForPoseEstimation']]); + +// NOTE: This is custom to Transformers.js, and is necessary because certain models +// (e.g., CLIP) are split into vision and text components +const MODEL_FOR_IMAGE_FEATURE_EXTRACTION_MAPPING_NAMES = new Map([ + ['clip', 'CLIPVisionModelWithProjection'], + ['siglip', 'SiglipVisionModel'], + ['jina_clip', 'JinaCLIPVisionModel'], +]); + +export const MODEL_CLASS_TYPE_MAPPING = [ + // MODEL_MAPPING_NAMES: + [MODEL_MAPPING_NAMES_ENCODER_ONLY, MODEL_TYPES.EncoderOnly], + [MODEL_MAPPING_NAMES_ENCODER_DECODER, MODEL_TYPES.EncoderDecoder], + [MODEL_MAPPING_NAMES_DECODER_ONLY, MODEL_TYPES.DecoderOnlyWithoutHead], + [MODEL_MAPPING_NAMES_AUTO_ENCODER, MODEL_TYPES.AutoEncoder], + + [MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES, MODEL_TYPES.Seq2Seq], + [MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES, MODEL_TYPES.Seq2Seq], + [MODEL_FOR_CAUSAL_LM_MAPPING_NAMES, MODEL_TYPES.DecoderOnly], + [MODEL_FOR_MULTIMODALITY_MAPPING_NAMES, MODEL_TYPES.MultiModality], + [MODEL_FOR_MASKED_LM_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES, MODEL_TYPES.Vision2Seq], + [MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES, MODEL_TYPES.ImageTextToText], + [MODEL_FOR_AUDIO_TEXT_TO_TEXT_MAPPING_NAMES, MODEL_TYPES.AudioTextToText], + [MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_IMAGE_MATTING_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_TIME_SERIES_PREDICTION_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_IMAGE_TO_IMAGE_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_NORMAL_ESTIMATION_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_POSE_ESTIMATION_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_MASK_GENERATION_MAPPING_NAMES, MODEL_TYPES.MaskGeneration], + [MODEL_FOR_CTC_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES, MODEL_TYPES.Seq2Seq], + [MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + [MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], + + // Custom: + [MODEL_FOR_IMAGE_FEATURE_EXTRACTION_MAPPING_NAMES, MODEL_TYPES.EncoderOnly], +]; + +for (const [mappings, type] of MODEL_CLASS_TYPE_MAPPING) { + // @ts-ignore + for (const name of mappings.values()) { + MODEL_TYPE_MAPPING.set(name, type); + const model = ALL_MODEL_FILES[name]; + MODEL_CLASS_TO_NAME_MAPPING.set(model, name); + MODEL_NAME_TO_CLASS_MAPPING.set(name, model); + } +} + +const CUSTOM_MAPPING = [ + // OVERRIDE: + // TODO: Refactor to allow class to specify model + ['MusicgenForConditionalGeneration', ALL_MODEL_FILES.MusicgenForConditionalGeneration, MODEL_TYPES.Musicgen], + ['Phi3VForCausalLM', ALL_MODEL_FILES.Phi3VForCausalLM, MODEL_TYPES.Phi3V], + + ['CLIPTextModelWithProjection', ALL_MODEL_FILES.CLIPTextModelWithProjection, MODEL_TYPES.EncoderOnly], + ['SiglipTextModel', ALL_MODEL_FILES.SiglipTextModel, MODEL_TYPES.EncoderOnly], + ['JinaCLIPTextModel', ALL_MODEL_FILES.JinaCLIPTextModel, MODEL_TYPES.EncoderOnly], + ['ClapTextModelWithProjection', ALL_MODEL_FILES.ClapTextModelWithProjection, MODEL_TYPES.EncoderOnly], + ['ClapAudioModelWithProjection', ALL_MODEL_FILES.ClapAudioModelWithProjection, MODEL_TYPES.EncoderOnly], + + ['DacEncoderModel', ALL_MODEL_FILES.DacEncoderModel, MODEL_TYPES.EncoderOnly], + ['DacDecoderModel', ALL_MODEL_FILES.DacDecoderModel, MODEL_TYPES.EncoderOnly], + ['MimiEncoderModel', ALL_MODEL_FILES.MimiEncoderModel, MODEL_TYPES.EncoderOnly], + ['MimiDecoderModel', ALL_MODEL_FILES.MimiDecoderModel, MODEL_TYPES.EncoderOnly], + ['SnacEncoderModel', ALL_MODEL_FILES.SnacEncoderModel, MODEL_TYPES.EncoderOnly], + ['SnacDecoderModel', ALL_MODEL_FILES.SnacDecoderModel, MODEL_TYPES.EncoderOnly], + + [ + 'Gemma3nForConditionalGeneration', + ALL_MODEL_FILES.Gemma3nForConditionalGeneration, + MODEL_TYPES.ImageAudioTextToText, + ], + [ + 'Gemma4ForConditionalGeneration', + ALL_MODEL_FILES.Gemma4ForConditionalGeneration, + MODEL_TYPES.ImageAudioTextToText, + ], + ['SupertonicForConditionalGeneration', ALL_MODEL_FILES.SupertonicForConditionalGeneration, MODEL_TYPES.Supertonic], + ['ChatterboxModel', ALL_MODEL_FILES.ChatterboxModel, MODEL_TYPES.Chatterbox], + + [ + 'VoxtralRealtimeForConditionalGeneration', + ALL_MODEL_FILES.VoxtralRealtimeForConditionalGeneration, + MODEL_TYPES.VoxtralRealtime, + ], +]; +for (const [name, model, type] of CUSTOM_MAPPING) { + MODEL_TYPE_MAPPING.set(name, type); + MODEL_CLASS_TO_NAME_MAPPING.set(model, name); + MODEL_NAME_TO_CLASS_MAPPING.set(name, model); +} + +export const CUSTOM_ARCHITECTURES_MAPPING = new Map([ + ['modnet', MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES], + ['birefnet', MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES], + ['isnet', MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES], + ['ben', MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES], +]); +for (const [name, mapping] of CUSTOM_ARCHITECTURES_MAPPING.entries()) { + mapping.set(name, 'PreTrainedModel'); + MODEL_TYPE_MAPPING.set(name, MODEL_TYPES.EncoderOnly); + MODEL_NAME_TO_CLASS_MAPPING.set(name, PreTrainedModel); +} +export const CUSTOM_ARCHITECTURES = new Set(CUSTOM_ARCHITECTURES_MAPPING.keys()); + +// Default mappings +MODEL_TYPE_MAPPING.set('PreTrainedModel', MODEL_TYPES.EncoderOnly); +MODEL_CLASS_TO_NAME_MAPPING.set(PreTrainedModel, 'PreTrainedModel'); + +export const MODEL_MAPPINGS = { + MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES, + MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES, + MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES, + MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES, + MODEL_FOR_MASKED_LM_MAPPING_NAMES, + MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES, + MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES, + MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES, + MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES, + MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING_NAMES, + MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES, + MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES, + MODEL_FOR_MASK_GENERATION_MAPPING_NAMES, + MODEL_FOR_CTC_MAPPING_NAMES, + MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES, + MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES, + MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES, + MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES, + MODEL_FOR_IMAGE_MATTING_MAPPING_NAMES, + MODEL_FOR_IMAGE_TO_IMAGE_MAPPING_NAMES, + MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES, + MODEL_FOR_NORMAL_ESTIMATION_MAPPING_NAMES, + MODEL_FOR_POSE_ESTIMATION_MAPPING_NAMES, + MODEL_FOR_IMAGE_FEATURE_EXTRACTION_MAPPING_NAMES, + MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES, + MODEL_FOR_AUDIO_TEXT_TO_TEXT_MAPPING_NAMES, + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES, + MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES, + MODEL_FOR_CAUSAL_LM_MAPPING_NAMES, + MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES, +}; +registerTaskMappings(MODEL_MAPPINGS); diff --git a/packages/transformers/src/models/resnet/modeling_resnet.js b/packages/transformers/src/models/resnet/modeling_resnet.js new file mode 100644 index 0000000..00b7506 --- /dev/null +++ b/packages/transformers/src/models/resnet/modeling_resnet.js @@ -0,0 +1,24 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +/** + * An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. + */ +export class ResNetPreTrainedModel extends PreTrainedModel {} + +/** + * The bare ResNet model outputting raw features without any specific head on top. + */ +export class ResNetModel extends ResNetPreTrainedModel {} + +/** + * ResNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. + */ +export class ResNetForImageClassification extends ResNetPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/rf_detr/modeling_rf_detr.js b/packages/transformers/src/models/rf_detr/modeling_rf_detr.js new file mode 100644 index 0000000..2bd887e --- /dev/null +++ b/packages/transformers/src/models/rf_detr/modeling_rf_detr.js @@ -0,0 +1,15 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { RTDetrObjectDetectionOutput } from '../rt_detr/modeling_rt_detr.js'; + +export class RFDetrPreTrainedModel extends PreTrainedModel {} +export class RFDetrModel extends RFDetrPreTrainedModel {} +export class RFDetrForObjectDetection extends RFDetrPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new RFDetrObjectDetectionOutput(await super._call(model_inputs)); + } +} + +export class RFDetrObjectDetectionOutput extends RTDetrObjectDetectionOutput {} diff --git a/packages/transformers/src/models/roberta/modeling_roberta.js b/packages/transformers/src/models/roberta/modeling_roberta.js new file mode 100644 index 0000000..8ca55ba --- /dev/null +++ b/packages/transformers/src/models/roberta/modeling_roberta.js @@ -0,0 +1,70 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { + MaskedLMOutput, + QuestionAnsweringModelOutput, + SequenceClassifierOutput, + TokenClassifierOutput, +} from '../modeling_outputs.js'; + +export class RobertaPreTrainedModel extends PreTrainedModel {} +export class RobertaModel extends RobertaPreTrainedModel {} + +/** + * RobertaForMaskedLM class for performing masked language modeling on Roberta models. + */ +export class RobertaForMaskedLM extends RobertaPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +/** + * RobertaForSequenceClassification class for performing sequence classification on Roberta models. + */ +export class RobertaForSequenceClassification extends RobertaPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * RobertaForTokenClassification class for performing token classification on Roberta models. + */ +export class RobertaForTokenClassification extends RobertaPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * RobertaForQuestionAnswering class for performing question answering on Roberta models. + */ +export class RobertaForQuestionAnswering extends RobertaPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/roberta/tokenization_roberta.js b/packages/transformers/src/models/roberta/tokenization_roberta.js new file mode 100644 index 0000000..01651f9 --- /dev/null +++ b/packages/transformers/src/models/roberta/tokenization_roberta.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class RobertaTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/roformer/modeling_roformer.js b/packages/transformers/src/models/roformer/modeling_roformer.js new file mode 100644 index 0000000..e823c64 --- /dev/null +++ b/packages/transformers/src/models/roformer/modeling_roformer.js @@ -0,0 +1,77 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { + QuestionAnsweringModelOutput, + SequenceClassifierOutput, + TokenClassifierOutput, + MaskedLMOutput, +} from '../modeling_outputs.js'; + +export class RoFormerPreTrainedModel extends PreTrainedModel {} + +/** + * The bare RoFormer Model transformer outputting raw hidden-states without any specific head on top. + */ +export class RoFormerModel extends RoFormerPreTrainedModel {} + +/** + * RoFormer Model with a `language modeling` head on top. + */ +export class RoFormerForMaskedLM extends RoFormerPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for masked language modeling. + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +/** + * RoFormer Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) + */ +export class RoFormerForSequenceClassification extends RoFormerPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * RoFormer Model with a token classification head on top (a linear layer on top of the hidden-states output) + * e.g. for Named-Entity-Recognition (NER) tasks. + */ +export class RoFormerForTokenClassification extends RoFormerPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * RoFormer Model with a span classification head on top for extractive question-answering tasks like SQuAD + * (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). + */ +export class RoFormerForQuestionAnswering extends RoFormerPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for question answering. + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} +// TODO: Add RoFormerForCausalLM and RoFormerForMultipleChoice diff --git a/packages/transformers/src/models/roformer/tokenization_roformer.js b/packages/transformers/src/models/roformer/tokenization_roformer.js new file mode 100644 index 0000000..079e61d --- /dev/null +++ b/packages/transformers/src/models/roformer/tokenization_roformer.js @@ -0,0 +1,5 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class RoFormerTokenizer extends PreTrainedTokenizer { + return_token_type_ids = true; +} diff --git a/packages/transformers/src/models/rt_detr/image_processing_rt_detr.js b/packages/transformers/src/models/rt_detr/image_processing_rt_detr.js new file mode 100644 index 0000000..62e85be --- /dev/null +++ b/packages/transformers/src/models/rt_detr/image_processing_rt_detr.js @@ -0,0 +1,8 @@ +import { ImageProcessor, post_process_object_detection } from '../../image_processors_utils.js'; + +export class RTDetrImageProcessor extends ImageProcessor { + /** @type {typeof post_process_object_detection} */ + post_process_object_detection(...args) { + return post_process_object_detection(...args); + } +} diff --git a/packages/transformers/src/models/rt_detr/modeling_rt_detr.js b/packages/transformers/src/models/rt_detr/modeling_rt_detr.js new file mode 100644 index 0000000..81191d8 --- /dev/null +++ b/packages/transformers/src/models/rt_detr/modeling_rt_detr.js @@ -0,0 +1,28 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { ModelOutput } from '../modeling_outputs.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class RTDetrPreTrainedModel extends PreTrainedModel {} +export class RTDetrModel extends RTDetrPreTrainedModel {} +export class RTDetrForObjectDetection extends RTDetrPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new RTDetrObjectDetectionOutput(await super._call(model_inputs)); + } +} + +export class RTDetrObjectDetectionOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.logits Classification logits (including no-object) for all queries. + * @param {Tensor} output.pred_boxes Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). + * These values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding possible padding). + */ + constructor({ logits, pred_boxes }) { + super(); + this.logits = logits; + this.pred_boxes = pred_boxes; + } +} diff --git a/packages/transformers/src/models/rt_detr_v2/modeling_rt_detr_v2.js b/packages/transformers/src/models/rt_detr_v2/modeling_rt_detr_v2.js new file mode 100644 index 0000000..f862ef9 --- /dev/null +++ b/packages/transformers/src/models/rt_detr_v2/modeling_rt_detr_v2.js @@ -0,0 +1,15 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { RTDetrObjectDetectionOutput } from '../rt_detr/modeling_rt_detr.js'; + +export class RTDetrV2PreTrainedModel extends PreTrainedModel {} +export class RTDetrV2Model extends RTDetrV2PreTrainedModel {} +export class RTDetrV2ForObjectDetection extends RTDetrV2PreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new RTDetrV2ObjectDetectionOutput(await super._call(model_inputs)); + } +} + +export class RTDetrV2ObjectDetectionOutput extends RTDetrObjectDetectionOutput {} diff --git a/packages/transformers/src/models/sam/image_processing_sam.js b/packages/transformers/src/models/sam/image_processing_sam.js new file mode 100644 index 0000000..4200350 --- /dev/null +++ b/packages/transformers/src/models/sam/image_processing_sam.js @@ -0,0 +1,230 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; +import { calculateDimensions } from '../../utils/core.js'; + +import { interpolate_4d, Tensor } from '../../utils/tensor.js'; + +/** + * @typedef {Object} SamImageProcessorResult + * @property {Tensor} pixel_values + * @property {import("../../image_processors_utils.js").HeightWidth[]} original_sizes + * @property {import("../../image_processors_utils.js").HeightWidth[]} reshaped_input_sizes + * @property {Tensor} [input_points] + * @property {Tensor} [input_labels] + * @property {Tensor} [input_boxes] + */ + +export class SamImageProcessor extends ImageProcessor { + /** + * + * @param {any} input_points + * @param {import("../../image_processors_utils.js").HeightWidth[]} original_sizes + * @param {import("../../image_processors_utils.js").HeightWidth[]} reshaped_input_sizes + * @returns {Tensor} + */ + reshape_input_points(input_points, original_sizes, reshaped_input_sizes, is_bounding_box = false) { + // Make deep copy to avoid altering user's input + input_points = structuredClone(input_points); + let shape = calculateDimensions(input_points); + + // TODO: add support for 2D input_points + if (shape.length === 3) { + // Correct user's input + if (!is_bounding_box) { + shape = [1, ...shape]; + } + input_points = [input_points]; + } else if (shape.length !== 4) { + throw Error( + 'The input_points must be a 4D tensor of shape `batch_size`, `point_batch_size`, `nb_points_per_image`, `2`.', + ); + } + + // Reshape input points + for (let i = 0; i < input_points.length; ++i) { + // batch_size + const [originalHeight, originalWidth] = original_sizes[i]; + const [reshapedHeight, reshapedWidth] = reshaped_input_sizes[i]; + + const resizeFactors = [reshapedWidth / originalWidth, reshapedHeight / originalHeight]; + + for (let j = 0; j < input_points[i].length; ++j) { + // point_batch_size + for (let k = 0; k < input_points[i][j].length; ++k) { + // nb_points_per_image + for (let w = 0; w < input_points[i][j][k].length; ++w) { + // 2 or 4 + input_points[i][j][k][w] *= resizeFactors[w % 2]; + } + } + } + } + + return new Tensor('float32', Float32Array.from(input_points.flat(Infinity)), shape); + } + + /** + * + * @param {any} input_labels + * @param {Tensor} input_points + * @returns {Tensor} + */ + add_input_labels(input_labels, input_points) { + let shape = calculateDimensions(input_labels); + if (shape.length === 2) { + // Correct user's input + shape = [1, ...shape]; + input_labels = [input_labels]; + } else if (shape.length !== 3) { + throw Error( + 'The input_points must be a 4D tensor of shape `batch_size`, `point_batch_size`, `nb_points_per_image`, `2`.', + ); + } + + if (shape.some((x, i) => x !== input_points.dims[i])) { + throw Error(`The first ${shape.length} dimensions of 'input_points' and 'input_labels' must be the same.`); + } + return new Tensor('int64', input_labels.flat(Infinity).map(BigInt), shape); + } + /** + * @param {any[]} images The URL(s) of the image(s) to extract features from. + * @param {Object} [options] Additional options for the processor. + * @param {any} [options.input_points=null] A 3D or 4D array, representing the input points provided by the user. + * - 3D: `[point_batch_size, nb_points_per_image, 2]`. In this case, `batch_size` is assumed to be 1. + * - 4D: `[batch_size, point_batch_size, nb_points_per_image, 2]`. + * @param {any} [options.input_labels=null] A 2D or 3D array, representing the input labels for the points, used by the prompt encoder to encode the prompt. + * - 2D: `[point_batch_size, nb_points_per_image]`. In this case, `batch_size` is assumed to be 1. + * - 3D: `[batch_size, point_batch_size, nb_points_per_image]`. + * @param {number[][][]} [options.input_boxes=null] A 3D array of shape `(batch_size, num_boxes, 4)`, representing the input boxes provided by the user. + * This is used by the prompt encoder to encode the prompt. Generally yields to much better generated masks. + * The processor will generate a tensor, with each dimension corresponding respectively to the image batch size, + * the number of boxes per image and the coordinates of the top left and botton right point of the box. + * In the order (`x1`, `y1`, `x2`, `y2`): + * - `x1`: the x coordinate of the top left point of the input box + * - `y1`: the y coordinate of the top left point of the input box + * - `x2`: the x coordinate of the bottom right point of the input box + * - `y2`: the y coordinate of the bottom right point of the input box + * @returns {Promise} + */ + async _call(images, { input_points = null, input_labels = null, input_boxes = null } = {}) { + // TODO allow user to use preprocessed images + /** @type {SamImageProcessorResult} */ + const processed = await super._call(images); + + if (input_points) { + processed.input_points = this.reshape_input_points( + input_points, + processed.original_sizes, + processed.reshaped_input_sizes, + ); + } + + if (input_labels) { + if (!processed.input_points) { + throw Error('`input_points` must be provided if `input_labels` are provided.'); + } + processed.input_labels = this.add_input_labels(input_labels, processed.input_points); + } + + if (input_boxes) { + processed.input_boxes = this.reshape_input_points( + input_boxes, + processed.original_sizes, + processed.reshaped_input_sizes, + true, + ); + } + + return processed; + } + + /** + * Remove padding and upscale masks to the original image size. + * @param {Tensor} masks Batched masks from the mask_decoder in (batch_size, num_channels, height, width) format. + * @param {[number, number][]} original_sizes The original sizes of each image before it was resized to the model's expected input shape, in (height, width) format. + * @param {[number, number][]} reshaped_input_sizes The size of each image as it is fed to the model, in (height, width) format. Used to remove padding. + * @param {Object} options Optional parameters for post-processing. + * @param {number} [options.mask_threshold] The threshold to use for binarizing the masks. + * @param {boolean} [options.binarize] Whether to binarize the masks. + * @param {Object} [options.pad_size] The target size the images were padded to before being passed to the model. If `null`, the target size is assumed to be the processor's `pad_size`. + * @param {number} [options.pad_size.height] The height the images were padded to. + * @param {number} [options.pad_size.width] The width the images were padded to. + * @returns {Promise} Batched masks in batch_size, num_channels, height, width) format, where (height, width) is given by original_size. + */ + async post_process_masks( + masks, + original_sizes, + reshaped_input_sizes, + { mask_threshold = 0.0, binarize = true, pad_size = null } = {}, + ) { + // masks: [1, 1, 3, 256, 256] + + const output_masks = []; + + pad_size = pad_size ?? this.pad_size ?? this.size; + + /** @type {[number, number]} */ + const target_image_size = [pad_size.height, pad_size.width]; + + for (let i = 0; i < original_sizes.length; ++i) { + const original_size = original_sizes[i]; + const reshaped_input_size = reshaped_input_sizes[i]; + + // Upscale mask to padded size + let interpolated_mask = await interpolate_4d(masks[i], { mode: 'bilinear', size: target_image_size }); + + // Crop mask + interpolated_mask = interpolated_mask.slice( + null, + null, + [0, reshaped_input_size[0]], + [0, reshaped_input_size[1]], + ); + + // Downscale mask + interpolated_mask = await interpolate_4d(interpolated_mask, { mode: 'bilinear', size: original_size }); + + if (binarize) { + const data = interpolated_mask.data; + const binarizedMaskData = new Uint8Array(data.length); + for (let i = 0; i < data.length; ++i) { + if (data[i] > mask_threshold) { + binarizedMaskData[i] = 1; + } + } + interpolated_mask = new Tensor('bool', binarizedMaskData, interpolated_mask.dims); + } + + output_masks.push(interpolated_mask); + } + + return output_masks; + } + + /** + * Generates a list of crop boxes of different sizes. Each layer has (2**i)**2 boxes for the ith layer. + * @param {import("../../utils/image.js").RawImage} image Input original image + * @param {number} target_size Target size of the resized image + * @param {Object} options Options for generating crop boxes + * @param {number} [options.crop_n_layers] If >0, mask prediction will be run again on crops of the image. + * Sets the number of layers to run, where each layer has 2**i_layer number of image crops. + * @param {number} [options.overlap_ratio] Sets the degree to which crops overlap. In the first crop layer, + * crops will overlap by this fraction of the image length. Later layers with more crops scale down this overlap. + * @param {number} [options.points_per_crop] Number of points to sample from each crop. + * @param {number} [options.crop_n_points_downscale_factor] The number of points-per-side sampled in layer n is + * scaled down by crop_n_points_downscale_factor**n. + * @returns {Object} An object containing the crop boxes, number of points per crop, cropped images, and input labels. + */ + generate_crop_boxes( + image, + target_size, + { + crop_n_layers = 0, + overlap_ratio = 512 / 1500, + points_per_crop = 32, + crop_n_points_downscale_factor = 1, + } = {}, + ) { + // TODO: Implement + // return { crop_boxes, points_per_crop, cropped_images, input_labels } + } +} diff --git a/packages/transformers/src/models/sam/modeling_sam.js b/packages/transformers/src/models/sam/modeling_sam.js new file mode 100644 index 0000000..4ec0340 --- /dev/null +++ b/packages/transformers/src/models/sam/modeling_sam.js @@ -0,0 +1,145 @@ +import { PreTrainedModel, encoder_forward } from '../modeling_utils.js'; +import { ones } from '../../utils/tensor.js'; +import { sessionRun } from '../session.js'; +import { ModelOutput } from '../modeling_outputs.js'; +import { Tensor } from '../../utils/tensor.js'; + +/** + * Base class for Segment-Anything model's output. + */ +export class SamImageSegmentationOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.iou_scores The output logits of the model. + * @param {Tensor} output.pred_masks Predicted boxes. + */ + constructor({ iou_scores, pred_masks }) { + super(); + this.iou_scores = iou_scores; + this.pred_masks = pred_masks; + } +} + +export class SamPreTrainedModel extends PreTrainedModel {} + +/** + * Segment Anything Model (SAM) for generating segmentation masks, given an input image + * and optional 2D location and bounding boxes. + * + * **Example:** Perform mask generation w/ `Xenova/sam-vit-base`. + * ```javascript + * import { SamModel, AutoProcessor, RawImage } from '@huggingface/transformers'; + * + * const model = await SamModel.from_pretrained('Xenova/sam-vit-base'); + * const processor = await AutoProcessor.from_pretrained('Xenova/sam-vit-base'); + * + * const img_url = 'https://huggingface.co/ybelkada/segment-anything/resolve/main/assets/car.png'; + * const raw_image = await RawImage.read(img_url); + * const input_points = [[[450, 600]]] // 2D localization of a window + * + * const inputs = await processor(raw_image, { input_points }); + * const outputs = await model(inputs); + * + * const masks = await processor.post_process_masks(outputs.pred_masks, inputs.original_sizes, inputs.reshaped_input_sizes); + * // [ + * // Tensor { + * // dims: [ 1, 3, 1764, 2646 ], + * // type: 'bool', + * // data: Uint8Array(14002632) [ ... ], + * // size: 14002632 + * // } + * // ] + * const scores = outputs.iou_scores; + * // Tensor { + * // dims: [ 1, 1, 3 ], + * // type: 'float32', + * // data: Float32Array(3) [ + * // 0.8892380595207214, + * // 0.9311248064041138, + * // 0.983696699142456 + * // ], + * // size: 3 + * // } + * ``` + */ +export class SamModel extends SamPreTrainedModel { + /** + * Compute image embeddings and positional image embeddings, given the pixel values of an image. + * @param {Object} model_inputs Object containing the model inputs. + * @param {Tensor} model_inputs.pixel_values Pixel values obtained using a `SamProcessor`. + * @returns {Promise<{ image_embeddings: Tensor, image_positional_embeddings: Tensor }>} The image embeddings and positional image embeddings. + */ + async get_image_embeddings({ pixel_values }) { + // in: + // - pixel_values: tensor.float32[batch_size,3,1024,1024] + // + // out: + // - image_embeddings: tensor.float32[batch_size,256,64,64] + // - image_positional_embeddings: tensor.float32[batch_size,256,64,64] + return await encoder_forward(this, { pixel_values }); + } + + /** + * @typedef {Object} SamModelInputs Object containing the model inputs. + * @property {Tensor} pixel_values Pixel values as a Tensor with shape `(batch_size, num_channels, height, width)`. + * These can be obtained using a `SamProcessor`. + * @property {Tensor} [input_points] Input 2D spatial points with shape `(batch_size, num_points, 2)`. + * This is used by the prompt encoder to encode the prompt. + * @property {Tensor} [input_labels] Input labels for the points, as a Tensor of shape `(batch_size, point_batch_size, num_points)`. + * This is used by the prompt encoder to encode the prompt. There are 4 types of labels: + * - `1`: the point is a point that contains the object of interest + * - `0`: the point is a point that does not contain the object of interest + * - `-1`: the point corresponds to the background + * - `-10`: the point is a padding point, thus should be ignored by the prompt encoder + * @property {Tensor} [input_boxes] Input bounding boxes with shape `(batch_size, num_boxes, 4)`. + * @property {Tensor} [image_embeddings] Image embeddings used by the mask decoder. + * @property {Tensor} [image_positional_embeddings] Image positional embeddings used by the mask decoder. + */ + + /** + * @param {SamModelInputs} model_inputs Object containing the model inputs. + * @returns {Promise} The output of the model. + */ + async forward(model_inputs) { + if (!model_inputs.image_embeddings || !model_inputs.image_positional_embeddings) { + // Compute the image embeddings if they are missing + model_inputs = { + ...model_inputs, + ...(await this.get_image_embeddings(model_inputs)), + }; + } else { + model_inputs = { ...model_inputs }; + } + + // Set default input labels if they are missing + model_inputs.input_labels ??= ones(model_inputs.input_points.dims.slice(0, -1)); + + const decoder_inputs = { + image_embeddings: model_inputs.image_embeddings, + image_positional_embeddings: model_inputs.image_positional_embeddings, + }; + if (model_inputs.input_points) { + decoder_inputs.input_points = model_inputs.input_points; + } + if (model_inputs.input_labels) { + decoder_inputs.input_labels = model_inputs.input_labels; + } + if (model_inputs.input_boxes) { + decoder_inputs.input_boxes = model_inputs.input_boxes; + } + + // Returns: + // - iou_scores: tensor.float32[batch_size,point_batch_size,3] + // - pred_masks: tensor.float32[batch_size,point_batch_size,3,256,256] + return await sessionRun(this.sessions['prompt_encoder_mask_decoder'], decoder_inputs); + } + + /** + * Runs the model with the provided inputs + * @param {Object} model_inputs Model inputs + * @returns {Promise} Object containing segmentation outputs + */ + async _call(model_inputs) { + return new SamImageSegmentationOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/sam/processing_sam.js b/packages/transformers/src/models/sam/processing_sam.js new file mode 100644 index 0000000..2db258c --- /dev/null +++ b/packages/transformers/src/models/sam/processing_sam.js @@ -0,0 +1,20 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoImageProcessor } from '../auto/image_processing_auto.js'; + +export class SamProcessor extends Processor { + static image_processor_class = AutoImageProcessor; + + async _call(...args) { + return await this.image_processor(...args); + } + + post_process_masks(...args) { + // @ts-ignore + return this.image_processor.post_process_masks(...args); + } + + reshape_input_points(...args) { + // @ts-ignore + return this.image_processor.reshape_input_points(...args); + } +} diff --git a/packages/transformers/src/models/sam2/image_processing_sam2.js b/packages/transformers/src/models/sam2/image_processing_sam2.js new file mode 100644 index 0000000..14e7707 --- /dev/null +++ b/packages/transformers/src/models/sam2/image_processing_sam2.js @@ -0,0 +1 @@ +export { SamImageProcessor as Sam2ImageProcessor } from '../sam/image_processing_sam.js'; diff --git a/packages/transformers/src/models/sam2/modeling_sam2.js b/packages/transformers/src/models/sam2/modeling_sam2.js new file mode 100644 index 0000000..fb4e891 --- /dev/null +++ b/packages/transformers/src/models/sam2/modeling_sam2.js @@ -0,0 +1,95 @@ +import { PreTrainedModel, encoder_forward } from '../modeling_utils.js'; +import { sessionRun } from '../session.js'; +import { ModelOutput } from '../modeling_outputs.js'; +import { ones, full } from '../../utils/tensor.js'; +import { pick } from '../../utils/core.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class Sam2ImageSegmentationOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.iou_scores The output logits of the model. + * @param {Tensor} output.pred_masks Predicted boxes. + * @param {Tensor} output.object_score_logits Logits for the object score, indicating if an object is present. + */ + constructor({ iou_scores, pred_masks, object_score_logits }) { + super(); + this.iou_scores = iou_scores; + this.pred_masks = pred_masks; + this.object_score_logits = object_score_logits; + } +} + +export class Sam2PreTrainedModel extends PreTrainedModel {} +export class Sam2Model extends Sam2PreTrainedModel { + /** + * Compute image embeddings and positional image embeddings, given the pixel values of an image. + * @param {Object} model_inputs Object containing the model inputs. + * @param {Tensor} model_inputs.pixel_values Pixel values obtained using a `Sam2Processor`. + * @returns {Promise>} The image embeddings. + */ + async get_image_embeddings({ pixel_values }) { + // in: + // - pixel_values: tensor.float32[batch_size,3,1024,1024] + // + // out: + // - image_embeddings.0: tensor.float32[batch_size,32,256,256] + // - image_embeddings.1: tensor.float32[batch_size,64,128,128] + // - image_embeddings.2: tensor.float32[batch_size,256,64,64] + return await encoder_forward(this, { pixel_values }); + } + + async forward(model_inputs) { + // @ts-expect-error ts(2339) + const { num_feature_levels } = this.config.vision_config; + const image_embeddings_name = Array.from({ length: num_feature_levels }, (_, i) => `image_embeddings.${i}`); + + if (image_embeddings_name.some((name) => !model_inputs[name])) { + // Compute the image embeddings if they are missing + model_inputs = { + ...model_inputs, + ...(await this.get_image_embeddings(model_inputs)), + }; + } else { + model_inputs = { ...model_inputs }; + } + + if (model_inputs.input_points) { + if (model_inputs.input_boxes && model_inputs.input_boxes.dims[1] !== 1) { + throw new Error( + 'When both `input_points` and `input_boxes` are provided, the number of boxes per image must be 1.', + ); + } + const shape = model_inputs.input_points.dims; + model_inputs.input_labels ??= ones(shape.slice(0, -1)); + model_inputs.input_boxes ??= full([shape[0], 0, 4], 0.0); + } else if (model_inputs.input_boxes) { + // only boxes + const shape = model_inputs.input_boxes.dims; + model_inputs.input_labels = full([shape[0], shape[1], 0], -1n); + model_inputs.input_points = full([shape[0], 1, 0, 2], 0.0); + } else { + throw new Error('At least one of `input_points` or `input_boxes` must be provided.'); + } + + const prompt_encoder_mask_decoder_session = this.sessions['prompt_encoder_mask_decoder']; + const decoder_inputs = pick(model_inputs, prompt_encoder_mask_decoder_session.inputNames); + + // Returns: + // - iou_scores: tensor.float32[batch_size,num_boxes_or_points,3] + // - pred_masks: tensor.float32[batch_size,num_boxes_or_points,3,256,256] + // - object_score_logits: tensor.float32[batch_size,num_boxes_or_points,1] + return await sessionRun(prompt_encoder_mask_decoder_session, decoder_inputs); + } + + /** + * Runs the model with the provided inputs + * @param {Object} model_inputs Model inputs + * @returns {Promise} Object containing segmentation outputs + */ + async _call(model_inputs) { + return new Sam2ImageSegmentationOutput(await super._call(model_inputs)); + } +} +export class EdgeTamModel extends Sam2Model {} // NOTE: extends Sam2Model +export class Sam3TrackerModel extends Sam2Model {} // NOTE: extends Sam2Model diff --git a/packages/transformers/src/models/sam2/processing_sam2.js b/packages/transformers/src/models/sam2/processing_sam2.js new file mode 100644 index 0000000..30622bf --- /dev/null +++ b/packages/transformers/src/models/sam2/processing_sam2.js @@ -0,0 +1,4 @@ +import { SamProcessor } from '../sam/processing_sam.js'; + +export class Sam2Processor extends SamProcessor {} +export class Sam2VideoProcessor extends Sam2Processor {} diff --git a/packages/transformers/src/models/sam3/image_processing_sam3.js b/packages/transformers/src/models/sam3/image_processing_sam3.js new file mode 100644 index 0000000..6c2f60f --- /dev/null +++ b/packages/transformers/src/models/sam3/image_processing_sam3.js @@ -0,0 +1 @@ +export { Sam2ImageProcessor as Sam3ImageProcessor } from '../sam2/image_processing_sam2.js'; diff --git a/packages/transformers/src/models/sapiens/image_processing_sapiens.js b/packages/transformers/src/models/sapiens/image_processing_sapiens.js new file mode 100644 index 0000000..08746f9 --- /dev/null +++ b/packages/transformers/src/models/sapiens/image_processing_sapiens.js @@ -0,0 +1,9 @@ +import { ImageProcessor, post_process_semantic_segmentation } from '../../image_processors_utils.js'; + +export class SapiensImageProcessor extends ImageProcessor { + /** @type {typeof post_process_semantic_segmentation} */ + post_process_semantic_segmentation(...args) { + return post_process_semantic_segmentation(...args); + } +} +export class SapiensFeatureExtractor extends SapiensImageProcessor {} diff --git a/packages/transformers/src/models/sapiens/modeling_sapiens.js b/packages/transformers/src/models/sapiens/modeling_sapiens.js new file mode 100644 index 0000000..98910a7 --- /dev/null +++ b/packages/transformers/src/models/sapiens/modeling_sapiens.js @@ -0,0 +1,6 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class SapiensPreTrainedModel extends PreTrainedModel {} +export class SapiensForSemanticSegmentation extends SapiensPreTrainedModel {} +export class SapiensForDepthEstimation extends SapiensPreTrainedModel {} +export class SapiensForNormalEstimation extends SapiensPreTrainedModel {} diff --git a/packages/transformers/src/models/seamless_m4t/feature_extraction_seamless_m4t.js b/packages/transformers/src/models/seamless_m4t/feature_extraction_seamless_m4t.js new file mode 100644 index 0000000..43f364d --- /dev/null +++ b/packages/transformers/src/models/seamless_m4t/feature_extraction_seamless_m4t.js @@ -0,0 +1,159 @@ +import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { Tensor } from '../../utils/tensor.js'; +import { mel_filter_bank, spectrogram, window_function } from '../../utils/audio.js'; + +export class SeamlessM4TFeatureExtractor extends FeatureExtractor { + constructor(config) { + super(config); + + const sampling_rate = this.config.sampling_rate; + const mel_filters = mel_filter_bank( + 257, // num_frequency_bins + this.config.num_mel_bins, // num_mel_filters + 20, // min_frequency + Math.floor(sampling_rate / 2), // max_frequency + sampling_rate, // sampling_rate + null, // norm + 'kaldi', // mel_scale + true, // triangularize_in_mel_space + ); + this.mel_filters = mel_filters; + + this.window = window_function(400, 'povey', { + periodic: false, + }); + } + + /** + * Computes the log-Mel spectrogram of the provided audio waveform. + * @param {Float32Array|Float64Array} waveform The audio waveform to process. + * @param {number} max_length The maximum number of frames to return. + * @returns {Promise} An object containing the log-Mel spectrogram data as a Float32Array and its dimensions as an array of numbers. + */ + async _extract_fbank_features(waveform, max_length) { + // NOTE: We don't pad/truncate since that is passed in as `max_num_frames` + + // Kaldi compliance: 16-bit signed integers + // 32768 == 2 ** 15 + waveform = waveform.map((/** @type {number} */ x) => x * 32768); + + return spectrogram( + waveform, + this.window, // window + 400, // frame_length + 160, // hop_length + { + fft_length: 512, + power: 2.0, + center: false, + preemphasis: 0.97, + mel_filters: this.mel_filters, + log_mel: 'log', + mel_floor: 1.192092955078125e-7, + remove_dc_offset: true, + + // Custom + max_num_frames: max_length, + transpose: true, + }, + ); + } + + /** + * Asynchronously extracts features from a given audio using the provided configuration. + * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array. + * @param {Object} options Optional parameters for feature extraction. + * @param {boolean} [options.padding=true] Whether to pad the sequence to a multiple of `pad_to_multiple_of`. + * @param {number} [options.pad_to_multiple_of=2] The number to pad the sequence to a multiple of. + * @param {boolean} [options.do_normalize_per_mel_bins=true] Whether or not to zero-mean unit-variance normalize the input per mel-channel. + * @param {boolean} [options.return_attention_mask=true] Whether to return the attention mask. + * @returns {Promise<{ input_features: Tensor, attention_mask?: Tensor }>} A Promise resolving to an object containing the extracted input features and attention masks as Tensors. + */ + async _call( + audio, + { padding = true, pad_to_multiple_of = 2, do_normalize_per_mel_bins = true, return_attention_mask = true } = {}, + ) { + validate_audio_inputs(audio, 'SeamlessM4TFeatureExtractor'); + + let features = await this._extract_fbank_features(audio, this.config.max_length); + + if (do_normalize_per_mel_bins) { + const [num_features, feature_size] = features.dims; + const data = features.data; + for (let i = 0; i < feature_size; ++i) { + let sum = 0; + for (let j = 0; j < num_features; ++j) { + sum += data[j * feature_size + i]; + } + + const mean = sum / num_features; + + let variance = 0; + for (let j = 0; j < num_features; ++j) { + variance += (data[j * feature_size + i] - mean) ** 2; + } + variance /= num_features - 1; // NOTE: We use ddof=1 + + const std = Math.sqrt(variance + 1e-7); + for (let j = 0; j < num_features; ++j) { + const index = j * feature_size + i; + data[index] = (data[index] - mean) / std; + } + } + } + + let padded_attention_mask; + if (padding) { + const [num_frames, num_channels] = features.dims; + const data = /** @type {Float32Array} */ (features.data); + + const pad_size = num_frames % pad_to_multiple_of; + if (pad_size > 0) { + const padded_data = new Float32Array(num_channels * (num_frames + pad_size)); + padded_data.set(data); + padded_data.fill(this.config.padding_value, data.length); + + const numPaddedFrames = num_frames + pad_size; + features = new Tensor(features.type, padded_data, [numPaddedFrames, num_channels]); + + if (return_attention_mask) { + padded_attention_mask = new Tensor('int64', new BigInt64Array(numPaddedFrames), [ + 1, + numPaddedFrames, + ]); + /** @type {BigInt64Array} */ (padded_attention_mask.data).fill(1n, 0, num_frames); + } + } + } + + const [num_frames, num_channels] = features.dims; + + const stride = this.config.stride; + const remainder = num_frames % stride; + if (remainder !== 0) { + throw new Error(`The number of frames (${num_frames}) must be a multiple of the stride (${stride}).`); + } + + const input_features = features.view(1, Math.floor(num_frames / stride), num_channels * stride); + + const result = { input_features }; + + if (return_attention_mask) { + const reshapedNumFrames = input_features.dims[1]; + + const attention_mask_data = new BigInt64Array(reshapedNumFrames); + + if (padded_attention_mask) { + const padded_attention_mask_data = padded_attention_mask.data; + for (let i = 1, j = 0; i < num_frames; i += stride, ++j) { + attention_mask_data[j] = padded_attention_mask_data[i]; + } + } else { + attention_mask_data.fill(1n); + } + result.attention_mask = new Tensor('int64', attention_mask_data, [1, reshapedNumFrames]); + } + + return result; + } +} diff --git a/packages/transformers/src/models/segformer/image_processing_segformer.js b/packages/transformers/src/models/segformer/image_processing_segformer.js new file mode 100644 index 0000000..aec208d --- /dev/null +++ b/packages/transformers/src/models/segformer/image_processing_segformer.js @@ -0,0 +1,9 @@ +import { ImageProcessor, post_process_semantic_segmentation } from '../../image_processors_utils.js'; + +export class SegformerImageProcessor extends ImageProcessor { + /** @type {typeof post_process_semantic_segmentation} */ + post_process_semantic_segmentation(...args) { + return post_process_semantic_segmentation(...args); + } +} +export class SegformerFeatureExtractor extends SegformerImageProcessor {} diff --git a/packages/transformers/src/models/segformer/modeling_segformer.js b/packages/transformers/src/models/segformer/modeling_segformer.js new file mode 100644 index 0000000..c05f4c8 --- /dev/null +++ b/packages/transformers/src/models/segformer/modeling_segformer.js @@ -0,0 +1,18 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class SegformerPreTrainedModel extends PreTrainedModel {} + +/** + * The bare SegFormer encoder (Mix-Transformer) outputting raw hidden-states without any specific head on top. + */ +export class SegformerModel extends SegformerPreTrainedModel {} + +/** + * SegFormer Model transformer with an image classification head on top (a linear layer on top of the final hidden states) e.g. for ImageNet. + */ +export class SegformerForImageClassification extends SegformerPreTrainedModel {} + +/** + * SegFormer Model transformer with an all-MLP decode head on top e.g. for ADE20k, CityScapes. + */ +export class SegformerForSemanticSegmentation extends SegformerPreTrainedModel {} diff --git a/packages/transformers/src/models/session.js b/packages/transformers/src/models/session.js new file mode 100644 index 0000000..04f6c4c --- /dev/null +++ b/packages/transformers/src/models/session.js @@ -0,0 +1,285 @@ +import { + createInferenceSession, + deviceToExecutionProviders, + isONNXProxy, + isONNXTensor, + runInferenceSession, +} from '../backends/onnx.js'; +import { getCacheNames } from '../configs.js'; +import { DATA_TYPES, DEFAULT_DTYPE_SUFFIX_MAPPING, isWebGpuFp16Supported, selectDtype } from '../utils/dtypes.js'; +import { selectDevice } from '../utils/devices.js'; +import { apis } from '../env.js'; +import { getCoreModelFile, getModelDataFiles } from '../utils/model-loader.js'; +import { Tensor } from '../utils/tensor.js'; +import { logger } from '../utils/logger.js'; + +/** + * Constructs an InferenceSession using a model file located at the specified path. + * @param {string} pretrained_model_name_or_path The path to the directory containing the model file. + * @param {string} fileName The name of the model file. + * @param {import('../utils/hub.js').PretrainedModelOptions} options Additional options for loading the model. + * @param {boolean} [cache_config=false] Whether to compute cache shapes for GPU-pinned outputs. + * @param {string} [session_name] The name of the session (used to determine cache shapes). + * @returns {Promise<{buffer_or_path: Uint8Array|string, session_options: Object, session_config: Object}>} A Promise that resolves to the data needed to create an InferenceSession object. + * @private + */ +async function getSession( + pretrained_model_name_or_path, + fileName, + options, + cache_config = false, + session_name = undefined, +) { + let custom_config = options.config?.['transformers.js_config'] ?? {}; + + // If the device is not specified, we use the default (supported) execution providers. + const selectedDevice = /** @type {import("../utils/devices.js").DeviceType} */ ( + selectDevice(options.device ?? custom_config.device, fileName, { + warn: (msg) => logger.info(msg), + }) + ); + + const executionProviders = deviceToExecutionProviders(selectedDevice); + + // Update custom config with the selected device's config, if it exists + const device_config = custom_config.device_config ?? {}; + if (device_config.hasOwnProperty(selectedDevice)) { + custom_config = { + ...custom_config, + ...device_config[selectedDevice], + }; + } + + // If options.dtype is specified, we use it to choose the suffix for the model file. + // Otherwise, we use the default dtype for the device. + const selectedDtype = /** @type {import("../utils/dtypes.js").DataType} */ ( + selectDtype(options.dtype ?? custom_config.dtype, fileName, selectedDevice, { + configDtype: custom_config.dtype, + warn: (msg) => logger.info(msg), + }) + ); + + if (!DEFAULT_DTYPE_SUFFIX_MAPPING.hasOwnProperty(selectedDtype)) { + throw new Error(`Invalid dtype: ${selectedDtype}. Should be one of: ${Object.keys(DATA_TYPES).join(', ')}`); + } else if ( + selectedDevice === 'webgpu' && + // NOTE: Currently, we assume that the Native WebGPU EP always supports fp16. In future, we will add a check for this. + !apis.IS_NODE_ENV && + selectedDtype === DATA_TYPES.fp16 && + !(await isWebGpuFp16Supported()) + ) { + throw new Error(`The device (${selectedDevice}) does not support fp16.`); + } + + // Construct the model file suffix + const suffix = DEFAULT_DTYPE_SUFFIX_MAPPING[selectedDtype]; + + const session_options = { ...options.session_options }; + + // Overwrite `executionProviders` if not specified + session_options.executionProviders ??= executionProviders; + + // Overwrite `freeDimensionOverrides` if specified in config and not set in session options + const free_dimension_overrides = custom_config.free_dimension_overrides; + if (free_dimension_overrides) { + session_options.freeDimensionOverrides ??= free_dimension_overrides; + } else if (selectedDevice.startsWith('webnn') && !session_options.freeDimensionOverrides) { + logger.warn( + `WebNN does not currently support dynamic shapes and requires 'free_dimension_overrides' to be set in config.json, preferably as a field within config["transformers.js_config"]["device_config"]["${selectedDevice}"]. ` + + `When 'free_dimension_overrides' is not set, you may experience significant performance degradation.`, + ); + } + + const bufferOrPathPromise = getCoreModelFile(pretrained_model_name_or_path, fileName, options, suffix); + + // Handle onnx external data files + const use_external_data_format = options.use_external_data_format ?? custom_config.use_external_data_format; + const externalData = await getModelDataFiles( + pretrained_model_name_or_path, + fileName, + suffix, + options, + use_external_data_format, + session_options, + ); + + if (externalData.length > 0 && (!apis.IS_NODE_ENV || externalData.some((data) => typeof data !== 'string'))) { + session_options.externalData = externalData; + } + + if (cache_config && selectedDevice === 'webgpu') { + const names = getCacheNames(options.config, { + prefix: 'present', + session_name, + }); + if (names.size > 0 && !isONNXProxy()) { + // Only set preferredOutputLocation if names are present and we aren't proxying ONNX + /** @type {Record} */ + const preferredOutputLocation = {}; + for (const key of names) { + preferredOutputLocation[key] = 'gpu-buffer'; + } + session_options.preferredOutputLocation = preferredOutputLocation; + } + } + + const buffer_or_path = await bufferOrPathPromise; + const session_config = { + dtype: selectedDtype, + device: selectedDevice, + }; + return { buffer_or_path, session_options, session_config }; +} + +/** + * Helper function to create multiple InferenceSession objects. + * + * @param {string} pretrained_model_name_or_path The path to the directory containing the model file. + * @param {Record} names The names of the model files to load. + * @param {import('../utils/hub.js').PretrainedModelOptions} options Additional options for loading the model. + * @param {Record} [cache_sessions] A map from session name to `true`, indicating which + * sessions should have GPU-pinned KV cache outputs. + * @returns {Promise>} A Promise that resolves to a dictionary of InferenceSession objects. + * @private + */ +export async function constructSessions(pretrained_model_name_or_path, names, options, cache_sessions = undefined) { + return Object.fromEntries( + await Promise.all( + Object.keys(names).map(async (name) => { + const cache_config = cache_sessions?.[name] ?? false; + const { buffer_or_path, session_options, session_config } = await getSession( + pretrained_model_name_or_path, + names[name], + options, + cache_config, + name, + ); + const session = await createInferenceSession(buffer_or_path, session_options, session_config); + return [name, session]; + }), + ), + ); +} + +/** + * Replaces ONNX Tensor objects with custom Tensor objects to support additional functions. + * @param {Object} obj The object to replace tensor objects in. + * @returns {Object} The object with tensor objects replaced by custom Tensor objects. + * @private + */ +function replaceTensors(obj) { + for (let prop in obj) { + if (isONNXTensor(obj[prop])) { + obj[prop] = new Tensor(obj[prop]); + } else if (typeof obj[prop] === 'object') { + replaceTensors(obj[prop]); + } + } + return obj; +} + +/** + * Executes an InferenceSession using the specified inputs. + * NOTE: `inputs` must contain at least the input names of the model. + * - If additional inputs are passed, they will be ignored. + * - If inputs are missing, an error will be thrown. + * + * @param {Object} session The InferenceSession object to run. + * @param {Object} inputs An object that maps input names to input tensors. + * @returns {Promise} A Promise that resolves to an object that maps output names to output tensors. + * @private + */ +export async function sessionRun(session, inputs) { + const checkedInputs = validateInputs(session, inputs); + try { + // pass the original ort tensor + const ortFeed = Object.fromEntries( + Object.entries(checkedInputs).map(([k, v]) => { + const tensor = /** @type {any} */ (v.ort_tensor); + if (apis.IS_NODE_ENV) { + // In recent versions of Node.js, which support Float16Array, we need to convert + // the Float16Array to Uint16Array for ONNX Runtime to accept it. + if (typeof Float16Array !== 'undefined' && tensor.cpuData instanceof Float16Array) { + tensor.cpuData = new Uint16Array(tensor.cpuData.buffer); // reinterpret as Uint16Array + } + } + return [k, tensor]; + }), + ); + + const output = await runInferenceSession(session, ortFeed); + return replaceTensors(output); + } catch (e) { + // Error messages can be long (nested) and uninformative. For this reason, + // we apply minor formatting to show the most important information + const formatted = Object.fromEntries( + Object.entries(checkedInputs).map(([k, tensor]) => { + // Extract these properties from the underlying ORT tensor + const unpacked = { + type: tensor.type, + dims: tensor.dims, + location: tensor.location, + }; + if (unpacked.location !== 'gpu-buffer') { + // Only return the data if it's not a GPU buffer + unpacked.data = tensor.data; + } + return [k, unpacked]; + }), + ); + + // This usually occurs when the inputs are of the wrong type. + logger.error(`An error occurred during model execution: "${e}".`); + logger.error('Inputs given to model:', formatted); + throw e; + } +} + +/** + * Validate model inputs + * @param {Object} session The InferenceSession object that will be run. + * @param {Object} inputs The inputs to check. + * @returns {Record} The checked inputs. + * @throws {Error} If any inputs are missing. + * @private + */ +function validateInputs(session, inputs) { + /** + * NOTE: Create either a shallow or deep copy based on `onnx.wasm.proxy` + * @type {Record} + */ + const checkedInputs = Object.create(null); + const missingInputs = []; + for (const inputName of session.inputNames) { + const tensor = inputs[inputName]; + // Rare case where one of the model's input names corresponds to a built-in + // object name (e.g., toString), which would cause a simple (!tensor) check to fail, + // because it's not undefined but a function. + if (!(tensor instanceof Tensor)) { + missingInputs.push(inputName); + continue; + } + // NOTE: When `env.wasm.proxy is true` the tensor is moved across the Worker + // boundary, transferring ownership to the worker and invalidating the tensor. + // So, in this case, we simply sacrifice a clone for it. + checkedInputs[inputName] = isONNXProxy() ? tensor.clone() : tensor; + } + if (missingInputs.length > 0) { + throw new Error( + `An error occurred during model execution: "Missing the following inputs: ${missingInputs.join(', ')}.`, + ); + } + + const numInputsProvided = Object.keys(inputs).length; + const numInputsNeeded = session.inputNames.length; + if (numInputsProvided > numInputsNeeded) { + // No missing inputs, but too many inputs were provided. + // Warn the user and ignore the extra inputs. + let ignored = Object.keys(inputs).filter((inputName) => !session.inputNames.includes(inputName)); + logger.warn( + `WARNING: Too many inputs were provided (${numInputsProvided} > ${numInputsNeeded}). The following inputs will be ignored: "${ignored.join(', ')}".`, + ); + } + + return checkedInputs; +} diff --git a/packages/transformers/src/models/session_config.js b/packages/transformers/src/models/session_config.js new file mode 100644 index 0000000..93eedd6 --- /dev/null +++ b/packages/transformers/src/models/session_config.js @@ -0,0 +1,170 @@ +export const MODEL_TYPES = { + EncoderOnly: 0, + EncoderDecoder: 1, + Seq2Seq: 2, + Vision2Seq: 3, + DecoderOnly: 4, + DecoderOnlyWithoutHead: 5, + MaskGeneration: 6, + ImageTextToText: 7, + Musicgen: 8, + MultiModality: 9, + Phi3V: 10, + AudioTextToText: 11, + AutoEncoder: 12, + ImageAudioTextToText: 13, + Supertonic: 14, + Chatterbox: 15, + VoxtralRealtime: 16, +}; + +export const MODEL_SESSION_CONFIG = { + [MODEL_TYPES.DecoderOnly]: { + sessions: (config, options) => ({ model: options.model_file_name ?? 'model' }), + cache_sessions: { model: true }, + optional_configs: { generation_config: 'generation_config.json' }, + }, + [MODEL_TYPES.DecoderOnlyWithoutHead]: { + sessions: (config, options) => ({ model: options.model_file_name ?? 'model' }), + }, + [MODEL_TYPES.Seq2Seq]: { + sessions: () => ({ model: 'encoder_model', decoder_model_merged: 'decoder_model_merged' }), + cache_sessions: { decoder_model_merged: true }, + optional_configs: { generation_config: 'generation_config.json' }, + }, + [MODEL_TYPES.Vision2Seq]: { + sessions: () => ({ model: 'encoder_model', decoder_model_merged: 'decoder_model_merged' }), + cache_sessions: { decoder_model_merged: true }, + optional_configs: { generation_config: 'generation_config.json' }, + }, + [MODEL_TYPES.Musicgen]: { + sessions: () => ({ + model: 'text_encoder', + decoder_model_merged: 'decoder_model_merged', + encodec_decode: 'encodec_decode', + }), + cache_sessions: { decoder_model_merged: true }, + optional_configs: { generation_config: 'generation_config.json' }, + }, + [MODEL_TYPES.EncoderDecoder]: { + sessions: () => ({ model: 'encoder_model', decoder_model_merged: 'decoder_model_merged' }), + cache_sessions: { decoder_model_merged: true }, + }, + [MODEL_TYPES.MaskGeneration]: { + sessions: () => ({ model: 'vision_encoder', prompt_encoder_mask_decoder: 'prompt_encoder_mask_decoder' }), + }, + [MODEL_TYPES.ImageTextToText]: { + text_only_sessions: { embed_tokens: 'embed_tokens', decoder_model_merged: 'decoder_model_merged' }, + sessions: (config, options, textOnly) => { + const s = { ...MODEL_SESSION_CONFIG[MODEL_TYPES.ImageTextToText].text_only_sessions }; + if (!textOnly) s['vision_encoder'] = 'vision_encoder'; + if (config.is_encoder_decoder) s['model'] = 'encoder_model'; + return s; + }, + cache_sessions: { decoder_model_merged: true }, + optional_configs: { generation_config: 'generation_config.json' }, + }, + [MODEL_TYPES.AudioTextToText]: { + text_only_sessions: { embed_tokens: 'embed_tokens', decoder_model_merged: 'decoder_model_merged' }, + sessions: (config, options, textOnly) => { + const s = { ...MODEL_SESSION_CONFIG[MODEL_TYPES.AudioTextToText].text_only_sessions }; + if (!textOnly) s['audio_encoder'] = 'audio_encoder'; + return s; + }, + cache_sessions: { decoder_model_merged: true }, + optional_configs: { generation_config: 'generation_config.json' }, + }, + [MODEL_TYPES.ImageAudioTextToText]: { + text_only_sessions: { embed_tokens: 'embed_tokens', decoder_model_merged: 'decoder_model_merged' }, + sessions: (config, options, textOnly) => { + const s = { ...MODEL_SESSION_CONFIG[MODEL_TYPES.ImageAudioTextToText].text_only_sessions }; + if (!textOnly) { + s['audio_encoder'] = 'audio_encoder'; + s['vision_encoder'] = 'vision_encoder'; + } + return s; + }, + optional_configs: { generation_config: 'generation_config.json' }, + }, + [MODEL_TYPES.Phi3V]: { + sessions: () => ({ + prepare_inputs_embeds: 'prepare_inputs_embeds', + model: 'model', + vision_encoder: 'vision_encoder', + }), + cache_sessions: { model: true }, + optional_configs: { generation_config: 'generation_config.json' }, + }, + [MODEL_TYPES.MultiModality]: { + sessions: () => ({ + prepare_inputs_embeds: 'prepare_inputs_embeds', + model: 'language_model', + lm_head: 'lm_head', + gen_head: 'gen_head', + gen_img_embeds: 'gen_img_embeds', + image_decode: 'image_decode', + }), + cache_sessions: { model: true }, + optional_configs: { generation_config: 'generation_config.json' }, + }, + [MODEL_TYPES.AutoEncoder]: { + sessions: () => ({ encoder_model: 'encoder_model', decoder_model: 'decoder_model' }), + }, + [MODEL_TYPES.Supertonic]: { + sessions: () => ({ + text_encoder: 'text_encoder', + latent_denoiser: 'latent_denoiser', + voice_decoder: 'voice_decoder', + }), + }, + [MODEL_TYPES.Chatterbox]: { + sessions: () => ({ + embed_tokens: 'embed_tokens', + speech_encoder: 'speech_encoder', + model: 'language_model', + conditional_decoder: 'conditional_decoder', + }), + cache_sessions: { model: true }, + optional_configs: { generation_config: 'generation_config.json' }, + }, + [MODEL_TYPES.VoxtralRealtime]: { + text_only_sessions: { embed_tokens: 'embed_tokens', decoder_model_merged: 'decoder_model_merged' }, + sessions: (config, options, textOnly) => { + const s = { ...MODEL_SESSION_CONFIG[MODEL_TYPES.VoxtralRealtime].text_only_sessions }; + if (!textOnly) s['audio_encoder'] = 'audio_encoder'; + return s; + }, + cache_sessions: { decoder_model_merged: true, audio_encoder: true }, + optional_configs: { generation_config: 'generation_config.json' }, + }, + default: { + sessions: (config, options) => ({ model: options.model_file_name ?? 'model' }), + }, +}; + +/** + * Returns the text-only session names for a given model type, or `null` if + * the model type does not define a text-only session set. + * @param {number} modelType The model type enum value. + * @returns {Record|null} + */ +export function getTextOnlySessions(modelType) { + const typeConfig = MODEL_SESSION_CONFIG[modelType]; + return typeConfig?.text_only_sessions ?? null; +} + +/** + * Get the session configuration for a given model type. + * @param {number} modelType The model type enum value. + * @param {Object} config The model config. + * @param {Object} [options] Loading options. + * @returns {{ sessions: Record, cache_sessions?: Record, optional_configs?: Record }} + */ +export function getSessionsConfig(modelType, config, options = {}) { + const typeConfig = MODEL_SESSION_CONFIG[modelType] ?? MODEL_SESSION_CONFIG.default; + return { + sessions: typeConfig.sessions(config, options, options.textOnly ?? false), + cache_sessions: typeConfig.cache_sessions, + optional_configs: typeConfig.optional_configs, + }; +} diff --git a/packages/transformers/src/models/siglip/image_processing_siglip.js b/packages/transformers/src/models/siglip/image_processing_siglip.js new file mode 100644 index 0000000..060d5cf --- /dev/null +++ b/packages/transformers/src/models/siglip/image_processing_siglip.js @@ -0,0 +1,3 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class SiglipImageProcessor extends ImageProcessor {} diff --git a/packages/transformers/src/models/siglip/modeling_siglip.js b/packages/transformers/src/models/siglip/modeling_siglip.js new file mode 100644 index 0000000..29d8457 --- /dev/null +++ b/packages/transformers/src/models/siglip/modeling_siglip.js @@ -0,0 +1,123 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { CLIPPreTrainedModel } from '../clip/modeling_clip.js'; + +export class SiglipPreTrainedModel extends PreTrainedModel {} + +/** + * SigLIP Text and Vision Model with a projection layers on top + * + * **Example:** Perform zero-shot image classification with a `SiglipModel`. + * + * ```javascript + * import { AutoTokenizer, AutoProcessor, SiglipModel, RawImage } from '@huggingface/transformers'; + * + * // Load tokenizer, processor, and model + * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/siglip-base-patch16-224'); + * const processor = await AutoProcessor.from_pretrained('Xenova/siglip-base-patch16-224'); + * const model = await SiglipModel.from_pretrained('Xenova/siglip-base-patch16-224'); + * + * // Run tokenization + * const texts = ['a photo of 2 cats', 'a photo of 2 dogs']; + * const text_inputs = tokenizer(texts, { padding: 'max_length', truncation: true }); + * + * // Read image and run processor + * const image = await RawImage.read('http://images.cocodataset.org/val2017/000000039769.jpg'); + * const image_inputs = await processor(image); + * + * // Run model with both text and pixel inputs + * const output = await model({ ...text_inputs, ...image_inputs }); + * // { + * // logits_per_image: Tensor { + * // dims: [ 1, 2 ], + * // data: Float32Array(2) [ -1.6019744873046875, -10.720091819763184 ], + * // }, + * // logits_per_text: Tensor { + * // dims: [ 2, 1 ], + * // data: Float32Array(2) [ -1.6019744873046875, -10.720091819763184 ], + * // }, + * // text_embeds: Tensor { + * // dims: [ 2, 768 ], + * // data: Float32Array(1536) [ ... ], + * // }, + * // image_embeds: Tensor { + * // dims: [ 1, 768 ], + * // data: Float32Array(768) [ ... ], + * // } + * // } + * ``` + */ +export class SiglipModel extends SiglipPreTrainedModel {} + +/** + * The text model from SigLIP without any head or projection on top. + * + * **Example:** Compute text embeddings with `SiglipTextModel`. + * + * ```javascript + * import { AutoTokenizer, SiglipTextModel } from '@huggingface/transformers'; + * + * // Load tokenizer and text model + * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/siglip-base-patch16-224'); + * const text_model = await SiglipTextModel.from_pretrained('Xenova/siglip-base-patch16-224'); + * + * // Run tokenization + * const texts = ['a photo of 2 cats', 'a photo of 2 dogs']; + * const text_inputs = tokenizer(texts, { padding: 'max_length', truncation: true }); + * + * // Compute embeddings + * const { pooler_output } = await text_model(text_inputs); + * // Tensor { + * // dims: [ 2, 768 ], + * // type: 'float32', + * // data: Float32Array(1536) [ ... ], + * // size: 1536 + * // } + * ``` + */ +export class SiglipTextModel extends SiglipPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'text_model', + }); + } +} + +/** + * The vision model from SigLIP without any head or projection on top. + * + * **Example:** Compute vision embeddings with `SiglipVisionModel`. + * + * ```javascript + * import { AutoProcessor, SiglipVisionModel, RawImage } from '@huggingface/transformers'; + * + * // Load processor and vision model + * const processor = await AutoProcessor.from_pretrained('Xenova/siglip-base-patch16-224'); + * const vision_model = await SiglipVisionModel.from_pretrained('Xenova/siglip-base-patch16-224'); + * + * // Read image and run processor + * const image = await RawImage.read('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg'); + * const image_inputs = await processor(image); + * + * // Compute embeddings + * const { pooler_output } = await vision_model(image_inputs); + * // Tensor { + * // dims: [ 1, 768 ], + * // type: 'float32', + * // data: Float32Array(768) [ ... ], + * // size: 768 + * // } + * ``` + */ +export class SiglipVisionModel extends CLIPPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'vision_model', + }); + } +} diff --git a/packages/transformers/src/models/siglip/tokenization_siglip.js b/packages/transformers/src/models/siglip/tokenization_siglip.js new file mode 100644 index 0000000..81407b1 --- /dev/null +++ b/packages/transformers/src/models/siglip/tokenization_siglip.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class SiglipTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/smollm3/modeling_smollm3.js b/packages/transformers/src/models/smollm3/modeling_smollm3.js new file mode 100644 index 0000000..3021aae --- /dev/null +++ b/packages/transformers/src/models/smollm3/modeling_smollm3.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class SmolLM3PreTrainedModel extends PreTrainedModel {} +export class SmolLM3Model extends SmolLM3PreTrainedModel {} +export class SmolLM3ForCausalLM extends SmolLM3PreTrainedModel {} diff --git a/packages/transformers/src/models/smolvlm/image_processing_smolvlm.js b/packages/transformers/src/models/smolvlm/image_processing_smolvlm.js new file mode 100644 index 0000000..84bc346 --- /dev/null +++ b/packages/transformers/src/models/smolvlm/image_processing_smolvlm.js @@ -0,0 +1 @@ +export { Idefics3ImageProcessor as SmolVLMImageProcessor } from '../idefics3/image_processing_idefics3.js'; diff --git a/packages/transformers/src/models/smolvlm/modeling_smolvlm.js b/packages/transformers/src/models/smolvlm/modeling_smolvlm.js new file mode 100644 index 0000000..bb50013 --- /dev/null +++ b/packages/transformers/src/models/smolvlm/modeling_smolvlm.js @@ -0,0 +1,7 @@ +import { Idefics3ForConditionalGeneration } from '../idefics3/modeling_idefics3.js'; + +/** + * The SmolVLM Model with a language modeling head. + * It is made up a SigLIP vision encoder, with a language modeling head on top. + */ +export class SmolVLMForConditionalGeneration extends Idefics3ForConditionalGeneration {} diff --git a/packages/transformers/src/models/smolvlm/processing_smolvlm.js b/packages/transformers/src/models/smolvlm/processing_smolvlm.js new file mode 100644 index 0000000..2bcd4e9 --- /dev/null +++ b/packages/transformers/src/models/smolvlm/processing_smolvlm.js @@ -0,0 +1 @@ +export { Idefics3Processor as SmolVLMProcessor } from '../idefics3/processing_idefics3.js'; diff --git a/packages/transformers/src/models/snac/feature_extraction_snac.js b/packages/transformers/src/models/snac/feature_extraction_snac.js new file mode 100644 index 0000000..679b489 --- /dev/null +++ b/packages/transformers/src/models/snac/feature_extraction_snac.js @@ -0,0 +1,3 @@ +import { DacFeatureExtractor } from '../dac/feature_extraction_dac.js'; + +export class SnacFeatureExtractor extends DacFeatureExtractor {} diff --git a/packages/transformers/src/models/snac/modeling_snac.js b/packages/transformers/src/models/snac/modeling_snac.js new file mode 100644 index 0000000..2f1d91e --- /dev/null +++ b/packages/transformers/src/models/snac/modeling_snac.js @@ -0,0 +1,53 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { sessionRun } from '../session.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class SnacPreTrainedModel extends PreTrainedModel { + main_input_name = 'input_values'; + forward_params = ['input_values']; +} + +/** + * The SNAC (Multi-Scale Neural Audio Codec) model. + */ +export class SnacModel extends SnacPreTrainedModel { + /** + * Encodes the input audio waveform into discrete codes. + * @param {Object} inputs Model inputs + * @param {Tensor} [inputs.input_values] Float values of the input audio waveform, of shape `(batch_size, channels, sequence_length)`). + * @returns {Promise>} The output tensors of shape `(batch_size, num_codebooks, sequence_length)`. + */ + async encode(inputs) { + return await sessionRun(this.sessions['encoder_model'], inputs); + } + + /** + * Decodes the given frames into an output audio waveform. + * @param {Record} inputs The encoded audio codes. + * @returns {Promise<{audio_values: Tensor}>} The output tensor of shape `(batch_size, num_channels, sequence_length)`. + */ + async decode(inputs) { + return await sessionRun(this.sessions['decoder_model'], inputs); + } +} + +export class SnacEncoderModel extends SnacPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'encoder_model', + }); + } +} +export class SnacDecoderModel extends SnacPreTrainedModel { + /** @type {typeof PreTrainedModel.from_pretrained} */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + return super.from_pretrained(pretrained_model_name_or_path, { + ...options, + // Update default model file name if not provided + model_file_name: options.model_file_name ?? 'decoder_model', + }); + } +} diff --git a/packages/transformers/src/models/solar_open/modeling_solar_open.js b/packages/transformers/src/models/solar_open/modeling_solar_open.js new file mode 100644 index 0000000..82a64ed --- /dev/null +++ b/packages/transformers/src/models/solar_open/modeling_solar_open.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class SolarOpenPreTrainedModel extends PreTrainedModel {} +export class SolarOpenModel extends SolarOpenPreTrainedModel {} +export class SolarOpenForCausalLM extends SolarOpenPreTrainedModel {} diff --git a/packages/transformers/src/models/speecht5/feature_extraction_speecht5.js b/packages/transformers/src/models/speecht5/feature_extraction_speecht5.js new file mode 100644 index 0000000..57e3e4e --- /dev/null +++ b/packages/transformers/src/models/speecht5/feature_extraction_speecht5.js @@ -0,0 +1,3 @@ +import { FeatureExtractor } from '../../feature_extraction_utils.js'; + +export class SpeechT5FeatureExtractor extends FeatureExtractor {} diff --git a/packages/transformers/src/models/speecht5/modeling_speecht5.js b/packages/transformers/src/models/speecht5/modeling_speecht5.js new file mode 100644 index 0000000..970031e --- /dev/null +++ b/packages/transformers/src/models/speecht5/modeling_speecht5.js @@ -0,0 +1,164 @@ +import { PreTrainedModel, encoder_forward, boolTensor, addPastKeyValues, getPastKeyValues } from '../modeling_utils.js'; +import { sessionRun } from '../session.js'; +import { Tensor, cat } from '../../utils/tensor.js'; + +/** + * An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. + */ +export class SpeechT5PreTrainedModel extends PreTrainedModel {} + +/** + * The bare SpeechT5 Encoder-Decoder Model outputting raw hidden-states without any specific pre- or post-nets. + */ +export class SpeechT5Model extends SpeechT5PreTrainedModel {} + +/** + * SpeechT5 Model with a speech encoder and a text decoder. + * + * **Example:** Generate speech from text with `SpeechT5ForSpeechToText`. + * ```javascript + * import { AutoTokenizer, AutoProcessor, SpeechT5ForTextToSpeech, SpeechT5HifiGan, Tensor } from '@huggingface/transformers'; + * + * // Load the tokenizer and processor + * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/speecht5_tts'); + * const processor = await AutoProcessor.from_pretrained('Xenova/speecht5_tts'); + * + * // Load the models + * // NOTE: We use the full-precision versions as they are more accurate + * const model = await SpeechT5ForTextToSpeech.from_pretrained('Xenova/speecht5_tts', { dtype: 'fp32' }); + * const vocoder = await SpeechT5HifiGan.from_pretrained('Xenova/speecht5_hifigan', { dtype: 'fp32' }); + * + * // Load speaker embeddings from URL + * const speaker_embeddings_data = new Float32Array( + * await (await fetch('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/speaker_embeddings.bin')).arrayBuffer() + * ); + * const speaker_embeddings = new Tensor( + * 'float32', + * speaker_embeddings_data, + * [1, speaker_embeddings_data.length] + * ) + * + * // Run tokenization + * const { input_ids } = tokenizer('Hello, my dog is cute'); + * + * // Generate waveform + * const { waveform } = await model.generate_speech(input_ids, speaker_embeddings, { vocoder }); + * console.log(waveform) + * // Tensor { + * // dims: [ 26112 ], + * // type: 'float32', + * // size: 26112, + * // data: Float32Array(26112) [ -0.00043630177970044315, -0.00018082228780258447, ... ], + * // } + * ``` + */ +export class SpeechT5ForSpeechToText extends SpeechT5PreTrainedModel {} + +/** + * SpeechT5 Model with a text encoder and a speech decoder. + */ +export class SpeechT5ForTextToSpeech extends SpeechT5PreTrainedModel { + /** + * @typedef {Object} SpeechOutput + * @property {Tensor} [spectrogram] The predicted log-mel spectrogram of shape + * `(output_sequence_length, config.num_mel_bins)`. Returned when no `vocoder` is provided + * @property {Tensor} [waveform] The predicted waveform of shape `(num_frames,)`. Returned when a `vocoder` is provided. + * @property {Tensor} [cross_attentions] The outputs of the decoder's cross-attention layers of shape + * `(config.decoder_layers, config.decoder_attention_heads, output_sequence_length, input_sequence_length)`. returned when `output_cross_attentions` is `true`. + */ + + /** + * Converts a sequence of input tokens into a sequence of mel spectrograms, which are subsequently turned into a speech waveform using a vocoder. + * @param {Tensor} input_values Indices of input sequence tokens in the vocabulary. + * @param {Tensor} speaker_embeddings Tensor containing the speaker embeddings. + * @param {Object} options Optional parameters for generating speech. + * @param {number} [options.threshold=0.5] The generated sequence ends when the predicted stop token probability exceeds this value. + * @param {number} [options.minlenratio=0.0] Used to calculate the minimum required length for the output sequence. + * @param {number} [options.maxlenratio=20.0] Used to calculate the maximum allowed length for the output sequence. + * @param {Object} [options.vocoder=null] The vocoder that converts the mel spectrogram into a speech waveform. If `null`, the output is the mel spectrogram. + * @param {boolean} [options.output_cross_attentions=false] Whether or not to return the attentions tensors of the decoder's cross-attention layers. + * @returns {Promise} A promise which resolves to an object containing the spectrogram, waveform, and cross-attention tensors. + */ + async generate_speech( + input_values, + speaker_embeddings, + { + threshold = 0.5, + minlenratio = 0.0, + maxlenratio = 20.0, + vocoder = null, + // output_cross_attentions = false, // TODO add + } = {}, + ) { + const model_inputs = { + input_ids: input_values, + }; + + const { encoder_outputs, encoder_attention_mask } = await encoder_forward(this, model_inputs); + + // @ts-expect-error TS2339 + const r = encoder_outputs.dims[1] / this.config.reduction_factor; + const maxlen = Math.floor(r * maxlenratio); + const minlen = Math.floor(r * minlenratio); + + // @ts-expect-error TS2339 + const num_mel_bins = this.config.num_mel_bins; + + let spectrogramParts = []; + let past_key_values = null; + let decoder_outputs = null; + let idx = 0; + + while (true) { + ++idx; + + const use_cache_branch = boolTensor(!!decoder_outputs); + let output_sequence; + if (decoder_outputs) { + output_sequence = decoder_outputs.output_sequence_out; + } else { + output_sequence = new Tensor('float32', new Float32Array(num_mel_bins), [1, 1, num_mel_bins]); + } + let decoderFeeds = { + use_cache_branch, + output_sequence, + encoder_attention_mask: encoder_attention_mask, + speaker_embeddings: speaker_embeddings, + encoder_hidden_states: encoder_outputs, + }; + + addPastKeyValues(this, decoderFeeds, past_key_values); + decoder_outputs = await sessionRun(this.sessions['decoder_model_merged'], decoderFeeds); + past_key_values = getPastKeyValues(decoder_outputs, past_key_values); + + const { prob, spectrum } = decoder_outputs; + spectrogramParts.push(spectrum); + + if ( + idx >= minlen && + // Finished when stop token or maximum length is reached. + (Array.from(prob.data).filter((p) => p >= threshold).length > 0 || idx >= maxlen) + ) { + break; + } + } + + const spectrogram = cat(spectrogramParts); + const { waveform } = await sessionRun(vocoder.sessions['model'], { spectrogram }); + + return { + spectrogram, + waveform, + // cross_attentions: null, // TODO add + }; + } +} + +/** + * HiFi-GAN vocoder. + * + * See [SpeechT5ForSpeechToText](./models#module_models.SpeechT5ForSpeechToText) for example usage. + */ +export class SpeechT5HifiGan extends PreTrainedModel { + main_input_name = 'spectrogram'; +} diff --git a/packages/transformers/src/models/speecht5/processing_speecht5.js b/packages/transformers/src/models/speecht5/processing_speecht5.js new file mode 100644 index 0000000..580f880 --- /dev/null +++ b/packages/transformers/src/models/speecht5/processing_speecht5.js @@ -0,0 +1,17 @@ +import { Processor } from '../../processing_utils.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { AutoFeatureExtractor } from '../auto/feature_extraction_auto.js'; + +export class SpeechT5Processor extends Processor { + static tokenizer_class = AutoTokenizer; + static feature_extractor_class = AutoFeatureExtractor; + + /** + * Calls the feature_extractor function with the given input. + * @param {any} input The input to extract features from. + * @returns {Promise} A Promise that resolves with the extracted features. + */ + async _call(input) { + return await this.feature_extractor(input); + } +} diff --git a/packages/transformers/src/models/speecht5/tokenization_speecht5.js b/packages/transformers/src/models/speecht5/tokenization_speecht5.js new file mode 100644 index 0000000..95c2d2e --- /dev/null +++ b/packages/transformers/src/models/speecht5/tokenization_speecht5.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class SpeechT5Tokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/squeezebert/modeling_squeezebert.js b/packages/transformers/src/models/squeezebert/modeling_squeezebert.js new file mode 100644 index 0000000..178987e --- /dev/null +++ b/packages/transformers/src/models/squeezebert/modeling_squeezebert.js @@ -0,0 +1,38 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { MaskedLMOutput, QuestionAnsweringModelOutput, SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class SqueezeBertPreTrainedModel extends PreTrainedModel {} +export class SqueezeBertModel extends SqueezeBertPreTrainedModel {} +export class SqueezeBertForMaskedLM extends SqueezeBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} +export class SqueezeBertForSequenceClassification extends SqueezeBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} +export class SqueezeBertForQuestionAnswering extends SqueezeBertPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/squeezebert/tokenization_squeezebert.js b/packages/transformers/src/models/squeezebert/tokenization_squeezebert.js new file mode 100644 index 0000000..2f249b4 --- /dev/null +++ b/packages/transformers/src/models/squeezebert/tokenization_squeezebert.js @@ -0,0 +1,5 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class SqueezeBertTokenizer extends PreTrainedTokenizer { + return_token_type_ids = true; +} diff --git a/packages/transformers/src/models/stablelm/modeling_stablelm.js b/packages/transformers/src/models/stablelm/modeling_stablelm.js new file mode 100644 index 0000000..bb75295 --- /dev/null +++ b/packages/transformers/src/models/stablelm/modeling_stablelm.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class StableLmPreTrainedModel extends PreTrainedModel {} + +/** + * The bare StableLm Model transformer outputting raw hidden-states without any specific head on top. + */ +export class StableLmModel extends StableLmPreTrainedModel {} + +/** + * StableLm Model with a `language modeling` head on top for Causal Language Modeling (with past). + */ +export class StableLmForCausalLM extends StableLmPreTrainedModel {} diff --git a/packages/transformers/src/models/starcoder2/modeling_starcoder2.js b/packages/transformers/src/models/starcoder2/modeling_starcoder2.js new file mode 100644 index 0000000..8122990 --- /dev/null +++ b/packages/transformers/src/models/starcoder2/modeling_starcoder2.js @@ -0,0 +1,10 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +/** + * The bare Starcoder2 Model outputting raw hidden-states without any specific head on top. + */ +export class Starcoder2PreTrainedModel extends PreTrainedModel {} + +export class Starcoder2Model extends Starcoder2PreTrainedModel {} + +export class Starcoder2ForCausalLM extends Starcoder2PreTrainedModel {} diff --git a/packages/transformers/src/models/style_text_to_speech_2/modeling_style_text_to_speech_2.js b/packages/transformers/src/models/style_text_to_speech_2/modeling_style_text_to_speech_2.js new file mode 100644 index 0000000..2261034 --- /dev/null +++ b/packages/transformers/src/models/style_text_to_speech_2/modeling_style_text_to_speech_2.js @@ -0,0 +1,4 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class StyleTextToSpeech2PreTrainedModel extends PreTrainedModel {} +export class StyleTextToSpeech2Model extends StyleTextToSpeech2PreTrainedModel {} diff --git a/packages/transformers/src/models/supertonic/modeling_supertonic.js b/packages/transformers/src/models/supertonic/modeling_supertonic.js new file mode 100644 index 0000000..2e8f125 --- /dev/null +++ b/packages/transformers/src/models/supertonic/modeling_supertonic.js @@ -0,0 +1,86 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { Tensor, full, randn } from '../../utils/tensor.js'; +import { sessionRun } from '../session.js'; + +export class SupertonicPreTrainedModel extends PreTrainedModel {} +export class SupertonicForConditionalGeneration extends SupertonicPreTrainedModel { + async generate_speech({ + // Required inputs + input_ids, + attention_mask, + style, + + // Optional inputs + num_inference_steps = 5, + speed = 1.05, + }) { + // @ts-expect-error TS2339 + const { sampling_rate, chunk_compress_factor, base_chunk_size, latent_dim } = this.config; + + // 1. Text Encoder + const { last_hidden_state, durations: raw_durations } = await sessionRun(this.sessions['text_encoder'], { + input_ids, + attention_mask, + style, + }); + + // Convert durations to sample counts + const durations = raw_durations.div(speed).mul_(sampling_rate); + + // 2. Latent Preparation + // Calculate latent lengths: ceil(durations / latent_size) + const latent_size = base_chunk_size * chunk_compress_factor; + const durationsData = /** @type {Float32Array} */ (durations.data); + const latentLengths = Int32Array.from(durationsData, (d) => Math.ceil(d / latent_size)); + const maxLatentLen = Math.max(...latentLengths); + + // Create latent mask: (arange(max_len) < latent_lengths[:, None]) + const batch_size = input_ids.dims[0]; + const latentMaskData = new BigInt64Array(batch_size * maxLatentLen); + for (let i = 0; i < batch_size; ++i) { + latentMaskData.fill(1n, i * maxLatentLen, i * maxLatentLen + latentLengths[i]); + } + const latent_mask = new Tensor('int64', latentMaskData, [batch_size, maxLatentLen]); + + // Create initial noise and apply mask: latents *= latent_mask[:, None, :] + const latentChannels = latent_dim * chunk_compress_factor; + const latentStride = latentChannels * maxLatentLen; + let noisy_latents = randn([batch_size, latentChannels, maxLatentLen]); + const latentsData = /** @type {Float32Array} */ (noisy_latents.data); + for (let i = 0; i < batch_size; ++i) { + if (latentLengths[i] === maxLatentLen) continue; // No padding for this item + for (let c = 0; c < latentChannels; ++c) { + latentsData.fill( + 0, + i * latentStride + c * maxLatentLen + latentLengths[i], + i * latentStride + (c + 1) * maxLatentLen, + ); + } + } + + // 3. Denoising Loop + const num_steps = full([batch_size], num_inference_steps); + for (let step = 0; step < num_inference_steps; ++step) { + const timestep = full([batch_size], step); + ({ denoised_latents: noisy_latents } = await sessionRun(this.sessions['latent_denoiser'], { + style, + noisy_latents, + latent_mask, + encoder_outputs: last_hidden_state, + attention_mask, + timestep, + num_inference_steps: num_steps, + })); + } + + // 4. Voice Decoder + const { waveform } = await sessionRun(this.sessions['voice_decoder'], { + latents: noisy_latents, + }); + + return { + waveform, + durations, + }; + } +} diff --git a/packages/transformers/src/models/swin/modeling_swin.js b/packages/transformers/src/models/swin/modeling_swin.js new file mode 100644 index 0000000..9f43c89 --- /dev/null +++ b/packages/transformers/src/models/swin/modeling_swin.js @@ -0,0 +1,14 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class SwinPreTrainedModel extends PreTrainedModel {} +export class SwinModel extends SwinPreTrainedModel {} +export class SwinForImageClassification extends SwinPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} +export class SwinForSemanticSegmentation extends SwinPreTrainedModel {} diff --git a/packages/transformers/src/models/swin2sr/image_processing_swin2sr.js b/packages/transformers/src/models/swin2sr/image_processing_swin2sr.js new file mode 100644 index 0000000..b2e5a58 --- /dev/null +++ b/packages/transformers/src/models/swin2sr/image_processing_swin2sr.js @@ -0,0 +1,27 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class Swin2SRImageProcessor extends ImageProcessor { + pad_image(pixelData, imgDims, padSize, options = {}) { + // NOTE: In this case, `padSize` represents the size of the sliding window for the local attention. + // In other words, the image is padded so that its width and height are multiples of `padSize`. + const [imageHeight, imageWidth, imageChannels] = imgDims; + + return super.pad_image( + pixelData, + imgDims, + { + // NOTE: For Swin2SR models, the original python implementation adds padding even when the image's width/height is already + // a multiple of `pad_size`. However, this is most likely a bug (PR: https://github.com/mv-lab/swin2sr/pull/19). + // For this reason, we only add padding when the image's width/height is not a multiple of `pad_size`. + width: imageWidth + ((padSize - (imageWidth % padSize)) % padSize), + height: imageHeight + ((padSize - (imageHeight % padSize)) % padSize), + }, + { + mode: 'symmetric', + center: false, + constant_values: -1, + ...options, + }, + ); + } +} diff --git a/packages/transformers/src/models/swin2sr/modeling_swin2sr.js b/packages/transformers/src/models/swin2sr/modeling_swin2sr.js new file mode 100644 index 0000000..a4865ae --- /dev/null +++ b/packages/transformers/src/models/swin2sr/modeling_swin2sr.js @@ -0,0 +1,42 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class Swin2SRPreTrainedModel extends PreTrainedModel {} + +/** + * The bare Swin2SR Model transformer outputting raw hidden-states without any specific head on top. + */ +export class Swin2SRModel extends Swin2SRPreTrainedModel {} + +/** + * Swin2SR Model transformer with an upsampler head on top for image super resolution and restoration. + * + * **Example:** Super-resolution w/ `Xenova/swin2SR-classical-sr-x2-64`. + * + * ```javascript + * import { AutoProcessor, Swin2SRForImageSuperResolution, RawImage } from '@huggingface/transformers'; + * + * // Load processor and model + * const model_id = 'Xenova/swin2SR-classical-sr-x2-64'; + * const processor = await AutoProcessor.from_pretrained(model_id); + * const model = await Swin2SRForImageSuperResolution.from_pretrained(model_id); + * + * // Prepare model inputs + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/butterfly.jpg'; + * const image = await RawImage.fromURL(url); + * const inputs = await processor(image); + * + * // Run model + * const outputs = await model(inputs); + * + * // Convert Tensor to RawImage + * const output = outputs.reconstruction.squeeze().clamp_(0, 1).mul_(255).round_().to('uint8'); + * const outputImage = RawImage.fromTensor(output); + * // RawImage { + * // data: Uint8Array(786432) [ 41, 31, 24, ... ], + * // width: 512, + * // height: 512, + * // channels: 3 + * // } + * ``` + */ +export class Swin2SRForImageSuperResolution extends Swin2SRPreTrainedModel {} diff --git a/packages/transformers/src/models/t5/modeling_t5.js b/packages/transformers/src/models/t5/modeling_t5.js new file mode 100644 index 0000000..91474e5 --- /dev/null +++ b/packages/transformers/src/models/t5/modeling_t5.js @@ -0,0 +1,19 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class T5PreTrainedModel extends PreTrainedModel { + forward_params = [ + 'input_ids', + 'attention_mask', + 'encoder_outputs', + 'decoder_input_ids', + 'decoder_attention_mask', + 'past_key_values', + ]; +} + +export class T5Model extends T5PreTrainedModel {} + +/** + * T5Model is a class representing a T5 model for conditional generation. + */ +export class T5ForConditionalGeneration extends T5PreTrainedModel {} diff --git a/packages/transformers/src/models/t5/tokenization_t5.js b/packages/transformers/src/models/t5/tokenization_t5.js new file mode 100644 index 0000000..8363a56 --- /dev/null +++ b/packages/transformers/src/models/t5/tokenization_t5.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class T5Tokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/table_transformer/modeling_table_transformer.js b/packages/transformers/src/models/table_transformer/modeling_table_transformer.js new file mode 100644 index 0000000..4d35db8 --- /dev/null +++ b/packages/transformers/src/models/table_transformer/modeling_table_transformer.js @@ -0,0 +1,24 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { DetrObjectDetectionOutput } from '../detr/modeling_detr.js'; + +export class TableTransformerPreTrainedModel extends PreTrainedModel {} + +/** + * The bare Table Transformer Model (consisting of a backbone and encoder-decoder Transformer) + * outputting raw hidden-states without any specific head on top. + */ +export class TableTransformerModel extends TableTransformerPreTrainedModel {} + +/** + * Table Transformer Model (consisting of a backbone and encoder-decoder Transformer) + * with object detection heads on top, for tasks such as COCO detection. + */ +export class TableTransformerForObjectDetection extends TableTransformerPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new TableTransformerObjectDetectionOutput(await super._call(model_inputs)); + } +} +export class TableTransformerObjectDetectionOutput extends DetrObjectDetectionOutput {} diff --git a/packages/transformers/src/models/tokenizers.js b/packages/transformers/src/models/tokenizers.js new file mode 100644 index 0000000..4fe335f --- /dev/null +++ b/packages/transformers/src/models/tokenizers.js @@ -0,0 +1,48 @@ +export * from './albert/tokenization_albert.js'; +export * from './auto/tokenization_auto.js'; +export * from './bart/tokenization_bart.js'; +export * from './bert/tokenization_bert.js'; +export * from './blenderbot_small/tokenization_blenderbot_small.js'; +export * from './blenderbot/tokenization_blenderbot.js'; +export * from './bloom/tokenization_bloom.js'; +export * from './camembert/tokenization_camembert.js'; +export * from './clip/tokenization_clip.js'; +export * from './code_llama/tokenization_code_llama.js'; +export * from './codegen/tokenization_codegen.js'; +export * from './cohere/tokenization_cohere.js'; +export * from './cohere_asr/tokenization_cohere_asr.js'; +export * from './convbert/tokenization_convbert.js'; +export * from './deberta_v2/tokenization_deberta_v2.js'; +export * from './deberta/tokenization_deberta.js'; +export * from './distilbert/tokenization_distilbert.js'; +export * from './electra/tokenization_electra.js'; +export * from './esm/tokenization_esm.js'; +export * from './falcon/tokenization_falcon.js'; +export * from './gemma/tokenization_gemma.js'; +export * from './gpt_neox/tokenization_gpt_neox.js'; +export * from './gpt2/tokenization_gpt2.js'; +export * from './herbert/tokenization_herbert.js'; +export * from './llama/tokenization_llama.js'; +export * from './m2m_100/tokenization_m2m_100.js'; +export * from './marian/tokenization_marian.js'; +export * from './mbart/tokenization_mbart.js'; +export * from './mbart50/tokenization_mbart50.js'; +export * from './mgp_str/tokenization_mgp_str.js'; +export * from './mobilebert/tokenization_mobilebert.js'; +export * from './mpnet/tokenization_mpnet.js'; +export * from './nllb/tokenization_nllb.js'; +export * from './nougat/tokenization_nougat.js'; +export * from './qwen2/tokenization_qwen2.js'; +export * from './roberta/tokenization_roberta.js'; +export * from './roformer/tokenization_roformer.js'; +export * from './siglip/tokenization_siglip.js'; +export * from './speecht5/tokenization_speecht5.js'; +export * from './squeezebert/tokenization_squeezebert.js'; +export * from './t5/tokenization_t5.js'; +export * from './vits/tokenization_vits.js'; +export * from './wav2vec2/tokenization_wav2vec2.js'; +export * from './whisper/tokenization_whisper.js'; +export * from './xlm_roberta/tokenization_xlm_roberta.js'; +export * from './xlm/tokenization_xlm.js'; + +export { PreTrainedTokenizer, PreTrainedTokenizer as TokenizersBackend } from '../tokenization_utils.js'; diff --git a/packages/transformers/src/models/trocr/modeling_trocr.js b/packages/transformers/src/models/trocr/modeling_trocr.js new file mode 100644 index 0000000..62674ce --- /dev/null +++ b/packages/transformers/src/models/trocr/modeling_trocr.js @@ -0,0 +1,8 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class TrOCRPreTrainedModel extends PreTrainedModel {} + +/** + * The TrOCR Decoder with a language modeling head. + */ +export class TrOCRForCausalLM extends TrOCRPreTrainedModel {} diff --git a/packages/transformers/src/models/ultravox/modeling_ultravox.js b/packages/transformers/src/models/ultravox/modeling_ultravox.js new file mode 100644 index 0000000..7752237 --- /dev/null +++ b/packages/transformers/src/models/ultravox/modeling_ultravox.js @@ -0,0 +1,19 @@ +import { PreTrainedModel, default_merge_input_ids_with_audio_features } from '../modeling_utils.js'; + +export class UltravoxPreTrainedModel extends PreTrainedModel { + forward_params = ['input_ids', 'attention_mask', 'position_ids', 'audio_values', 'past_key_values']; +} + +export class UltravoxModel extends UltravoxPreTrainedModel { + _merge_input_ids_with_audio_features(kwargs) { + const audio_hidden_size = kwargs.audio_features.dims.at(-1); + const reshaped_audio_features = kwargs.audio_features.view(-1, audio_hidden_size); + + return default_merge_input_ids_with_audio_features({ + // @ts-ignore + audio_token_id: this.config.ignore_index ?? this.config.audio_token_id ?? this.config.audio_token_index, + ...kwargs, + audio_features: reshaped_audio_features, + }); + } +} diff --git a/packages/transformers/src/models/ultravox/processing_ultravox.js b/packages/transformers/src/models/ultravox/processing_ultravox.js new file mode 100644 index 0000000..67dacfb --- /dev/null +++ b/packages/transformers/src/models/ultravox/processing_ultravox.js @@ -0,0 +1,54 @@ +import { AutoFeatureExtractor } from '../auto/feature_extraction_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { Processor } from '../../processing_utils.js'; + +/** + * Represents a UltravoxProcessor that extracts features from an audio input. + */ +export class UltravoxProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static feature_extractor_class = AutoFeatureExtractor; + static uses_processor_config = true; + + /** + * @param {string} text The text input to process. + * @param {Float32Array} audio The audio input to process. + */ + async _call(text, audio = null, kwargs = {}) { + // TODO: Support batched inputs + if (Array.isArray(text)) { + throw new Error('Batched inputs are not supported yet.'); + } + + let audio_inputs = {}; + if (audio) { + const audio_len = audio.length; + const { input_features } = await this.feature_extractor(audio, { + ...kwargs, + max_length: audio_len, + }); + const nb_encoder_frames = Math.round(audio_len / this.config.encoder_ds_factor + 1e-4); + + // NOTE: The python version appears to have an off-by-one error. + const audio_embed_frames = 1 + Math.ceil(nb_encoder_frames / this.config.stack_factor); + audio_inputs['audio_token_len'] = [audio_embed_frames]; + audio_inputs['audio_values'] = input_features; + + const image_token = this.config.audio_placeholder; + if (!text.includes(image_token)) { + throw new Error(`The input text does not contain the image token ${image_token}.`); + } + text = text.replaceAll(image_token, image_token.repeat(audio_embed_frames)); + } + + const text_inputs = this.tokenizer(text, { + add_special_tokens: false, + ...kwargs, + }); + + return { + ...text_inputs, + ...audio_inputs, + }; + } +} diff --git a/packages/transformers/src/models/unispeech/modeling_unispeech.js b/packages/transformers/src/models/unispeech/modeling_unispeech.js new file mode 100644 index 0000000..4711aaa --- /dev/null +++ b/packages/transformers/src/models/unispeech/modeling_unispeech.js @@ -0,0 +1,38 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { CausalLMOutput, SequenceClassifierOutput } from '../modeling_outputs.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class UniSpeechPreTrainedModel extends PreTrainedModel {} + +/** + * The bare UniSpeech Model transformer outputting raw hidden-states without any specific head on top. + */ +export class UniSpeechModel extends UniSpeechPreTrainedModel {} + +/** + * UniSpeech Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). + */ +export class UniSpeechForCTC extends UniSpeechPreTrainedModel { + /** + * @param {Object} model_inputs + * @param {Tensor} model_inputs.input_values Float values of input raw speech waveform. + * @param {Tensor} model_inputs.attention_mask Mask to avoid performing convolution and attention on padding token indices. Mask values selected in [0, 1] + */ + async _call(model_inputs) { + return new CausalLMOutput(await super._call(model_inputs)); + } +} + +/** + * UniSpeech Model with a sequence classification head on top (a linear layer over the pooled output). + */ +export class UniSpeechForSequenceClassification extends UniSpeechPreTrainedModel { + /** + * Calls the model on new inputs. + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/unispeech_sat/modeling_unispeech_sat.js b/packages/transformers/src/models/unispeech_sat/modeling_unispeech_sat.js new file mode 100644 index 0000000..bfe1ef4 --- /dev/null +++ b/packages/transformers/src/models/unispeech_sat/modeling_unispeech_sat.js @@ -0,0 +1,52 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { CausalLMOutput, SequenceClassifierOutput, TokenClassifierOutput } from '../modeling_outputs.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class UniSpeechSatPreTrainedModel extends PreTrainedModel {} + +/** + * The bare UniSpeechSat Model transformer outputting raw hidden-states without any specific head on top. + */ +export class UniSpeechSatModel extends UniSpeechSatPreTrainedModel {} + +/** + * UniSpeechSat Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). + */ +export class UniSpeechSatForCTC extends UniSpeechSatPreTrainedModel { + /** + * @param {Object} model_inputs + * @param {Tensor} model_inputs.input_values Float values of input raw speech waveform. + * @param {Tensor} model_inputs.attention_mask Mask to avoid performing convolution and attention on padding token indices. Mask values selected in [0, 1] + */ + async _call(model_inputs) { + return new CausalLMOutput(await super._call(model_inputs)); + } +} + +/** + * UniSpeechSat Model with a sequence classification head on top (a linear layer over the pooled output). + */ +export class UniSpeechSatForSequenceClassification extends UniSpeechSatPreTrainedModel { + /** + * Calls the model on new inputs. + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * UniSpeechSat Model with a frame classification head on top for tasks like Speaker Diarization. + */ +export class UniSpeechSatForAudioFrameClassification extends UniSpeechSatPreTrainedModel { + /** + * Calls the model on new inputs. + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/vaultgemma/modeling_vaultgemma.js b/packages/transformers/src/models/vaultgemma/modeling_vaultgemma.js new file mode 100644 index 0000000..0e8badd --- /dev/null +++ b/packages/transformers/src/models/vaultgemma/modeling_vaultgemma.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class VaultGemmaPreTrainedModel extends PreTrainedModel {} +export class VaultGemmaModel extends VaultGemmaPreTrainedModel {} +export class VaultGemmaForCausalLM extends VaultGemmaPreTrainedModel {} diff --git a/packages/transformers/src/models/vision_encoder_decoder/modeling_vision_encoder_decoder.js b/packages/transformers/src/models/vision_encoder_decoder/modeling_vision_encoder_decoder.js new file mode 100644 index 0000000..5d38ff3 --- /dev/null +++ b/packages/transformers/src/models/vision_encoder_decoder/modeling_vision_encoder_decoder.js @@ -0,0 +1,17 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +/** + * Vision Encoder-Decoder model based on OpenAI's GPT architecture for image captioning and other vision tasks + */ +export class VisionEncoderDecoderModel extends PreTrainedModel { + main_input_name = 'pixel_values'; + forward_params = [ + // Encoder inputs + 'pixel_values', + + // Decoder inpputs + 'decoder_input_ids', + 'encoder_hidden_states', + 'past_key_values', + ]; +} diff --git a/packages/transformers/src/models/vit/image_processing_vit.js b/packages/transformers/src/models/vit/image_processing_vit.js new file mode 100644 index 0000000..5466a1e --- /dev/null +++ b/packages/transformers/src/models/vit/image_processing_vit.js @@ -0,0 +1,4 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class ViTImageProcessor extends ImageProcessor {} +export class ViTFeatureExtractor extends ViTImageProcessor {} diff --git a/packages/transformers/src/models/vit/modeling_vit.js b/packages/transformers/src/models/vit/modeling_vit.js new file mode 100644 index 0000000..741157a --- /dev/null +++ b/packages/transformers/src/models/vit/modeling_vit.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class ViTPreTrainedModel extends PreTrainedModel {} +export class ViTModel extends ViTPreTrainedModel {} +export class ViTForImageClassification extends ViTPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/vit_mae/modeling_vit_mae.js b/packages/transformers/src/models/vit_mae/modeling_vit_mae.js new file mode 100644 index 0000000..6d62a31 --- /dev/null +++ b/packages/transformers/src/models/vit_mae/modeling_vit_mae.js @@ -0,0 +1,4 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class ViTMAEPreTrainedModel extends PreTrainedModel {} +export class ViTMAEModel extends ViTMAEPreTrainedModel {} diff --git a/packages/transformers/src/models/vit_msn/modeling_vit_msn.js b/packages/transformers/src/models/vit_msn/modeling_vit_msn.js new file mode 100644 index 0000000..74f80ef --- /dev/null +++ b/packages/transformers/src/models/vit_msn/modeling_vit_msn.js @@ -0,0 +1,13 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { SequenceClassifierOutput } from '../modeling_outputs.js'; + +export class ViTMSNPreTrainedModel extends PreTrainedModel {} +export class ViTMSNModel extends ViTMSNPreTrainedModel {} +export class ViTMSNForImageClassification extends ViTMSNPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/vitmatte/image_processing_vitmatte.js b/packages/transformers/src/models/vitmatte/image_processing_vitmatte.js new file mode 100644 index 0000000..13941ca --- /dev/null +++ b/packages/transformers/src/models/vitmatte/image_processing_vitmatte.js @@ -0,0 +1,51 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +import { stack, cat } from '../../utils/tensor.js'; + +export class VitMatteImageProcessor extends ImageProcessor { + /** + * Calls the feature extraction process on an array of images, preprocesses + * each image, and concatenates the resulting features into a single Tensor. + * @param {import("../../utils/image.js").RawImage[]} images The image(s) to extract features from. + * @param {import("../../utils/image.js").RawImage[]} trimaps The trimaps(s) to extract features from. + * @returns {Promise} An object containing the concatenated pixel values of the preprocessed images. + */ + async _call(images, trimaps) { + if (!Array.isArray(images)) { + images = [images]; + } + if (!Array.isArray(trimaps)) { + trimaps = [trimaps]; + } + + const imageData = await Promise.all(images.map((x) => this.preprocess(x))); + const trimapData = await Promise.all( + trimaps.map((x) => + this.preprocess(x, { + do_normalize: false, + do_convert_rgb: false, + do_convert_grayscale: true, + }), + ), + ); + + // Stack pixel values + const pixel_values = stack( + imageData.map( + // Concatenate images and trimaps + (x, i) => cat([x.pixel_values, trimapData[i].pixel_values], 0), + ), + 0, + ); + + return { + pixel_values, + + // Original sizes of images + original_sizes: imageData.map((x) => x.original_size), + + // Reshaped sizes of images, before padding or cropping + reshaped_input_sizes: imageData.map((x) => x.reshaped_input_size), + }; + } +} diff --git a/packages/transformers/src/models/vitmatte/modeling_vitmatte.js b/packages/transformers/src/models/vitmatte/modeling_vitmatte.js new file mode 100644 index 0000000..844db32 --- /dev/null +++ b/packages/transformers/src/models/vitmatte/modeling_vitmatte.js @@ -0,0 +1,64 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { ImageMattingOutput } from '../modeling_outputs.js'; + +export class VitMattePreTrainedModel extends PreTrainedModel {} + +/** + * ViTMatte framework leveraging any vision backbone e.g. for ADE20k, CityScapes. + * + * **Example:** Perform image matting with a `VitMatteForImageMatting` model. + * ```javascript + * import { AutoProcessor, VitMatteForImageMatting, RawImage } from '@huggingface/transformers'; + * + * // Load processor and model + * const processor = await AutoProcessor.from_pretrained('Xenova/vitmatte-small-distinctions-646'); + * const model = await VitMatteForImageMatting.from_pretrained('Xenova/vitmatte-small-distinctions-646'); + * + * // Load image and trimap + * const image = await RawImage.fromURL('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/vitmatte_image.png'); + * const trimap = await RawImage.fromURL('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/vitmatte_trimap.png'); + * + * // Prepare image + trimap for the model + * const inputs = await processor(image, trimap); + * + * // Predict alpha matte + * const { alphas } = await model(inputs); + * // Tensor { + * // dims: [ 1, 1, 640, 960 ], + * // type: 'float32', + * // size: 614400, + * // data: Float32Array(614400) [ 0.9894027709960938, 0.9970508813858032, ... ] + * // } + * ``` + * + * You can visualize the alpha matte as follows: + * ```javascript + * import { Tensor, cat } from '@huggingface/transformers'; + * + * // Visualize predicted alpha matte + * const imageTensor = image.toTensor(); + * + * // Convert float (0-1) alpha matte to uint8 (0-255) + * const alphaChannel = alphas + * .squeeze(0) + * .mul_(255) + * .clamp_(0, 255) + * .round_() + * .to('uint8'); + * + * // Concatenate original image with predicted alpha + * const imageData = cat([imageTensor, alphaChannel], 0); + * + * // Save output image + * const outputImage = RawImage.fromTensor(imageData); + * outputImage.save('output.png'); + * ``` + */ +export class VitMatteForImageMatting extends VitMattePreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new ImageMattingOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/vitpose/image_processing_vitpose.js b/packages/transformers/src/models/vitpose/image_processing_vitpose.js new file mode 100644 index 0000000..d3a14c2 --- /dev/null +++ b/packages/transformers/src/models/vitpose/image_processing_vitpose.js @@ -0,0 +1,87 @@ +import { ImageProcessor } from '../../image_processors_utils.js'; + +export class VitPoseImageProcessor extends ImageProcessor { + /** + * Transform the heatmaps into keypoint predictions and transform them back to the image. + * NOTE: This is a naive implementation and does not include advanced post-processing techniques, + * so the results may not be as accurate as the original implementation. + * @param {import('../../utils/tensor.js').Tensor} outputs The model outputs. + * @param {[number, number, number, number][][]} boxes List or array of bounding boxes for each image. + * Each box should be a list of 4 floats representing the bounding box coordinates in COCO format (top_left_x, top_left_y, width, height). + * @returns {{ + * bbox: [number, number, number, number], + * scores: number[], + * labels: number[], + * keypoints: [number, number][] + * }[][]} List of keypoints predictions for each image. + */ + post_process_pose_estimation( + outputs, + boxes, + { + threshold = null, + // TODO: + // kernel_size = 11, + // target_sizes = null, + } = {}, + ) { + // NOTE: boxes are 3D (batch_size, num_boxes, 4) + const heatmaps = outputs.tolist(); + const [batch_size, num_classes, height, width] = outputs.dims; + + const results = []; + for (let b = 0; b < batch_size; ++b) { + const heatmap = heatmaps[b]; + const bboxes = boxes[b]; + + const batch_results = []; + for (let n = 0; n < bboxes.length; ++n) { + const bbox = bboxes[n]; + + const keypoints = []; + const scores = []; + const labels = []; + + const xScale = bbox.at(-2) / width; + const yScale = bbox.at(-1) / height; + for (let c = 0; c < heatmap.length; ++c) { + let [xWeightedSum, yWeightedSum] = [0, 0]; + let sum = 0; + let score = -Infinity; + const row = heatmap[c]; + for (let y = 0; y < row.length; ++y) { + const col = row[y]; + for (let x = 0; x < col.length; ++x) { + const value = col[x]; + sum += value; + + score = Math.max(score, value); + + // Get weighted sum of positions + // TODO: Determine best offsets + xWeightedSum += (x + 0.5) * value; + yWeightedSum += y * value; + } + } + + // Ignore low scores, if threshold is set + if (threshold != null && score < threshold) continue; + + /** @type {[number, number]} */ + const keypoint = [(xScale * xWeightedSum) / sum, (yScale * yWeightedSum) / sum]; + keypoints.push(keypoint); + labels.push(c); + scores.push(score); + } + batch_results.push({ + bbox, + scores, + labels, + keypoints, + }); + } + results.push(batch_results); + } + return results; + } +} diff --git a/packages/transformers/src/models/vitpose/modeling_vitpose.js b/packages/transformers/src/models/vitpose/modeling_vitpose.js new file mode 100644 index 0000000..a1bc2c6 --- /dev/null +++ b/packages/transformers/src/models/vitpose/modeling_vitpose.js @@ -0,0 +1,8 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class VitPosePreTrainedModel extends PreTrainedModel {} + +/** + * The VitPose model with a pose estimation head on top. + */ +export class VitPoseForPoseEstimation extends VitPosePreTrainedModel {} diff --git a/packages/transformers/src/models/vits/modeling_vits.js b/packages/transformers/src/models/vits/modeling_vits.js new file mode 100644 index 0000000..32b4588 --- /dev/null +++ b/packages/transformers/src/models/vits/modeling_vits.js @@ -0,0 +1,60 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { ModelOutput } from '../modeling_outputs.js'; + +/** + * Describes the outputs for the VITS model. + */ +export class VitsModelOutput extends ModelOutput { + /** + * @typedef {import('../../utils/tensor.js').Tensor} Tensor + */ + + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.waveform The final audio waveform predicted by the model, of shape `(batch_size, sequence_length)`. + * @param {Tensor} output.spectrogram The log-mel spectrogram predicted at the output of the flow model. + * This spectrogram is passed to the Hi-Fi GAN decoder model to obtain the final audio waveform. + */ + constructor({ waveform, spectrogram }) { + super(); + this.waveform = waveform; + this.spectrogram = spectrogram; + } +} + +export class VitsPreTrainedModel extends PreTrainedModel {} + +/** + * The complete VITS model, for text-to-speech synthesis. + * + * **Example:** Generate speech from text with `VitsModel`. + * ```javascript + * import { AutoTokenizer, VitsModel } from '@huggingface/transformers'; + * + * // Load the tokenizer and model + * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/mms-tts-eng'); + * const model = await VitsModel.from_pretrained('Xenova/mms-tts-eng'); + * + * // Run tokenization + * const inputs = tokenizer('I love transformers'); + * + * // Generate waveform + * const { waveform } = await model(inputs); + * // Tensor { + * // dims: [ 1, 35328 ], + * // type: 'float32', + * // data: Float32Array(35328) [ ... ], + * // size: 35328, + * // } + * ``` + */ +export class VitsModel extends VitsPreTrainedModel { + /** + * Calls the model on new inputs. + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} The outputs for the VITS model. + */ + async _call(model_inputs) { + return new VitsModelOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/vits/tokenization_vits.js b/packages/transformers/src/models/vits/tokenization_vits.js new file mode 100644 index 0000000..9f34a37 --- /dev/null +++ b/packages/transformers/src/models/vits/tokenization_vits.js @@ -0,0 +1,22 @@ +import { Decoder } from '@huggingface/tokenizers'; + +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +class VitsDecoder extends Decoder { + /** @type {Decoder['decode_chain']} */ + decode_chain(tokens) { + let decoded = ''; + for (let i = 1; i < tokens.length; i += 2) { + decoded += tokens[i]; + } + return [decoded]; + } +} +export class VitsTokenizer extends PreTrainedTokenizer { + constructor(tokenizerJSON, tokenizerConfig) { + super(tokenizerJSON, tokenizerConfig); + + // Custom decoder function + this._tokenizer.decoder = new VitsDecoder({ type: 'VitsDecoder' }); + } +} diff --git a/packages/transformers/src/models/voxtral/modeling_voxtral.js b/packages/transformers/src/models/voxtral/modeling_voxtral.js new file mode 100644 index 0000000..e45885c --- /dev/null +++ b/packages/transformers/src/models/voxtral/modeling_voxtral.js @@ -0,0 +1,3 @@ +import { UltravoxModel } from '../ultravox/modeling_ultravox.js'; + +export class VoxtralForConditionalGeneration extends UltravoxModel {} diff --git a/packages/transformers/src/models/voxtral/processing_voxtral.js b/packages/transformers/src/models/voxtral/processing_voxtral.js new file mode 100644 index 0000000..1d0d421 --- /dev/null +++ b/packages/transformers/src/models/voxtral/processing_voxtral.js @@ -0,0 +1,93 @@ +import { AutoFeatureExtractor } from '../auto/feature_extraction_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { Processor } from '../../processing_utils.js'; +import { cat } from '../../utils/tensor.js'; + +const AUDIO_TOKEN = '[AUDIO]'; +const BEGIN_AUDIO_TOKEN = '[BEGIN_AUDIO]'; +const NUM_AUDIO_TOKENS = 375; + +/** + * Helper function to split audio into non-overlapping chunks of n_samples + * @param {Float32Array} audio + * @param {number} n_samples + * @returns {Float32Array[]} + */ +function chunk(audio, n_samples) { + const chunks = []; + for (let i = 0; i < audio.length; i += n_samples) { + chunks.push(audio.subarray(i, Math.min(i + n_samples, audio.length))); + } + return chunks; +} + +/** + * Represents a VoxtralProcessor that extracts features from an audio input. + */ +export class VoxtralProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static feature_extractor_class = AutoFeatureExtractor; + static uses_processor_config = false; + + /** + * @param {string} text The text input to process. + * @param {Float32Array|Float32Array[]} audio The audio input(s) to process. + */ + async _call(text, audio = null, kwargs = {}) { + if (Array.isArray(text)) { + throw new Error('Batched inputs are not supported yet.'); + } + + const audio_inputs = {}; + if (audio) { + if (!text.includes(AUDIO_TOKEN)) { + throw new Error(`The input text does not contain the audio token ${AUDIO_TOKEN}.`); + } + if (!Array.isArray(audio)) { + audio = [audio]; + } + const text_parts = text.split(AUDIO_TOKEN); + const num_audio_tokens = text_parts.length - 1; + if (num_audio_tokens !== audio.length) { + throw new Error( + `The number of audio inputs (${audio.length}) does not match the number of audio tokens in the text (${num_audio_tokens}).`, + ); + } + + const n_samples = this.feature_extractor.config.n_samples; + + // Split each audio input into chunks and keep track of chunk counts + const audio_chunks = audio.map((a) => chunk(a, n_samples)); + const chunk_counts = audio_chunks.map((chunks) => chunks.length); + + // Flatten all chunks for feature extraction + const all_chunks = audio_chunks.flat(); + const features = ( + await Promise.all(all_chunks.map((audio_input) => this.feature_extractor(audio_input, kwargs))) + ).map((x) => x.input_features); + + audio_inputs['audio_values'] = features.length > 1 ? cat(features, 0) : features[0]; + + // Replace text tokens for each audio input, expanding for chunk count + let new_text = text_parts[0]; + for (let i = 0; i < chunk_counts.length; ++i) { + new_text += BEGIN_AUDIO_TOKEN; + for (let j = 0; j < chunk_counts[i]; ++j) { + new_text += AUDIO_TOKEN.repeat(NUM_AUDIO_TOKENS); + } + new_text += text_parts[i + 1]; + } + text = new_text; + } + + const text_inputs = this.tokenizer(text, { + add_special_tokens: false, + ...kwargs, + }); + + return { + ...text_inputs, + ...audio_inputs, + }; + } +} diff --git a/packages/transformers/src/models/voxtral_realtime/feature_extraction_voxtral_realtime.js b/packages/transformers/src/models/voxtral_realtime/feature_extraction_voxtral_realtime.js new file mode 100644 index 0000000..b5f174a --- /dev/null +++ b/packages/transformers/src/models/voxtral_realtime/feature_extraction_voxtral_realtime.js @@ -0,0 +1,71 @@ +import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { mel_filter_bank, spectrogram, window_function } from '../../utils/audio.js'; + +export class VoxtralRealtimeFeatureExtractor extends FeatureExtractor { + constructor(config) { + super(config); + + this.config.mel_filters ??= mel_filter_bank( + Math.floor(1 + this.config.n_fft / 2), // num_frequency_bins + this.config.feature_size, // num_mel_filters + 0.0, // min_frequency + 8000.0, // max_frequency + this.config.sampling_rate, // sampling_rate + 'slaney', // norm + 'slaney', // mel_scale + ); + + this.window = window_function(this.config.n_fft, 'hann'); + } + + /** + * Computes the log-Mel spectrogram of the provided audio waveform. + * @param {Float32Array|Float64Array} waveform The audio waveform to process. + * @param {Object} [options] + * @param {boolean} [options.center=true] Whether to center-pad the waveform for STFT. + * @returns {Promise} The log-Mel spectrogram tensor of shape [num_mel_bins, num_frames]. + */ + async _extract_fbank_features(waveform, { center = true } = {}) { + const { n_fft, hop_length, mel_filters, global_log_mel_max } = this.config; + + // torch.stft drops the last frame via [:-1] + // center=True: floor(signal_len / hop_length) frames + // center=False: floor((signal_len - n_fft) / hop_length) frames + const max_num_frames = center + ? Math.floor(waveform.length / hop_length) + : Math.floor((waveform.length - n_fft) / hop_length); + + return await spectrogram( + waveform, + this.window, + n_fft, // frame_length + hop_length, + { + power: 2.0, + mel_filters, + log_mel: 'log10_max_norm', + max_log_mel: global_log_mel_max, + center, + max_num_frames, + do_pad: false, + }, + ); + } + + /** + * Extract mel spectrogram features from audio. + * @param {Float32Array|Float64Array} audio The audio data. + * @param {Object} [options] + * @param {boolean} [options.center=true] Whether to center-pad the waveform. + * @returns {Promise<{ input_features: import('../../utils/tensor.js').Tensor }>} + */ + async _call(audio, { center = true } = {}) { + validate_audio_inputs(audio, 'VoxtralRealtimeFeatureExtractor'); + + const features = await this._extract_fbank_features(audio, { center }); + + return { + input_features: features.unsqueeze_(0), + }; + } +} diff --git a/packages/transformers/src/models/voxtral_realtime/modeling_voxtral_realtime.js b/packages/transformers/src/models/voxtral_realtime/modeling_voxtral_realtime.js new file mode 100644 index 0000000..75f2422 --- /dev/null +++ b/packages/transformers/src/models/voxtral_realtime/modeling_voxtral_realtime.js @@ -0,0 +1,248 @@ +import { PreTrainedModel, addPastKeyValues, resolveCacheShape } from '../modeling_utils.js'; +import { sessionRun } from '../session.js'; +import { getCacheNames } from '../../configs.js'; +import { Tensor, ones } from '../../utils/tensor.js'; +import { DataTypeMap } from '../../utils/dtypes.js'; +import { pick } from '../../utils/core.js'; +import { DynamicCache } from '../../cache_utils.js'; +import { StoppingCriteria, StoppingCriteriaList } from '../../generation/stopping_criteria.js'; + +// Causal conv padding constants +const CONV1_LEFT_PAD = 2; +const CONV2_LEFT_PAD = 1; + +/** + * WeakMap to hold encoder streaming states for each model instance during generation. + * This allows the state to be accessed and modified across the generation process + * without exposing it on the model instance itself. + * @private + * @type {WeakMap} + */ +const states = new WeakMap(); + +/** + * Creates encoder streaming state for a VoxtralRealtime generation session. + * @param {VoxtralRealtimeForConditionalGeneration} model + * @param {Iterable|AsyncIterable} input_features + * @returns {Object} Encoder state object. + * @private + */ +function createEncoderState(model, input_features) { + const { text_config, audio_config } = /** @type {any} */ (model.config); + const encoder_session = model.sessions['audio_encoder']; + + const { num_mel_bins, hidden_size: enc_hidden_size } = audio_config; + const PADDING_CACHE_CHANNELS = num_mel_bins + enc_hidden_size; + + // Initialize encoder KV cache + const enc_kv_cache = new DynamicCache(); + const enc_names = getCacheNames(audio_config); + const enc_symbols = { batch_size: 1 }; + /** @type {import('onnxruntime-common').Tensor.Type} */ + let padding_type = 'float32'; + for (const meta of encoder_session.inputMetadata) { + if (meta.name === 'past_padding_cache') { + padding_type = meta.type; + continue; + } + if (!enc_names.has(meta.name)) continue; + const shape = resolveCacheShape(meta.shape, enc_symbols); + const size = shape.reduce((a, b) => a * b, 1); + const cls = DataTypeMap[meta.type]; + enc_kv_cache[meta.name] = new Tensor(meta.type, new cls(size), shape); + } + + const padding_cls = DataTypeMap[padding_type]; + const enc_padding_cache = new Tensor(padding_type, new padding_cls(PADDING_CACHE_CHANNELS * CONV1_LEFT_PAD), [ + 1, + PADDING_CACHE_CHANNELS, + CONV1_LEFT_PAD, + ]); + + // Set up iterator from input_features + const chunks_iter = input_features[Symbol.asyncIterator]?.() ?? input_features[Symbol.iterator]?.(); + if (!chunks_iter) { + throw new Error('input_features must be iterable or async iterable'); + } + + return { + encoder_session, + enc_kv_cache, + enc_padding_cache, + enc_past_seq_len: 0, + audio_embed_queue: [], + audio_embed_total_tokens: 0, + audio_queue_offset: 0, + audio_consumed: 0, + stream_exhausted: false, + chunks_iter, + text_hidden_size: text_config.hidden_size, + }; +} + +/** + * Encodes one audio chunk through the audio encoder. + * @param {Object} s Encoder state. + * @param {Tensor} chunk_features Mel spectrogram chunk [1, num_mel_bins, seq_len]. + * @returns {Promise} Audio embeddings. + * @private + */ +async function encodeChunk(s, chunk_features) { + const audio_seq_len = chunk_features.dims[2]; + const conv2_output_len = Math.floor((CONV2_LEFT_PAD + audio_seq_len - 3) / 2) + 1; + + const position_ids = new Tensor( + 'int64', + BigInt64Array.from({ length: conv2_output_len }, (_, i) => BigInt(s.enc_past_seq_len + i)), + [1, conv2_output_len], + ); + + const total_seq_len = s.enc_past_seq_len + conv2_output_len; + const attention_mask = ones([1, total_seq_len]); + const { audio_embeds, present_padding_cache, ...present_cache } = await sessionRun(s.encoder_session, { + input_features: chunk_features, + attention_mask, + position_ids, + past_padding_cache: s.enc_padding_cache, + ...s.enc_kv_cache, + }); + // Dispose previous padding cache and update + if (s.enc_padding_cache.location === 'gpu-buffer') { + s.enc_padding_cache.dispose(); + } + s.enc_padding_cache = present_padding_cache; + + // Update encoder KV cache, disposing previous tensors + for (const name in present_cache) { + if (name.startsWith('present.')) { + const pastName = name.replace('present', 'past_key_values'); + const prev = s.enc_kv_cache[pastName]; + if (prev?.location === 'gpu-buffer') { + prev.dispose(); + } + s.enc_kv_cache[pastName] = present_cache[name]; + } + } + s.enc_past_seq_len = total_seq_len; + return audio_embeds; +} + +/** + * Fills the audio embedding buffer until it has enough tokens. + * @param {Object} s Encoder state. + * @param {number} needed Total number of audio tokens needed. + * @private + */ +async function fillAudioBuffer(s, needed) { + while (s.audio_embed_total_tokens < needed && !s.stream_exhausted) { + const result = await s.chunks_iter.next(); + if (result.done) { + s.stream_exhausted = true; + break; + } + const new_embeds = await encodeChunk(s, result.value); + s.audio_embed_queue.push({ data: new_embeds.data, tokens: new_embeds.dims[1] }); + s.audio_embed_total_tokens += new_embeds.dims[1]; + } +} + +/** + * Adds audio embeddings to text embeddings from the queue. + * @param {Object} s Encoder state. + * @param {Tensor} inputs_embeds Text embeddings tensor (modified in-place). + * @param {number} current_len Number of tokens to consume. + * @private + */ +function addAudioEmbeddings(s, inputs_embeds, current_len) { + if (s.audio_embed_queue.length === 0) return; + + const embed_data = inputs_embeds.data; + let embed_write_pos = 0; + let remaining = current_len; + + while (remaining > 0 && s.audio_embed_queue.length > 0) { + const front = s.audio_embed_queue[0]; + const available = front.tokens - s.audio_queue_offset; + const n = Math.min(remaining, available); + + const src_offset = s.audio_queue_offset * s.text_hidden_size; + for (let i = 0; i < n * s.text_hidden_size; ++i) { + embed_data[embed_write_pos * s.text_hidden_size + i] += front.data[src_offset + i]; + } + + embed_write_pos += n; + remaining -= n; + s.audio_queue_offset += n; + + if (s.audio_queue_offset >= front.tokens) { + s.audio_embed_queue.shift(); + s.audio_queue_offset = 0; + } + } + s.audio_consumed += current_len - remaining; +} + +/** + * Stopping criterion that triggers when the audio stream is exhausted + * and all buffered audio embeddings have been consumed. + * @private + */ +class AudioExhaustedCriteria extends StoppingCriteria { + constructor(enc_state) { + super(); + this._s = enc_state; + } + _call(input_ids) { + const done = this._s.stream_exhausted && this._s.audio_embed_queue.length === 0; + return input_ids.map(() => done); + } +} + +export class VoxtralRealtimePreTrainedModel extends PreTrainedModel { + forward_params = ['input_ids', 'attention_mask', 'position_ids', 'past_key_values']; +} + +export class VoxtralRealtimeForConditionalGeneration extends VoxtralRealtimePreTrainedModel { + async forward({ input_ids, past_key_values, ...kwargs }) { + const current_len = input_ids.dims[1]; + + const enc = states.get(this); + if (enc) { + // Fill audio buffer and embed tokens with audio + await fillAudioBuffer(enc, enc.audio_consumed + current_len); + } + + const { inputs_embeds } = await sessionRun(this.sessions['embed_tokens'], { input_ids }); + if (enc) { + addAudioEmbeddings(enc, inputs_embeds, current_len); + } + + const decoder_feeds = { inputs_embeds, ...kwargs }; + addPastKeyValues(this, decoder_feeds, past_key_values); + + const session = this.sessions['decoder_model_merged']; + const fixed = pick(decoder_feeds, session.inputNames); + return await sessionRun(session, fixed); + } + + async generate({ input_features, stopping_criteria: userStoppingCriteria, ...kwargs }) { + if (!input_features) { + throw new Error('input_features (generator/iterable) must be provided'); + } + + const enc_state = createEncoderState(this, input_features); + states.set(this, enc_state); + + const stopping_criteria = new StoppingCriteriaList(); + stopping_criteria.push(new AudioExhaustedCriteria(enc_state)); + if (userStoppingCriteria) stopping_criteria.extend(userStoppingCriteria); + + try { + return await super.generate({ ...kwargs, stopping_criteria }); + } finally { + // Cleanup encoder state + enc_state.enc_kv_cache.dispose(); + states.delete(this); + } + } +} diff --git a/packages/transformers/src/models/voxtral_realtime/processing_voxtral_realtime.js b/packages/transformers/src/models/voxtral_realtime/processing_voxtral_realtime.js new file mode 100644 index 0000000..5acca7d --- /dev/null +++ b/packages/transformers/src/models/voxtral_realtime/processing_voxtral_realtime.js @@ -0,0 +1,113 @@ +import { AutoFeatureExtractor } from '../auto/feature_extraction_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { Processor } from '../../processing_utils.js'; +import { Tensor } from '../../utils/tensor.js'; +import { validate_audio_inputs } from '../../feature_extraction_utils.js'; + +// Voxtral Realtime audio config constants (from mistral_common AudioConfig) +const NUM_LEFT_PAD_TOKENS = 32; +const NUM_DELAY_TOKENS = 6; +const AUDIO_LENGTH_PER_TOK = 8; +const OFFLINE_STREAMING_BUFFER_TOKENS = 10; + +/** Token ID for [STREAMING_PAD] in the Voxtral tokenizer. */ +const STREAMING_PAD_TOKEN_ID = 32; + +export class VoxtralRealtimeProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static feature_extractor_class = AutoFeatureExtractor; + static uses_processor_config = false; + + /** Number of mel frames in the first audio chunk. */ + get num_mel_frames_first_audio_chunk() { + return (NUM_DELAY_TOKENS + 1) * AUDIO_LENGTH_PER_TOK; + } + + /** Number of raw audio samples in the first audio chunk. */ + get num_samples_first_audio_chunk() { + const { hop_length, n_fft } = this.feature_extractor.config; + return (this.num_mel_frames_first_audio_chunk - 1) * hop_length + Math.floor(n_fft / 2); + } + + /** Number of raw audio samples per subsequent audio chunk. */ + get num_samples_per_audio_chunk() { + const { hop_length, n_fft } = this.feature_extractor.config; + return AUDIO_LENGTH_PER_TOK * hop_length + n_fft; + } + + /** Number of right-pad tokens for non-streaming mode. */ + get num_right_pad_tokens() { + return NUM_DELAY_TOKENS + 1 + OFFLINE_STREAMING_BUFFER_TOKENS; + } + + /** Number of mel frames per text token. */ + get audio_length_per_tok() { + return AUDIO_LENGTH_PER_TOK; + } + + /** Number of raw audio samples per token. */ + get raw_audio_length_per_tok() { + return AUDIO_LENGTH_PER_TOK * this.feature_extractor.config.hop_length; + } + + /** + * Process audio input for VoxtralRealtime. + * + * In streaming mode with `is_first_audio_chunk=true`, the audio is left-padded + * with silence and mel features are extracted with `center=true`. + * Returns `{ input_ids, input_features }`. + * + * In streaming mode with `is_first_audio_chunk=false`, the audio chunk is + * processed with `center=false` and only `{ input_features }` is returned. + * + * In non-streaming mode, the audio is right-padded to ensure the model + * transcribes the full audio, then processed with `center=true`. + * Returns `{ input_features }`. + * + * @param {Float32Array|Float64Array} audio The audio waveform. + * @param {Object} [options] + * @param {boolean} [options.is_streaming=false] Whether processing in streaming mode. + * @param {boolean} [options.is_first_audio_chunk=true] Whether this is the first audio chunk. + * @returns {Promise} + */ + async _call(audio, { is_streaming = false, is_first_audio_chunk = true } = {}) { + validate_audio_inputs(audio, 'VoxtralRealtimeProcessor'); + + if (!is_streaming && !is_first_audio_chunk) { + throw new Error('In non-streaming mode (`is_streaming=false`), `is_first_audio_chunk` must be `true`.'); + } + + if (is_first_audio_chunk) { + if (is_streaming) { + // Streaming first chunk: left-pad audio with silence, extract mel with center=true, build input_ids + const num_left_pad_samples = NUM_LEFT_PAD_TOKENS * this.raw_audio_length_per_tok; + const padded_audio = new Float32Array(num_left_pad_samples + audio.length); + padded_audio.set(audio, num_left_pad_samples); + + const audio_encoding = await this.feature_extractor(padded_audio, { center: true }); + + // Build input_ids: BOS + (num_left_pad_tokens + num_delay_tokens) * [STREAMING_PAD] + const num_pad_tokens = NUM_LEFT_PAD_TOKENS + NUM_DELAY_TOKENS; + const num_input_tokens = 1 + num_pad_tokens; + const input_ids_data = new BigInt64Array(num_input_tokens).fill(BigInt(STREAMING_PAD_TOKEN_ID)); + input_ids_data[0] = 1n; // BOS + const input_ids = new Tensor('int64', input_ids_data, [1, num_input_tokens]); + + return { + input_ids, + ...audio_encoding, + }; + } else { + // Non-streaming: right-pad audio to ensure full transcription, extract mel with center=true + const right_pad_samples = this.num_right_pad_tokens * this.raw_audio_length_per_tok; + const padded_audio = new Float32Array(audio.length + right_pad_samples); + padded_audio.set(audio); + + return await this.feature_extractor(padded_audio, { center: true }); + } + } else { + // Subsequent streaming chunks: extract mel with center=false + return await this.feature_extractor(audio, { center: false }); + } + } +} diff --git a/packages/transformers/src/models/wav2vec2/feature_extraction_wav2vec2.js b/packages/transformers/src/models/wav2vec2/feature_extraction_wav2vec2.js new file mode 100644 index 0000000..05e3390 --- /dev/null +++ b/packages/transformers/src/models/wav2vec2/feature_extraction_wav2vec2.js @@ -0,0 +1,43 @@ +import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class Wav2Vec2FeatureExtractor extends FeatureExtractor { + /** + * @param {Float32Array} input_values + * @returns {Float32Array} + */ + _zero_mean_unit_var_norm(input_values) { + // TODO support batch? + const sum = input_values.reduce((a, b) => a + b, 0); + const mean = sum / input_values.length; + const variance = input_values.reduce((a, b) => a + (b - mean) ** 2, 0) / input_values.length; + return input_values.map((x) => (x - mean) / Math.sqrt(variance + 1e-7)); + } + + /** + * Asynchronously extracts features from a given audio using the provided configuration. + * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array. + * @returns {Promise<{ input_values: Tensor; attention_mask: Tensor }>} A Promise resolving to an object containing the extracted input features and attention mask as Tensors. + */ + async _call(audio) { + validate_audio_inputs(audio, 'Wav2Vec2FeatureExtractor'); + + if (audio instanceof Float64Array) { + audio = new Float32Array(audio); + } + + let input_values = audio; + + // zero-mean and unit-variance normalization + if (this.config.do_normalize) { + input_values = this._zero_mean_unit_var_norm(input_values); + } + + // TODO: allow user to pass in attention mask + const shape = [1, input_values.length]; + return { + input_values: new Tensor('float32', input_values, shape), + attention_mask: new Tensor('int64', new BigInt64Array(input_values.length).fill(1n), shape), + }; + } +} diff --git a/packages/transformers/src/models/wav2vec2/modeling_wav2vec2.js b/packages/transformers/src/models/wav2vec2/modeling_wav2vec2.js new file mode 100644 index 0000000..fcd69cb --- /dev/null +++ b/packages/transformers/src/models/wav2vec2/modeling_wav2vec2.js @@ -0,0 +1,69 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { CausalLMOutput, SequenceClassifierOutput, TokenClassifierOutput } from '../modeling_outputs.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class Wav2Vec2PreTrainedModel extends PreTrainedModel {} + +/** + * The bare Wav2Vec2 Model transformer outputting raw hidden-states without any specific head on top. + * + * **Example:** Load and run a `Wav2Vec2Model` for feature extraction. + * + * ```javascript + * import { AutoProcessor, AutoModel, read_audio } from '@huggingface/transformers'; + * + * // Read and preprocess audio + * const processor = await AutoProcessor.from_pretrained('Xenova/mms-300m'); + * const audio = await read_audio('https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/mlk.flac', 16000); + * const inputs = await processor(audio); + * + * // Run model with inputs + * const model = await AutoModel.from_pretrained('Xenova/mms-300m'); + * const output = await model(inputs); + * // { + * // last_hidden_state: Tensor { + * // dims: [ 1, 1144, 1024 ], + * // type: 'float32', + * // data: Float32Array(1171456) [ ... ], + * // size: 1171456 + * // } + * // } + * ``` + */ +export class Wav2Vec2Model extends Wav2Vec2PreTrainedModel {} + +export class Wav2Vec2ForCTC extends Wav2Vec2PreTrainedModel { + /** + * @param {Object} model_inputs + * @param {Tensor} model_inputs.input_values Float values of input raw speech waveform. + * @param {Tensor} model_inputs.attention_mask Mask to avoid performing convolution and attention on padding token indices. Mask values selected in [0, 1] + */ + async _call(model_inputs) { + return new CausalLMOutput(await super._call(model_inputs)); + } +} + +export class Wav2Vec2ForSequenceClassification extends Wav2Vec2PreTrainedModel { + /** + * Calls the model on new inputs. + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * Wav2Vec2 Model with a frame classification head on top for tasks like Speaker Diarization. + */ +export class Wav2Vec2ForAudioFrameClassification extends Wav2Vec2PreTrainedModel { + /** + * Calls the model on new inputs. + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/wav2vec2/processing_wav2vec2.js b/packages/transformers/src/models/wav2vec2/processing_wav2vec2.js new file mode 100644 index 0000000..4042d75 --- /dev/null +++ b/packages/transformers/src/models/wav2vec2/processing_wav2vec2.js @@ -0,0 +1,17 @@ +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { AutoFeatureExtractor } from '../auto/feature_extraction_auto.js'; +import { Processor } from '../../processing_utils.js'; + +export class Wav2Vec2Processor extends Processor { + static tokenizer_class = AutoTokenizer; + static feature_extractor_class = AutoFeatureExtractor; + + /** + * Calls the feature_extractor function with the given audio input. + * @param {any} audio The audio input to extract features from. + * @returns {Promise} A Promise that resolves with the extracted features. + */ + async _call(audio) { + return await this.feature_extractor(audio); + } +} diff --git a/packages/transformers/src/models/wav2vec2/tokenization_wav2vec2.js b/packages/transformers/src/models/wav2vec2/tokenization_wav2vec2.js new file mode 100644 index 0000000..0664e41 --- /dev/null +++ b/packages/transformers/src/models/wav2vec2/tokenization_wav2vec2.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class Wav2Vec2CTCTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/wav2vec2_bert/modeling_wav2vec2_bert.js b/packages/transformers/src/models/wav2vec2_bert/modeling_wav2vec2_bert.js new file mode 100644 index 0000000..f58b9b9 --- /dev/null +++ b/packages/transformers/src/models/wav2vec2_bert/modeling_wav2vec2_bert.js @@ -0,0 +1,38 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { CausalLMOutput, SequenceClassifierOutput } from '../modeling_outputs.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class Wav2Vec2BertPreTrainedModel extends PreTrainedModel {} + +/** + * The bare Wav2Vec2Bert Model transformer outputting raw hidden-states without any specific head on top. + */ +export class Wav2Vec2BertModel extends Wav2Vec2BertPreTrainedModel {} + +/** + * Wav2Vec2Bert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). + */ +export class Wav2Vec2BertForCTC extends Wav2Vec2BertPreTrainedModel { + /** + * @param {Object} model_inputs + * @param {Tensor} model_inputs.input_features Float values of input mel-spectrogram. + * @param {Tensor} model_inputs.attention_mask Mask to avoid performing convolution and attention on padding token indices. Mask values selected in [0, 1] + */ + async _call(model_inputs) { + return new CausalLMOutput(await super._call(model_inputs)); + } +} + +/** + * Wav2Vec2Bert Model with a sequence classification head on top (a linear layer over the pooled output). + */ +export class Wav2Vec2BertForSequenceClassification extends Wav2Vec2BertPreTrainedModel { + /** + * Calls the model on new inputs. + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.js b/packages/transformers/src/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.js new file mode 100644 index 0000000..7560ae9 --- /dev/null +++ b/packages/transformers/src/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.js @@ -0,0 +1,17 @@ +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { AutoFeatureExtractor } from '../auto/feature_extraction_auto.js'; +import { Processor } from '../../processing_utils.js'; + +export class Wav2Vec2ProcessorWithLM extends Processor { + static tokenizer_class = AutoTokenizer; + static feature_extractor_class = AutoFeatureExtractor; + + /** + * Calls the feature_extractor function with the given audio input. + * @param {any} audio The audio input to extract features from. + * @returns {Promise} A Promise that resolves with the extracted features. + */ + async _call(audio) { + return await this.feature_extractor(audio); + } +} diff --git a/packages/transformers/src/models/wavlm/modeling_wavlm.js b/packages/transformers/src/models/wavlm/modeling_wavlm.js new file mode 100644 index 0000000..2ec28f2 --- /dev/null +++ b/packages/transformers/src/models/wavlm/modeling_wavlm.js @@ -0,0 +1,171 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { CausalLMOutput, SequenceClassifierOutput, TokenClassifierOutput, ModelOutput } from '../modeling_outputs.js'; +import { Tensor } from '../../utils/tensor.js'; + +/** + * Base class for outputs of XVector models. + */ +export class XVectorOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.logits Classification hidden states before AMSoftmax, of shape `(batch_size, config.xvector_output_dim)`. + * @param {Tensor} output.embeddings Utterance embeddings used for vector similarity-based retrieval, of shape `(batch_size, config.xvector_output_dim)`. + */ + constructor({ logits, embeddings }) { + super(); + this.logits = logits; + this.embeddings = embeddings; + } +} + +/** + * An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. + */ +export class WavLMPreTrainedModel extends PreTrainedModel {} + +/** + * The bare WavLM Model transformer outputting raw hidden-states without any specific head on top. + * + * **Example:** Load and run a `WavLMModel` for feature extraction. + * + * ```javascript + * import { AutoProcessor, AutoModel, read_audio } from '@huggingface/transformers'; + * + * // Read and preprocess audio + * const processor = await AutoProcessor.from_pretrained('Xenova/wavlm-base'); + * const audio = await read_audio('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav', 16000); + * const inputs = await processor(audio); + * + * // Run model with inputs + * const model = await AutoModel.from_pretrained('Xenova/wavlm-base'); + * const output = await model(inputs); + * // { + * // last_hidden_state: Tensor { + * // dims: [ 1, 549, 768 ], + * // type: 'float32', + * // data: Float32Array(421632) [-0.349443256855011, -0.39341306686401367, 0.022836603224277496, ...], + * // size: 421632 + * // } + * // } + * ``` + */ +export class WavLMModel extends WavLMPreTrainedModel {} + +/** + * WavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). + */ +export class WavLMForCTC extends WavLMPreTrainedModel { + /** + * @param {Object} model_inputs + * @param {Tensor} model_inputs.input_values Float values of input raw speech waveform. + * @param {Tensor} model_inputs.attention_mask Mask to avoid performing convolution and attention on padding token indices. Mask values selected in [0, 1] + */ + async _call(model_inputs) { + return new CausalLMOutput(await super._call(model_inputs)); + } +} + +/** + * WavLM Model with a sequence classification head on top (a linear layer over the pooled output). + */ +export class WavLMForSequenceClassification extends WavLMPreTrainedModel { + /** + * Calls the model on new inputs. + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * WavLM Model with an XVector feature extraction head on top for tasks like Speaker Verification. + * + * **Example:** Extract speaker embeddings with `WavLMForXVector`. + * ```javascript + * import { AutoProcessor, AutoModel, read_audio } from '@huggingface/transformers'; + * + * // Read and preprocess audio + * const processor = await AutoProcessor.from_pretrained('Xenova/wavlm-base-plus-sv'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav'; + * const audio = await read_audio(url, 16000); + * const inputs = await processor(audio); + * + * // Run model with inputs + * const model = await AutoModel.from_pretrained('Xenova/wavlm-base-plus-sv'); + * const outputs = await model(inputs); + * // { + * // logits: Tensor { + * // dims: [ 1, 512 ], + * // type: 'float32', + * // data: Float32Array(512) [0.5847219228744507, ...], + * // size: 512 + * // }, + * // embeddings: Tensor { + * // dims: [ 1, 512 ], + * // type: 'float32', + * // data: Float32Array(512) [-0.09079201519489288, ...], + * // size: 512 + * // } + * // } + * ``` + */ +export class WavLMForXVector extends WavLMPreTrainedModel { + /** + * Calls the model on new inputs. + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits and speaker embeddings. + */ + async _call(model_inputs) { + return new XVectorOutput(await super._call(model_inputs)); + } +} + +/** + * WavLM Model with a frame classification head on top for tasks like Speaker Diarization. + * + * **Example:** Perform speaker diarization with `WavLMForAudioFrameClassification`. + * ```javascript + * import { AutoProcessor, AutoModelForAudioFrameClassification, read_audio } from '@huggingface/transformers'; + * + * // Read and preprocess audio + * const processor = await AutoProcessor.from_pretrained('Xenova/wavlm-base-plus-sd'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav'; + * const audio = await read_audio(url, 16000); + * const inputs = await processor(audio); + * + * // Run model with inputs + * const model = await AutoModelForAudioFrameClassification.from_pretrained('Xenova/wavlm-base-plus-sd'); + * const { logits } = await model(inputs); + * // { + * // logits: Tensor { + * // dims: [ 1, 549, 2 ], // [batch_size, num_frames, num_speakers] + * // type: 'float32', + * // data: Float32Array(1098) [-3.5301010608673096, ...], + * // size: 1098 + * // } + * // } + * + * const labels = logits[0].sigmoid().tolist().map( + * frames => frames.map(speaker => speaker > 0.5 ? 1 : 0) + * ); + * console.log(labels); // labels is a one-hot array of shape (num_frames, num_speakers) + * // [ + * // [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], + * // [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], + * // [0, 0], [0, 1], [0, 1], [0, 1], [0, 1], [0, 1], + * // ... + * // ] + * ``` + */ +export class WavLMForAudioFrameClassification extends WavLMPreTrainedModel { + /** + * Calls the model on new inputs. + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for sequence classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/wespeaker/feature_extraction_wespeaker.js b/packages/transformers/src/models/wespeaker/feature_extraction_wespeaker.js new file mode 100644 index 0000000..1fb1154 --- /dev/null +++ b/packages/transformers/src/models/wespeaker/feature_extraction_wespeaker.js @@ -0,0 +1,92 @@ +import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { Tensor } from '../../utils/tensor.js'; +import { mel_filter_bank, spectrogram, window_function } from '../../utils/audio.js'; + +export class WeSpeakerFeatureExtractor extends FeatureExtractor { + constructor(config) { + super(config); + + const sampling_rate = this.config.sampling_rate; + const mel_filters = mel_filter_bank( + 257, // num_frequency_bins + this.config.num_mel_bins, // num_mel_filters + 20, // min_frequency + Math.floor(sampling_rate / 2), // max_frequency + sampling_rate, // sampling_rate + null, // norm + 'kaldi', // mel_scale + true, // triangularize_in_mel_space + ); + this.mel_filters = mel_filters; + + this.window = window_function(400, 'hamming', { + periodic: false, + }); + this.min_num_frames = this.config.min_num_frames; + } + + /** + * Computes the log-Mel spectrogram of the provided audio waveform. + * @param {Float32Array|Float64Array} waveform The audio waveform to process. + * @returns {Promise} An object containing the log-Mel spectrogram data as a Float32Array and its dimensions as an array of numbers. + */ + async _extract_fbank_features(waveform) { + // Kaldi compliance: 16-bit signed integers + // 32768 == 2 ** 15 + waveform = waveform.map((/** @type {number} */ x) => x * 32768); + + return spectrogram( + waveform, + this.window, // window + 400, // frame_length + 160, // hop_length + { + fft_length: 512, + power: 2.0, + center: false, + preemphasis: 0.97, + mel_filters: this.mel_filters, + log_mel: 'log', + mel_floor: 1.192092955078125e-7, + remove_dc_offset: true, + + // Custom + transpose: true, + min_num_frames: this.min_num_frames, + }, + ); + } + + /** + * Asynchronously extracts features from a given audio using the provided configuration. + * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array. + * @returns {Promise<{ input_features: Tensor }>} A Promise resolving to an object containing the extracted input features as a Tensor. + */ + async _call(audio) { + validate_audio_inputs(audio, 'WeSpeakerFeatureExtractor'); + + const features = (await this._extract_fbank_features(audio)).unsqueeze_(0); + + if (this.config.fbank_centering_span === null) { + // center features with global average + const meanData = /** @type {Float32Array} */ (features.mean(1).data); + const featuresData = /** @type {Float32Array} */ (features.data); + const [batch_size, num_frames, feature_size] = features.dims; + + for (let i = 0; i < batch_size; ++i) { + const offset1 = i * num_frames * feature_size; + const offset2 = i * feature_size; + for (let j = 0; j < num_frames; ++j) { + const offset3 = offset1 + j * feature_size; + for (let k = 0; k < feature_size; ++k) { + featuresData[offset3 + k] -= meanData[offset2 + k]; + } + } + } + } + + return { + input_features: features, + }; + } +} diff --git a/packages/transformers/src/models/wespeaker_resnet/modeling_wespeaker_resnet.js b/packages/transformers/src/models/wespeaker_resnet/modeling_wespeaker_resnet.js new file mode 100644 index 0000000..9b597d6 --- /dev/null +++ b/packages/transformers/src/models/wespeaker_resnet/modeling_wespeaker_resnet.js @@ -0,0 +1,4 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class WeSpeakerResNetPreTrainedModel extends PreTrainedModel {} +export class WeSpeakerResNetModel extends WeSpeakerResNetPreTrainedModel {} diff --git a/packages/transformers/src/models/whisper/common_whisper.js b/packages/transformers/src/models/whisper/common_whisper.js new file mode 100644 index 0000000..2802528 --- /dev/null +++ b/packages/transformers/src/models/whisper/common_whisper.js @@ -0,0 +1,156 @@ +const WHISPER_LANGUAGES = [ + ['en', 'english'], + ['zh', 'chinese'], + ['de', 'german'], + ['es', 'spanish'], + ['ru', 'russian'], + ['ko', 'korean'], + ['fr', 'french'], + ['ja', 'japanese'], + ['pt', 'portuguese'], + ['tr', 'turkish'], + ['pl', 'polish'], + ['ca', 'catalan'], + ['nl', 'dutch'], + ['ar', 'arabic'], + ['sv', 'swedish'], + ['it', 'italian'], + ['id', 'indonesian'], + ['hi', 'hindi'], + ['fi', 'finnish'], + ['vi', 'vietnamese'], + ['he', 'hebrew'], + ['uk', 'ukrainian'], + ['el', 'greek'], + ['ms', 'malay'], + ['cs', 'czech'], + ['ro', 'romanian'], + ['da', 'danish'], + ['hu', 'hungarian'], + ['ta', 'tamil'], + ['no', 'norwegian'], + ['th', 'thai'], + ['ur', 'urdu'], + ['hr', 'croatian'], + ['bg', 'bulgarian'], + ['lt', 'lithuanian'], + ['la', 'latin'], + ['mi', 'maori'], + ['ml', 'malayalam'], + ['cy', 'welsh'], + ['sk', 'slovak'], + ['te', 'telugu'], + ['fa', 'persian'], + ['lv', 'latvian'], + ['bn', 'bengali'], + ['sr', 'serbian'], + ['az', 'azerbaijani'], + ['sl', 'slovenian'], + ['kn', 'kannada'], + ['et', 'estonian'], + ['mk', 'macedonian'], + ['br', 'breton'], + ['eu', 'basque'], + ['is', 'icelandic'], + ['hy', 'armenian'], + ['ne', 'nepali'], + ['mn', 'mongolian'], + ['bs', 'bosnian'], + ['kk', 'kazakh'], + ['sq', 'albanian'], + ['sw', 'swahili'], + ['gl', 'galician'], + ['mr', 'marathi'], + ['pa', 'punjabi'], + ['si', 'sinhala'], + ['km', 'khmer'], + ['sn', 'shona'], + ['yo', 'yoruba'], + ['so', 'somali'], + ['af', 'afrikaans'], + ['oc', 'occitan'], + ['ka', 'georgian'], + ['be', 'belarusian'], + ['tg', 'tajik'], + ['sd', 'sindhi'], + ['gu', 'gujarati'], + ['am', 'amharic'], + ['yi', 'yiddish'], + ['lo', 'lao'], + ['uz', 'uzbek'], + ['fo', 'faroese'], + ['ht', 'haitian creole'], + ['ps', 'pashto'], + ['tk', 'turkmen'], + ['nn', 'nynorsk'], + ['mt', 'maltese'], + ['sa', 'sanskrit'], + ['lb', 'luxembourgish'], + ['my', 'myanmar'], + ['bo', 'tibetan'], + ['tl', 'tagalog'], + ['mg', 'malagasy'], + ['as', 'assamese'], + ['tt', 'tatar'], + ['haw', 'hawaiian'], + ['ln', 'lingala'], + ['ha', 'hausa'], + ['ba', 'bashkir'], + ['jw', 'javanese'], + ['su', 'sundanese'], +]; + +// @ts-ignore +export const WHISPER_LANGUAGE_MAPPING = new Map(WHISPER_LANGUAGES); +// @ts-ignore +export const WHISPER_TO_LANGUAGE_CODE_MAPPING = new Map([ + ...WHISPER_LANGUAGES.map(([k, v]) => [v, k]), + ...[ + ['burmese', 'my'], + ['valencian', 'ca'], + ['flemish', 'nl'], + ['haitian', 'ht'], + ['letzeburgesch', 'lb'], + ['pushto', 'ps'], + ['panjabi', 'pa'], + ['moldavian', 'ro'], + ['moldovan', 'ro'], + ['sinhalese', 'si'], + ['castilian', 'es'], + ], +]); + +/** + * @param {string} language The language name or code + * @returns {string} The language code + */ +export function whisper_language_to_code(language) { + language = language.toLowerCase(); + + // Map to code from user-friendly name (e.g., "english" -> "en") + let language_code = WHISPER_TO_LANGUAGE_CODE_MAPPING.get(language); + + if (language_code === undefined) { + // User provided something that is not a language name + + // Perhaps the user passed the special token itself + const language_special_token = language.match(/^<\|([a-z]{2})\|>$/); + if (language_special_token) { + language = language_special_token[1]; + } + + if (WHISPER_LANGUAGE_MAPPING.has(language)) { + // User provided the language code directly (e.g., "en") + language_code = language; + } else { + // User provided something that is not a language code or name + const is_language_code = language.length === 2; + const langs = is_language_code ? WHISPER_LANGUAGE_MAPPING.keys() : WHISPER_LANGUAGE_MAPPING.values(); + + throw new Error( + `Language "${language}" is not supported. Must be one of: ${JSON.stringify(Array.from(langs))}`, + ); + } + } + return language_code; +} diff --git a/packages/transformers/src/models/whisper/feature_extraction_whisper.js b/packages/transformers/src/models/whisper/feature_extraction_whisper.js new file mode 100644 index 0000000..7849c01 --- /dev/null +++ b/packages/transformers/src/models/whisper/feature_extraction_whisper.js @@ -0,0 +1,80 @@ +import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js'; +import { Tensor } from '../../utils/tensor.js'; +import { mel_filter_bank, spectrogram, window_function } from '../../utils/audio.js'; +import { logger } from '../../utils/logger.js'; + +export class WhisperFeatureExtractor extends FeatureExtractor { + constructor(config) { + super(config); + + // Prefer given `mel_filters` from preprocessor_config.json, or calculate them if they don't exist. + this.config.mel_filters ??= mel_filter_bank( + Math.floor(1 + this.config.n_fft / 2), // num_frequency_bins + this.config.feature_size, // num_mel_filters + 0.0, // min_frequency + 8000.0, // max_frequency + this.config.sampling_rate, // sampling_rate + 'slaney', // norm + 'slaney', // mel_scale + ); + + this.window = window_function(this.config.n_fft, 'hann'); + } + + /** + * Computes the log-Mel spectrogram of the provided audio waveform. + * @param {Float32Array|Float64Array} waveform The audio waveform to process. + * @returns {Promise} An object containing the log-Mel spectrogram data as a Float32Array and its dimensions as an array of numbers. + */ + async _extract_fbank_features(waveform) { + return await spectrogram( + waveform, + this.window, // window + this.config.n_fft, // frame_length + this.config.hop_length, // hop_length + { + power: 2.0, + mel_filters: this.config.mel_filters, + log_mel: 'log10_max_norm', + + // Custom + max_num_frames: Math.min( + Math.floor(waveform.length / this.config.hop_length), + this.config.nb_max_frames, // 3000 + ), + }, + ); + } + + /** + * Asynchronously extracts features from a given audio using the provided configuration. + * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array. + * @returns {Promise<{ input_features: Tensor }>} A Promise resolving to an object containing the extracted input features as a Tensor. + */ + async _call(audio, { max_length = null } = {}) { + validate_audio_inputs(audio, 'WhisperFeatureExtractor'); + + let waveform; + const length = max_length ?? this.config.n_samples; + if (audio.length > length) { + if (audio.length > this.config.n_samples) { + logger.warn( + 'Attempting to extract features for audio longer than 30 seconds. ' + + 'If using a pipeline to extract transcript from a long audio clip, ' + + 'remember to specify `chunk_length_s` and/or `stride_length_s`.', + ); + } + waveform = audio.slice(0, length); + } else { + // pad with zeros + waveform = new Float32Array(length); + waveform.set(audio); + } + + const features = await this._extract_fbank_features(waveform); + + return { + input_features: features.unsqueeze_(0), + }; + } +} diff --git a/packages/transformers/src/models/whisper/generation_whisper.js b/packages/transformers/src/models/whisper/generation_whisper.js new file mode 100644 index 0000000..0fd1daa --- /dev/null +++ b/packages/transformers/src/models/whisper/generation_whisper.js @@ -0,0 +1,88 @@ +import { GenerationConfig } from '../../generation/configuration_utils.js'; + +export class WhisperGenerationConfig extends GenerationConfig { + /** + * Whether to return the timestamps with the text. This enables the `WhisperTimestampsLogitsProcessor`. + * @type {boolean} + */ + return_timestamps = null; + + /** + * Whether to return token-level timestamps + * with the text. This can be used with or without the `return_timestamps` option. To get word-level + * timestamps, use the tokenizer to group the tokens into words. + * @type {boolean} + */ + return_token_timestamps = null; + + /** + * The number of audio frames available in this chunk. This is only used generating word-level timestamps. + * @type {number} + */ + num_frames = null; + + /** + * Alignment heads to predict word-level timestamps. This is a list of [layer, head] pairs that + * select the cross-attention heads that are highly correlated to word-level timing. + * @type {[number, number][]} + */ + alignment_heads = null; + + /** + * Task to use for generation, either "translate" or "transcribe". + * @type {string} + */ + task = null; + + /** + * Language token to use for generation, can be either in the form of `<|en|>`, `en` or `english`. + * You can find all the possible language tokens in the `model.generation_config.lang_to_id` dictionary. + * @type {string} + */ + language = null; + + /** + * The id of the `"<|notimestamps|>"` token. + * @type {number} + */ + no_timestamps_token_id = null; + + /** + * Rank-1 list of token IDs created by passing text to [`~WhisperProcessor.get_prompt_ids`] that is + * provided as a prompt to each chunk. This can be used to provide or "prompt-engineer" a context for + * transcription, e.g. custom vocabularies or proper nouns to make it more likely to predict those words + * correctly. It cannot be used in conjunction with `decoder_start_token_id` as it overwrites this value. + * @type {number[]} + */ + prompt_ids = null; + + /** + * Whether the model is multilingual or not. + * @type {boolean} + */ + is_multilingual = null; + + /** + * (Optional) A mapping from language tokens to their corresponding IDs. + * Only required if the model is multilingual. + * @type {Record|null} + */ + lang_to_id = null; + + /** + * (Optional) A mapping from task tokens to their corresponding IDs. + * @type {Record|null} + */ + task_to_id = null; + + /** + * Used to set the maximum value of the initial timestamp. This is used to prevent the model from + * predicting timestamps that are too far in the future. + * @type {number} + */ + max_initial_timestamp_index = 1; +} + +/** + * @typedef {import('../../generation/parameters.js').GenerationFunctionParameters & {generation_config: WhisperGenerationConfig} & WhisperGenerationConfig} WhisperGenerationFunctionParameters + */ diff --git a/packages/transformers/src/models/whisper/modeling_whisper.js b/packages/transformers/src/models/whisper/modeling_whisper.js new file mode 100644 index 0000000..d0d6f0b --- /dev/null +++ b/packages/transformers/src/models/whisper/modeling_whisper.js @@ -0,0 +1,514 @@ +import { cat, mean, Tensor, stack, std_mean } from '../../utils/tensor.js'; +import { PreTrainedModel } from '../modeling_utils.js'; +import { WhisperGenerationConfig } from './generation_whisper.js'; +import { whisper_language_to_code } from './common_whisper.js'; +import { prepareTensorForDecode } from '../../tokenization_utils.js'; +import { + LogitsProcessorList, + SuppressTokensAtBeginLogitsProcessor, + WhisperTimeStampLogitsProcessor, +} from '../../generation/logits_process.js'; +import { medianFilter, dynamic_time_warping } from '../../utils/maths.js'; +import { mergeArrays } from '../../utils/core.js'; +import { ModelOutput } from '../modeling_outputs.js'; +import { logger } from '../../utils/logger.js'; + +export class WhisperPreTrainedModel extends PreTrainedModel { + requires_attention_mask = false; + main_input_name = 'input_features'; + forward_params = [ + 'input_features', + 'attention_mask', + 'decoder_input_ids', + 'decoder_attention_mask', + 'past_key_values', + ]; +} + +/** + * WhisperModel class for training Whisper models without a language model head. + */ +export class WhisperModel extends WhisperPreTrainedModel {} + +/** + * WhisperForConditionalGeneration class for generating conditional outputs from Whisper models. + */ +export class WhisperForConditionalGeneration extends WhisperPreTrainedModel { + _prepare_generation_config(generation_config, kwargs) { + return /** @type {WhisperGenerationConfig} */ ( + super._prepare_generation_config(generation_config, kwargs, WhisperGenerationConfig) + ); + } + + /** + * + * @param {WhisperGenerationConfig} generation_config + */ + _retrieve_init_tokens(generation_config) { + // prefix tokens are of the form: + // - Multilingual: <|startoftranscript|> <|lang_id|> <|task|> [<|notimestamps|>] + // - English-only: <|startoftranscript|> [<|notimestamps|>] + + // 1. Handle <|startoftranscript|> token + const init_tokens = [generation_config.decoder_start_token_id]; + + // 2. Handle <|lang_id|> and <|task> tokens + let language = generation_config.language; + const task = generation_config.task; + if (generation_config.is_multilingual) { + if (!language) { + // TODO: Implement language detection + logger.warn('No language specified - defaulting to English (en).'); + language = 'en'; + } + + // Add language token + const language_code = whisper_language_to_code(language); + const language_token = `<|${language_code}|>`; + init_tokens.push(generation_config.lang_to_id[language_token]); + + // Add task token + // NOTE: Defaults to 'transcribe' if no task is specified + init_tokens.push(generation_config.task_to_id[task ?? 'transcribe']); + } else if (language || task) { + throw new Error( + 'Cannot specify `task` or `language` for an English-only model. If the model is intended to be multilingual, pass `is_multilingual=true` to generate, or update the generation config.', + ); + } + + // 3. Handle <|notimestamps|> token + if ( + !generation_config.return_timestamps && + generation_config.no_timestamps_token_id && + init_tokens.at(-1) !== generation_config.no_timestamps_token_id + ) { + init_tokens.push(generation_config.no_timestamps_token_id); + } else if ( + generation_config.return_timestamps && + init_tokens.at(-1) === generation_config.no_timestamps_token_id + ) { + logger.warn( + '<|notimestamps|> prompt token is removed from generation_config since `return_timestamps` is set to `true`.', + ); + init_tokens.pop(); + } + + // let's make sure we don't pass `null` tokens as prompt tokens + return init_tokens.filter((token) => token != null); + } + + /** + * Transcribes or translates log-mel input features to a sequence of auto-regressively generated token ids. + * @param {import('./generation_whisper.js').WhisperGenerationFunctionParameters} options + * @returns {Promise} The output of the model, which can contain the generated token ids, attentions, and scores. + */ + async generate({ + inputs = null, + generation_config = null, + logits_processor = null, + stopping_criteria = null, + + // Whisper-specific options (passed to kwargs) + // prompt_ids = null, + // language = null, + // task = null, + + ...kwargs + }) { + generation_config = this._prepare_generation_config(generation_config, kwargs); + + const init_tokens = + kwargs.decoder_input_ids instanceof Tensor + ? prepareTensorForDecode(kwargs.decoder_input_ids) + : (kwargs.decoder_input_ids ?? this._retrieve_init_tokens(generation_config)); + + if (generation_config.return_timestamps) { + logits_processor ??= new LogitsProcessorList(); + logits_processor.push(new WhisperTimeStampLogitsProcessor(generation_config, init_tokens)); + } + + if (generation_config.begin_suppress_tokens) { + logits_processor ??= new LogitsProcessorList(); + logits_processor.push( + new SuppressTokensAtBeginLogitsProcessor(generation_config.begin_suppress_tokens, init_tokens.length), + ); + } + + if (generation_config.return_token_timestamps) { + if (!generation_config.alignment_heads) { + throw new Error( + 'Model generation config has no `alignment_heads`, token-level timestamps not available. ' + + 'See https://gist.github.com/hollance/42e32852f24243b748ae6bc1f985b13a on how to add this property to the generation config.', + ); + } + + if (generation_config.task === 'translate') { + logger.warn("Token-level timestamps may not be reliable for task 'translate'."); + } + + generation_config.output_attentions = true; + generation_config.return_dict_in_generate = true; + } + + // For timestamp mode, use seek-based sequential generation. + // This matches Python's WhisperForConditionalGeneration.generate() which uses a seek loop + // to handle audio that the model doesn't fully transcribe in a single pass. + // Skip the seek loop when max_new_tokens is explicitly set (e.g., for prefix token tests). + if (generation_config.return_timestamps && !kwargs.max_new_tokens) { + return this._generate_with_seek({ + inputs, + generation_config, + logits_processor, + init_tokens, + kwargs, + }); + } + + const outputs = await super.generate({ + inputs, + generation_config, + logits_processor, + decoder_input_ids: init_tokens, + ...kwargs, + }); + + if (generation_config.return_token_timestamps) { + outputs['token_timestamps'] = this._extract_token_timestamps( + // @ts-expect-error TS2345 + outputs, + generation_config.alignment_heads, + generation_config.num_frames, + 0.02, + init_tokens.length, + ); + } + + return outputs; + } + + /** + * Generates with a seek loop for timestamp mode, re-encoding and generating + * for each segment until all audio frames are consumed. + * This matches Python's WhisperForConditionalGeneration.generate() behavior. + * @private + */ + async _generate_with_seek({ inputs, generation_config, logits_processor, init_tokens, kwargs }) { + const timestamp_begin = generation_config.no_timestamps_token_id + 1; + const eos_token_id = Array.isArray(generation_config.eos_token_id) + ? generation_config.eos_token_id[0] + : generation_config.eos_token_id; + const return_token_timestamps = generation_config.return_token_timestamps; + + // input_features shape: [batch=1, n_mels, total_frames] + const input_features = inputs; + const total_frames = input_features.dims[2]; + + // The encoder downsamples by input_stride (=2 for whisper), so: + // num_segment_frames = input_stride * max_source_positions = 3000 mel frames per segment + // Timestamp token T maps to mel frame position T * input_stride + const input_stride = 2; + // @ts-expect-error ts(2339) + const max_source_positions = /** @type {number} */ (this.config.max_source_positions); + const num_segment_frames = input_stride * max_source_positions; + + let seek = 0; + const allTokens = []; + const allTokenTimestamps = []; + + while (seek < total_frames) { + // Slice input features for this segment + const seek_end = Math.min(seek + num_segment_frames, total_frames); + const segment_input = input_features.slice(null, null, [seek, seek_end]); + + // Pad to full segment size if needed (whisper expects fixed-length input) + let segment_features; + const segment_frames = segment_input.dims[2]; + if (segment_frames < num_segment_frames) { + const n_mels = input_features.dims[1]; + const padded_data = new Float32Array(n_mels * num_segment_frames); + const src = /** @type {Float32Array} */ (segment_input.data); + // Copy each mel band row separately to handle the stride difference + for (let m = 0; m < n_mels; ++m) { + padded_data.set(src.subarray(m * segment_frames, (m + 1) * segment_frames), m * num_segment_frames); + } + segment_features = new Tensor('float32', padded_data, [1, n_mels, num_segment_frames]); + } else { + segment_features = segment_input; + } + + // Reset logits processor begin_index for each segment + if (logits_processor) { + for (const proc of logits_processor) { + if ('begin_index' in proc) { + proc.begin_index = init_tokens.length; + } + } + } + + const outputs = /** @type {any} */ ( + await super.generate({ + inputs: segment_features, + generation_config, + logits_processor, + decoder_input_ids: init_tokens, + ...kwargs, + }) + ); + + // Extract tokens (skip init_tokens prefix) + const raw_sequence = return_token_timestamps ? outputs.sequences : /** @type {Tensor} */ (outputs); + const generated_tokens = raw_sequence[0].tolist().map(Number).slice(init_tokens.length); + + // Extract token-level timestamps for this seek pass if needed + let seek_token_timestamps; + if (return_token_timestamps) { + outputs['token_timestamps'] = this._extract_token_timestamps( + outputs, + generation_config.alignment_heads, + Math.floor((seek_end - seek) / input_stride), + 0.02, + init_tokens.length, + ); + const time_offset = (seek / input_stride) * 0.02; + seek_token_timestamps = outputs.token_timestamps[0] + .tolist() + .slice(init_tokens.length) + .map((/** @type {number} */ t) => t + time_offset); + } + + // Remove trailing EOS + if (generated_tokens.length > 0 && generated_tokens.at(-1) === eos_token_id) { + generated_tokens.pop(); + } + + if (generated_tokens.length === 0) { + // No tokens generated — skip the rest of the audio + break; + } + + // Determine seek advancement using the same logic as Python's _retrieve_segment: + // 1. Find consecutive timestamp token pairs (segment boundaries) + // 2. If the sequence ends with a single timestamp (no speech after it), + // consume all remaining frames in this segment + // 3. Otherwise, seek to the last complete segment boundary + const is_timestamp = generated_tokens.map((t) => t >= timestamp_begin); + + // Check for single_timestamp_ending: last token is timestamp, second-to-last is not + const single_timestamp_ending = + generated_tokens.length >= 2 && + is_timestamp[generated_tokens.length - 1] && + !is_timestamp[generated_tokens.length - 2]; + + // Find consecutive timestamp pairs (segment boundaries) + const segment_boundary_indices = []; + for (let i = 0; i < generated_tokens.length - 1; ++i) { + if (is_timestamp[i] && is_timestamp[i + 1]) { + segment_boundary_indices.push(i + 1); // index of the second token in the pair + } + } + + let segment_offset; + let tokens_to_keep = generated_tokens.length; + if (segment_boundary_indices.length > 0) { + if (single_timestamp_ending) { + // Ends with a single timestamp after the last pair — no more speech + segment_offset = seek_end - seek; + } else { + // Ends mid-segment — seek to the last pair's end timestamp + // Discard tokens after the last pair (they're from an incomplete segment) + // Keep up to the first token of the last pair (the end-of-segment timestamp), + // excluding the second token (the start-of-next-segment marker) + const last_boundary = segment_boundary_indices.at(-1); + const last_ts_pos = generated_tokens[last_boundary - 1] - timestamp_begin; + segment_offset = last_ts_pos * input_stride; + tokens_to_keep = last_boundary; + } + } else { + // No consecutive pairs found — consume entire segment + segment_offset = seek_end - seek; + } + + // Offset timestamp tokens by the current seek position so they're + // monotonically increasing across segments. Cap at the maximum valid + // timestamp token (30.00s = 1500 positions) to stay within the token vocab. + const timestamp_offset = Math.floor(seek / input_stride); + const max_timestamp_token = timestamp_begin + 1500; + for (let i = 0; i < tokens_to_keep; ++i) { + if (generated_tokens[i] >= timestamp_begin) { + generated_tokens[i] = Math.min(generated_tokens[i] + timestamp_offset, max_timestamp_token); + } + } + + allTokens.push(...generated_tokens.slice(0, tokens_to_keep)); + if (seek_token_timestamps) { + allTokenTimestamps.push(...seek_token_timestamps.slice(0, tokens_to_keep)); + } + seek += segment_offset; + } + + // Add EOS back + allTokens.push(eos_token_id); + + // Reconstruct output + const full_sequence = [...init_tokens, ...allTokens]; + if (return_token_timestamps) { + // Return dict format with sequences and token_timestamps + const sequences = new Tensor('int64', full_sequence.map(BigInt), [1, full_sequence.length]); + // Pad token_timestamps to match full_sequence (init_tokens get 0.0) + const full_timestamps = [...new Array(init_tokens.length).fill(0), ...allTokenTimestamps, 0]; + const token_timestamps = new Tensor('float32', new Float32Array(full_timestamps), [ + 1, + full_timestamps.length, + ]); + return { sequences, token_timestamps }; + } + return new Tensor('int64', full_sequence.map(BigInt), [1, full_sequence.length]); + } + + /** + * Calculates token-level timestamps using the encoder-decoder cross-attentions and + * dynamic time-warping (DTW) to map each output token to a position in the input audio. + * If `num_frames` is specified, the encoder-decoder cross-attentions will be cropped before applying DTW. + * @param {Object} generate_outputs Outputs generated by the model + * @param {Tensor[][]} generate_outputs.cross_attentions The cross attentions output by the model + * @param {Tensor} generate_outputs.sequences The sequences output by the model + * @param {number[][]} alignment_heads Alignment heads of the model + * @param {number} [num_frames=null] Number of frames in the input audio. + * @param {number} [time_precision=0.02] Precision of the timestamps in seconds + * @param {number} [num_input_ids=0] Number of decoder input ids (prefix tokens) to skip in DTW + * @returns {Tensor} tensor containing the timestamps in seconds for each predicted token + */ + _extract_token_timestamps( + generate_outputs, + alignment_heads, + num_frames = null, + time_precision = 0.02, + num_input_ids = 0, + ) { + if (!generate_outputs.cross_attentions) { + throw new Error( + 'Model outputs must contain cross attentions to extract timestamps. ' + + 'This is most likely because the model was not exported with `output_attentions=True`.', + ); + } + if (num_frames == null) { + logger.warn( + '`num_frames` has not been set, meaning the entire audio will be analyzed. ' + + 'This may lead to inaccurate token-level timestamps for short audios (< 30 seconds).', + ); + } + + // @ts-expect-error TS2339 + let median_filter_width = this.config.median_filter_width; + if (median_filter_width === undefined) { + logger.warn('Model config has no `median_filter_width`, using default value of 7.'); + median_filter_width = 7; + } + + // TODO: Improve batch processing + const batch = generate_outputs.cross_attentions; + // Create a list with `decoder_layers` elements, each a tensor of shape + // (batch size, attention_heads, output length, input length). + const cross_attentions = Array.from( + // @ts-expect-error TS2339 + { length: this.config.decoder_layers }, + // Concatenate the cross attentions for each layer across sequence length dimension. + (_, i) => + cat( + batch.map((x) => x[i]), + 2, + ), + ); + + const weights = stack( + alignment_heads.map(([l, h]) => { + if (l >= cross_attentions.length) { + throw new Error( + `Layer index ${l} is out of bounds for cross attentions (length ${cross_attentions.length}).`, + ); + } + return num_frames + ? cross_attentions[l].slice(null, h, null, [0, num_frames]) + : cross_attentions[l].slice(null, h); + }), + ).transpose(1, 0, 2, 3); + + const [std, calculatedMean] = std_mean(weights, -2, 0, true); + + // Normalize and smoothen the weights. + const smoothedWeights = weights.clone(); // [1, 8, seqLength, 1500] + + for (let a = 0; a < smoothedWeights.dims[0]; ++a) { + const aTensor = smoothedWeights[a]; // [8, seqLength, 1500] + + for (let b = 0; b < aTensor.dims[0]; ++b) { + const bTensor = aTensor[b]; // [seqLength, 1500] + + const stdTensorData = std[a][b][0].data; // [1500] + const meanTensorData = calculatedMean[a][b][0].data; // [1500] + + for (let c = 0; c < bTensor.dims[0]; ++c) { + let cTensorData = bTensor[c].data; // [1500] + for (let d = 0; d < cTensorData.length; ++d) { + cTensorData[d] = (cTensorData[d] - meanTensorData[d]) / stdTensorData[d]; + } + + // Apply median filter. + cTensorData.set(medianFilter(cTensorData, median_filter_width)); + } + } + } + + // Skip decoder_input_ids in the cross-attention weights + const croppedWeights = + num_input_ids > 0 + ? smoothedWeights.slice(null, null, [num_input_ids, smoothedWeights.dims[2]], null) + : smoothedWeights; + + // Average the different cross-attention heads. + const batchedMatrices = [mean(croppedWeights, 1)]; + + const timestampsShape = generate_outputs.sequences.dims; + + const timestamps = new Tensor( + 'float32', + new Float32Array(timestampsShape[0] * timestampsShape[1]), + timestampsShape, + ); + + // Perform dynamic time warping on each element of the batch. + for (let batch_idx = 0; batch_idx < timestampsShape[0]; ++batch_idx) { + // NOTE: Since we run only one batch at a time, we can squeeze to get the same dimensions + // as the python implementation + const matrix = batchedMatrices[batch_idx].neg().squeeze_(0); + const [text_indices, time_indices] = dynamic_time_warping(matrix.tolist()); + + const diffs = Array.from( + { length: text_indices.length - 1 }, + (v, i) => text_indices[i + 1] - text_indices[i], + ); + const jumps = mergeArrays([1], diffs).map((x) => !!x); // convert to boolean + + const jump_times = []; + for (let i = 0; i < jumps.length; ++i) { + if (jumps[i]) { + // NOTE: No point in rounding here, since we set to Float32Array later + jump_times.push(time_indices[i] * time_precision); + } + } + + // Pad with num_input_ids zeros at the start (for prefix tokens), + // then DTW jump_times, then duplicate last value (for eos token) + const padded = new Array(num_input_ids).fill(0); + padded.push(...jump_times); + if (jump_times.length > 0) { + padded.push(jump_times.at(-1)); + } + timestamps[batch_idx].data.set(padded); + } + + return timestamps; + } +} + +export class LiteWhisperForConditionalGeneration extends WhisperForConditionalGeneration {} diff --git a/packages/transformers/src/models/whisper/processing_whisper.js b/packages/transformers/src/models/whisper/processing_whisper.js new file mode 100644 index 0000000..a4cd9a6 --- /dev/null +++ b/packages/transformers/src/models/whisper/processing_whisper.js @@ -0,0 +1,20 @@ +import { AutoFeatureExtractor } from '../auto/feature_extraction_auto.js'; +import { AutoTokenizer } from '../auto/tokenization_auto.js'; +import { Processor } from '../../processing_utils.js'; + +/** + * Represents a WhisperProcessor that extracts features from an audio input. + */ +export class WhisperProcessor extends Processor { + static tokenizer_class = AutoTokenizer; + static feature_extractor_class = AutoFeatureExtractor; + + /** + * Calls the feature_extractor function with the given audio input. + * @param {any} audio The audio input to extract features from. + * @returns {Promise} A Promise that resolves with the extracted features. + */ + async _call(audio) { + return await this.feature_extractor(audio); + } +} diff --git a/packages/transformers/src/models/whisper/tokenization_whisper.js b/packages/transformers/src/models/whisper/tokenization_whisper.js new file mode 100644 index 0000000..43aaaf9 --- /dev/null +++ b/packages/transformers/src/models/whisper/tokenization_whisper.js @@ -0,0 +1,696 @@ +import { PreTrainedTokenizer, prepareTensorForDecode } from '../../tokenization_utils.js'; +import { WHISPER_LANGUAGE_MAPPING } from './common_whisper.js'; + +import { round } from '../../utils/maths.js'; +import { Tensor } from '../../utils/tensor.js'; +import { mergeArrays } from '../../utils/core.js'; + +const PUNCTUATION_REGEX = '\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E'; +const PUNCTUATION_ONLY_REGEX = new RegExp(`^[${PUNCTUATION_REGEX}]+$`, 'gu'); + +/** + * Small tolerance (in seconds) for timestamp ordering in chunk overlap matching. + * DTW can produce slightly different timestamps for the same word across overlapping + * chunks, so we allow a small tolerance to avoid rejecting valid token matches. + */ +const TIMESTAMP_MERGE_TOLERANCE = 0.1; + +export class WhisperTokenizer extends PreTrainedTokenizer { + get timestamp_begin() { + return this._tokenizer.token_to_id('<|notimestamps|>') + 1; + } + + /** + * Decodes automatic speech recognition (ASR) sequences. + * @param {Array<{tokens: bigint[], token_timestamps?: number[], stride: number[]}>} sequences The sequences to decode. + * @param {Object} options The options to use for decoding. + * @returns {Array, text: string}>}>} The decoded sequences. + */ + _decode_asr( + sequences, + { return_timestamps = false, return_language = false, time_precision = null, force_full_sequences = true } = {}, + ) { + // Set force_full_sequences=false if you want streaming + // TODO add support for `return_language` + + // Internal method meant to only be used by asr pipeline. + // Handles all the little quirks specific to whisper to handle + // the various options not allowed in other seq2seq models + + // =========== Overview ============ + // - iterate over all outputs + // - all tokens within output + // - Each token can be + // - language token + // - special token + // - timestamp token + // - text token + // - We accumulate the text tokens. + // - We split on end timestamps + // - Lots of complexity comes from stride and timestamps + + if (time_precision === null) { + throw Error('Must specify time_precision'); + } + let last_language = null; + + const returnWordTimestamps = return_timestamps === 'word'; + + function new_chunk() { + return { language: last_language, timestamp: [null, null], text: '' }; + } + + // Welcome to the state machine! + const chunks = []; + let chunk = new_chunk(); + let time_offset = 0.0; + const timestamp_begin = this.timestamp_begin; + // Whisper timestamp tokens start from 0.00 and go to timestamp 30.00 in 0.02 increments. + // We can calculate the last time stamp token as timestamp_begin plus the number of tokens + // tokens from 0.00 to 30.00 which is 1500. + const total_timestamp_tokens = 1500; // (30.00 - 0.00) / 0.02 + const timestamp_end = timestamp_begin + total_timestamp_tokens; + + let previous_tokens = []; + let previous_token_timestamps = []; + + let skip = false; + let right_stride_start = null; + + const all_special_ids = new Set(this.all_special_ids); + + for (const output of sequences) { + // NOTE: python version has batches, so it uses [0] + const token_ids = output.tokens; + const token_timestamps = returnWordTimestamps ? output.token_timestamps : null; + + // These keep track of timestamps within strides, which need + // to be skipped and resolve all tokens in a single chunk. + let last_timestamp = null; + let first_timestamp = timestamp_begin; + + if ('stride' in output) { + const [chunk_len, stride_left, stride_right] = output.stride; + + // Offset the timings to account for the other `model_outputs`. + time_offset -= stride_left; + right_stride_start = chunk_len - stride_right; + + // Keeping track of timestamps within strides + // We're going to NOT split on those, and delay until we're + // out of BOTH stride. Otherwise lots of issues occur and + // corner cases + if (stride_left) { + first_timestamp = stride_left / time_precision + timestamp_begin; + } + + if (stride_right) { + for (let i = token_ids.length - 1; i >= 0; --i) { + const token = Number(token_ids[i]); + if (token >= timestamp_begin) { + // There can be several token in the right stride + // But the last one is ALWAYS going to be skipped + if ( + last_timestamp !== null && + (token - timestamp_begin) * time_precision < right_stride_start + ) { + break; + } + last_timestamp = token; + } + } + } + } + + let current_tokens = []; + let current_token_timestamps = []; + + // - all tokens within output + for (let i = 0; i < token_ids.length; ++i) { + const token = Number(token_ids[i]); + // 4 possible states for each token + // - 1/ Language code + // - 2/ all other special tokens (which we ignore) + // - 3/ Timestamp + // - 4/ Regular text + + if (all_special_ids.has(token)) { + const text = this.decode([token]); + const language = WHISPER_LANGUAGE_MAPPING.get(text.slice(2, -2)); + + if (language !== undefined) { + // 1/ Indeed some language + // TODO Handle when language is different from the previous + // one, and we cannot use timestamped tokens to create chunks + if (last_language !== null && language !== last_language && !return_timestamps) { + previous_tokens.push(current_tokens); + const resolved_tokens = this.findLongestCommonSequence(previous_tokens)[0]; + const resolved_text = this.decode(resolved_tokens); + chunk.text = resolved_text; + chunks.push(chunk); + + // Flush all our temporary context + previous_tokens = []; + current_tokens = []; + chunk = new_chunk(); + } + + last_language = chunk.language = language; + } else { + // 2/ This is a regular special token, ignoring it + } + } else if (token >= timestamp_begin && token <= timestamp_end) { + // 3/ Timestamp token + const time = (token - timestamp_begin) * time_precision + time_offset; + const rounded_time = round(time, 2); + + if (last_timestamp !== null && token >= last_timestamp) { + // Whisper outputted a timestamp token, but it falls within + // our stride, so we're going to skip it for the time being + // and resolve this later + // Skip is necessary because timestamp tokens always come + // by pair, so we need to skip the next one too (which would mark the start of another chunk). + skip = true; + } else if (skip || (previous_tokens.length > 0 && token < first_timestamp)) { + skip = false; + } else if (chunk.timestamp[0] === null) { + chunk.timestamp[0] = rounded_time; + } else { + // This is the end of the timestamp chunk + if (rounded_time === chunk.timestamp[0]) { + // This is a bug in timestamp token output + // where we're taking the duplicate token + // as a stop where it should be a start. + // This is an issue in the underlying model output + // Let's just skip it so it becomes de-factor a start agin + } else { + chunk.timestamp[1] = rounded_time; + + // Handling merges + previous_tokens.push(current_tokens); + + if (returnWordTimestamps) { + previous_token_timestamps.push(current_token_timestamps); + } + const [resolved_tokens, resolved_token_timestamps] = this.findLongestCommonSequence( + previous_tokens, + previous_token_timestamps, + ); + + const resolved_text = this.decode(resolved_tokens); + chunk.text = resolved_text; + + if (returnWordTimestamps) { + chunk.words = this.collateWordTimestamps( + resolved_tokens, + resolved_token_timestamps, + last_language, + ); + + // Cap word end timestamps to the chunk's end timestamp, + // but only if it wouldn't create an inverted range (end < start) + if (chunk.words.length > 0 && chunk.timestamp[1] !== null) { + for (const word of chunk.words) { + if ( + word.timestamp[1] > chunk.timestamp[1] && + chunk.timestamp[1] >= word.timestamp[0] + ) { + word.timestamp[1] = chunk.timestamp[1]; + } + } + } + } + + chunks.push(chunk); + + // Flush all our temporary context + previous_tokens = []; + current_tokens = []; + previous_token_timestamps = []; + current_token_timestamps = []; + chunk = new_chunk(); + } + } + } else { + // 4/ Regular token + // We just append to the list of all tokens so we can handle + // merges later and decode into text. + current_tokens.push(token); + + if (returnWordTimestamps) { + let start_time = round(token_timestamps[i] + time_offset, 2); + + let end_time; + if (i + 1 < token_timestamps.length) { + end_time = round(token_timestamps[i + 1] + time_offset, 2); + + // Do not allow punctuation-only tokens to have a duration. + // This prevents long pauses from messing up the timestamps. + const decoded_text = this.decode([token]); + if (PUNCTUATION_ONLY_REGEX.test(decoded_text)) { + // Add `time_precision` to avoid overlapping timestamps + end_time = round(Math.min(start_time + time_precision, end_time), 2); + } + } else { + // should never happen + end_time = null; + } + current_token_timestamps.push([start_time, end_time]); + } + } + } + + if ('stride' in output) { + const [chunk_len, stride_left, stride_right] = output.stride; + time_offset += chunk_len - stride_right; + } + + // Leftover tokens + if (current_tokens.length > 0) { + previous_tokens.push(current_tokens); + if (returnWordTimestamps) { + previous_token_timestamps.push(current_token_timestamps); + } + } else if (previous_tokens.every((p) => p.length === 0)) { + // Flushing previous tokens (END)" + chunk = new_chunk(); + previous_tokens = []; + current_tokens = []; + previous_token_timestamps = []; + current_token_timestamps = []; + } + } + + if (previous_tokens.length > 0) { + if (force_full_sequences && return_timestamps) { + // Last token should always be timestamps, so there shouldn't be + // leftover + throw new Error( + 'Whisper did not predict an ending timestamp, which can happen if audio is cut off in the middle of a word. ' + + 'Also make sure WhisperTimeStampLogitsProcessor was used during generation.', + ); + } + + // Happens when we don't use timestamps + const [resolved_tokens, resolved_token_timestamps] = this.findLongestCommonSequence( + previous_tokens, + previous_token_timestamps, + ); + + // Flushing previous tokens (FINAL) + const resolved_text = this.decode(resolved_tokens); + chunk.text = resolved_text; + if (returnWordTimestamps) { + chunk.words = this.collateWordTimestamps(resolved_tokens, resolved_token_timestamps, last_language); + } + chunks.push(chunk); + } + + let optional = Object.create(null); + + // Preparing and cleaning up the pipeline output + const full_text = chunks.map((chunk) => chunk.text).join(''); + if (return_timestamps || return_language) { + for (let i = 0; i < chunks.length; ++i) { + const chunk = chunks[i]; + if (!return_timestamps) { + delete chunk['timestamp']; + } + + if (!return_language) { + delete chunk['language']; + } + } + if (returnWordTimestamps) { + const new_chunks = []; + for (const chunk of chunks) { + for (const word of chunk.words) { + new_chunks.push(word); + } + } + optional = { chunks: new_chunks }; + } else { + optional = { chunks: chunks }; + } + } + return [full_text, optional]; + } + + /** + * Finds the longest common sequence among the provided sequences. + * @param {number[][]} sequences An array of sequences of token ids to compare. + * @returns {number[][]} The longest common sequence found. + * @throws {Error} If there is a bug within the function. + * @private + */ + findLongestCommonSequence(sequences, token_timestamp_sequences = null) { + // It would be much harder to do O(n) because of fault tolerance. + // We actually have a really good property which is that the total sequence + // MUST be those subsequences in order. + // If token_timestamp_sequences is provided, will split those sequences in + // exactly the same way. + + let leftSequence = sequences[0]; + let leftLength = leftSequence.length; + let totalSequence = []; + + const use_token_timestamp_sequences = + Array.isArray(token_timestamp_sequences) && token_timestamp_sequences.length > 0; + let total_token_timestamp_sequence = use_token_timestamp_sequences ? [] : null; + let left_token_timestamp_sequence = use_token_timestamp_sequences ? token_timestamp_sequences[0] : null; + for (let i = 1; i < sequences.length; ++i) { + const rightSequence = sequences[i]; + let max = 0.0; + let maxIndices = [leftLength, leftLength, 0, 0]; + // Here we're sliding matches + // [a, b, c, d] + // [c, d, f] + // = [c] == [d] + + // [a, b, c, d] + // [c, d, f] + // = [c, d] == [c, d] + + // [a, b, c, d] + // [c, d, f] + + // = [b, c, d] == [c, d, f] + + // [a, b, c, d] + // [c, d, f] + + // [a, b, c] == [c, d, f] + + // [a, b, c, d] + // [d, f] + + // [a, b] == [d, f] + + // [a, b, c, d] + // [f] + + // [a] == [f] + + const rightLength = rightSequence.length; + for (let j = 1; j < leftLength + rightLength; ++j) { + // Slightly convoluted because we don't want out of bound indices + // This will be necessary for a small conflict resolution optimization + // later + const leftStart = Math.max(0, leftLength - j); + const leftStop = Math.min(leftLength, leftLength + rightLength - j); + const left = leftSequence.slice(leftStart, leftStop); + const rightStart = Math.max(0, j - leftLength); + const rightStop = Math.min(rightLength, j); + const right = rightSequence.slice(rightStart, rightStop); + if (left.length !== right.length) { + throw new Error( + 'There is a bug within whisper `decode_asr` function, please report it. Dropping to prevent bad inference.', + ); + } + + let matches; + if (use_token_timestamp_sequences) { + // Get length of longest subsequence of tokens that match + // and have timestamps that are in order. + // Use a small tolerance since DTW can produce slightly different + // timestamps for the same word across overlapping chunks. + matches = left.filter( + (elem, idx) => + elem === right[idx] && + left_token_timestamp_sequence[leftStart + idx][0] - TIMESTAMP_MERGE_TOLERANCE <= + token_timestamp_sequences[i][rightStart + idx][0], + ).length; + } else { + matches = left.filter((elem, idx) => elem === right[idx]).length; + } + + // epsilon to favor long perfect matches + const eps = j / 10000.0; + const matching = matches / j + eps; + if (matches > 1 && matching > max) { + max = matching; + maxIndices = [leftStart, leftStop, rightStart, rightStop]; + } + } + const [leftStart, leftStop, rightStart, rightStop] = maxIndices; + const leftMid = Math.floor((leftStop + leftStart) / 2); + let rightMid = Math.floor((rightStop + rightStart) / 2); + + // When no overlap is found and we have timestamps, skip right-side tokens + // that precede the left's last timestamp to avoid backwards-jumping timestamps. + if (use_token_timestamp_sequences && max === 0.0 && leftLength > 0) { + const lastLeftTs = left_token_timestamp_sequence[leftLength - 1][0]; + const idx = token_timestamp_sequences[i].findIndex((ts) => ts[0] >= lastLeftTs); + rightMid = idx === -1 ? rightSequence.length : idx; + } + + totalSequence.push(...leftSequence.slice(0, leftMid)); + leftSequence = rightSequence.slice(rightMid); + leftLength = leftSequence.length; + + if (use_token_timestamp_sequences) { + total_token_timestamp_sequence.push(...left_token_timestamp_sequence.slice(0, leftMid)); + left_token_timestamp_sequence = token_timestamp_sequences[i].slice(rightMid); + } + } + totalSequence.push(...leftSequence); + + if (use_token_timestamp_sequences) { + total_token_timestamp_sequence.push(...left_token_timestamp_sequence); + return [totalSequence, total_token_timestamp_sequence]; + } else { + return [totalSequence, []]; + } + } + + /** @private */ + collateWordTimestamps(tokens, token_timestamps, language) { + const [words, _, token_indices] = this.combineTokensIntoWords(tokens, language); + + const timings = []; + for (let i = 0; i < words.length; ++i) { + const indices = token_indices[i]; + timings.push({ + text: words[i], + timestamp: [token_timestamps[indices.at(0)][0], token_timestamps[indices.at(-1)][1]], + }); + } + return timings; + } + + /** + * Groups tokens by word. Returns a tuple containing a list of strings with the words, + * and a list of `token_id` sequences with the tokens making up each word. + * @param {number[]} tokens + * @param {string} [language] + * @param {string} prepend_punctionations + * @param {string} append_punctuations + * + * @private + */ + combineTokensIntoWords( + tokens, + language, + prepend_punctionations = '"\'“¡¿([{-', + append_punctuations = '"\'.。,，!！?？:：”)]}、', + ) { + language = language ?? 'english'; + + let words, word_tokens, token_indices; + + if (['chinese', 'japanese', 'thai', 'lao', 'myanmar'].includes(language)) { + // These languages don't typically use spaces. + [words, word_tokens, token_indices] = this.splitTokensOnUnicode(tokens); + } else { + [words, word_tokens, token_indices] = this.splitTokensOnSpaces(tokens); + } + + return this.mergePunctuations(words, word_tokens, token_indices, prepend_punctionations, append_punctuations); + } + + /** @type {PreTrainedTokenizer['decode']} */ + decode(token_ids, decode_args) { + let text; + // @ts-ignore + if (decode_args?.decode_with_timestamps) { + if (token_ids instanceof Tensor) { + token_ids = prepareTensorForDecode(token_ids); + } + text = this.decodeWithTimestamps(token_ids, decode_args); + } else { + text = super.decode(token_ids, decode_args); + } + // TODO: implement offsets + // if (decode_args.output_offsets) { + // let offsets = this.computeOffsets + // } + return text; + } + + /** + * @param {number[]|bigint[]} token_ids List of token IDs to decode. + * @param {Object} decode_args Optional arguments for decoding + * @private + */ + decodeWithTimestamps(token_ids, decode_args) { + const time_precision = decode_args?.time_precision ?? 0.02; + + const timestamp_begin = this.all_special_ids.at(-1) + 1; + /**@type {any[]} */ + let outputs = [[]]; + for (let token of token_ids) { + token = Number(token); + if (token >= timestamp_begin) { + const timestamp = ((token - timestamp_begin) * time_precision).toFixed(2); + outputs.push(`<|${timestamp}|>`); + outputs.push([]); + } else { + outputs[outputs.length - 1].push(token); + } + } + outputs = outputs.map((s) => (typeof s === 'string' ? s : super.decode(s, decode_args))); + + return outputs.join(''); + } + + /** + * Combine tokens into words by splitting at any position where the tokens are decoded as valid unicode points. + * @param {number[]} tokens + * @returns {*} + * @private + */ + splitTokensOnUnicode(tokens) { + const decoded_full = this.decode(tokens, { + // @ts-ignore + decode_with_timestamps: true, + }); + const replacement_char = '\uFFFD'; + + const words = []; + const word_tokens = []; + const token_indices = []; + let current_tokens = []; + let current_indices = []; + let unicode_offset = 0; + + for (let token_idx = 0; token_idx < tokens.length; ++token_idx) { + const token = tokens[token_idx]; + + current_tokens.push(token); + current_indices.push(token_idx); + + const decoded = this.decode(current_tokens, { + // @ts-ignore + decode_with_timestamps: true, + }); + + if ( + !decoded.includes(replacement_char) || + decoded_full[unicode_offset + decoded.indexOf(replacement_char)] === replacement_char + ) { + words.push(decoded); + word_tokens.push(current_tokens); + token_indices.push(current_indices); + current_tokens = []; + current_indices = []; + unicode_offset += decoded.length; + } + } + + return [words, word_tokens, token_indices]; + } + + /** + * Combine tokens into words by splitting at whitespace and punctuation tokens. + * @param {number[]} tokens + * @private + */ + splitTokensOnSpaces(tokens) { + const [subwords, subword_tokens_list, subword_indices_list] = this.splitTokensOnUnicode(tokens); + + const words = []; + const word_tokens = []; + const token_indices = []; + + for (let i = 0; i < subwords.length; ++i) { + const subword = subwords[i]; + const subword_tokens = subword_tokens_list[i]; + const subword_indices = subword_indices_list[i]; + + // @ts-ignore + const special = subword_tokens[0] >= this._tokenizer.token_to_id('<|endoftext|>'); + const with_space = subword.startsWith(' '); + const trimmed = subword.trim(); + const punctuation = PUNCTUATION_ONLY_REGEX.test(trimmed); + + if (special || with_space || punctuation || words.length === 0) { + words.push(subword); + word_tokens.push(subword_tokens); + token_indices.push(subword_indices); + } else { + const ix = words.length - 1; + words[ix] += subword; + word_tokens[ix].push(...subword_tokens); + token_indices[ix].push(...subword_indices); + } + } + + return [words, word_tokens, token_indices]; + } + + /** + * Merges punctuation tokens with neighboring words. + * @param {string[]} words + * @param {number[][]} tokens + * @param {number[][]} indices + * @param {string} prepended + * @param {string} appended + * @private + */ + mergePunctuations(words, tokens, indices, prepended, appended) { + const newWords = structuredClone(words); + const newTokens = structuredClone(tokens); + const newIndices = structuredClone(indices); + + // prepend punctuations + let i = newWords.length - 2; + let j = newWords.length - 1; + + while (i >= 0) { + if (newWords[i].startsWith(' ') && prepended.includes(newWords[i].trim())) { + newWords[j] = newWords[i] + newWords[j]; + newTokens[j] = mergeArrays(newTokens[i], newTokens[j]); + newIndices[j] = mergeArrays(newIndices[i], newIndices[j]); + newWords[i] = ''; + newTokens[i] = []; + newIndices[i] = []; + } else { + j = i; + } + --i; + } + + // append punctuations + i = 0; + j = 1; + while (j < newWords.length) { + if (!newWords[i].endsWith(' ') && appended.includes(newWords[j])) { + newWords[i] += newWords[j]; + newTokens[i] = mergeArrays(newTokens[i], newTokens[j]); + newIndices[i] = mergeArrays(newIndices[i], newIndices[j]); + newWords[j] = ''; + newTokens[j] = []; + newIndices[j] = []; + } else { + i = j; + } + ++j; + } + + return [ + newWords.filter((x) => x), + newTokens.filter((x) => x.length > 0), + newIndices.filter((x) => x.length > 0), + ]; + } +} diff --git a/packages/transformers/src/models/xlm/modeling_xlm.js b/packages/transformers/src/models/xlm/modeling_xlm.js new file mode 100644 index 0000000..59f719f --- /dev/null +++ b/packages/transformers/src/models/xlm/modeling_xlm.js @@ -0,0 +1,74 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { + MaskedLMOutput, + QuestionAnsweringModelOutput, + SequenceClassifierOutput, + TokenClassifierOutput, +} from '../modeling_outputs.js'; + +export class XLMPreTrainedModel extends PreTrainedModel {} + +/** + * The bare XLM Model transformer outputting raw hidden-states without any specific head on top. + */ +export class XLMModel extends XLMPreTrainedModel {} + +/** + * The XLM Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). + */ +export class XLMWithLMHeadModel extends XLMPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +/** + * XLM Model with a sequence classification/regression head on top (a linear layer on top of the pooled output) + */ +export class XLMForSequenceClassification extends XLMPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * XLM Model with a token classification head on top (a linear layer on top of the hidden-states output) + */ +export class XLMForTokenClassification extends XLMPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * XLM Model with a span classification head on top for extractive question-answering tasks + */ +export class XLMForQuestionAnswering extends XLMPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/xlm/tokenization_xlm.js b/packages/transformers/src/models/xlm/tokenization_xlm.js new file mode 100644 index 0000000..0139b77 --- /dev/null +++ b/packages/transformers/src/models/xlm/tokenization_xlm.js @@ -0,0 +1,13 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; +import { logger } from '../../utils/logger.js'; + +export class XLMTokenizer extends PreTrainedTokenizer { + return_token_type_ids = true; + + constructor(tokenizerJSON, tokenizerConfig) { + super(tokenizerJSON, tokenizerConfig); + logger.warn( + 'WARNING: `XLMTokenizer` is not yet supported by Hugging Face\'s "fast" tokenizers library. Therefore, you may experience slightly inaccurate results.', + ); + } +} diff --git a/packages/transformers/src/models/xlm_roberta/modeling_xlm_roberta.js b/packages/transformers/src/models/xlm_roberta/modeling_xlm_roberta.js new file mode 100644 index 0000000..b4ac198 --- /dev/null +++ b/packages/transformers/src/models/xlm_roberta/modeling_xlm_roberta.js @@ -0,0 +1,70 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { + MaskedLMOutput, + QuestionAnsweringModelOutput, + SequenceClassifierOutput, + TokenClassifierOutput, +} from '../modeling_outputs.js'; + +export class XLMRobertaPreTrainedModel extends PreTrainedModel {} +export class XLMRobertaModel extends XLMRobertaPreTrainedModel {} + +/** + * XLMRobertaForMaskedLM class for performing masked language modeling on XLMRoberta models. + */ +export class XLMRobertaForMaskedLM extends XLMRobertaPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new MaskedLMOutput(await super._call(model_inputs)); + } +} + +/** + * XLMRobertaForSequenceClassification class for performing sequence classification on XLMRoberta models. + */ +export class XLMRobertaForSequenceClassification extends XLMRobertaPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new SequenceClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * XLMRobertaForTokenClassification class for performing token classification on XLMRoberta models. + */ +export class XLMRobertaForTokenClassification extends XLMRobertaPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} An object containing the model's output logits for token classification. + */ + async _call(model_inputs) { + return new TokenClassifierOutput(await super._call(model_inputs)); + } +} + +/** + * XLMRobertaForQuestionAnswering class for performing question answering on XLMRoberta models. + */ +export class XLMRobertaForQuestionAnswering extends XLMRobertaPreTrainedModel { + /** + * Calls the model on new inputs. + * + * @param {Object} model_inputs The inputs to the model. + * @returns {Promise} returned object + */ + async _call(model_inputs) { + return new QuestionAnsweringModelOutput(await super._call(model_inputs)); + } +} diff --git a/packages/transformers/src/models/xlm_roberta/tokenization_xlm_roberta.js b/packages/transformers/src/models/xlm_roberta/tokenization_xlm_roberta.js new file mode 100644 index 0000000..4a01371 --- /dev/null +++ b/packages/transformers/src/models/xlm_roberta/tokenization_xlm_roberta.js @@ -0,0 +1,3 @@ +import { PreTrainedTokenizer } from '../../tokenization_utils.js'; + +export class XLMRobertaTokenizer extends PreTrainedTokenizer {} diff --git a/packages/transformers/src/models/yolos/image_processing_yolos.js b/packages/transformers/src/models/yolos/image_processing_yolos.js new file mode 100644 index 0000000..22487cf --- /dev/null +++ b/packages/transformers/src/models/yolos/image_processing_yolos.js @@ -0,0 +1,9 @@ +import { ImageProcessor, post_process_object_detection } from '../../image_processors_utils.js'; + +export class YolosImageProcessor extends ImageProcessor { + /** @type {typeof post_process_object_detection} */ + post_process_object_detection(...args) { + return post_process_object_detection(...args); + } +} +export class YolosFeatureExtractor extends YolosImageProcessor {} diff --git a/packages/transformers/src/models/yolos/modeling_yolos.js b/packages/transformers/src/models/yolos/modeling_yolos.js new file mode 100644 index 0000000..06e964e --- /dev/null +++ b/packages/transformers/src/models/yolos/modeling_yolos.js @@ -0,0 +1,28 @@ +import { PreTrainedModel } from '../modeling_utils.js'; +import { ModelOutput } from '../modeling_outputs.js'; +import { Tensor } from '../../utils/tensor.js'; + +export class YolosPreTrainedModel extends PreTrainedModel {} +export class YolosModel extends YolosPreTrainedModel {} +export class YolosForObjectDetection extends YolosPreTrainedModel { + /** + * @param {any} model_inputs + */ + async _call(model_inputs) { + return new YolosObjectDetectionOutput(await super._call(model_inputs)); + } +} + +export class YolosObjectDetectionOutput extends ModelOutput { + /** + * @param {Object} output The output of the model. + * @param {Tensor} output.logits Classification logits (including no-object) for all queries. + * @param {Tensor} output.pred_boxes Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). + * These values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding possible padding). + */ + constructor({ logits, pred_boxes }) { + super(); + this.logits = logits; + this.pred_boxes = pred_boxes; + } +} diff --git a/packages/transformers/src/models/youtu/modeling_youtu.js b/packages/transformers/src/models/youtu/modeling_youtu.js new file mode 100644 index 0000000..5548a61 --- /dev/null +++ b/packages/transformers/src/models/youtu/modeling_youtu.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class YoutuPreTrainedModel extends PreTrainedModel {} +export class YoutuModel extends YoutuPreTrainedModel {} +export class YoutuForCausalLM extends YoutuPreTrainedModel {} diff --git a/packages/transformers/src/models/zaya/modeling_zaya.js b/packages/transformers/src/models/zaya/modeling_zaya.js new file mode 100644 index 0000000..501229f --- /dev/null +++ b/packages/transformers/src/models/zaya/modeling_zaya.js @@ -0,0 +1,5 @@ +import { PreTrainedModel } from '../modeling_utils.js'; + +export class ZayaPreTrainedModel extends PreTrainedModel {} +export class ZayaModel extends ZayaPreTrainedModel {} +export class ZayaForCausalLM extends ZayaPreTrainedModel {} diff --git a/packages/transformers/src/ops/registry.js b/packages/transformers/src/ops/registry.js new file mode 100644 index 0000000..4250826 --- /dev/null +++ b/packages/transformers/src/ops/registry.js @@ -0,0 +1,176 @@ +import { createInferenceSession, runInferenceSession, isONNXProxy } from '../backends/onnx.js'; +import { Tensor } from '../utils/tensor.js'; + +/** + * Asynchronously creates a wrapper function for running an ONNX inference session. + * + * @param {number[]} session_bytes The session data in bytes. + * @param {import('onnxruntime-common').InferenceSession.SessionOptions} session_options The options for the ONNX session. + * @template {string | [string] | string[]} T + * @param {T} names The name(s) of the output tensor(s). + * + * @returns {Promise): Promise>} + * The wrapper function for running the ONNX inference session. + */ +const wrap = async (session_bytes, session_options, names) => { + const session = await createInferenceSession(new Uint8Array(session_bytes), session_options); + + return /** @type {any} */ ( + async (/** @type {Record} */ inputs) => { + const proxied = isONNXProxy(); + const ortFeed = Object.fromEntries( + Object.entries(inputs).map(([k, v]) => [k, (proxied ? v.clone() : v).ort_tensor]), + ); + const outputs = await runInferenceSession(session, ortFeed); + if (Array.isArray(names)) { + return names.map((n) => new Tensor(outputs[n])); + } else { + return new Tensor(outputs[/** @type {string} */ (names)]); + } + } + ); +}; + +// In-memory registry of initialized ONNX operators +export class TensorOpRegistry { + static session_options = { + // TODO: Allow for multiple execution providers + // executionProviders: ['webgpu'], + }; + + static get nearest_interpolate_4d() { + if (!this._nearest_interpolate_4d) { + this._nearest_interpolate_4d = wrap( + [ + 8, 10, 18, 0, 58, 129, 1, 10, 41, 10, 1, 120, 10, 0, 10, 0, 10, 1, 115, 18, 1, 121, 34, 6, 82, 101, + 115, 105, 122, 101, 42, 18, 10, 4, 109, 111, 100, 101, 34, 7, 110, 101, 97, 114, 101, 115, 116, 160, + 1, 3, 18, 1, 114, 90, 31, 10, 1, 120, 18, 26, 10, 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, + 99, 10, 3, 18, 1, 104, 10, 3, 18, 1, 119, 90, 15, 10, 1, 115, 18, 10, 10, 8, 8, 7, 18, 4, 10, 2, 8, + 4, 98, 31, 10, 1, 121, 18, 26, 10, 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 99, 10, 3, 18, + 1, 104, 10, 3, 18, 1, 119, 66, 2, 16, 21, + ], + this.session_options, + 'y', + ); + } + return this._nearest_interpolate_4d; + } + static get bilinear_interpolate_4d() { + if (!this._bilinear_interpolate_4d) { + this._bilinear_interpolate_4d = wrap( + [ + 8, 9, 18, 0, 58, 128, 1, 10, 40, 10, 1, 120, 10, 0, 10, 0, 10, 1, 115, 18, 1, 121, 34, 6, 82, 101, + 115, 105, 122, 101, 42, 17, 10, 4, 109, 111, 100, 101, 34, 6, 108, 105, 110, 101, 97, 114, 160, 1, + 3, 18, 1, 114, 90, 31, 10, 1, 120, 18, 26, 10, 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 99, + 10, 3, 18, 1, 104, 10, 3, 18, 1, 119, 90, 15, 10, 1, 115, 18, 10, 10, 8, 8, 7, 18, 4, 10, 2, 8, 4, + 98, 31, 10, 1, 121, 18, 26, 10, 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 99, 10, 3, 18, 1, + 104, 10, 3, 18, 1, 119, 66, 2, 16, 20, + ], + this.session_options, + 'y', + ); + } + return this._bilinear_interpolate_4d; + } + + static get bicubic_interpolate_4d() { + if (!this._bicubic_interpolate_4d) { + this._bicubic_interpolate_4d = wrap( + [ + 8, 9, 18, 0, 58, 127, 10, 39, 10, 1, 120, 10, 0, 10, 0, 10, 1, 115, 18, 1, 121, 34, 6, 82, 101, 115, + 105, 122, 101, 42, 16, 10, 4, 109, 111, 100, 101, 34, 5, 99, 117, 98, 105, 99, 160, 1, 3, 18, 1, + 114, 90, 31, 10, 1, 120, 18, 26, 10, 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 99, 10, 3, + 18, 1, 104, 10, 3, 18, 1, 119, 90, 15, 10, 1, 115, 18, 10, 10, 8, 8, 7, 18, 4, 10, 2, 8, 4, 98, 31, + 10, 1, 121, 18, 26, 10, 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 99, 10, 3, 18, 1, 104, 10, + 3, 18, 1, 119, 66, 2, 16, 20, + ], + this.session_options, + 'y', + ); + } + return this._bicubic_interpolate_4d; + } + + static get matmul() { + if (!this._matmul) { + this._matmul = wrap( + [ + 8, 9, 18, 0, 58, 55, 10, 17, 10, 1, 97, 10, 1, 98, 18, 1, 99, 34, 6, 77, 97, 116, 77, 117, 108, 18, + 1, 114, 90, 9, 10, 1, 97, 18, 4, 10, 2, 8, 1, 90, 9, 10, 1, 98, 18, 4, 10, 2, 8, 1, 98, 9, 10, 1, + 99, 18, 4, 10, 2, 8, 1, 66, 2, 16, 20, + ], + this.session_options, + 'c', + ); + } + return this._matmul; + } + + static get stft() { + if (!this._stft) { + this._stft = wrap( + [ + 8, 7, 18, 0, 58, 148, 1, 10, 38, 10, 1, 115, 10, 1, 106, 10, 1, 119, 10, 1, 108, 18, 1, 111, 34, 4, + 83, 84, 70, 84, 42, 15, 10, 8, 111, 110, 101, 115, 105, 100, 101, 100, 24, 1, 160, 1, 2, 18, 1, 115, + 90, 26, 10, 1, 115, 18, 21, 10, 19, 8, 1, 18, 15, 10, 3, 18, 1, 98, 10, 3, 18, 1, 115, 10, 3, 18, 1, + 99, 90, 11, 10, 1, 106, 18, 6, 10, 4, 8, 7, 18, 0, 90, 16, 10, 1, 119, 18, 11, 10, 9, 8, 1, 18, 5, + 10, 3, 18, 1, 119, 90, 11, 10, 1, 108, 18, 6, 10, 4, 8, 7, 18, 0, 98, 31, 10, 1, 111, 18, 26, 10, + 24, 8, 1, 18, 20, 10, 3, 18, 1, 98, 10, 3, 18, 1, 102, 10, 3, 18, 1, 100, 10, 3, 18, 1, 99, 66, 2, + 16, 17, + ], + this.session_options, + 'o', + ); + } + return this._stft; + } + + static get rfft() { + if (!this._rfft) { + this._rfft = wrap( + [ + 8, 9, 18, 0, 58, 97, 10, 33, 10, 1, 120, 10, 0, 10, 1, 97, 18, 1, 121, 34, 3, 68, 70, 84, 42, 15, + 10, 8, 111, 110, 101, 115, 105, 100, 101, 100, 24, 1, 160, 1, 2, 18, 1, 100, 90, 21, 10, 1, 120, 18, + 16, 10, 14, 8, 1, 18, 10, 10, 3, 18, 1, 115, 10, 3, 18, 1, 99, 90, 11, 10, 1, 97, 18, 6, 10, 4, 8, + 7, 18, 0, 98, 21, 10, 1, 121, 18, 16, 10, 14, 8, 1, 18, 10, 10, 3, 18, 1, 115, 10, 3, 18, 1, 99, 66, + 2, 16, 20, + ], + this.session_options, + 'y', + ); + } + return this._rfft; + } + + static get top_k() { + if (!this._top_k) { + this._top_k = wrap( + [ + 8, 10, 18, 0, 58, 73, 10, 18, 10, 1, 120, 10, 1, 107, 18, 1, 118, 18, 1, 105, 34, 4, 84, 111, 112, + 75, 18, 1, 116, 90, 9, 10, 1, 120, 18, 4, 10, 2, 8, 1, 90, 15, 10, 1, 107, 18, 10, 10, 8, 8, 7, 18, + 4, 10, 2, 8, 1, 98, 9, 10, 1, 118, 18, 4, 10, 2, 8, 1, 98, 9, 10, 1, 105, 18, 4, 10, 2, 8, 7, 66, 2, + 16, 21, + ], + this.session_options, + [/* Values */ 'v', /* Indices */ 'i'], + ); + } + return this._top_k; + } + + static get slice() { + if (!this._slice) { + this._slice = wrap( + [ + 8, 7, 18, 0, 58, 96, 10, 25, 10, 1, 120, 10, 1, 115, 10, 1, 101, 10, 1, 97, 10, 1, 116, 18, 1, 121, + 34, 5, 83, 108, 105, 99, 101, 18, 1, 114, 90, 9, 10, 1, 120, 18, 4, 10, 2, 8, 1, 90, 9, 10, 1, 115, + 18, 4, 10, 2, 8, 7, 90, 9, 10, 1, 101, 18, 4, 10, 2, 8, 7, 90, 9, 10, 1, 97, 18, 4, 10, 2, 8, 7, 90, + 9, 10, 1, 116, 18, 4, 10, 2, 8, 7, 98, 9, 10, 1, 121, 18, 4, 10, 2, 8, 1, 66, 2, 16, 13, + ], + this.session_options, + 'y', + ); + } + return this._slice; + } +} diff --git a/packages/transformers/src/pipelines.js b/packages/transformers/src/pipelines.js new file mode 100644 index 0000000..98c10c6 --- /dev/null +++ b/packages/transformers/src/pipelines.js @@ -0,0 +1,266 @@ +/** + * @file Pipelines provide a high-level, easy to use, API for running machine learning models. + * + * **Example:** Instantiate pipeline using the `pipeline` function. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const classifier = await pipeline('sentiment-analysis'); + * const output = await classifier('I love transformers!'); + * // [{'label': 'POSITIVE', 'score': 0.999817686}] + * ``` + * + * @module pipelines + */ + +import { DefaultProgressCallback, dispatchCallback } from './utils/core.js'; +import { logger } from './utils/logger.js'; + +import { AutoTokenizer } from './models/auto/tokenization_auto.js'; +import { AutoProcessor } from './models/auto/processing_auto.js'; +import { AutoConfig } from './configs.js'; + +import { + SUPPORTED_TASKS, + TASK_ALIASES, + TextClassificationPipeline, + TokenClassificationPipeline, + QuestionAnsweringPipeline, + FillMaskPipeline, + SummarizationPipeline, + TranslationPipeline, + Text2TextGenerationPipeline, + TextGenerationPipeline, + ZeroShotClassificationPipeline, + AudioClassificationPipeline, + ZeroShotAudioClassificationPipeline, + AutomaticSpeechRecognitionPipeline, + TextToAudioPipeline, + ImageToTextPipeline, + ImageClassificationPipeline, + ImageSegmentationPipeline, + BackgroundRemovalPipeline, + ZeroShotImageClassificationPipeline, + ObjectDetectionPipeline, + ZeroShotObjectDetectionPipeline, + DocumentQuestionAnsweringPipeline, + ImageToImagePipeline, + DepthEstimationPipeline, + FeatureExtractionPipeline, + ImageFeatureExtractionPipeline, +} from './pipelines/index.js'; +import { get_pipeline_files } from './utils/model_registry/get_pipeline_files.js'; +import { get_file_metadata } from './utils/model_registry/get_file_metadata.js'; + +/** + * @typedef {keyof typeof SUPPORTED_TASKS} TaskType + * @typedef {keyof typeof TASK_ALIASES} AliasType + * @typedef {TaskType | AliasType} PipelineType All possible pipeline types. + * @typedef {{[K in TaskType]: InstanceType}} SupportedTasks A mapping of pipeline names to their corresponding pipeline classes. + * @typedef {{[K in AliasType]: InstanceType}} AliasTasks A mapping from pipeline aliases to their corresponding pipeline classes. + * @typedef {SupportedTasks & AliasTasks} AllTasks A mapping from all pipeline names and aliases to their corresponding pipeline classes. + */ + +/** + * Utility factory method to build a `Pipeline` object. + * + * @template {PipelineType} T The type of pipeline to return. + * @param {T} task The task defining which pipeline will be returned. Currently accepted tasks are: + * - `"audio-classification"`: will return a `AudioClassificationPipeline`. + * - `"automatic-speech-recognition"`: will return a `AutomaticSpeechRecognitionPipeline`. + * - `"background-removal"`: will return a `BackgroundRemovalPipeline`. + * - `"depth-estimation"`: will return a `DepthEstimationPipeline`. + * - `"document-question-answering"`: will return a `DocumentQuestionAnsweringPipeline`. + * - `"feature-extraction"`: will return a `FeatureExtractionPipeline`. + * - `"fill-mask"`: will return a `FillMaskPipeline`. + * - `"image-classification"`: will return a `ImageClassificationPipeline`. + * - `"image-segmentation"`: will return a `ImageSegmentationPipeline`. + * - `"image-to-text"`: will return a `ImageToTextPipeline`. + * - `"object-detection"`: will return a `ObjectDetectionPipeline`. + * - `"question-answering"`: will return a `QuestionAnsweringPipeline`. + * - `"summarization"`: will return a `SummarizationPipeline`. + * - `"text2text-generation"`: will return a `Text2TextGenerationPipeline`. + * - `"text-classification"` (alias "sentiment-analysis" available): will return a `TextClassificationPipeline`. + * - `"text-generation"`: will return a `TextGenerationPipeline`. + * - `"token-classification"` (alias "ner" available): will return a `TokenClassificationPipeline`. + * - `"translation"`: will return a `TranslationPipeline`. + * - `"translation_xx_to_yy"`: will return a `TranslationPipeline`. + * - `"zero-shot-classification"`: will return a `ZeroShotClassificationPipeline`. + * - `"zero-shot-audio-classification"`: will return a `ZeroShotAudioClassificationPipeline`. + * - `"zero-shot-image-classification"`: will return a `ZeroShotImageClassificationPipeline`. + * - `"zero-shot-object-detection"`: will return a `ZeroShotObjectDetectionPipeline`. + * @param {string} [model=null] The name of the pre-trained model to use. If not specified, the default model for the task will be used. + * @param {import('./utils/hub.js').PretrainedModelOptions} [options] Optional parameters for the pipeline. + * @returns {Promise} A Pipeline object for the specified task. + * @throws {Error} If an unsupported pipeline is requested. + */ +export async function pipeline( + task, + model = null, + { + progress_callback = null, + config = null, + cache_dir = null, + local_files_only = false, + revision = 'main', + device = null, + dtype = null, + subfolder = 'onnx', + use_external_data_format = null, + model_file_name = null, + session_options = {}, + } = {}, +) { + // Apply aliases + // @ts-ignore + task = TASK_ALIASES[task] ?? task; + + // Get pipeline info + const pipelineInfo = SUPPORTED_TASKS[task.split('_', 1)[0]]; + if (!pipelineInfo) { + throw Error(`Unsupported pipeline: ${task}. Must be one of [${Object.keys(SUPPORTED_TASKS)}]`); + } + + // Use model if specified, otherwise, use default + if (!model) { + model = pipelineInfo.default.model; + logger.info(`No model specified. Using default model: "${model}".`); + if (!dtype && pipelineInfo.default.dtype) { + dtype = pipelineInfo.default.dtype; + } + } + + // Determine which files the model needs + const expected_files = await get_pipeline_files(task, model, { + device, + dtype, + }); + + /** @type {import('./utils/core.js').FilesLoadingMap} */ + let files_loading = {}; + if (progress_callback) { + /** @type {Array<{exists: boolean, size?: number, contentType?: string, fromCache?: boolean}>} */ + const metadata = await Promise.all(expected_files.map(async (file) => get_file_metadata(model, file))); + metadata.forEach((m, i) => { + if (m.exists) { + files_loading[expected_files[i]] = { + loaded: 0, + total: m.size ?? 0, + }; + } + }); + } + + const pretrainedOptions = { + progress_callback: progress_callback + ? new DefaultProgressCallback(progress_callback, files_loading) + : undefined, + config, + cache_dir, + local_files_only, + revision, + device, + dtype, + subfolder, + use_external_data_format, + model_file_name, + session_options, + }; + + // Determine which components to load based on the expected files + const hasTokenizer = expected_files.includes('tokenizer.json'); + const hasProcessor = expected_files.includes('preprocessor_config.json'); + + // Resolve the correct model class (needs config when multiple candidates exist) + const modelClasses = pipelineInfo.model; + let modelPromise; + if (Array.isArray(modelClasses)) { + const resolvedConfig = config ?? (await AutoConfig.from_pretrained(model, pretrainedOptions)); + const { model_type } = resolvedConfig; + const matchedClass = modelClasses.find((cls) => cls.supports(model_type)); + if (!matchedClass) { + throw Error( + `Unsupported model type "${model_type}" for task "${task}". ` + + `None of the candidate model classes support this type.`, + ); + } + modelPromise = matchedClass.from_pretrained(model, { ...pretrainedOptions, config: resolvedConfig }); + } else { + modelPromise = modelClasses.from_pretrained(model, pretrainedOptions); + } + + // Load all components in parallel + const [tokenizer, processor, model_loaded] = await Promise.all([ + hasTokenizer ? AutoTokenizer.from_pretrained(model, pretrainedOptions) : null, + hasProcessor ? AutoProcessor.from_pretrained(model, pretrainedOptions) : null, + modelPromise, + ]); + + const results = { task, model: model_loaded }; + if (tokenizer) results.tokenizer = tokenizer; + if (processor) results.processor = processor; + + dispatchCallback(progress_callback, { + status: 'ready', + task: task, + model: model, + }); + + const pipelineClass = pipelineInfo.pipeline; + return new pipelineClass(results); +} + +export { + TextClassificationPipeline, + TokenClassificationPipeline, + QuestionAnsweringPipeline, + FillMaskPipeline, + SummarizationPipeline, + TranslationPipeline, + Text2TextGenerationPipeline, + TextGenerationPipeline, + ZeroShotClassificationPipeline, + AudioClassificationPipeline, + ZeroShotAudioClassificationPipeline, + AutomaticSpeechRecognitionPipeline, + TextToAudioPipeline, + ImageToTextPipeline, + ImageClassificationPipeline, + ImageSegmentationPipeline, + BackgroundRemovalPipeline, + ZeroShotImageClassificationPipeline, + ObjectDetectionPipeline, + ZeroShotObjectDetectionPipeline, + DocumentQuestionAnsweringPipeline, + ImageToImagePipeline, + DepthEstimationPipeline, + FeatureExtractionPipeline, + ImageFeatureExtractionPipeline, +}; + +// Export pipeline output types +/** + * @typedef {import('./pipelines/fill-mask.js').FillMaskOutput} FillMaskOutput + * @typedef {import('./pipelines/text-classification.js').TextClassificationOutput} TextClassificationOutput + * @typedef {import('./pipelines/token-classification.js').TokenClassificationOutput} TokenClassificationOutput + * @typedef {import('./pipelines/question-answering.js').QuestionAnsweringOutput} QuestionAnsweringOutput + * @typedef {import('./pipelines/summarization.js').SummarizationOutput} SummarizationOutput + * @typedef {import('./pipelines/translation.js').TranslationOutput} TranslationOutput + * @typedef {import('./pipelines/text2text-generation.js').Text2TextGenerationOutput} Text2TextGenerationOutput + * @typedef {import('./pipelines/text-generation.js').TextGenerationOutput} TextGenerationOutput + * @typedef {import('./pipelines/text-generation.js').TextGenerationStringOutput} TextGenerationStringOutput + * @typedef {import('./pipelines/text-generation.js').TextGenerationChatOutput} TextGenerationChatOutput + * @typedef {import('./pipelines/zero-shot-classification.js').ZeroShotClassificationOutput} ZeroShotClassificationOutput + * @typedef {import('./pipelines/audio-classification.js').AudioClassificationOutput} AudioClassificationOutput + * @typedef {import('./pipelines/zero-shot-audio-classification.js').ZeroShotAudioClassificationOutput} ZeroShotAudioClassificationOutput + * @typedef {import('./pipelines/automatic-speech-recognition.js').AutomaticSpeechRecognitionOutput} AutomaticSpeechRecognitionOutput + * @typedef {import('./pipelines/text-to-audio.js').TextToAudioOutput} TextToAudioOutput + * @typedef {import('./pipelines/image-classification.js').ImageClassificationOutput} ImageClassificationOutput + * @typedef {import('./pipelines/image-segmentation.js').ImageSegmentationOutput} ImageSegmentationOutput + * @typedef {import('./pipelines/image-to-text.js').ImageToTextOutput} ImageToTextOutput + * @typedef {import('./pipelines/object-detection.js').ObjectDetectionOutput} ObjectDetectionOutput + * @typedef {import('./pipelines/zero-shot-object-detection.js').ZeroShotObjectDetectionOutput} ZeroShotObjectDetectionOutput + * @typedef {import('./pipelines/zero-shot-image-classification.js').ZeroShotImageClassificationOutput} ZeroShotImageClassificationOutput + * @typedef {import('./pipelines/document-question-answering.js').DocumentQuestionAnsweringOutput} DocumentQuestionAnsweringOutput + * @typedef {import('./pipelines/depth-estimation.js').DepthEstimationOutput} DepthEstimationOutput + */ diff --git a/packages/transformers/src/pipelines/_base.js b/packages/transformers/src/pipelines/_base.js new file mode 100644 index 0000000..495c10c --- /dev/null +++ b/packages/transformers/src/pipelines/_base.js @@ -0,0 +1,144 @@ +import { PreTrainedTokenizer } from '../tokenization_utils.js'; +import { PreTrainedModel } from '../models/modeling_utils.js'; +import { Processor } from '../processing_utils.js'; + +import { Callable } from '../utils/generic.js'; + +import { read_audio } from '../utils/audio.js'; +import { RawImage } from '../utils/image.js'; + +/** + * @typedef {string | RawImage | URL | Blob | HTMLCanvasElement | OffscreenCanvas} ImageInput + * @typedef {ImageInput|ImageInput[]} ImagePipelineInputs + */ + +/** + * Prepare images for further tasks. + * @param {ImagePipelineInputs} images images to prepare. + * @returns {Promise} returns processed images. + */ +export async function prepareImages(images) { + if (!Array.isArray(images)) { + images = [images]; + } + + // Possibly convert any non-images to images + return await Promise.all(images.map((x) => RawImage.read(x))); +} + +/** + * @typedef {string | URL | Float32Array | Float64Array} AudioInput + * @typedef {AudioInput|AudioInput[]} AudioPipelineInputs + */ + +/** + * Prepare audios for further tasks. + * @param {AudioPipelineInputs} audios audios to prepare. + * @param {number} sampling_rate sampling rate of the audios. + * @returns {Promise} The preprocessed audio data. + */ +export async function prepareAudios(audios, sampling_rate) { + if (!Array.isArray(audios)) { + audios = [audios]; + } + + return await Promise.all( + audios.map((x) => { + if (typeof x === 'string' || x instanceof URL) { + return read_audio(x, sampling_rate); + } else if (x instanceof Float64Array) { + return new Float32Array(x); + } + return x; + }), + ); +} + +/** + * @typedef {Object} BoundingBox + * @property {number} xmin The minimum x coordinate of the bounding box. + * @property {number} ymin The minimum y coordinate of the bounding box. + * @property {number} xmax The maximum x coordinate of the bounding box. + * @property {number} ymax The maximum y coordinate of the bounding box. + */ + +/** + * Helper function to convert list [xmin, xmax, ymin, ymax] into object { "xmin": xmin, ... } + * @param {number[]} box The bounding box as a list. + * @param {boolean} asInteger Whether to cast to integers. + * @returns {BoundingBox} The bounding box as an object. + * @private + */ +export function get_bounding_box(box, asInteger) { + if (asInteger) { + box = box.map((x) => x | 0); + } + const [xmin, ymin, xmax, ymax] = box; + + return { xmin, ymin, xmax, ymax }; +} + +/** + * @callback DisposeType Disposes the item. + * @returns {Promise} A promise that resolves when the item has been disposed. + * + * @typedef {Object} Disposable + * @property {DisposeType} dispose A promise that resolves when the pipeline has been disposed. + */ + +/** + * The Pipeline class is the class from which all pipelines inherit. + * Refer to this class for methods shared across different pipelines. + */ +export class Pipeline extends Callable { + /** + * Create a new Pipeline. + * @param {Object} options An object containing the following properties: + * @param {string} [options.task] The task of the pipeline. Useful for specifying subtasks. + * @param {PreTrainedModel} [options.model] The model used by the pipeline. + * @param {PreTrainedTokenizer} [options.tokenizer=null] The tokenizer used by the pipeline (if any). + * @param {Processor} [options.processor=null] The processor used by the pipeline (if any). + */ + constructor({ task, model, tokenizer = null, processor = null }) { + super(); + this.task = task; + this.model = model; + this.tokenizer = tokenizer; + this.processor = processor; + } + + /** @type {DisposeType} */ + async dispose() { + await this.model.dispose(); + } +} + +/** + * @typedef {Object} ModelTokenizerConstructorArgs + * @property {string} task The task of the pipeline. Useful for specifying subtasks. + * @property {PreTrainedModel} model The model used by the pipeline. + * @property {PreTrainedTokenizer} tokenizer The tokenizer used by the pipeline. + * + * @typedef {ModelTokenizerConstructorArgs} TextPipelineConstructorArgs An object used to instantiate a text-based pipeline. + */ + +/** + * @typedef {Object} ModelProcessorConstructorArgs + * @property {string} task The task of the pipeline. Useful for specifying subtasks. + * @property {PreTrainedModel} model The model used by the pipeline. + * @property {Processor} processor The processor used by the pipeline. + * + * @typedef {ModelProcessorConstructorArgs} AudioPipelineConstructorArgs An object used to instantiate an audio-based pipeline. + * @typedef {ModelProcessorConstructorArgs} ImagePipelineConstructorArgs An object used to instantiate an image-based pipeline. + */ + +/** + * @typedef {Object} ModelTokenizerProcessorConstructorArgs + * @property {string} task The task of the pipeline. Useful for specifying subtasks. + * @property {PreTrainedModel} model The model used by the pipeline. + * @property {PreTrainedTokenizer} tokenizer The tokenizer used by the pipeline. + * @property {Processor} processor The processor used by the pipeline. + * + * @typedef {ModelTokenizerProcessorConstructorArgs} TextAudioPipelineConstructorArgs An object used to instantiate a text- and audio-based pipeline. + * @typedef {ModelTokenizerProcessorConstructorArgs} TextImagePipelineConstructorArgs An object used to instantiate a text- and image-based pipeline. + */ diff --git a/packages/transformers/src/pipelines/audio-classification.js b/packages/transformers/src/pipelines/audio-classification.js new file mode 100644 index 0000000..60cd9ff --- /dev/null +++ b/packages/transformers/src/pipelines/audio-classification.js @@ -0,0 +1,97 @@ +import { Pipeline, prepareAudios } from './_base.js'; + +import { Tensor, topk } from '../utils/tensor.js'; +import { softmax } from '../utils/maths.js'; + +/** + * @typedef {import('./_base.js').AudioPipelineConstructorArgs} AudioPipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + * @typedef {import('./_base.js').AudioInput} AudioInput + */ + +/** + * @typedef {Object} AudioClassificationSingle + * @property {string} label The label predicted. + * @property {number} score The corresponding probability. + * @typedef {AudioClassificationSingle[]} AudioClassificationOutput + * + * @typedef {Object} AudioClassificationPipelineOptions Parameters specific to audio classification pipelines. + * @property {number} [top_k=5] The number of top labels that will be returned by the pipeline. + * If the provided number is `null` or higher than the number of labels available in the model configuration, + * it will default to the number of labels. + * + * @typedef {AudioPipelineConstructorArgs & AudioClassificationPipelineCallback & Disposable} AudioClassificationPipelineType + */ + +/** + * @template T + * @typedef {T extends AudioInput[] ? AudioClassificationOutput[] : AudioClassificationOutput} AudioClassificationPipelineResult + */ + +/** + * @typedef {(audio: T, options?: AudioClassificationPipelineOptions) => Promise>} AudioClassificationPipelineCallback + */ + +/** + * Audio classification pipeline using any `AutoModelForAudioClassification`. + * This pipeline predicts the class of a raw waveform or an audio file. + * + * **Example:** Perform audio classification with `Xenova/wav2vec2-large-xlsr-53-gender-recognition-librispeech`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const classifier = await pipeline('audio-classification', 'Xenova/wav2vec2-large-xlsr-53-gender-recognition-librispeech'); + * const audio = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav'; + * const output = await classifier(audio); + * // [ + * // { label: 'male', score: 0.9981542229652405 }, + * // { label: 'female', score: 0.001845747814513743 } + * // ] + * ``` + * + * **Example:** Perform audio classification with `Xenova/ast-finetuned-audioset-10-10-0.4593` and return top 4 results. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const classifier = await pipeline('audio-classification', 'Xenova/ast-finetuned-audioset-10-10-0.4593'); + * const audio = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cat_meow.wav'; + * const output = await classifier(audio, { top_k: 4 }); + * // [ + * // { label: 'Meow', score: 0.5617874264717102 }, + * // { label: 'Cat', score: 0.22365376353263855 }, + * // { label: 'Domestic animals, pets', score: 0.1141069084405899 }, + * // { label: 'Animal', score: 0.08985692262649536 }, + * // ] + * ``` + */ +export class AudioClassificationPipeline + extends /** @type {new (options: AudioPipelineConstructorArgs) => AudioClassificationPipelineType} */ (Pipeline) +{ + async _call(audio, { top_k = 5 } = {}) { + const sampling_rate = this.processor.feature_extractor.config.sampling_rate; + const preparedAudios = await prepareAudios(audio, sampling_rate); + + // @ts-expect-error TS2339 + const id2label = this.model.config.id2label; + + const toReturn = []; + for (const aud of preparedAudios) { + const inputs = await this.processor(aud); + const output = await this.model(inputs); + const logits = output.logits[0]; + + const scores = await topk(new Tensor('float32', softmax(logits.data), logits.dims), top_k); + + const values = scores[0].tolist(); + const indices = scores[1].tolist(); + + const vals = indices.map((x, i) => ({ + label: /** @type {string} */ (id2label ? id2label[x] : `LABEL_${x}`), + score: /** @type {number} */ (values[i]), + })); + + toReturn.push(vals); + } + return Array.isArray(audio) ? toReturn : toReturn[0]; + } +} diff --git a/packages/transformers/src/pipelines/automatic-speech-recognition.js b/packages/transformers/src/pipelines/automatic-speech-recognition.js new file mode 100644 index 0000000..238bb45 --- /dev/null +++ b/packages/transformers/src/pipelines/automatic-speech-recognition.js @@ -0,0 +1,374 @@ +import { Pipeline, prepareAudios } from './_base.js'; + +import { Tensor } from '../utils/tensor.js'; +import { max, round } from '../utils/maths.js'; +import { logger } from '../utils/logger.js'; + +/** + * @typedef {import('./_base.js').TextAudioPipelineConstructorArgs} TextAudioPipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + * @typedef {import('./_base.js').AudioInput} AudioInput + */ + +/** + * @typedef {Object} Chunk + * @property {[number, number]} timestamp The start and end timestamp of the chunk in seconds. + * @property {string} text The recognized text. + */ + +/** + * @typedef {Object} AutomaticSpeechRecognitionOutput + * @property {string} text The recognized text. + * @property {Chunk[]} [chunks] When using `return_timestamps`, the `chunks` will become a list + * containing all the various text chunks identified by the model. + * + * @typedef {Object} AutomaticSpeechRecognitionSpecificParams Parameters specific to automatic-speech-recognition pipelines. + * @property {boolean|'word'} [return_timestamps] Whether to return timestamps or not. Default is `false`. + * @property {number} [chunk_length_s] The length of audio chunks to process in seconds. Default is 0 (no chunking). + * @property {number} [stride_length_s] The length of overlap between consecutive audio chunks in seconds. If not provided, defaults to `chunk_length_s / 6`. + * @property {boolean} [force_full_sequences] Whether to force outputting full sequences or not. Default is `false`. + * @property {string} [language] The source language. Default is `null`, meaning it should be auto-detected. Use this to potentially improve performance if the source language is known. + * @property {string} [task] The task to perform. Default is `null`, meaning it should be auto-detected. + * @property {number} [num_frames] The number of frames in the input audio. + * @typedef {import('../generation/parameters.js').GenerationFunctionParameters & AutomaticSpeechRecognitionSpecificParams} AutomaticSpeechRecognitionConfig + * + * @typedef {TextAudioPipelineConstructorArgs & AutomaticSpeechRecognitionPipelineCallback & Disposable} AutomaticSpeechRecognitionPipelineType + */ + +/** + * @template T + * @typedef {T extends AudioInput[] ? AutomaticSpeechRecognitionOutput[] : AutomaticSpeechRecognitionOutput} AutomaticSpeechRecognitionPipelineResult + */ + +/** + * @typedef {(audio: T, options?: Partial) => Promise>} AutomaticSpeechRecognitionPipelineCallback + */ + +/** + * Pipeline that aims at extracting spoken text contained within some audio. + * + * **Example:** Transcribe English. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav'; + * const output = await transcriber(url); + * // { text: " And so my fellow Americans ask not what your country can do for you, ask what you can do for your country." } + * ``` + * + * **Example:** Transcribe English w/ timestamps. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav'; + * const output = await transcriber(url, { return_timestamps: true }); + * // { + * // text: " And so my fellow Americans ask not what your country can do for you, ask what you can do for your country." + * // chunks: [ + * // { timestamp: [0, 8], text: " And so my fellow Americans ask not what your country can do for you" } + * // { timestamp: [8, 11], text: " ask what you can do for your country." } + * // ] + * // } + * ``` + * + * **Example:** Transcribe English w/ word-level timestamps. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav'; + * const output = await transcriber(url, { return_timestamps: 'word' }); + * // { + * // "text": " And so my fellow Americans ask not what your country can do for you ask what you can do for your country.", + * // "chunks": [ + * // { "text": " And", "timestamp": [0, 0.78] }, + * // { "text": " so", "timestamp": [0.78, 1.06] }, + * // { "text": " my", "timestamp": [1.06, 1.46] }, + * // ... + * // { "text": " for", "timestamp": [9.72, 9.92] }, + * // { "text": " your", "timestamp": [9.92, 10.22] }, + * // { "text": " country.", "timestamp": [10.22, 13.5] } + * // ] + * // } + * ``` + * + * **Example:** Transcribe French. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-small'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/french-audio.mp3'; + * const output = await transcriber(url, { language: 'french', task: 'transcribe' }); + * // { text: " J'adore, j'aime, je n'aime pas, je déteste." } + * ``` + * + * **Example:** Translate French to English. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-small'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/french-audio.mp3'; + * const output = await transcriber(url, { language: 'french', task: 'translate' }); + * // { text: " I love, I like, I don't like, I hate." } + * ``` + * + * **Example:** Transcribe/translate audio longer than 30 seconds. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/ted_60.wav'; + * const output = await transcriber(url, { chunk_length_s: 30, stride_length_s: 5 }); + * // { text: " So in college, I was a government major, which means [...] So I'd start off light and I'd bump it up" } + * ``` + */ +export class AutomaticSpeechRecognitionPipeline + extends /** @type {new (options: TextAudioPipelineConstructorArgs) => AutomaticSpeechRecognitionPipelineType} */ ( + Pipeline + ) +{ + _default_generation_config = { + // TODO: figure out good defaults for ASR generation parameters + // max_new_tokens: 256, + // num_beams: 5, + }; + async _call(audio, kwargs = {}) { + kwargs = { + ...this._default_generation_config, + ...kwargs, + }; + switch (this.model.config.model_type) { + case 'whisper': + case 'lite-whisper': + return this._call_whisper(audio, kwargs); + case 'wav2vec2': + case 'wav2vec2-bert': + case 'unispeech': + case 'unispeech-sat': + case 'hubert': + case 'parakeet_ctc': + return this._call_wav2vec2(audio, kwargs); + case 'moonshine': + return this._call_moonshine(audio, kwargs); + case 'cohere_asr': + return this._call_cohere_asr(audio, kwargs); + default: + throw new Error( + `AutomaticSpeechRecognitionPipeline does not support model type '${this.model.config.model_type}'.`, + ); + } + } + + async _call_wav2vec2(audio, kwargs) { + // TODO use kwargs + + if (kwargs.language) { + logger.warn('`language` parameter is not yet supported for `wav2vec2` models, defaulting to "English".'); + } + if (kwargs.task) { + logger.warn('`task` parameter is not yet supported for `wav2vec2` models, defaulting to "transcribe".'); + } + + const single = !Array.isArray(audio); + const batchedAudio = single ? [audio] : audio; + + const sampling_rate = this.processor.feature_extractor.config.sampling_rate; + const preparedAudios = await prepareAudios(batchedAudio, sampling_rate); + + const toReturn = []; + for (const aud of preparedAudios) { + const inputs = await this.processor(aud); + const output = await this.model(inputs); + const logits = output.logits[0]; + + const predicted_ids = []; + for (const item of logits) { + predicted_ids.push(max(item.data)[1]); + } + const predicted_sentences = this.tokenizer.decode(predicted_ids, { skip_special_tokens: true }).trim(); + toReturn.push({ text: predicted_sentences }); + } + return single ? toReturn[0] : toReturn; + } + + async _call_whisper(audio, kwargs) { + const return_timestamps = kwargs.return_timestamps ?? false; + const chunk_length_s = kwargs.chunk_length_s ?? 0; + const force_full_sequences = kwargs.force_full_sequences ?? false; + let stride_length_s = kwargs.stride_length_s ?? null; + + const generation_config = { ...kwargs }; + + if (return_timestamps === 'word') { + generation_config['return_token_timestamps'] = true; + generation_config['return_timestamps'] = true; + } + + const single = !Array.isArray(audio); + const batchedAudio = single ? [audio] : audio; + const feature_extractor_config = this.processor.feature_extractor.config; + + // @ts-expect-error TS2339 + const time_precision = feature_extractor_config.chunk_length / this.model.config.max_source_positions; + const hop_length = feature_extractor_config.hop_length; + + const sampling_rate = feature_extractor_config.sampling_rate; + const preparedAudios = await prepareAudios(batchedAudio, sampling_rate); + + const toReturn = []; + for (const aud of preparedAudios) { + /** @type {{stride: number[], input_features: Tensor, is_last: boolean, tokens?: bigint[], token_timestamps?: number[]}[]} */ + let chunks = []; + if (chunk_length_s > 0) { + if (stride_length_s === null) { + stride_length_s = chunk_length_s / 6; + } else if (chunk_length_s <= stride_length_s) { + throw Error('`chunk_length_s` must be larger than `stride_length_s`.'); + } + + // TODO support different stride_length_s (for left and right) + + const window = sampling_rate * chunk_length_s; + const stride = sampling_rate * stride_length_s; + const jump = window - 2 * stride; + let offset = 0; + + // Create subarrays of audio with overlaps + while (true) { + const offset_end = offset + window; + const subarr = aud.subarray(offset, offset_end); + const feature = await this.processor(subarr); + + const is_first = offset === 0; + const is_last = offset_end >= aud.length; + chunks.push({ + stride: [subarr.length, is_first ? 0 : stride, is_last ? 0 : stride], + input_features: feature.input_features, + is_last, + }); + if (is_last) break; + offset += jump; + } + } else { + chunks = [ + { + stride: [aud.length, 0, 0], + input_features: (await this.processor(aud)).input_features, + is_last: true, + }, + ]; + } + + // Generate for each set of input features + for (const chunk of chunks) { + generation_config.num_frames = Math.floor(chunk.stride[0] / hop_length); + + // NOTE: doing sequentially for now + const data = await this.model.generate({ + inputs: chunk.input_features, + ...generation_config, + }); + + // TODO: Right now we only get top beam + if (return_timestamps === 'word') { + // @ts-expect-error TS2339 + const sequences = data.sequences.tolist()[0]; + // @ts-expect-error TS2339 + const token_ts = data.token_timestamps.tolist()[0]; + + // Strip decoder_input_ids prefix from sequences and token_timestamps + // to match Python's behavior (where generate() returns sequences without the prefix) + // @ts-expect-error ts(2339) + const timestamp_begin = this.tokenizer.timestamp_begin; + const prefixLength = Math.max( + sequences.findIndex((/** @type {bigint} */ t) => Number(t) >= timestamp_begin), + 0, + ); + + chunk.tokens = sequences.slice(prefixLength); + chunk.token_timestamps = token_ts.slice(prefixLength).map((/** @type {number} */ x) => round(x, 2)); + } else { + chunk.tokens = /** @type {Tensor} */ (data)[0].tolist(); + } + + // convert stride to seconds + chunk.stride = chunk.stride.map((x) => x / sampling_rate); + } + + // Merge text chunks + // @ts-ignore + const [full_text, optional] = this.tokenizer._decode_asr(chunks, { + time_precision, + return_timestamps, + force_full_sequences, + }); + + toReturn.push({ text: full_text, ...optional }); + } + return single ? toReturn[0] : toReturn; + } + + async _call_moonshine(audio, kwargs) { + const single = !Array.isArray(audio); + const batchedAudio = single ? [audio] : audio; + const sampling_rate = this.processor.feature_extractor.config.sampling_rate; + const preparedAudios = await prepareAudios(batchedAudio, sampling_rate); + const toReturn = []; + for (const aud of preparedAudios) { + const inputs = await this.processor(aud); + + // According to the [paper](https://huggingface.co/papers/2410.15608): + // "We use greedy decoding, with a heuristic limit of 6 output tokens + // per second of audio to avoid repeated output sequences." + const max_new_tokens = Math.floor(aud.length / sampling_rate) * 6; + const outputs = await this.model.generate({ max_new_tokens, ...kwargs, ...inputs }); + + const text = this.processor.batch_decode(/** @type {Tensor} */ (outputs), { skip_special_tokens: true })[0]; + toReturn.push({ text }); + } + return single ? toReturn[0] : toReturn; + } + + async _call_cohere_asr(audio, kwargs) { + const single = !Array.isArray(audio); + const batchedAudio = single ? [audio] : audio; + + const feature_extractor = this.processor.feature_extractor; + const sampling_rate = feature_extractor.config.sampling_rate; + const preparedAudios = await prepareAudios(batchedAudio, sampling_rate); + + const language = kwargs.language ?? 'en'; + // @ts-expect-error TS2339 + const decoder_input_ids = this.processor.get_decoder_prompt_ids(language); + + const toReturn = []; + for (const aud of preparedAudios) { + // Split long audio at energy-based boundaries + // @ts-expect-error TS2339 + const audioChunks = feature_extractor.split_audio(aud); + + const chunk_texts = []; + for (const chunk of audioChunks) { + const inputs = await this.processor(chunk); + + const outputs = await this.model.generate({ + ...inputs, + decoder_input_ids, + ...kwargs, + }); + + const text = this.tokenizer + .decode(/** @type {Tensor} */ (outputs)[0].tolist(), { skip_special_tokens: true }) + .trim(); + chunk_texts.push(text); + } + + // @ts-expect-error TS2339 + const full_text = this.processor.constructor.join_chunks(chunk_texts, language); + toReturn.push({ text: full_text }); + } + return single ? toReturn[0] : toReturn; + } +} diff --git a/packages/transformers/src/pipelines/background-removal.js b/packages/transformers/src/pipelines/background-removal.js new file mode 100644 index 0000000..37b8470 --- /dev/null +++ b/packages/transformers/src/pipelines/background-removal.js @@ -0,0 +1,58 @@ +import { ImageSegmentationPipeline } from './image-segmentation.js'; +import { prepareImages } from './_base.js'; +import { RawImage } from '../utils/image.js'; + +/** + * @typedef {import('./_base.js').ImagePipelineConstructorArgs} ImagePipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + * @typedef {import('./_base.js').ImageInput} ImageInput + */ + +/** + * @typedef {Object} BackgroundRemovalPipelineOptions Parameters specific to background removal pipelines. + * + * @typedef {ImagePipelineConstructorArgs & BackgroundRemovalPipelineCallback & Disposable} BackgroundRemovalPipelineType + */ + +/** + * @template T + * @typedef {T extends ImageInput[] ? RawImage[] : RawImage} BackgroundRemovalPipelineResult + */ + +/** + * @typedef {(images: T, options?: BackgroundRemovalPipelineOptions) => Promise>} BackgroundRemovalPipelineCallback + */ + +/** + * Background removal pipeline using certain `AutoModelForXXXSegmentation`. + * This pipeline removes the backgrounds of images. + * + * **Example:** Perform background removal with `Xenova/modnet`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const segmenter = await pipeline('background-removal', 'Xenova/modnet'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/portrait-of-woman_small.jpg'; + * const output = await segmenter(url); + * // RawImage { data: Uint8ClampedArray(648000) [ ... ], width: 360, height: 450, channels: 4 } + * ``` + */ +export class BackgroundRemovalPipeline + extends /** @type {new (options: ImagePipelineConstructorArgs) => BackgroundRemovalPipelineType} */ ( + /** @type {any} */ (ImageSegmentationPipeline) + ) +{ + async _call(images, options = {}) { + const preparedImages = await prepareImages(images); + + // @ts-expect-error TS2339 + const masks = await super._call(images, options); + const result = preparedImages.map((img, i) => { + const cloned = img.clone(); + cloned.putAlpha(masks[i].mask); + return cloned; + }); + + return Array.isArray(images) ? result : result[0]; + } +} diff --git a/packages/transformers/src/pipelines/depth-estimation.js b/packages/transformers/src/pipelines/depth-estimation.js new file mode 100644 index 0000000..fd60aac --- /dev/null +++ b/packages/transformers/src/pipelines/depth-estimation.js @@ -0,0 +1,94 @@ +import { Pipeline, prepareImages } from './_base.js'; +import { RawImage } from '../utils/image.js'; + +import { interpolate_4d } from '../utils/tensor.js'; + +/** + * @typedef {import('./_base.js').ImagePipelineConstructorArgs} ImagePipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + * @typedef {import('./_base.js').ImageInput} ImageInput + */ + +/** + * @typedef {Object} DepthEstimationOutput + * @property {import('../utils/tensor.js').Tensor} predicted_depth The raw depth map predicted by the model. + * @property {RawImage} depth The processed depth map as an image (with the same size as the input image). + * + * @typedef {ImagePipelineConstructorArgs & DepthEstimationPipelineCallback & Disposable} DepthEstimationPipelineType + */ + +/** + * @template T + * @typedef {T extends ImageInput[] ? DepthEstimationOutput[] : DepthEstimationOutput} DepthEstimationPipelineResult + */ + +/** + * @typedef {(images: T) => Promise>} DepthEstimationPipelineCallback + */ + +/** + * Depth estimation pipeline using any `AutoModelForDepthEstimation`. This pipeline predicts the depth of an image. + * + * **Example:** Depth estimation w/ `onnx-community/depth-anything-v2-small` + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const depth_estimator = await pipeline('depth-estimation', 'onnx-community/depth-anything-v2-small'); + * const image = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg'; + * const output = await depth_estimator(image); + * // { + * // predicted_depth: Tensor { + * // dims: [ 480, 640 ], + * // type: 'float32', + * // data: Float32Array(307200) [ 2.6300313472747803, 2.5856235027313232, 2.620532751083374, ... ], + * // size: 307200 + * // }, + * // depth: RawImage { + * // data: Uint8Array(307200) [ 106, 104, 106, ... ], + * // width: 640, + * // height: 480, + * // channels: 1 + * // } + * // } + * ``` + */ +export class DepthEstimationPipeline + extends /** @type {new (options: ImagePipelineConstructorArgs) => DepthEstimationPipelineType} */ (Pipeline) +{ + async _call(images) { + const preparedImages = await prepareImages(images); + + const inputs = await this.processor(preparedImages); + const { predicted_depth } = await this.model(inputs); + + const toReturn = []; + for (let i = 0; i < preparedImages.length; ++i) { + const batch = predicted_depth[i]; + const [height, width] = batch.dims.slice(-2); + const [new_width, new_height] = preparedImages[i].size; + + // Interpolate to original size + const prediction = ( + await interpolate_4d(batch.view(1, 1, height, width), { + size: [new_height, new_width], + mode: 'bilinear', + }) + ).view(new_height, new_width); + + const minval = /** @type {number} */ (prediction.min().item()); + const maxval = /** @type {number} */ (prediction.max().item()); + const formatted = prediction + .sub(minval) + .div_(maxval - minval) + .mul_(255) + .to('uint8') + .unsqueeze(0); + const depth = RawImage.fromTensor(formatted); + toReturn.push({ + predicted_depth: prediction, + depth, + }); + } + return Array.isArray(images) ? toReturn : toReturn[0]; + } +} diff --git a/packages/transformers/src/pipelines/document-question-answering.js b/packages/transformers/src/pipelines/document-question-answering.js new file mode 100644 index 0000000..0914811 --- /dev/null +++ b/packages/transformers/src/pipelines/document-question-answering.js @@ -0,0 +1,91 @@ +import { Pipeline, prepareImages } from './_base.js'; + +import { Tensor } from '../utils/tensor.js'; + +/** + * @typedef {import('./_base.js').TextImagePipelineConstructorArgs} TextImagePipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + * @typedef {import('./_base.js').ImageInput} ImageInput + */ + +/** + * @typedef {Object} DocumentQuestionAnsweringSingle + * @property {string} answer The generated text. + * @typedef {DocumentQuestionAnsweringSingle[]} DocumentQuestionAnsweringOutput + * + * @callback DocumentQuestionAnsweringPipelineCallback Answer the question given as input by using the document. + * @param {ImageInput|ImageInput[]} image The image of the document to use. + * @param {string} question A question to ask of the document. + * @param {Partial} [options] Additional keyword arguments to pass along to the generate method of the model. + * @returns {Promise} An object (or array of objects) containing the answer(s). + * + * @typedef {TextImagePipelineConstructorArgs & DocumentQuestionAnsweringPipelineCallback & Disposable} DocumentQuestionAnsweringPipelineType + */ + +/** + * Document Question Answering pipeline using any `AutoModelForDocumentQuestionAnswering`. + * The inputs/outputs are similar to the (extractive) question answering pipeline; however, + * the pipeline takes an image (and optional OCR'd words/boxes) as input instead of text context. + * + * **Example:** Answer questions about a document with `Xenova/donut-base-finetuned-docvqa`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const qa_pipeline = await pipeline('document-question-answering', 'Xenova/donut-base-finetuned-docvqa'); + * const image = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/invoice.png'; + * const question = 'What is the invoice number?'; + * const output = await qa_pipeline(image, question); + * // [{ answer: 'us-001' }] + * ``` + */ +export class DocumentQuestionAnsweringPipeline + extends /** @type {new (options: TextImagePipelineConstructorArgs) => DocumentQuestionAnsweringPipelineType} */ ( + Pipeline + ) +{ + _default_generation_config = { + max_new_tokens: 256, + }; + async _call(image, question, generate_kwargs = {}) { + if (Array.isArray(image)) { + if (image.length !== 1) { + throw Error('Document Question Answering pipeline currently only supports a batch size of 1.'); + } + image = image[0]; + } + + // Preprocess image + const preparedImage = (await prepareImages(image))[0]; + const { pixel_values } = await this.processor(preparedImage); + + // Run tokenization + const task_prompt = `${question}`; + const decoder_input_ids = this.tokenizer(task_prompt, { + add_special_tokens: false, + padding: true, + truncation: true, + }).input_ids; + + // Run model + const output = await this.model.generate({ + inputs: pixel_values, + // @ts-expect-error Ts2339 + max_length: this.model.config.decoder.max_position_embeddings, + decoder_input_ids, + + ...this._default_generation_config, + ...generate_kwargs, + }); + + // Decode output + const decoded = this.tokenizer.batch_decode(/** @type {Tensor} */ (output))[0]; + + // Parse answer + const match = decoded.match(/(.*?)<\/s_answer>/); + let answer = null; + if (match && match.length >= 2) { + answer = match[1].trim(); + } + return [{ answer }]; + } +} diff --git a/packages/transformers/src/pipelines/feature-extraction.js b/packages/transformers/src/pipelines/feature-extraction.js new file mode 100644 index 0000000..4a2214e --- /dev/null +++ b/packages/transformers/src/pipelines/feature-extraction.js @@ -0,0 +1,134 @@ +import { Pipeline } from './_base.js'; + +import { Tensor, mean_pooling, quantize_embeddings } from '../utils/tensor.js'; + +/** + * @typedef {import('./_base.js').TextPipelineConstructorArgs} TextPipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + */ + +/** + * @typedef {Object} FeatureExtractionPipelineOptions Parameters specific to feature extraction pipelines. + * @property {'none'|'mean'|'cls'|'first_token'|'eos'|'last_token'} [pooling="none"] The pooling method to use. + * @property {boolean} [normalize=false] Whether or not to normalize the embeddings in the last dimension. + * @property {boolean} [quantize=false] Whether or not to quantize the embeddings. + * @property {'binary'|'ubinary'} [precision='binary'] The precision to use for quantization. + * + * @callback FeatureExtractionPipelineCallback Extract the features of the input(s). + * @param {string|string[]} texts One or several texts (or one list of texts) to get the features of. + * @param {FeatureExtractionPipelineOptions} [options] The options to use for feature extraction. + * @returns {Promise} The features computed by the model. + * + * @typedef {TextPipelineConstructorArgs & FeatureExtractionPipelineCallback & Disposable} FeatureExtractionPipelineType + */ + +/** + * Feature extraction pipeline using no model head. This pipeline extracts the hidden + * states from the base transformer, which can be used as features in downstream tasks. + * + * **Example:** Run feature extraction using `onnx-community/all-MiniLM-L6-v2-ONNX` (without pooling or normalization). + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const extractor = await pipeline('feature-extraction', 'onnx-community/all-MiniLM-L6-v2-ONNX'); + * const output = await extractor('This is a simple test.'); + * // Tensor { + * // type: 'float32', + * // data: Float32Array [0.2157987803220749, -0.09140099585056305, ...], + * // dims: [1, 8, 384] + * // } + * + * // You can convert this Tensor to a nested JavaScript array using `.tolist()`: + * console.log(output.tolist()); + * ``` + * + * **Example:** Run feature extraction using `onnx-community/all-MiniLM-L6-v2-ONNX` (with pooling and normalization). + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const extractor = await pipeline('feature-extraction', 'onnx-community/all-MiniLM-L6-v2-ONNX'); + * const output = await extractor('This is a simple test.', { pooling: 'mean', normalize: true }); + * // Tensor { + * // type: 'float32', + * // data: Float32Array [0.09528215229511261, -0.024730168282985687, ...], + * // dims: [1, 384] + * // } + * + * // You can convert this Tensor to a nested JavaScript array using `.tolist()`: + * console.log(output.tolist()); + * ``` + * + * **Example:** Run feature extraction using `onnx-community/all-MiniLM-L6-v2-ONNX` models (with pooling and binary quantization). + * ```javascript + * const extractor = await pipeline('feature-extraction', 'onnx-community/all-MiniLM-L6-v2-ONNX'); + * const output = await extractor('This is a simple test.', { pooling: 'mean', quantize: true, precision: 'binary' }); + * // Tensor { + * // type: 'int8', + * // data: Int8Array [49, 108, 25, ...], + * // dims: [1, 48] + * // } + * + * // You can convert this Tensor to a nested JavaScript array using `.tolist()`: + * console.log(output.tolist()); + * ``` + */ +export class FeatureExtractionPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => FeatureExtractionPipelineType} */ (Pipeline) +{ + /** @type {FeatureExtractionPipelineCallback} */ + async _call( + texts, + { + pooling = /** @type {'none'} */ ('none'), + normalize = false, + quantize = false, + precision = /** @type {'binary'} */ ('binary'), + } = {}, + ) { + // Run tokenization + const model_inputs = this.tokenizer(texts, { + padding: true, + truncation: true, + }); + + // Run model + const outputs = await this.model(model_inputs); + + // TODO: Provide warning to the user that they might be using model which was not exported + // specifically for feature extraction + // console.log(this.model.config) + // console.log(outputs) + + /** @type {Tensor} */ + let result = outputs.last_hidden_state ?? outputs.logits ?? outputs.token_embeddings; + + switch (pooling) { + case 'none': + // Skip pooling + break; + case 'mean': + result = mean_pooling(result, model_inputs.attention_mask); + break; + case 'first_token': + case 'cls': + result = result.slice(null, 0); + break; + case 'last_token': + case 'eos': + result = result.slice(null, -1); + break; + default: + throw Error(`Pooling method '${pooling}' not supported.`); + } + + if (normalize) { + result = result.normalize(2, -1); + } + + if (quantize) { + result = quantize_embeddings(result, precision); + } + + return result; + } +} diff --git a/packages/transformers/src/pipelines/fill-mask.js b/packages/transformers/src/pipelines/fill-mask.js new file mode 100644 index 0000000..da2d1a8 --- /dev/null +++ b/packages/transformers/src/pipelines/fill-mask.js @@ -0,0 +1,130 @@ +import { Pipeline } from './_base.js'; + +import { Tensor, topk } from '../utils/tensor.js'; +import { softmax } from '../utils/maths.js'; + +/** + * @typedef {import('./_base.js').TextPipelineConstructorArgs} TextPipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + */ + +/** + * @typedef {Object} FillMaskSingle + * @property {string} sequence The corresponding input with the mask token prediction. + * @property {number} score The corresponding probability. + * @property {number} token The predicted token id (to replace the masked one). + * @property {string} token_str The predicted token (to replace the masked one). + * @typedef {FillMaskSingle[]} FillMaskOutput + * + * @typedef {Object} FillMaskPipelineOptions Parameters specific to fill mask pipelines. + * @property {number} [top_k=5] When passed, overrides the number of predictions to return. + * + * @typedef {TextPipelineConstructorArgs & FillMaskPipelineCallback & Disposable} FillMaskPipelineType + */ + +/** + * @template T + * @typedef {T extends string[] ? FillMaskOutput[] : FillMaskOutput} FillMaskPipelineResult + */ + +/** + * @typedef {(texts: T, options?: FillMaskPipelineOptions) => Promise>} FillMaskPipelineCallback + */ + +/** + * Masked language modeling prediction pipeline using any `ModelWithLMHead`. + * + * **Example:** Perform masked language modelling (a.k.a. "fill-mask") with `onnx-community/ettin-encoder-32m-ONNX`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const unmasker = await pipeline('fill-mask', 'onnx-community/ettin-encoder-32m-ONNX'); + * const output = await unmasker('The capital of France is [MASK].'); + * // [ + * // { score: 0.5151872038841248, token: 7785, token_str: ' Paris', sequence: 'The capital of France is Paris.' }, + * // { score: 0.033725105226039886, token: 42268, token_str: ' Lyon', sequence: 'The capital of France is Lyon.' }, + * // { score: 0.031234024092555046, token: 23397, token_str: ' Nancy', sequence: 'The capital of France is Nancy.' }, + * // { score: 0.02075139433145523, token: 30167, token_str: ' Brussels', sequence: 'The capital of France is Brussels.' }, + * // { score: 0.018962178379297256, token: 31955, token_str: ' Geneva', sequence: 'The capital of France is Geneva.' } + * // ] + * ``` + * + * **Example:** Perform masked language modelling (a.k.a. "fill-mask") with `Xenova/bert-base-uncased`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const unmasker = await pipeline('fill-mask', 'Xenova/bert-base-cased'); + * const output = await unmasker('The goal of life is [MASK].'); + * // [ + * // { score: 0.11368396878242493, sequence: "The goal of life is survival.", token: 8115, token_str: "survival" }, + * // { score: 0.053510840982198715, sequence: "The goal of life is love.", token: 1567, token_str: "love" }, + * // { score: 0.05041185021400452, sequence: "The goal of life is happiness.", token: 9266, token_str: "happiness" }, + * // { score: 0.033218126744031906, sequence: "The goal of life is freedom.", token: 4438, token_str: "freedom" }, + * // { score: 0.03301157429814339, sequence: "The goal of life is success.", token: 2244, token_str: "success" }, + * // ] + * ``` + * + * **Example:** Perform masked language modelling (a.k.a. "fill-mask") with `Xenova/bert-base-cased` (and return top result). + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const unmasker = await pipeline('fill-mask', 'Xenova/bert-base-cased'); + * const output = await unmasker('The Milky Way is a [MASK] galaxy.', { top_k: 1 }); + * // [{ score: 0.5982972383499146, sequence: "The Milky Way is a spiral galaxy.", token: 14061, token_str: "spiral" }] + * ``` + */ +export class FillMaskPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => FillMaskPipelineType} */ (Pipeline) +{ + async _call(texts, { top_k = 5 } = {}) { + const { mask_token_id, mask_token } = this.tokenizer; + + // Run tokenization + const model_inputs = this.tokenizer(texts, { + padding: true, + truncation: true, + }); + + // Run model + const { logits } = await this.model(model_inputs); + + const toReturn = []; + + /** @type {bigint[][]} */ + const input_ids = model_inputs.input_ids.tolist(); + for (let i = 0; i < input_ids.length; ++i) { + const ids = input_ids[i]; + const mask_token_index = ids.findIndex( + (x) => + // We use == to match bigint with number + // @ts-ignore - TS2367: Intentional loose equality for bigint/number comparison + x == mask_token_id, + ); + if (mask_token_index === -1) { + throw Error(`Mask token (${mask_token}) not found in text.`); + } + const itemLogits = logits[i][mask_token_index]; + + const scores = await topk(new Tensor('float32', softmax(itemLogits.data), itemLogits.dims), top_k); + const values = scores[0].tolist(); + const indices = scores[1].tolist(); + + toReturn.push( + indices.map((x, i) => { + const sequence = ids.slice(); + sequence[mask_token_index] = x; + + return { + score: values[i], + token: Number(x), + token_str: this.tokenizer.decode([x]), + sequence: this.tokenizer.decode(sequence, { + skip_special_tokens: true, + }), + }; + }), + ); + } + return Array.isArray(texts) ? toReturn : toReturn[0]; + } +} diff --git a/packages/transformers/src/pipelines/image-classification.js b/packages/transformers/src/pipelines/image-classification.js new file mode 100644 index 0000000..f9374e8 --- /dev/null +++ b/packages/transformers/src/pipelines/image-classification.js @@ -0,0 +1,108 @@ +import { Pipeline, prepareImages } from './_base.js'; + +import { Tensor, topk } from '../utils/tensor.js'; +import { softmax } from '../utils/maths.js'; + +/** + * @typedef {import('./_base.js').ImagePipelineConstructorArgs} ImagePipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + * @typedef {import('./_base.js').ImageInput} ImageInput + */ + +/** + * @typedef {Object} ImageClassificationSingle + * @property {string} label The label identified by the model. + * @property {number} score The score attributed by the model for that label. + * @typedef {ImageClassificationSingle[]} ImageClassificationOutput + * + * @typedef {Object} ImageClassificationPipelineOptions Parameters specific to image classification pipelines. + * @property {number} [top_k=1] The number of top labels that will be returned by the pipeline. + * + * @typedef {ImagePipelineConstructorArgs & ImageClassificationPipelineCallback & Disposable} ImageClassificationPipelineType + */ + +/** + * @template T + * @typedef {T extends ImageInput[] ? ImageClassificationOutput[] : ImageClassificationOutput} ImageClassificationPipelineResult + */ + +/** + * @typedef {(images: T, options?: ImageClassificationPipelineOptions) => Promise>} ImageClassificationPipelineCallback + */ + +/** + * Image classification pipeline using any `AutoModelForImageClassification`. + * This pipeline predicts the class of an image. + * + * **Example:** Classify an image. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const classifier = await pipeline('image-classification', 'Xenova/vit-base-patch16-224'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/tiger.jpg'; + * const output = await classifier(url); + * // [ + * // { label: 'tiger, Panthera tigris', score: 0.632695734500885 }, + * // ] + * ``` + * + * **Example:** Classify an image and return top `n` classes. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const classifier = await pipeline('image-classification', 'Xenova/vit-base-patch16-224'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/tiger.jpg'; + * const output = await classifier(url, { top_k: 3 }); + * // [ + * // { label: 'tiger, Panthera tigris', score: 0.632695734500885 }, + * // { label: 'tiger cat', score: 0.3634825646877289 }, + * // { label: 'lion, king of beasts, Panthera leo', score: 0.00045060308184474707 }, + * // ] + * ``` + * + * **Example:** Classify an image and return all classes. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const classifier = await pipeline('image-classification', 'Xenova/vit-base-patch16-224'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/tiger.jpg'; + * const output = await classifier(url, { top_k: 0 }); + * // [ + * // { label: 'tiger, Panthera tigris', score: 0.632695734500885 }, + * // { label: 'tiger cat', score: 0.3634825646877289 }, + * // { label: 'lion, king of beasts, Panthera leo', score: 0.00045060308184474707 }, + * // { label: 'jaguar, panther, Panthera onca, Felis onca', score: 0.00035465499968267977 }, + * // ... + * // ] + * ``` + */ +export class ImageClassificationPipeline + extends /** @type {new (options: ImagePipelineConstructorArgs) => ImageClassificationPipelineType} */ (Pipeline) +{ + async _call(images, { top_k = 5 } = {}) { + const preparedImages = await prepareImages(images); + + const { pixel_values } = await this.processor(preparedImages); + const output = await this.model({ pixel_values }); + + // @ts-expect-error TS2339 + const { id2label } = this.model.config; + + /** @type {ImageClassificationOutput[]} */ + const toReturn = []; + for (const batch of output.logits) { + const scores = await topk(new Tensor('float32', softmax(batch.data), batch.dims), top_k); + + const values = scores[0].tolist(); + const indices = scores[1].tolist(); + + const vals = indices.map((x, i) => ({ + label: /** @type {string} */ (id2label ? id2label[x] : `LABEL_${x}`), + score: /** @type {number} */ (values[i]), + })); + toReturn.push(vals); + } + + return Array.isArray(images) ? toReturn : toReturn[0]; + } +} diff --git a/packages/transformers/src/pipelines/image-feature-extraction.js b/packages/transformers/src/pipelines/image-feature-extraction.js new file mode 100644 index 0000000..25687f3 --- /dev/null +++ b/packages/transformers/src/pipelines/image-feature-extraction.js @@ -0,0 +1,80 @@ +import { Pipeline, prepareImages } from './_base.js'; + +import { Tensor } from '../utils/tensor.js'; + +/** + * @typedef {import('./_base.js').ImagePipelineConstructorArgs} ImagePipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + * @typedef {import('./_base.js').ImagePipelineInputs} ImagePipelineInputs + */ + +/** + * @typedef {Object} ImageFeatureExtractionPipelineOptions Parameters specific to image feature extraction pipelines. + * @property {boolean} [pool=null] Whether or not to return the pooled output. If set to `false`, the model will return the raw hidden states. + * + * @callback ImageFeatureExtractionPipelineCallback Extract the features of the input(s). + * @param {ImagePipelineInputs} images One or several images (or one list of images) to get the features of. + * @param {ImageFeatureExtractionPipelineOptions} [options] The options to use for image feature extraction. + * @returns {Promise} The image features computed by the model. + * + * @typedef {ImagePipelineConstructorArgs & ImageFeatureExtractionPipelineCallback & Disposable} ImageFeatureExtractionPipelineType + */ + +/** + * Image feature extraction pipeline using no model head. This pipeline extracts the hidden + * states from the base transformer, which can be used as features in downstream tasks. + * + * **Example:** Perform image feature extraction with `onnx-community/dinov3-vits16-pretrain-lvd1689m-ONNX`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const image_feature_extractor = await pipeline('image-feature-extraction', 'onnx-community/dinov3-vits16-pretrain-lvd1689m-ONNX'); + * const image = 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/cats.png'; + * const features = await image_feature_extractor(image); + * // Tensor { + * // dims: [ 1, 201, 384 ], + * // type: 'float32', + * // data: Float32Array(77184) [ ... ], + * // size: 77184 + * // } + * ``` + * + * **Example:** Compute image embeddings with `Xenova/clip-vit-base-patch32`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const image_feature_extractor = await pipeline('image-feature-extraction', 'Xenova/clip-vit-base-patch32'); + * const image = 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/cats.png'; + * const features = await image_feature_extractor(image); + * // Tensor { + * // dims: [ 1, 512 ], + * // type: 'float32', + * // data: Float32Array(512) [ ... ], + * // size: 512 + * // } + * ``` + */ +export class ImageFeatureExtractionPipeline + extends /** @type {new (options: ImagePipelineConstructorArgs) => ImageFeatureExtractionPipelineType} */ (Pipeline) +{ + /** @type {ImageFeatureExtractionPipelineCallback} */ + async _call(images, { pool = null } = {}) { + const preparedImages = await prepareImages(images); + const { pixel_values } = await this.processor(preparedImages); + const outputs = await this.model({ pixel_values }); + + /** @type {Tensor} */ + let result; + if (pool) { + if (!('pooler_output' in outputs)) { + throw Error( + `No pooled output was returned. Make sure the model has a 'pooler' layer when using the 'pool' option.`, + ); + } + result = outputs.pooler_output; + } else { + result = outputs.last_hidden_state ?? outputs.logits ?? outputs.image_embeds; + } + return result; + } +} diff --git a/packages/transformers/src/pipelines/image-segmentation.js b/packages/transformers/src/pipelines/image-segmentation.js new file mode 100644 index 0000000..566ced4 --- /dev/null +++ b/packages/transformers/src/pipelines/image-segmentation.js @@ -0,0 +1,195 @@ +import { Pipeline, prepareImages } from './_base.js'; + +import { RawImage } from '../utils/image.js'; + +/** + * @typedef {import('./_base.js').ImagePipelineConstructorArgs} ImagePipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + * @typedef {import('./_base.js').ImagePipelineInputs} ImagePipelineInputs + */ + +const SUBTASKS_MAPPING = { + // Mapping of subtasks to their corresponding post-processing function names. + panoptic: 'post_process_panoptic_segmentation', + instance: 'post_process_instance_segmentation', + semantic: 'post_process_semantic_segmentation', +}; + +/** + * @typedef {Object} ImageSegmentationOutputSingle + * @property {string|null} label The label of the segment. + * @property {number|null} score The score of the segment. + * @property {RawImage} mask The mask of the segment. + * + * @typedef {ImageSegmentationOutputSingle[]} ImageSegmentationOutput + * + * @typedef {Object} ImageSegmentationPipelineOptions Parameters specific to image segmentation pipelines. + * @property {number} [threshold=0.5] Probability threshold to filter out predicted masks. + * @property {number} [mask_threshold=0.5] Threshold to use when turning the predicted masks into binary values. + * @property {number} [overlap_mask_area_threshold=0.8] Mask overlap threshold to eliminate small, disconnected segments. + * @property {null|string} [subtask=null] Segmentation task to be performed. One of [`panoptic`, `instance`, and `semantic`], + * depending on model capabilities. If not set, the pipeline will attempt to resolve (in that order). + * @property {number[]} [label_ids_to_fuse=null] List of label ids to fuse. If not set, do not fuse any labels. + * @property {number[][]} [target_sizes=null] List of target sizes for the input images. If not set, use the original image sizes. + * + * @callback ImageSegmentationPipelineCallback Segment the input images. + * @param {ImagePipelineInputs} images The input images. + * @param {ImageSegmentationPipelineOptions} [options] The options to use for image segmentation. + * @returns {Promise} The annotated segments. + * + * @typedef {ImagePipelineConstructorArgs & ImageSegmentationPipelineCallback & Disposable} ImageSegmentationPipelineType + */ + +/** + * Image segmentation pipeline using any `AutoModelForXXXSegmentation`. + * This pipeline predicts masks of objects and their classes. + * + * **Example:** Perform image segmentation with `Xenova/detr-resnet-50-panoptic`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const segmenter = await pipeline('image-segmentation', 'Xenova/detr-resnet-50-panoptic'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg'; + * const output = await segmenter(url); + * // [ + * // { label: 'remote', score: 0.9984649419784546, mask: RawImage { ... } }, + * // { label: 'cat', score: 0.9994316101074219, mask: RawImage { ... } } + * // ] + * ``` + */ +export class ImageSegmentationPipeline + extends /** @type {new (options: ImagePipelineConstructorArgs) => ImageSegmentationPipelineType} */ (Pipeline) +{ + /** @type {ImageSegmentationPipelineCallback} */ + async _call( + images, + { + threshold = 0.5, + mask_threshold = 0.5, + overlap_mask_area_threshold = 0.8, + label_ids_to_fuse = null, + target_sizes = null, + subtask = null, + } = {}, + ) { + const isBatched = Array.isArray(images); + + if (isBatched && images.length !== 1) { + throw Error('Image segmentation pipeline currently only supports a batch size of 1.'); + } + + const preparedImages = await prepareImages(images); + const imageSizes = preparedImages.map((x) => [x.height, x.width]); + + const inputs = await this.processor(preparedImages); + + const { inputNames, outputNames } = this.model.sessions['model']; + if (!inputNames.includes('pixel_values')) { + if (inputNames.length !== 1) { + throw Error(`Expected a single input name, but got ${inputNames.length} inputs: ${inputNames}.`); + } + + const newName = inputNames[0]; + if (newName in inputs) { + throw Error(`Input name ${newName} already exists in the inputs.`); + } + // To ensure compatibility with certain background-removal models, + // we may need to perform a mapping of input to output names + inputs[newName] = inputs.pixel_values; + } + + const output = await this.model(inputs); + + let fn = null; + if (subtask !== null) { + fn = SUBTASKS_MAPPING[subtask]; + } else if (this.processor.image_processor) { + for (const [task, func] of Object.entries(SUBTASKS_MAPPING)) { + if (func in this.processor.image_processor) { + fn = this.processor.image_processor[func].bind(this.processor.image_processor); + subtask = task; + break; + } + } + } + + // @ts-expect-error TS2339 + const id2label = this.model.config.id2label; + + /** @type {ImageSegmentationOutput} */ + const annotation = []; + if (!subtask) { + // We define an epsilon to safeguard against numerical/precision issues when detecting + // the normalization mode of the output (i.e., sigmoid already applied, or not). + // See https://github.com/microsoft/onnxruntime/issues/23943 for more information. + const epsilon = 1e-5; + + // Perform standard image segmentation + const result = output[outputNames[0]]; + for (let i = 0; i < imageSizes.length; ++i) { + const size = imageSizes[i]; + const item = result[i]; + if (item.data.some((x) => x < -epsilon || x > 1 + epsilon)) { + item.sigmoid_(); + } + const mask = await RawImage.fromTensor(item.mul_(255).to('uint8')).resize(size[1], size[0]); + annotation.push({ + label: null, + score: null, + mask, + }); + } + } else if (subtask === 'panoptic' || subtask === 'instance') { + const processed = fn( + output, + threshold, + mask_threshold, + overlap_mask_area_threshold, + label_ids_to_fuse, + target_sizes ?? imageSizes, // TODO FIX? + )[0]; + + const segmentation = processed.segmentation; + + for (const segment of processed.segments_info) { + const maskData = new Uint8ClampedArray(segmentation.data.length); + for (let i = 0; i < segmentation.data.length; ++i) { + if (segmentation.data[i] === segment.id) { + maskData[i] = 255; + } + } + + const mask = new RawImage(maskData, segmentation.dims[1], segmentation.dims[0], 1); + + annotation.push({ + score: segment.score, + label: id2label[segment.label_id], + mask: mask, + }); + } + } else if (subtask === 'semantic') { + const { segmentation, labels } = fn(output, target_sizes ?? imageSizes)[0]; + + for (const label of labels) { + const maskData = new Uint8ClampedArray(segmentation.data.length); + for (let i = 0; i < segmentation.data.length; ++i) { + if (segmentation.data[i] === label) { + maskData[i] = 255; + } + } + + const mask = new RawImage(maskData, segmentation.dims[1], segmentation.dims[0], 1); + + annotation.push({ + score: null, + label: id2label[label], + mask: mask, + }); + } + } else { + throw Error(`Subtask ${subtask} not supported.`); + } + + return annotation; + } +} diff --git a/packages/transformers/src/pipelines/image-to-image.js b/packages/transformers/src/pipelines/image-to-image.js new file mode 100644 index 0000000..66324e4 --- /dev/null +++ b/packages/transformers/src/pipelines/image-to-image.js @@ -0,0 +1,58 @@ +import { Pipeline, prepareImages } from './_base.js'; +import { RawImage } from '../utils/image.js'; + +/** + * @typedef {import('./_base.js').ImagePipelineConstructorArgs} ImagePipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + * @typedef {import('./_base.js').ImageInput} ImageInput + */ + +/** + * @typedef {ImagePipelineConstructorArgs & ImageToImagePipelineCallback & Disposable} ImageToImagePipelineType + */ + +/** + * @template T + * @typedef {T extends ImageInput[] ? RawImage[] : RawImage} ImageToImagePipelineResult + */ + +/** + * @typedef {(images: T) => Promise>} ImageToImagePipelineCallback + */ + +/** + * Image to Image pipeline using any `AutoModelForImageToImage`. This pipeline generates an image based on a previous image input. + * + * **Example:** Super-resolution w/ `Xenova/swin2SR-classical-sr-x2-64` + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const upscaler = await pipeline('image-to-image', 'Xenova/swin2SR-classical-sr-x2-64'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/butterfly.jpg'; + * const output = await upscaler(url); + * // RawImage { + * // data: Uint8Array(786432) [ 41, 31, 24, 43, ... ], + * // width: 512, + * // height: 512, + * // channels: 3 + * // } + * ``` + */ +export class ImageToImagePipeline + extends /** @type {new (options: ImagePipelineConstructorArgs) => ImageToImagePipelineType} */ (Pipeline) +{ + async _call(images) { + const preparedImages = await prepareImages(images); + const inputs = await this.processor(preparedImages); + const outputs = await this.model(inputs); + + /** @type {RawImage[]} */ + const toReturn = []; + for (const batch of outputs.reconstruction) { + const output = batch.squeeze().clamp_(0, 1).mul_(255).round_().to('uint8'); + toReturn.push(RawImage.fromTensor(output)); + } + + return Array.isArray(images) ? toReturn : toReturn[0]; + } +} diff --git a/packages/transformers/src/pipelines/image-to-text.js b/packages/transformers/src/pipelines/image-to-text.js new file mode 100644 index 0000000..ddd0e3d --- /dev/null +++ b/packages/transformers/src/pipelines/image-to-text.js @@ -0,0 +1,74 @@ +import { Pipeline, prepareImages } from './_base.js'; + +import { Tensor } from '../utils/tensor.js'; + +/** + * @typedef {import('./_base.js').TextImagePipelineConstructorArgs} TextImagePipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + * @typedef {import('./_base.js').ImageInput} ImageInput + */ + +/** + * @typedef {Object} ImageToTextSingle + * @property {string} generated_text The generated text. + * @typedef {ImageToTextSingle[]} ImageToTextOutput + * + * @typedef {TextImagePipelineConstructorArgs & ImageToTextPipelineCallback & Disposable} ImageToTextPipelineType + */ + +/** + * @template T + * @typedef {T extends ImageInput[] ? ImageToTextOutput[] : ImageToTextOutput} ImageToTextPipelineResult + */ + +/** + * @typedef {(texts: T, options?: Partial) => Promise>} ImageToTextPipelineCallback + */ + +/** + * Image To Text pipeline using a `AutoModelForVision2Seq`. This pipeline predicts a caption for a given image. + * + * **Example:** Generate a caption for an image w/ `Xenova/vit-gpt2-image-captioning`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const captioner = await pipeline('image-to-text', 'Xenova/vit-gpt2-image-captioning'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg'; + * const output = await captioner(url); + * // [{ generated_text: 'a cat laying on a couch with another cat' }] + * ``` + * + * **Example:** Optical Character Recognition (OCR) w/ `Xenova/trocr-small-handwritten`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const captioner = await pipeline('image-to-text', 'Xenova/trocr-small-handwritten'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/handwriting.jpg'; + * const output = await captioner(url); + * // [{ generated_text: 'Mr. Brown commented icily.' }] + * ``` + */ +export class ImageToTextPipeline + extends /** @type {new (options: TextImagePipelineConstructorArgs) => ImageToTextPipelineType} */ (Pipeline) +{ + async _call(images, generate_kwargs = {}) { + const isBatched = Array.isArray(images); + const preparedImages = await prepareImages(images); + + const { pixel_values } = await this.processor(preparedImages); + + const toReturn = []; + for (const batch of pixel_values) { + batch.dims = [1, ...batch.dims]; + const output = await this.model.generate({ inputs: batch, ...generate_kwargs }); + const decoded = this.tokenizer + .batch_decode(/** @type {Tensor} */ (output), { + skip_special_tokens: true, + }) + .map((x) => ({ generated_text: x.trim() })); + toReturn.push(decoded); + } + + return isBatched ? toReturn : toReturn[0]; + } +} diff --git a/packages/transformers/src/pipelines/index.js b/packages/transformers/src/pipelines/index.js new file mode 100644 index 0000000..918fe51 --- /dev/null +++ b/packages/transformers/src/pipelines/index.js @@ -0,0 +1,313 @@ +/** + * @file Pipeline task configurations and aliases + * + * Defines which pipeline class and model class(es) each pipeline task needs. + * Tokenizer and processor loading is determined automatically from the model's files. + */ + +import { + AutoModel, + AutoModelForSequenceClassification, + AutoModelForAudioClassification, + AutoModelForTokenClassification, + AutoModelForQuestionAnswering, + AutoModelForMaskedLM, + AutoModelForSeq2SeqLM, + AutoModelForSpeechSeq2Seq, + AutoModelForTextToWaveform, + AutoModelForTextToSpectrogram, + AutoModelForCTC, + AutoModelForCausalLM, + AutoModelForVision2Seq, + AutoModelForImageClassification, + AutoModelForImageSegmentation, + AutoModelForSemanticSegmentation, + AutoModelForUniversalSegmentation, + AutoModelForObjectDetection, + AutoModelForZeroShotObjectDetection, + AutoModelForDocumentQuestionAnswering, + AutoModelForImageToImage, + AutoModelForDepthEstimation, + AutoModelForImageFeatureExtraction, +} from '../models/auto/modeling_auto.js'; + +import { TextClassificationPipeline } from './text-classification.js'; +import { TokenClassificationPipeline } from './token-classification.js'; +import { QuestionAnsweringPipeline } from './question-answering.js'; +import { FillMaskPipeline } from './fill-mask.js'; +import { SummarizationPipeline } from './summarization.js'; +import { TranslationPipeline } from './translation.js'; +import { Text2TextGenerationPipeline } from './text2text-generation.js'; +import { TextGenerationPipeline } from './text-generation.js'; +import { ZeroShotClassificationPipeline } from './zero-shot-classification.js'; +import { AudioClassificationPipeline } from './audio-classification.js'; +import { ZeroShotAudioClassificationPipeline } from './zero-shot-audio-classification.js'; +import { AutomaticSpeechRecognitionPipeline } from './automatic-speech-recognition.js'; +import { TextToAudioPipeline } from './text-to-audio.js'; +import { ImageToTextPipeline } from './image-to-text.js'; +import { ImageClassificationPipeline } from './image-classification.js'; +import { ImageSegmentationPipeline } from './image-segmentation.js'; +import { BackgroundRemovalPipeline } from './background-removal.js'; +import { ZeroShotImageClassificationPipeline } from './zero-shot-image-classification.js'; +import { ObjectDetectionPipeline } from './object-detection.js'; +import { ZeroShotObjectDetectionPipeline } from './zero-shot-object-detection.js'; +import { DocumentQuestionAnsweringPipeline } from './document-question-answering.js'; +import { ImageToImagePipeline } from './image-to-image.js'; +import { DepthEstimationPipeline } from './depth-estimation.js'; +import { FeatureExtractionPipeline } from './feature-extraction.js'; +import { ImageFeatureExtractionPipeline } from './image-feature-extraction.js'; + +export const SUPPORTED_TASKS = Object.freeze({ + 'text-classification': { + pipeline: TextClassificationPipeline, + model: AutoModelForSequenceClassification, + default: { + model: 'Xenova/distilbert-base-uncased-finetuned-sst-2-english', + }, + type: 'text', + }, + 'token-classification': { + pipeline: TokenClassificationPipeline, + model: AutoModelForTokenClassification, + default: { + model: 'Xenova/bert-base-multilingual-cased-ner-hrl', + }, + type: 'text', + }, + 'question-answering': { + pipeline: QuestionAnsweringPipeline, + model: AutoModelForQuestionAnswering, + default: { + model: 'Xenova/distilbert-base-cased-distilled-squad', + }, + type: 'text', + }, + 'fill-mask': { + pipeline: FillMaskPipeline, + model: AutoModelForMaskedLM, + default: { + model: 'onnx-community/ettin-encoder-32m-ONNX', + dtype: 'fp32', + }, + type: 'text', + }, + summarization: { + pipeline: SummarizationPipeline, + model: AutoModelForSeq2SeqLM, + default: { + model: 'Xenova/distilbart-cnn-6-6', + }, + type: 'text', + }, + translation: { + pipeline: TranslationPipeline, + model: AutoModelForSeq2SeqLM, + default: { + model: 'Xenova/t5-small', + }, + type: 'text', + }, + 'text2text-generation': { + pipeline: Text2TextGenerationPipeline, + model: AutoModelForSeq2SeqLM, + default: { + model: 'Xenova/flan-t5-small', + }, + type: 'text', + }, + 'text-generation': { + pipeline: TextGenerationPipeline, + model: AutoModelForCausalLM, + default: { + model: 'onnx-community/Qwen3-0.6B-ONNX', + dtype: 'q4', + }, + type: 'text', + }, + 'zero-shot-classification': { + pipeline: ZeroShotClassificationPipeline, + model: AutoModelForSequenceClassification, + default: { + model: 'Xenova/distilbert-base-uncased-mnli', + }, + type: 'text', + }, + 'audio-classification': { + pipeline: AudioClassificationPipeline, + model: AutoModelForAudioClassification, + default: { + model: 'Xenova/wav2vec2-base-superb-ks', + }, + type: 'audio', + }, + 'zero-shot-audio-classification': { + pipeline: ZeroShotAudioClassificationPipeline, + model: AutoModel, + default: { + model: 'Xenova/clap-htsat-unfused', + }, + type: 'multimodal', + }, + 'automatic-speech-recognition': { + pipeline: AutomaticSpeechRecognitionPipeline, + model: [AutoModelForSpeechSeq2Seq, AutoModelForCTC], + default: { + model: 'Xenova/whisper-tiny.en', + }, + type: 'multimodal', + }, + 'text-to-audio': { + pipeline: TextToAudioPipeline, + model: [AutoModelForTextToWaveform, AutoModelForTextToSpectrogram], + default: { + model: 'onnx-community/Supertonic-TTS-ONNX', + dtype: 'fp32', + }, + type: 'text', + }, + 'image-to-text': { + pipeline: ImageToTextPipeline, + model: AutoModelForVision2Seq, + default: { + model: 'Xenova/vit-gpt2-image-captioning', + }, + type: 'multimodal', + }, + 'image-classification': { + pipeline: ImageClassificationPipeline, + model: AutoModelForImageClassification, + default: { + model: 'Xenova/vit-base-patch16-224', + }, + type: 'multimodal', + }, + 'image-segmentation': { + pipeline: ImageSegmentationPipeline, + model: [AutoModelForImageSegmentation, AutoModelForSemanticSegmentation, AutoModelForUniversalSegmentation], + default: { + model: 'Xenova/detr-resnet-50-panoptic', + }, + type: 'multimodal', + }, + 'background-removal': { + pipeline: BackgroundRemovalPipeline, + model: [AutoModelForImageSegmentation, AutoModelForSemanticSegmentation, AutoModelForUniversalSegmentation], + default: { + model: 'Xenova/modnet', + }, + type: 'image', + }, + 'zero-shot-image-classification': { + pipeline: ZeroShotImageClassificationPipeline, + model: AutoModel, + default: { + model: 'Xenova/clip-vit-base-patch32', + }, + type: 'multimodal', + }, + 'object-detection': { + pipeline: ObjectDetectionPipeline, + model: AutoModelForObjectDetection, + default: { + model: 'Xenova/detr-resnet-50', + }, + type: 'multimodal', + }, + 'zero-shot-object-detection': { + pipeline: ZeroShotObjectDetectionPipeline, + model: AutoModelForZeroShotObjectDetection, + default: { + model: 'Xenova/owlvit-base-patch32', + }, + type: 'multimodal', + }, + 'document-question-answering': { + pipeline: DocumentQuestionAnsweringPipeline, + model: AutoModelForDocumentQuestionAnswering, + default: { + model: 'Xenova/donut-base-finetuned-docvqa', + }, + type: 'multimodal', + }, + 'image-to-image': { + pipeline: ImageToImagePipeline, + model: AutoModelForImageToImage, + default: { + model: 'Xenova/swin2SR-classical-sr-x2-64', + }, + type: 'image', + }, + 'depth-estimation': { + pipeline: DepthEstimationPipeline, + model: AutoModelForDepthEstimation, + default: { + model: 'onnx-community/depth-anything-v2-small', + }, + type: 'image', + }, + 'feature-extraction': { + pipeline: FeatureExtractionPipeline, + model: AutoModel, + default: { + model: 'onnx-community/all-MiniLM-L6-v2-ONNX', + dtype: 'fp32', + }, + type: 'text', + }, + 'image-feature-extraction': { + pipeline: ImageFeatureExtractionPipeline, + model: [AutoModelForImageFeatureExtraction, AutoModel], + default: { + model: 'onnx-community/dinov3-vits16-pretrain-lvd1689m-ONNX', + dtype: 'fp32', + }, + type: 'image', + }, +}); + +// TODO: Add types for TASK_ALIASES + +export const TASK_ALIASES = Object.freeze({ + 'sentiment-analysis': 'text-classification', + ner: 'token-classification', + // "vqa": "visual-question-answering", // TODO: Add + asr: 'automatic-speech-recognition', + 'text-to-speech': 'text-to-audio', + + // Add for backwards compatibility + embeddings: 'feature-extraction', +}); + +export { + TextClassificationPipeline, + TokenClassificationPipeline, + QuestionAnsweringPipeline, + FillMaskPipeline, + SummarizationPipeline, + TranslationPipeline, + Text2TextGenerationPipeline, + TextGenerationPipeline, + ZeroShotClassificationPipeline, + AudioClassificationPipeline, + ZeroShotAudioClassificationPipeline, + AutomaticSpeechRecognitionPipeline, + TextToAudioPipeline, + ImageToTextPipeline, + ImageClassificationPipeline, + ImageSegmentationPipeline, + BackgroundRemovalPipeline, + ZeroShotImageClassificationPipeline, + ObjectDetectionPipeline, + ZeroShotObjectDetectionPipeline, + DocumentQuestionAnsweringPipeline, + ImageToImagePipeline, + DepthEstimationPipeline, + FeatureExtractionPipeline, + ImageFeatureExtractionPipeline, +}; + +/** + * @typedef {keyof typeof SUPPORTED_TASKS} TaskType + * @typedef {keyof typeof TASK_ALIASES} AliasType + * @typedef {TaskType | AliasType} PipelineType All possible pipeline types. + */ diff --git a/packages/transformers/src/pipelines/object-detection.js b/packages/transformers/src/pipelines/object-detection.js new file mode 100644 index 0000000..a1fed17 --- /dev/null +++ b/packages/transformers/src/pipelines/object-detection.js @@ -0,0 +1,92 @@ +import { Pipeline, prepareImages, get_bounding_box } from './_base.js'; + +/** + * @typedef {import('./_base.js').ImagePipelineConstructorArgs} ImagePipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + * @typedef {import('./_base.js').ImageInput} ImageInput + * @typedef {import('./_base.js').BoundingBox} BoundingBox + */ + +/** + * @typedef {Object} ObjectDetectionPipelineSingle + * @property {string} label The class label identified by the model. + * @property {number} score The score attributed by the model for that label. + * @property {BoundingBox} box The bounding box of detected object in image's original size, or as a percentage if `percentage` is set to true. + * @typedef {ObjectDetectionPipelineSingle[]} ObjectDetectionOutput + * + * @typedef {Object} ObjectDetectionPipelineOptions Parameters specific to object detection pipelines. + * @property {number} [threshold=0.9] The threshold used to filter boxes by score. + * @property {boolean} [percentage=false] Whether to return the boxes coordinates in percentage (true) or in pixels (false). + * + * @typedef {ImagePipelineConstructorArgs & ObjectDetectionPipelineCallback & Disposable} ObjectDetectionPipelineType + */ + +/** + * @template T + * @typedef {T extends ImageInput[] ? ObjectDetectionOutput[] : ObjectDetectionOutput} ObjectDetectionPipelineResult + */ + +/** + * @typedef {(images: T, options?: ObjectDetectionPipelineOptions) => Promise>} ObjectDetectionPipelineCallback + */ + +/** + * Object detection pipeline using any `AutoModelForObjectDetection`. + * This pipeline predicts bounding boxes of objects and their classes. + * + * **Example:** Run object-detection with `Xenova/detr-resnet-50`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const detector = await pipeline('object-detection', 'Xenova/detr-resnet-50'); + * const img = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/cats.jpg'; + * const output = await detector(img, { threshold: 0.9 }); + * // [{ + * // score: 0.9976370930671692, + * // label: "remote", + * // box: { xmin: 31, ymin: 68, xmax: 190, ymax: 118 } + * // }, + * // ... + * // { + * // score: 0.9984092116355896, + * // label: "cat", + * // box: { xmin: 331, ymin: 19, xmax: 649, ymax: 371 } + * // }] + * ``` + */ +export class ObjectDetectionPipeline + extends /** @type {new (options: ImagePipelineConstructorArgs) => ObjectDetectionPipelineType} */ (Pipeline) +{ + async _call(images, { threshold = 0.9, percentage = false } = {}) { + const isBatched = Array.isArray(images); + + if (isBatched && images.length !== 1) { + throw Error('Object detection pipeline currently only supports a batch size of 1.'); + } + const preparedImages = await prepareImages(images); + + const imageSizes = percentage ? null : preparedImages.map((x) => [x.height, x.width]); + + const { pixel_values, pixel_mask } = await this.processor(preparedImages); + const output = await this.model({ pixel_values, pixel_mask }); + + // @ts-ignore + const processed = this.processor.image_processor.post_process_object_detection(output, threshold, imageSizes); + + // Add labels + // @ts-expect-error TS2339 + const { id2label } = this.model.config; + + // Format output + /** @type {ObjectDetectionOutput[]} */ + const result = processed.map((batch) => + batch.boxes.map((box, i) => ({ + score: batch.scores[i], + label: id2label[batch.classes[i]], + box: get_bounding_box(box, !percentage), + })), + ); + + return isBatched ? result : result[0]; + } +} diff --git a/packages/transformers/src/pipelines/question-answering.js b/packages/transformers/src/pipelines/question-answering.js new file mode 100644 index 0000000..fdfe9e1 --- /dev/null +++ b/packages/transformers/src/pipelines/question-answering.js @@ -0,0 +1,141 @@ +import { Pipeline } from './_base.js'; + +import { product } from '../utils/core.js'; +import { softmax } from '../utils/maths.js'; + +/** + * @typedef {import('./_base.js').TextPipelineConstructorArgs} TextPipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + */ + +/** + * @typedef {Object} QuestionAnsweringOutput + * @property {number} score The probability associated to the answer. + * @property {number} [start] The character start index of the answer (in the tokenized version of the input). + * @property {number} [end] The character end index of the answer (in the tokenized version of the input). + * @property {string} answer The answer to the question. + * + * @typedef {Object} QuestionAnsweringPipelineOptions Parameters specific to question answering pipelines. + * @property {number} [top_k=1] The number of top answer predictions to be returned. + * + * @typedef {TextPipelineConstructorArgs & QuestionAnsweringPipelineCallback & Disposable} QuestionAnsweringPipelineType + */ + +/** + * @template O + * @typedef {O extends { top_k: infer K } ? (1 extends K ? false : true) : false} QuestionAnsweringIsTopK + */ + +/** + * @template Q, O + * @typedef {Q extends string[] ? (QuestionAnsweringIsTopK extends true ? QuestionAnsweringOutput[][] : QuestionAnsweringOutput[]) : (QuestionAnsweringIsTopK extends true ? QuestionAnsweringOutput[] : QuestionAnsweringOutput)} QuestionAnsweringPipelineResult + */ + +/** + * @typedef {(question: Q, context: Q, options?: O) => Promise>} QuestionAnsweringPipelineCallback + */ + +/** + * Question Answering pipeline using any `ModelForQuestionAnswering`. + * + * **Example:** Run question answering with `Xenova/distilbert-base-uncased-distilled-squad`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const answerer = await pipeline('question-answering', 'Xenova/distilbert-base-uncased-distilled-squad'); + * const question = 'Who was Jim Henson?'; + * const context = 'Jim Henson was a nice puppet.'; + * const output = await answerer(question, context); + * // { + * // answer: "a nice puppet", + * // score: 0.5768911502526741 + * // } + * ``` + */ +export class QuestionAnsweringPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => QuestionAnsweringPipelineType} */ (Pipeline) +{ + async _call(question, context, { top_k = 1 } = {}) { + // Run tokenization + const inputs = this.tokenizer(question, { + text_pair: context, + padding: true, + truncation: true, + }); + const isBatched = Array.isArray(question); + + const { start_logits, end_logits } = await this.model(inputs); + const input_ids = inputs.input_ids.tolist(); + const attention_mask = inputs.attention_mask.tolist(); + + // TODO: add support for `return_special_tokens_mask` + const { all_special_ids, sep_token_id } = this.tokenizer; + + const batchedResults = []; + for (let j = 0; j < start_logits.dims[0]; ++j) { + const ids = input_ids[j]; + const sepIndex = ids.findIndex( + (x) => + // We use == to match bigint with number + // @ts-ignore + x == sep_token_id, + ); + + const start = start_logits[j].tolist(); + const end = end_logits[j].tolist(); + + // Now, we mask out values that can't be in the answer + // NOTE: We keep the cls_token unmasked (some models use it to indicate unanswerable questions) + for (let i = 1; i < start.length; ++i) { + if ( + attention_mask[j] == 0 || // is part of padding + i <= sepIndex || // is before the sep_token + all_special_ids.findIndex((x) => x == ids[i]) !== -1 // Is a special token + ) { + // Make sure non-context indexes in the tensor cannot contribute to the softmax + start[i] = -Infinity; + end[i] = -Infinity; + } + } + + // Normalize logits and spans to retrieve the answer + const start_scores = softmax(start).map((x, i) => [x, i]); + const end_scores = softmax(end).map((x, i) => [x, i]); + + // Mask CLS + start_scores[0][0] = 0; + end_scores[0][0] = 0; + + // Generate all valid spans and select best ones + const options = product(start_scores, end_scores) + .filter((x) => x[0][1] <= x[1][1]) + .map((x) => [x[0][1], x[1][1], x[0][0] * x[1][0]]) + .sort((a, b) => b[2] - a[2]); + + const sampleResults = []; + for (let k = 0; k < Math.min(options.length, top_k); ++k) { + const [start, end, score] = options[k]; + + const answer_tokens = ids.slice(start, end + 1); + + const answer = this.tokenizer.decode(answer_tokens, { + skip_special_tokens: true, + }); + + // TODO add start and end? + // NOTE: HF returns character index + sampleResults.push({ + answer, + score, + }); + } + if (top_k === 1) { + batchedResults.push(...sampleResults); + } else { + batchedResults.push(sampleResults); + } + } + + return isBatched ? batchedResults : batchedResults[0]; + } +} diff --git a/packages/transformers/src/pipelines/summarization.js b/packages/transformers/src/pipelines/summarization.js new file mode 100644 index 0000000..c35a871 --- /dev/null +++ b/packages/transformers/src/pipelines/summarization.js @@ -0,0 +1,50 @@ +import { Text2TextGenerationPipeline } from './text2text-generation.js'; + +/** + * @typedef {import('./_base.js').TextPipelineConstructorArgs} TextPipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + */ + +/** + * @typedef {Object} SummarizationSingle + * @property {string} summary_text The summary text. + * @typedef {SummarizationSingle[]} SummarizationOutput + * + * @callback SummarizationPipelineCallback Summarize the text(s) given as inputs. + * @param {string|string[]} texts One or several articles (or one list of articles) to summarize. + * @param {import('../generation/parameters.js').GenerationFunctionParameters} [options] Additional keyword arguments to pass along to the generate method of the model. + * @returns {Promise} + * + * @typedef {TextPipelineConstructorArgs & SummarizationPipelineCallback & Disposable} SummarizationPipelineType + */ + +/** + * A pipeline for summarization tasks, inheriting from Text2TextGenerationPipeline. + * + * **Example:** Summarization w/ `Xenova/distilbart-cnn-6-6`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const summarizer = await pipeline('summarization', 'Xenova/distilbart-cnn-6-6'); + * const text = 'The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, ' + + * 'and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. ' + + * 'During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest ' + + * 'man-made structure in the world, a title it held for 41 years until the Chrysler Building in New ' + + * 'York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to ' + + * 'the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the ' + + * 'Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second ' + + * 'tallest free-standing structure in France after the Millau Viaduct.'; + * const output = await summarizer(text, { + * max_new_tokens: 100, + * }); + * // [{ summary_text: ' The Eiffel Tower is about the same height as an 81-storey building and the tallest structure in Paris. It is the second tallest free-standing structure in France after the Millau Viaduct.' }] + * ``` + */ +export class SummarizationPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => SummarizationPipelineType} */ ( + /** @type {any} */ (Text2TextGenerationPipeline) + ) +{ + /** @type {'summary_text'} */ + _key = 'summary_text'; +} diff --git a/packages/transformers/src/pipelines/text-classification.js b/packages/transformers/src/pipelines/text-classification.js new file mode 100644 index 0000000..dc3a8ec --- /dev/null +++ b/packages/transformers/src/pipelines/text-classification.js @@ -0,0 +1,120 @@ +import { Pipeline } from './_base.js'; + +import { Tensor, topk } from '../utils/tensor.js'; +import { softmax } from '../utils/maths.js'; + +/** + * @typedef {import('./_base.js').TextPipelineConstructorArgs} TextPipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + */ + +/** + * @typedef {Object} TextClassificationSingle + * @property {string} label The label predicted. + * @property {number} score The corresponding probability. + * @typedef {TextClassificationSingle[]} TextClassificationOutput + * + * @typedef {Object} TextClassificationPipelineOptions Parameters specific to text classification pipelines. + * @property {number|null} [top_k=1] The number of top predictions to be returned. If set to `null`, all predictions are returned. + */ + +/** + * @template O + * @typedef {O extends { top_k: infer K } ? (1 extends K ? false : true) : false} TextClassificationIsTopK + */ + +/** + * @template Q, O + * @typedef {Q extends string[] ? (TextClassificationIsTopK extends true ? TextClassificationOutput[] : TextClassificationOutput) : TextClassificationOutput} TextClassificationPipelineResult + */ + +/** + * @typedef {(texts: Q, options?: O) => Promise>} TextClassificationPipelineCallback + */ + +/** + * @typedef {TextPipelineConstructorArgs & TextClassificationPipelineCallback & Disposable} TextClassificationPipelineType + */ + +/** + * Text classification pipeline using any `ModelForSequenceClassification`. + * + * **Example:** Sentiment-analysis w/ `Xenova/distilbert-base-uncased-finetuned-sst-2-english`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const classifier = await pipeline('sentiment-analysis', 'Xenova/distilbert-base-uncased-finetuned-sst-2-english'); + * const output = await classifier('I love transformers!'); + * // [{ label: 'POSITIVE', score: 0.999788761138916 }] + * ``` + * + * **Example:** Multilingual sentiment-analysis w/ `Xenova/bert-base-multilingual-uncased-sentiment` (and return top 5 classes). + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const classifier = await pipeline('sentiment-analysis', 'Xenova/bert-base-multilingual-uncased-sentiment'); + * const output = await classifier('Le meilleur film de tous les temps.', { top_k: 5 }); + * // [ + * // { label: '5 stars', score: 0.9610759615898132 }, + * // { label: '4 stars', score: 0.03323351591825485 }, + * // { label: '3 stars', score: 0.0036155181005597115 }, + * // { label: '1 star', score: 0.0011325967498123646 }, + * // { label: '2 stars', score: 0.0009423971059732139 } + * // ] + * ``` + * + * **Example:** Toxic comment classification w/ `Xenova/toxic-bert` (and return all classes). + * ```javascript + * const classifier = await pipeline('text-classification', 'Xenova/toxic-bert'); + * const output = await classifier('I hate you!', { top_k: null }); + * // [ + * // { label: 'toxic', score: 0.9593140482902527 }, + * // { label: 'insult', score: 0.16187334060668945 }, + * // { label: 'obscene', score: 0.03452680632472038 }, + * // { label: 'identity_hate', score: 0.0223250575363636 }, + * // { label: 'threat', score: 0.019197041168808937 }, + * // { label: 'severe_toxic', score: 0.005651099607348442 } + * // ] + * ``` + */ +export class TextClassificationPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => TextClassificationPipelineType} */ (Pipeline) +{ + async _call(texts, { top_k = 1 } = {}) { + // Run tokenization + const model_inputs = this.tokenizer(texts, { + padding: true, + truncation: true, + }); + + // Run model + const outputs = await this.model(model_inputs); + + // @ts-expect-error TS2339 + const { problem_type, id2label } = this.model.config; + + // TODO: Use softmax tensor function + const function_to_apply = + problem_type === 'multi_label_classification' + ? (batch) => batch.sigmoid() + : (batch) => new Tensor('float32', softmax(batch.data), batch.dims); // single_label_classification (default) + + const toReturn = []; + for (const batch of outputs.logits) { + const output = function_to_apply(batch); + const scores = await topk(output, top_k); + const values = scores[0].tolist(); + const indices = scores[1].tolist(); + const vals = indices.map((x, i) => ({ + label: id2label ? id2label[x] : `LABEL_${x}`, + score: values[i], + })); + if (top_k === 1) { + toReturn.push(...vals); + } else { + toReturn.push(vals); + } + } + return Array.isArray(texts) || top_k === 1 ? toReturn : toReturn[0]; + } +} diff --git a/packages/transformers/src/pipelines/text-generation.js b/packages/transformers/src/pipelines/text-generation.js new file mode 100644 index 0000000..fb9871a --- /dev/null +++ b/packages/transformers/src/pipelines/text-generation.js @@ -0,0 +1,214 @@ +import { Pipeline } from './_base.js'; + +import { Tensor } from '../utils/tensor.js'; +import { pick } from '../utils/core.js'; + +/** + * @typedef {import('./_base.js').TextPipelineConstructorArgs} TextPipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + * @typedef {import('../tokenization_utils.js').Message[]} Chat + */ + +function isChat(x) { + return Array.isArray(x) && x.every((x) => 'role' in x && 'content' in x); +} + +/** + * @typedef {Object} TextGenerationSingleString + * @property {string} generated_text The generated text. + * @typedef {TextGenerationSingleString[]} TextGenerationStringOutput + * + * @typedef {Object} TextGenerationSingleChat + * @property {Chat} generated_text The generated chat. + * @typedef {TextGenerationSingleChat[]} TextGenerationChatOutput + * + * @typedef {TextGenerationSingleString | TextGenerationSingleChat} TextGenerationSingle + * @typedef {TextGenerationSingle[]} TextGenerationOutput + * + * @typedef {Object} TextGenerationSpecificParams Parameters specific to text-generation pipelines. + * @property {boolean} [add_special_tokens] Whether or not to add special tokens when tokenizing the sequences. + * @property {boolean} [return_full_text=true] If set to `false` only added text is returned, otherwise the full text is returned. + * @property {Object[]|null} [tools=null] A list of tools to expose to chat templates that support tool use. + * @property {Record[]|null} [documents=null] A list of documents to expose to chat templates that support RAG. + * @property {string|null} [chat_template=null] A specific chat template (or template name) to apply. + * @property {Object} [tokenizer_encode_kwargs] Additional keyword arguments to pass along to the encoding step of the tokenizer. + * If the text input is a chat, it is passed to `apply_chat_template`. Otherwise, it is passed to the tokenizer's call function. + * @typedef {import('../generation/parameters.js').GenerationFunctionParameters & TextGenerationSpecificParams} TextGenerationConfig + * + * @typedef {TextPipelineConstructorArgs & TextGenerationPipelineCallback & Disposable} TextGenerationPipelineType + */ + +/** + * @template T + * @typedef {T extends string ? TextGenerationStringOutput : T extends Chat ? TextGenerationChatOutput : T extends string[] ? TextGenerationStringOutput[] : T extends Chat[] ? TextGenerationChatOutput[] : never} TextGenerationResult + */ + +/** + * @typedef {(texts: T, options?: Partial) => Promise>} TextGenerationPipelineCallback + */ + +/** + * Language generation pipeline using any `ModelWithLMHead` or `ModelForCausalLM`. + * This pipeline predicts the words that will follow a specified text prompt. + * NOTE: For the full list of generation parameters, see [`GenerationConfig`](./utils/generation#module_utils/generation.GenerationConfig). + * + * **Example:** Text generation with `HuggingFaceTB/SmolLM2-135M` (default settings). + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const generator = await pipeline('text-generation', 'onnx-community/SmolLM2-135M-ONNX'); + * const text = 'Once upon a time,'; + * const output = await generator(text, { max_new_tokens: 8 }); + * // [{ generated_text: 'Once upon a time, there was a little girl named Lily.' }] + * ``` + * + * **Example:** Chat completion with `onnx-community/Qwen3-0.6B-ONNX`. + * ```javascript + * import { pipeline, TextStreamer } from '@huggingface/transformers'; + * + * // Create a text generation pipeline + * const generator = await pipeline( + * 'text-generation', + * 'onnx-community/Qwen3-0.6B-ONNX', + * { dtype: 'q4f16' }, + * ); + * + * // Define the list of messages + * const messages = [ + * { role: 'system', content: 'You are a helpful assistant.' }, + * { role: 'user', content: 'Write me a poem about Machine Learning.' }, + * ]; + * + * // Generate a response + * const output = await generator(messages, { + * max_new_tokens: 512, + * do_sample: false, + * streamer: new TextStreamer(generator.tokenizer, { skip_prompt: true, skip_special_tokens: true }), + * }); + * console.log(output[0].generated_text.at(-1)?.content); + * ``` + */ +export class TextGenerationPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => TextGenerationPipelineType} */ (Pipeline) +{ + _default_generation_config = { + max_new_tokens: 256, + // do_sample: true, + // temperature: 0.7, + }; + + /** + * @param {string | string[] | import('../tokenization_utils.js').Message[] | import('../tokenization_utils.js').Message[][]} texts + * @param {Partial} generate_kwargs + */ + async _call(texts, generate_kwargs = {}) { + const { + add_special_tokens: add_special_tokens_arg, + return_full_text: return_full_text_arg, + tools, + documents, + chat_template, + tokenizer_encode_kwargs, + ...generation_kwargs + } = generate_kwargs; + + let isBatched = false; + let isChatInput = false; + + // By default, do not add special tokens, unless the tokenizer specifies otherwise + let add_special_tokens = + add_special_tokens_arg ?? (this.tokenizer.add_bos_token || this.tokenizer.add_eos_token) ?? false; + + let tokenizer_kwargs = tokenizer_encode_kwargs; + + // Normalize inputs + /** @type {string[]} */ + let inputs; + if (typeof texts === 'string') { + inputs = texts = [texts]; + } else if (Array.isArray(texts) && texts.every((x) => typeof x === 'string')) { + isBatched = true; + inputs = /** @type {string[]} */ (texts); + } else { + if (isChat(texts)) { + texts = [/** @type {Chat} */ (texts)]; + } else if (Array.isArray(texts) && texts.every(isChat)) { + isBatched = true; + } else { + throw new Error('Input must be a string, an array of strings, a Chat, or an array of Chats'); + } + isChatInput = true; + + // If the input is a chat, we need to apply the chat template + const chat_template_kwargs = { + tokenize: false, + add_generation_prompt: true, + ...pick({ tools, documents, chat_template }, ['tools', 'documents', 'chat_template']), + ...tokenizer_kwargs, + }; + + inputs = /** @type {string[]} */ ( + /** @type {Chat[]} */ (texts).map( + (x) => + /** @type {string} */ ( + /** @type {unknown} */ (this.tokenizer.apply_chat_template(x, chat_template_kwargs)) + ), + ) + ); + // Chat template handles these already + add_special_tokens = false; + tokenizer_kwargs = undefined; + } + + // By default, return full text + const return_full_text = isChatInput ? false : (return_full_text_arg ?? true); + + this.tokenizer.padding_side = 'left'; + const text_inputs = this.tokenizer(inputs, { + add_special_tokens, + padding: true, + truncation: true, + ...tokenizer_kwargs, + }); + + const outputTokenIds = /** @type {Tensor} */ ( + await this.model.generate({ + ...text_inputs, + ...this._default_generation_config, + ...generation_kwargs, + }) + ); + + const decoded = this.tokenizer.batch_decode(outputTokenIds, { + skip_special_tokens: true, + }); + + let promptLengths; + if (!return_full_text && text_inputs.input_ids.dims.at(-1) > 0) { + promptLengths = this.tokenizer + .batch_decode(text_inputs.input_ids, { + skip_special_tokens: true, + }) + .map((x) => x.length); + } + + /** @type {TextGenerationOutput[]} */ + const toReturn = Array.from({ length: texts.length }, (_) => []); + for (let i = 0; i < decoded.length; ++i) { + const textIndex = Math.floor((i / outputTokenIds.dims[0]) * texts.length); + + if (promptLengths) { + // Trim the decoded text to only include the generated part + decoded[i] = decoded[i].slice(promptLengths[textIndex]); + } + toReturn[textIndex].push( + /** @type {TextGenerationSingle} */ ({ + generated_text: isChatInput + ? [.../** @type {Chat[]} */ (texts)[textIndex], { role: 'assistant', content: decoded[i] }] + : decoded[i], + }), + ); + } + return !isBatched && toReturn.length === 1 ? toReturn[0] : toReturn; + } +} diff --git a/packages/transformers/src/pipelines/text-to-audio.js b/packages/transformers/src/pipelines/text-to-audio.js new file mode 100644 index 0000000..aab19b2 --- /dev/null +++ b/packages/transformers/src/pipelines/text-to-audio.js @@ -0,0 +1,216 @@ +import { Pipeline } from './_base.js'; + +import { Tensor } from '../utils/tensor.js'; +import { RawAudio } from '../utils/audio.js'; +import { logger } from '../utils/logger.js'; + +import { AutoModel } from '../models/auto/modeling_auto.js'; +import { env } from '../env.js'; + +/** + * @typedef {import('./_base.js').TextAudioPipelineConstructorArgs} TextAudioPipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + */ + +/** + * @typedef {Object} VocoderOptions + * @property {import('../models/modeling_utils.js').PreTrainedModel} [vocoder] The vocoder used by the pipeline (if the model uses one). If not provided, use the default HifiGan vocoder. + * @typedef {TextAudioPipelineConstructorArgs & VocoderOptions} TextToAudioPipelineConstructorArgs + */ + +/** + * @typedef {RawAudio[]} TextToAudioOutput + * + * @typedef {Object} TextToAudioPipelineOptions Parameters specific to text-to-audio pipelines. + * @property {Tensor|Float32Array|string|URL} [speaker_embeddings=null] The speaker embeddings (if the model requires it). + * @property {number} [num_inference_steps] The number of denoising steps (if the model supports it). + * More denoising steps usually lead to higher quality audio but slower inference. + * @property {number} [speed] The speed of the generated audio (if the model supports it). + * + * @typedef {TextToAudioPipelineConstructorArgs & TextToAudioPipelineCallback & Disposable} TextToAudioPipelineType + */ + +/** + * @template T + * @typedef {T extends string[] ? TextToAudioOutput : RawAudio} TextToAudioPipelineResult + */ + +/** + * @typedef {(text: T, options?: TextToAudioPipelineOptions) => Promise>} TextToAudioPipelineCallback + */ + +/** + * Text-to-audio generation pipeline using any `AutoModelForTextToWaveform` or `AutoModelForTextToSpectrogram`. + * This pipeline generates an audio file from an input text and optional other conditional inputs. + * + * **Example:** Generate audio from text with `onnx-community/Supertonic-TTS-ONNX`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const synthesizer = await pipeline('text-to-speech', 'onnx-community/Supertonic-TTS-ONNX'); + * const speaker_embeddings = 'https://huggingface.co/onnx-community/Supertonic-TTS-ONNX/resolve/main/voices/F1.bin'; + * const output = await synthesizer('Hello there, how are you doing?', { speaker_embeddings }); + * // RawAudio { + * // audio: Float32Array(95232) [-0.000482565927086398, -0.0004853440332226455, ...], + * // sampling_rate: 44100 + * // } + * + * // Optional: Save the audio to a .wav file or Blob + * await output.save('output.wav'); // You can also use `output.toBlob()` to access the audio as a Blob + * ``` + * + * **Example:** Multilingual speech generation with `Xenova/mms-tts-fra`. See [here](https://huggingface.co/models?pipeline_tag=text-to-speech&other=vits&sort=trending) for the full list of available languages (1107). + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const synthesizer = await pipeline('text-to-speech', 'Xenova/mms-tts-fra'); + * const output = await synthesizer('Bonjour'); + * // RawAudio { + * // audio: Float32Array(23808) [-0.00037693005288019776, 0.0003325853613205254, ...], + * // sampling_rate: 16000 + * // } + * ``` + */ +export class TextToAudioPipeline + extends /** @type {new (options: TextToAudioPipelineConstructorArgs) => TextToAudioPipelineType} */ (Pipeline) +{ + DEFAULT_VOCODER_ID = 'Xenova/speecht5_hifigan'; + + /** + * Create a new TextToAudioPipeline. + * @param {TextToAudioPipelineConstructorArgs} options An object used to instantiate the pipeline. + */ + constructor(options) { + super(options); + + // TODO: Find a better way for `pipeline` to set the default vocoder + this.vocoder = options.vocoder ?? null; + } + + async _prepare_speaker_embeddings(speaker_embeddings, batch_size) { + // Load speaker embeddings as Float32Array from path/URL + if (typeof speaker_embeddings === 'string' || speaker_embeddings instanceof URL) { + // Load from URL with fetch + speaker_embeddings = new Float32Array(await (await env.fetch(speaker_embeddings)).arrayBuffer()); + } + + if (speaker_embeddings instanceof Float32Array) { + speaker_embeddings = new Tensor('float32', speaker_embeddings, [speaker_embeddings.length]); + } else if (!(speaker_embeddings instanceof Tensor)) { + throw new Error('Speaker embeddings must be a `Tensor`, `Float32Array`, `string`, or `URL`.'); + } + + if (batch_size > 1) { + if (speaker_embeddings.dims[0] === 1) { + // Repeat speaker embeddings for batch size + speaker_embeddings = speaker_embeddings.repeat(batch_size, 1); + } else if (speaker_embeddings.dims[0] !== batch_size) { + throw new Error( + `Expected speaker embeddings batch size to be 1 or ${batch_size}, but got ${speaker_embeddings.dims[0]}.`, + ); + } + } + + return speaker_embeddings; + } + + /** + * Helper to convert batched waveform tensor to RawAudio output(s). + * @param {string|string[]} text_inputs Original text input(s) to determine return type. + * @param {Tensor} waveform The waveform tensor of shape [batch_size, waveform_length]. + * @param {number} sampling_rate The audio sampling rate. + * @param {Tensor} [durations] Optional durations tensor for trimming (used by Supertonic). + * @returns {RawAudio|RawAudio[]} Single RawAudio or array based on input type. + * @private + */ + _postprocess_waveform(text_inputs, waveform, sampling_rate, durations = null) { + const waveformData = /** @type {Float32Array} */ (waveform.data); + const [batch_size, waveformLength] = waveform.dims; + const durationsData = durations ? /** @type {Float32Array} */ (durations.data) : null; + + const results = []; + for (let i = 0; i < batch_size; ++i) { + const length = durationsData ? Math.min(Math.ceil(durationsData[i]), waveformLength) : waveformLength; + const start = i * waveformLength; + results.push(new RawAudio(waveformData.slice(start, start + length), sampling_rate)); + } + return Array.isArray(text_inputs) ? results : results[0]; + } + + async _call(text_inputs, options) { + // If this.processor is not set, we are using a `AutoModelForTextToWaveform` model + if (this.processor) { + return this._call_text_to_spectrogram(text_inputs, options); + } else if (this.model.config.model_type === 'supertonic') { + return this._call_supertonic(text_inputs, options); + } else { + return this._call_text_to_waveform(text_inputs); + } + } + + async _call_supertonic(text_inputs, { speaker_embeddings, num_inference_steps, speed }) { + if (!speaker_embeddings) { + throw new Error('Speaker embeddings must be provided for Supertonic models.'); + } + + // @ts-expect-error TS2339 + const { sampling_rate, style_dim } = this.model.config; + + const inputs = this.tokenizer(text_inputs, { + padding: true, + truncation: true, + }); + const batch_size = inputs.input_ids.dims[0]; + speaker_embeddings = await this._prepare_speaker_embeddings(speaker_embeddings, batch_size); + speaker_embeddings = /** @type {Tensor} */ (speaker_embeddings).view(batch_size, -1, style_dim); + + // @ts-expect-error TS2339 + const { waveform, durations } = await this.model.generate_speech({ + ...inputs, + style: speaker_embeddings, + num_inference_steps, + speed, + }); + + return this._postprocess_waveform(text_inputs, waveform, sampling_rate, durations); + } + + async _call_text_to_waveform(text_inputs) { + // Run tokenization + const inputs = this.tokenizer(text_inputs, { + padding: true, + truncation: true, + }); + + // Generate waveform + const { waveform } = await this.model(inputs); + + // @ts-expect-error TS2339 + const sampling_rate = this.model.config.sampling_rate; + return this._postprocess_waveform(text_inputs, waveform, sampling_rate); + } + + async _call_text_to_spectrogram(text_inputs, { speaker_embeddings }) { + // Load vocoder, if not provided + if (!this.vocoder) { + logger.info('No vocoder specified, using default HifiGan vocoder.'); + this.vocoder = await AutoModel.from_pretrained(this.DEFAULT_VOCODER_ID, { dtype: 'fp32' }); + } + + // Run tokenization + const { input_ids } = this.tokenizer(text_inputs, { + padding: true, + truncation: true, + }); + + const batch_size = input_ids.dims[0]; + speaker_embeddings = await this._prepare_speaker_embeddings(speaker_embeddings, batch_size); + speaker_embeddings = speaker_embeddings.view(batch_size, -1); + + // @ts-expect-error TS2339 + const { waveform } = await this.model.generate_speech(input_ids, speaker_embeddings, { vocoder: this.vocoder }); + + const sampling_rate = this.processor.feature_extractor.config.sampling_rate; + return this._postprocess_waveform(text_inputs, waveform, sampling_rate); + } +} diff --git a/packages/transformers/src/pipelines/text2text-generation.js b/packages/transformers/src/pipelines/text2text-generation.js new file mode 100644 index 0000000..b313dd9 --- /dev/null +++ b/packages/transformers/src/pipelines/text2text-generation.js @@ -0,0 +1,100 @@ +import { Pipeline } from './_base.js'; + +import { Tensor } from '../utils/tensor.js'; + +/** + * @typedef {import('./_base.js').TextPipelineConstructorArgs} TextPipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + */ + +/** + * @typedef {Object} Text2TextGenerationSingle + * @property {string} generated_text The generated text. + * @typedef {Text2TextGenerationSingle[]} Text2TextGenerationOutput + * + * @callback Text2TextGenerationPipelineCallback Generate the output text(s) using text(s) given as inputs. + * @param {string|string[]} texts Input text for the encoder. + * @param {Partial} [options] Additional keyword arguments to pass along to the generate method of the model. + * @returns {Promise} + * + * @typedef {TextPipelineConstructorArgs & Text2TextGenerationPipelineCallback & Disposable} Text2TextGenerationPipelineType + */ + +/** + * Text2TextGenerationPipeline class for generating text using a model that performs text-to-text generation tasks. + * + * **Example:** Text-to-text generation w/ `Xenova/LaMini-Flan-T5-783M`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const generator = await pipeline('text2text-generation', 'Xenova/LaMini-Flan-T5-783M'); + * const output = await generator('how can I become more healthy?', { + * max_new_tokens: 100, + * }); + * // [{ generated_text: "To become more healthy, you can: 1. Eat a balanced diet with plenty of fruits, vegetables, whole grains, lean proteins, and healthy fats. 2. Stay hydrated by drinking plenty of water. 3. Get enough sleep and manage stress levels. 4. Avoid smoking and excessive alcohol consumption. 5. Regularly exercise and maintain a healthy weight. 6. Practice good hygiene and sanitation. 7. Seek medical attention if you experience any health issues." }] + * ``` + */ +export class Text2TextGenerationPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => Text2TextGenerationPipelineType} */ (Pipeline) +{ + _default_generation_config = { + max_new_tokens: 256, + // do_sample: true, + // temperature: 0.7, + }; + + /** @type {'generated_text'} */ + _key = 'generated_text'; + + /** @type {Text2TextGenerationPipelineCallback} */ + async _call(texts, generate_kwargs = {}) { + if (!Array.isArray(texts)) { + texts = [texts]; + } + + // Add global prefix, if present + // @ts-expect-error TS2339 + if (this.model.config.prefix) { + // @ts-expect-error TS2339 + texts = texts.map((x) => this.model.config.prefix + x); + } + + // Handle task specific params: + // @ts-expect-error TS2339 + const task_specific_params = this.model.config.task_specific_params; + if (task_specific_params && task_specific_params[this.task]) { + // Add prefixes, if present + if (task_specific_params[this.task].prefix) { + texts = texts.map((x) => task_specific_params[this.task].prefix + x); + } + + // TODO update generation config + } + + const tokenizer = this.tokenizer; + const tokenizer_options = { + padding: true, + truncation: true, + }; + let inputs; + if (this.task === 'translation' && '_build_translation_inputs' in tokenizer) { + // TODO: move to Translation pipeline? + // Currently put here to avoid code duplication + // @ts-ignore + inputs = tokenizer._build_translation_inputs(texts, tokenizer_options, generate_kwargs); + } else { + inputs = tokenizer(texts, tokenizer_options); + } + + const outputTokenIds = await this.model.generate({ + ...inputs, + ...this._default_generation_config, + ...generate_kwargs, + }); + return tokenizer + .batch_decode(/** @type {Tensor} */ (outputTokenIds), { + skip_special_tokens: true, + }) + .map((text) => ({ [this._key]: text })); + } +} diff --git a/packages/transformers/src/pipelines/token-classification.js b/packages/transformers/src/pipelines/token-classification.js new file mode 100644 index 0000000..7091460 --- /dev/null +++ b/packages/transformers/src/pipelines/token-classification.js @@ -0,0 +1,227 @@ +import { Pipeline } from './_base.js'; + +import { max, softmax } from '../utils/maths.js'; + +/** + * @typedef {import('./_base.js').TextPipelineConstructorArgs} TextPipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + */ + +/** + * Strategy for fusing tokens based on the model prediction. + * - `"none"`: Return raw per-token predictions. + * - `"simple"`: Group entities using BIO / BIOES tags (see pipeline docs for details). + * @typedef {"none" | "simple"} AggregationStrategy + */ + +/** + * @typedef {Object} TokenClassificationPipelineOptions + * @property {string[]} [ignore_labels] A list of labels to ignore. + * @property {AggregationStrategy} [aggregation_strategy="none"] Token-fusion strategy. + * When set to anything other than `"none"`, results use `entity_group` instead of `entity`. + */ + +/** + * Single element of a token-classification result, parameterised by the options type `O` so that + * `entity` vs. `entity_group` is known statically based on `aggregation_strategy`. + * + * - Grouped (present when `O["aggregation_strategy"]` is `"simple"`): + * `{ word, score, entity_group }` + * - Raw (the default — when `aggregation_strategy` is missing, `"none"`, or `undefined`): + * `{ word, score, entity, index }` + * - Both variants also carry optional `start` / `end` character offsets. + * + * When `O` is the untyped `TokenClassificationPipelineOptions`, the element is the union of both shapes, + * narrowable via `if ("entity_group" in item)` / `if (item.entity !== undefined)`. + * + * @template {TokenClassificationPipelineOptions | undefined} [O=TokenClassificationPipelineOptions] + * @typedef {_PickElement[]} TokenClassificationOutput + */ + +/** + * @template {TokenClassificationPipelineOptions | undefined} O + * @typedef {O extends undefined + * ? _Raw + * : "aggregation_strategy" extends keyof O + * ? (O extends { aggregation_strategy?: infer A } + * ? ([A] extends ["simple"] ? _Grouped + * : [A] extends ["none" | undefined] ? _Raw + * : _Raw | _Grouped) + * : _Raw) + * : _Raw} _PickElement + */ + +/** + * @typedef {{ word: string, score: number, entity: string, index: number, entity_group?: undefined, start?: number, end?: number }} _Raw + * @typedef {{ word: string, score: number, entity_group: string, entity?: undefined, index?: undefined, start?: number, end?: number }} _Grouped + */ + +/** + * @typedef {TextPipelineConstructorArgs & TokenClassificationPipelineCallback & Disposable} TokenClassificationPipelineType + * + * @typedef {(texts: Q, options?: O) => Promise[] : TokenClassificationOutput>} TokenClassificationPipelineCallback + */ + +/** + * Named Entity Recognition pipeline using any `ModelForTokenClassification`. + * + * **Example:** Perform named entity recognition with `Xenova/bert-base-NER`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const classifier = await pipeline('token-classification', 'Xenova/bert-base-NER'); + * const output = await classifier('My name is Sarah and I live in London'); + * // [ + * // { entity: 'B-PER', score: 0.9980202913284302, index: 4, word: 'Sarah' }, + * // { entity: 'B-LOC', score: 0.9994474053382874, index: 9, word: 'London' } + * // ] + * ``` + * + * **Example:** Perform named entity recognition with `Xenova/bert-base-NER` (and return all labels). + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const classifier = await pipeline('token-classification', 'Xenova/bert-base-NER'); + * const output = await classifier('Sarah lives in the United States of America', { ignore_labels: [] }); + * // [ + * // { entity: 'B-PER', score: 0.9966587424278259, index: 1, word: 'Sarah' }, + * // { entity: 'O', score: 0.9987385869026184, index: 2, word: 'lives' }, + * // { entity: 'O', score: 0.9990072846412659, index: 3, word: 'in' }, + * // { entity: 'O', score: 0.9988298416137695, index: 4, word: 'the' }, + * // { entity: 'B-LOC', score: 0.9995510578155518, index: 5, word: 'United' }, + * // { entity: 'I-LOC', score: 0.9990395307540894, index: 6, word: 'States' }, + * // { entity: 'I-LOC', score: 0.9986724853515625, index: 7, word: 'of' }, + * // { entity: 'I-LOC', score: 0.9975294470787048, index: 8, word: 'America' } + * // ] + * ``` + * + * **Example:** Group adjacent BIO/BIOES tokens into entity spans using `aggregation_strategy: "simple"`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const classifier = await pipeline('token-classification', 'Xenova/bert-base-NER'); + * const output = await classifier('My name is Sarah and I live in London', { aggregation_strategy: 'simple' }); + * // [ + * // { entity_group: 'PER', score: 0.9985477924346924, word: 'Sarah' }, + * // { entity_group: 'LOC', score: 0.999621570110321, word: 'London' } + * // ] + * ``` + */ +export class TokenClassificationPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => TokenClassificationPipelineType} */ (Pipeline) +{ + async _call(texts, { ignore_labels = ['O'], aggregation_strategy = 'none' } = {}) { + if (aggregation_strategy !== 'none' && aggregation_strategy !== 'simple') { + throw new Error( + `Invalid aggregation_strategy: "${aggregation_strategy}". Must be one of "none" or "simple".`, + ); + } + + const isBatched = Array.isArray(texts); + + // Run tokenization + const model_inputs = this.tokenizer(isBatched ? texts : [texts], { + padding: true, + truncation: true, + }); + + // Run model + const outputs = await this.model(model_inputs); + + const logits = outputs.logits; + // @ts-expect-error TS2339 + const id2label = this.model.config.id2label; + + const toReturn = []; + for (let i = 0; i < logits.dims[0]; ++i) { + const ids = model_inputs.input_ids[i].tolist(); + const batch = logits[i]; + + const tokens = []; + for (let j = 0; j < batch.dims[0]; ++j) { + const tokenData = batch[j]; + const topScoreIndex = max(tokenData.data)[1]; + + const entity = id2label ? id2label[topScoreIndex] : `LABEL_${topScoreIndex}`; + if (ignore_labels.includes(entity)) continue; + + // TODO add option to keep special tokens? + const word = this.tokenizer.decode([ids[j]], { skip_special_tokens: true }); + if (word === '') continue; // Was a special token. + + const scores = softmax(tokenData.data); + tokens.push({ + entity, + score: scores[topScoreIndex], + index: j, + word, + // TODO: Add support for start and end + }); + } + + toReturn.push(aggregation_strategy === 'simple' ? groupEntities(tokens, ids, this.tokenizer) : tokens); + } + return isBatched ? toReturn : toReturn[0]; + } +} + +/** + * Split a raw entity label into its BIOES prefix and tag. + * + * @param {string} entity + * @returns {readonly [prefix: 'B'|'I'|'E'|'S', tag: string]} + */ +function getTag(entity) { + const p = entity[0]; + return entity[1] === '-' && (p === 'B' || p === 'I' || p === 'E' || p === 'S') + ? [p, entity.slice(2)] + : ['I', entity]; +} + +/** + * Group raw per-token predictions into entity spans using the SIMPLE strategy. + * + * The only "continue" predicate is: a non-`B`/non-`S` token whose tag matches + * the open group's tag, when that group hasn't been closed by an `E` / `S`. + * Everything else starts a fresh group. + * + * @param {_Raw[]} tokens + * @param {number[]} ids Full input_ids for the sequence (indexed by `token.index`), used to re-decode + * each group so the joined `word` matches what the tokenizer would produce. + * @param {any} tokenizer + * @returns {_Grouped[]} + */ +function groupEntities(tokens, ids, tokenizer) { + /** @type {{ tag: string, start: number, end: number }[]} */ + const groups = []; // each entry is a [start, end) slice into `tokens`, plus the shared tag + let openTag = null; // null = no open group + + for (let i = 0; i < tokens.length; ++i) { + const [prefix, tag] = getTag(tokens[i].entity); + const extend = openTag === tag && prefix !== 'B' && prefix !== 'S'; + if (extend) { + groups[groups.length - 1].end = i + 1; + // `E` terminates the group; subsequent `I`/`E`/`S` start fresh. + if (prefix === 'E') openTag = null; + } else { + groups.push({ tag, start: i, end: i + 1 }); + // `S` opens and immediately closes; anything else leaves the group open + // (including a leading `E` — best-effort recovery for a malformed sequence). + openTag = prefix === 'S' ? null : tag; + } + } + + return groups.map(({ tag, start, end }) => { + let scoreSum = 0; + const groupIds = []; + for (let i = start; i < end; ++i) { + scoreSum += tokens[i].score; + groupIds.push(ids[tokens[i].index]); + } + return { + entity_group: tag, + score: scoreSum / (end - start), + word: tokenizer.decode(groupIds, { skip_special_tokens: true }), + }; + }); +} diff --git a/packages/transformers/src/pipelines/translation.js b/packages/transformers/src/pipelines/translation.js new file mode 100644 index 0000000..a69115d --- /dev/null +++ b/packages/transformers/src/pipelines/translation.js @@ -0,0 +1,79 @@ +import { Text2TextGenerationPipeline } from './text2text-generation.js'; + +/** + * @typedef {import('./_base.js').TextPipelineConstructorArgs} TextPipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + */ + +/** + * @typedef {Object} TranslationSingle + * @property {string} translation_text The translated text. + * @typedef {TranslationSingle[]} TranslationOutput + * + * @callback TranslationPipelineCallback Translate the text(s) given as inputs. + * @param {string|string[]} texts Texts to be translated. + * @param {import('../generation/parameters.js').GenerationFunctionParameters} [options] Additional keyword arguments to pass along to the generate method of the model. + * @returns {Promise} + * + * @typedef {TextPipelineConstructorArgs & TranslationPipelineCallback & Disposable} TranslationPipelineType + */ + +/** + * Translates text from one language to another. + * + * **Example:** Multilingual translation w/ `Xenova/nllb-200-distilled-600M`. + * + * See [here](https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200) + * for the full list of languages and their corresponding codes. + * + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const translator = await pipeline('translation', 'Xenova/nllb-200-distilled-600M'); + * const output = await translator('जीवन एक चॉकलेट बॉक्स की तरह है।', { + * src_lang: 'hin_Deva', // Hindi + * tgt_lang: 'fra_Latn', // French + * }); + * // [{ translation_text: 'La vie est comme une boîte à chocolat.' }] + * ``` + * + * **Example:** Multilingual translation w/ `Xenova/m2m100_418M`. + * + * See [here](https://huggingface.co/facebook/m2m100_418M#languages-covered) + * for the full list of languages and their corresponding codes. + * + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const translator = await pipeline('translation', 'Xenova/m2m100_418M'); + * const output = await translator('生活就像一盒巧克力。', { + * src_lang: 'zh', // Chinese + * tgt_lang: 'en', // English + * }); + * // [{ translation_text: 'Life is like a box of chocolate.' }] + * ``` + * + * **Example:** Multilingual translation w/ `Xenova/mbart-large-50-many-to-many-mmt`. + * + * See [here](https://huggingface.co/facebook/mbart-large-50-many-to-many-mmt#languages-covered) + * for the full list of languages and their corresponding codes. + * + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const translator = await pipeline('translation', 'Xenova/mbart-large-50-many-to-many-mmt'); + * const output = await translator('संयुक्त राष्ट्र के प्रमुख का कहना है कि सीरिया में कोई सैन्य समाधान नहीं है', { + * src_lang: 'hi_IN', // Hindi + * tgt_lang: 'fr_XX', // French + * }); + * // [{ translation_text: 'Le chef des Nations affirme qu 'il n 'y a military solution in Syria.' }] + * ``` + */ +export class TranslationPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => TranslationPipelineType} */ ( + /** @type {any} */ (Text2TextGenerationPipeline) + ) +{ + /** @type {'translation_text'} */ + _key = 'translation_text'; +} diff --git a/packages/transformers/src/pipelines/zero-shot-audio-classification.js b/packages/transformers/src/pipelines/zero-shot-audio-classification.js new file mode 100644 index 0000000..20a9184 --- /dev/null +++ b/packages/transformers/src/pipelines/zero-shot-audio-classification.js @@ -0,0 +1,95 @@ +import { Pipeline, prepareAudios } from './_base.js'; + +import { softmax } from '../utils/maths.js'; + +/** + * @typedef {import('./_base.js').TextAudioPipelineConstructorArgs} TextAudioPipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + * @typedef {import('./_base.js').AudioInput} AudioInput + */ + +/** + * @typedef {Object} ZeroShotAudioClassificationOutputSingle + * @property {string} label The label identified by the model. It is one of the suggested `candidate_label`. + * @property {number} score The score attributed by the model for that label (between 0 and 1). + * + * @typedef {ZeroShotAudioClassificationOutputSingle[]} ZeroShotAudioClassificationOutput + * + * @typedef {Object} ZeroShotAudioClassificationPipelineOptions Parameters specific to zero-shot audio classification pipelines. + * @property {string} [hypothesis_template="This is a sound of {}."] The sentence used in conjunction with `candidate_labels` + * to attempt the audio classification by replacing the placeholder with the candidate_labels. + * Then likelihood is estimated by using `logits_per_audio`. + * + * @typedef {TextAudioPipelineConstructorArgs & ZeroShotAudioClassificationPipelineCallback & Disposable} ZeroShotAudioClassificationPipelineType + */ + +/** + * @template T + * @typedef {T extends AudioInput[] ? ZeroShotAudioClassificationOutput[] : ZeroShotAudioClassificationOutput} ZeroShotAudioClassificationPipelineResult + */ + +/** + * @typedef {(audio: T, candidate_labels: string[], options?: ZeroShotAudioClassificationPipelineOptions) => Promise>} ZeroShotAudioClassificationPipelineCallback + */ + +/** + * Zero shot audio classification pipeline using `ClapModel`. This pipeline predicts the class of an audio when you + * provide an audio and a set of `candidate_labels`. + * + * **Example**: Perform zero-shot audio classification with `Xenova/clap-htsat-unfused`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const classifier = await pipeline('zero-shot-audio-classification', 'Xenova/clap-htsat-unfused'); + * const audio = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/dog_barking.wav'; + * const candidate_labels = ['dog', 'vaccum cleaner']; + * const scores = await classifier(audio, candidate_labels); + * // [ + * // { score: 0.9993992447853088, label: 'dog' }, + * // { score: 0.0006007603369653225, label: 'vaccum cleaner' } + * // ] + * ``` + */ +export class ZeroShotAudioClassificationPipeline + extends /** @type {new (options: TextAudioPipelineConstructorArgs) => ZeroShotAudioClassificationPipelineType} */ ( + Pipeline + ) +{ + async _call(audio, candidate_labels, { hypothesis_template = 'This is a sound of {}.' } = {}) { + const single = !Array.isArray(audio); + if (single) { + audio = [/** @type {AudioInput} */ (audio)]; + } + + // Insert label into hypothesis template + const texts = candidate_labels.map((x) => hypothesis_template.replace('{}', x)); + + // Run tokenization + const text_inputs = this.tokenizer(texts, { + padding: true, + truncation: true, + }); + + const sampling_rate = this.processor.feature_extractor.config.sampling_rate; + const preparedAudios = await prepareAudios(audio, sampling_rate); + + const toReturn = []; + for (const aud of preparedAudios) { + const audio_inputs = await this.processor(aud); + + // Run model with both text and audio inputs + const output = await this.model({ ...text_inputs, ...audio_inputs }); + + // Compute softmax per audio + const probs = softmax(output.logits_per_audio.data); + + toReturn.push( + [...probs].map((x, i) => ({ + score: x, + label: candidate_labels[i], + })), + ); + } + return single ? toReturn[0] : toReturn; + } +} diff --git a/packages/transformers/src/pipelines/zero-shot-classification.js b/packages/transformers/src/pipelines/zero-shot-classification.js new file mode 100644 index 0000000..df0bbb9 --- /dev/null +++ b/packages/transformers/src/pipelines/zero-shot-classification.js @@ -0,0 +1,155 @@ +import { Pipeline } from './_base.js'; + +import { softmax } from '../utils/maths.js'; +import { logger } from '../utils/logger.js'; + +/** + * @typedef {import('./_base.js').TextPipelineConstructorArgs} TextPipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + */ + +/** + * @typedef {Object} ZeroShotClassificationOutput + * @property {string} sequence The sequence for which this is the output. + * @property {string[]} labels The labels sorted by order of likelihood. + * @property {number[]} scores The probabilities for each of the labels. + * + * @typedef {Object} ZeroShotClassificationPipelineOptions Parameters specific to zero-shot classification pipelines. + * @property {string} [hypothesis_template="This example is {}."] The template used to turn each + * candidate label into an NLI-style hypothesis. The candidate label will replace the {} placeholder. + * @property {boolean} [multi_label=false] Whether or not multiple candidate labels can be true. + * If `false`, the scores are normalized such that the sum of the label likelihoods for each sequence + * is 1. If `true`, the labels are considered independent and probabilities are normalized for each + * candidate by doing a softmax of the entailment score vs. the contradiction score. + * + * @typedef {TextPipelineConstructorArgs & ZeroShotClassificationPipelineCallback & Disposable} ZeroShotClassificationPipelineType + */ + +/** + * @template T + * @typedef {T extends string[] ? ZeroShotClassificationOutput[] : ZeroShotClassificationOutput} ZeroShotClassificationPipelineResult + */ + +/** + * @typedef {(texts: T, candidate_labels: string | string[], options?: ZeroShotClassificationPipelineOptions) => Promise>} ZeroShotClassificationPipelineCallback + */ + +/** + * NLI-based zero-shot classification pipeline using a `ModelForSequenceClassification` + * trained on NLI (natural language inference) tasks. Equivalent of `text-classification` + * pipelines, but these models don't require a hardcoded number of potential classes, they + * can be chosen at runtime. It usually means it's slower but it is **much** more flexible. + * + * **Example:** Zero shot classification with `Xenova/mobilebert-uncased-mnli`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const classifier = await pipeline('zero-shot-classification', 'Xenova/mobilebert-uncased-mnli'); + * const text = 'Last week I upgraded my iOS version and ever since then my phone has been overheating whenever I use your app.'; + * const labels = [ 'mobile', 'billing', 'website', 'account access' ]; + * const output = await classifier(text, labels); + * // { + * // sequence: 'Last week I upgraded my iOS version and ever since then my phone has been overheating whenever I use your app.', + * // labels: [ 'mobile', 'website', 'billing', 'account access' ], + * // scores: [ 0.5562091040482018, 0.1843621307860853, 0.13942646639336376, 0.12000229877234923 ] + * // } + * ``` + * + * **Example:** Zero shot classification with `Xenova/nli-deberta-v3-xsmall` (multi-label). + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const classifier = await pipeline('zero-shot-classification', 'Xenova/nli-deberta-v3-xsmall'); + * const text = 'I have a problem with my iphone that needs to be resolved asap!'; + * const labels = [ 'urgent', 'not urgent', 'phone', 'tablet', 'computer' ]; + * const output = await classifier(text, labels, { multi_label: true }); + * // { + * // sequence: 'I have a problem with my iphone that needs to be resolved asap!', + * // labels: [ 'urgent', 'phone', 'computer', 'tablet', 'not urgent' ], + * // scores: [ 0.9958870956360275, 0.9923963400697035, 0.002333537946160235, 0.0015134138567598765, 0.0010699384208377163 ] + * // } + * ``` + */ +export class ZeroShotClassificationPipeline + extends /** @type {new (options: TextPipelineConstructorArgs) => ZeroShotClassificationPipelineType} */ (Pipeline) +{ + /** + * Create a new ZeroShotClassificationPipeline. + * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline. + */ + constructor(options) { + super(options); + + // Use model config to get label2id mapping + this.label2id = Object.fromEntries( + Object.entries(/** @type {any} */ (this).model.config.label2id).map(([k, v]) => [k.toLowerCase(), v]), + ); + + this.entailment_id = this.label2id['entailment']; + if (this.entailment_id === undefined) { + logger.warn("Could not find 'entailment' in label2id mapping. Using 2 as entailment_id."); + this.entailment_id = 2; + } + + this.contradiction_id = this.label2id['contradiction'] ?? this.label2id['not_entailment']; + if (this.contradiction_id === undefined) { + logger.warn("Could not find 'contradiction' in label2id mapping. Using 0 as contradiction_id."); + this.contradiction_id = 0; + } + } + + async _call(texts, candidate_labels, { hypothesis_template = 'This example is {}.', multi_label = false } = {}) { + const isBatched = Array.isArray(texts); + if (!isBatched) { + texts = [/** @type {string} */ (texts)]; + } + if (!Array.isArray(candidate_labels)) { + candidate_labels = [candidate_labels]; + } + + // Insert labels into hypothesis template + const hypotheses = candidate_labels.map((x) => hypothesis_template.replace('{}', x)); + + // How to perform the softmax over the logits: + // - true: softmax over the entailment vs. contradiction dim for each label independently + // - false: softmax the "entailment" logits over all candidate labels + const softmaxEach = multi_label || candidate_labels.length === 1; + + /** @type {ZeroShotClassificationOutput[]} */ + const toReturn = []; + for (const premise of texts) { + const entails_logits = []; + + for (const hypothesis of hypotheses) { + const inputs = this.tokenizer(premise, { + text_pair: hypothesis, + padding: true, + truncation: true, + }); + const outputs = await this.model(inputs); + + if (softmaxEach) { + entails_logits.push([ + outputs.logits.data[this.contradiction_id], + outputs.logits.data[this.entailment_id], + ]); + } else { + entails_logits.push(outputs.logits.data[this.entailment_id]); + } + } + + /** @type {number[]} */ + const scores = softmaxEach ? entails_logits.map((x) => softmax(x)[1]) : softmax(entails_logits); + + // Sort by scores (desc) and return scores with indices + const scores_sorted = scores.map((x, i) => [x, i]).sort((a, b) => b[0] - a[0]); + + toReturn.push({ + sequence: premise, + labels: scores_sorted.map((x) => candidate_labels[x[1]]), + scores: scores_sorted.map((x) => x[0]), + }); + } + return isBatched ? toReturn : toReturn[0]; + } +} diff --git a/packages/transformers/src/pipelines/zero-shot-image-classification.js b/packages/transformers/src/pipelines/zero-shot-image-classification.js new file mode 100644 index 0000000..80a62f1 --- /dev/null +++ b/packages/transformers/src/pipelines/zero-shot-image-classification.js @@ -0,0 +1,99 @@ +import { Pipeline, prepareImages } from './_base.js'; + +import { softmax } from '../utils/maths.js'; + +/** + * @typedef {import('./_base.js').TextImagePipelineConstructorArgs} TextImagePipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + * @typedef {import('./_base.js').ImagePipelineInputs} ImagePipelineInputs + * @typedef {import('./_base.js').ImageInput} ImageInput + */ + +/** + * @typedef {Object} ZeroShotImageClassificationOutputSingle + * @property {string} label The label identified by the model. It is one of the suggested `candidate_label`. + * @property {number} score The score attributed by the model for that label (between 0 and 1). + * + * @typedef {ZeroShotImageClassificationOutputSingle[]} ZeroShotImageClassificationOutput + * + * @typedef {Object} ZeroShotImageClassificationPipelineOptions Parameters specific to zero-shot image classification pipelines. + * @property {string} [hypothesis_template="This is a photo of {}"] The sentence used in conjunction with `candidate_labels` + * to attempt the image classification by replacing the placeholder with the candidate_labels. + * Then likelihood is estimated by using `logits_per_image`. + * + * @typedef {TextImagePipelineConstructorArgs & ZeroShotImageClassificationPipelineCallback & Disposable} ZeroShotImageClassificationPipelineType + */ + +/** + * @template T + * @typedef {T extends ImageInput[] ? ZeroShotImageClassificationOutput[] : ZeroShotImageClassificationOutput} ZeroShotImageClassificationPipelineResult + */ + +/** + * @typedef {(images: T, candidate_labels: string[], options?: ZeroShotImageClassificationPipelineOptions) => Promise>} ZeroShotImageClassificationPipelineCallback + */ + +/** + * Zero shot image classification pipeline. This pipeline predicts the class of + * an image when you provide an image and a set of `candidate_labels`. + * + * **Example:** Zero shot image classification w/ `Xenova/clip-vit-base-patch32`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const classifier = await pipeline('zero-shot-image-classification', 'Xenova/clip-vit-base-patch32'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/tiger.jpg'; + * const output = await classifier(url, ['tiger', 'horse', 'dog']); + * // [ + * // { score: 0.9993917942047119, label: 'tiger' }, + * // { score: 0.0003519294841680676, label: 'horse' }, + * // { score: 0.0002562698791734874, label: 'dog' } + * // ] + * ``` + */ +export class ZeroShotImageClassificationPipeline + extends /** @type {new (options: TextImagePipelineConstructorArgs) => ZeroShotImageClassificationPipelineType} */ ( + Pipeline + ) +{ + async _call(images, candidate_labels, { hypothesis_template = 'This is a photo of {}' } = {}) { + const isBatched = Array.isArray(images); + const preparedImages = await prepareImages(images); + + // Insert label into hypothesis template + const texts = candidate_labels.map((x) => hypothesis_template.replace('{}', x)); + + // Run tokenization + const text_inputs = this.tokenizer(texts, { + padding: this.model.config.model_type === 'siglip' ? 'max_length' : true, + truncation: true, + }); + + // Run processor + const { pixel_values } = await this.processor(preparedImages); + + // Run model with both text and pixel inputs + const output = await this.model({ ...text_inputs, pixel_values }); + + const function_to_apply = + this.model.config.model_type === 'siglip' + ? (batch) => batch.sigmoid().data + : (batch) => softmax(batch.data); + + // Compare each image with each candidate label + const toReturn = []; + for (const batch of output.logits_per_image) { + // Compute softmax per image + const probs = function_to_apply(batch); + + const result = [...probs].map((x, i) => ({ + score: x, + label: candidate_labels[i], + })); + result.sort((a, b) => b.score - a.score); // sort by score in descending order + toReturn.push(result); + } + + return isBatched ? toReturn : toReturn[0]; + } +} diff --git a/packages/transformers/src/pipelines/zero-shot-object-detection.js b/packages/transformers/src/pipelines/zero-shot-object-detection.js new file mode 100644 index 0000000..b96c6a1 --- /dev/null +++ b/packages/transformers/src/pipelines/zero-shot-object-detection.js @@ -0,0 +1,173 @@ +import { Pipeline, prepareImages, get_bounding_box } from './_base.js'; + +/** + * @typedef {import('./_base.js').TextImagePipelineConstructorArgs} TextImagePipelineConstructorArgs + * @typedef {import('./_base.js').Disposable} Disposable + * @typedef {import('./_base.js').ImageInput} ImageInput + * @typedef {import('./_base.js').BoundingBox} BoundingBox + */ + +/** + * @typedef {Object} ZeroShotObjectDetectionOutputSingle + * @property {string} label Text query corresponding to the found object. + * @property {number} score Score corresponding to the object (between 0 and 1). + * @property {BoundingBox} box Bounding box of the detected object in image's original size, or as a percentage if `percentage` is set to true. + * + * @typedef {ZeroShotObjectDetectionOutputSingle[]} ZeroShotObjectDetectionOutput + * + * @typedef {Object} ZeroShotObjectDetectionPipelineOptions Parameters specific to zero-shot object detection pipelines. + * @property {number} [threshold=0.1] The probability necessary to make a prediction. + * @property {number} [top_k=null] The number of top predictions that will be returned by the pipeline. + * If the provided number is `null` or higher than the number of predictions available, it will default + * to the number of predictions. + * @property {boolean} [percentage=false] Whether to return the boxes coordinates in percentage (true) or in pixels (false). + * + * @typedef {TextImagePipelineConstructorArgs & ZeroShotObjectDetectionPipelineCallback & Disposable} ZeroShotObjectDetectionPipelineType + */ + +/** + * @template T + * @typedef {T extends ImageInput[] ? ZeroShotObjectDetectionOutput[] : ZeroShotObjectDetectionOutput} ZeroShotObjectDetectionPipelineResult + */ + +/** + * @typedef {(images: T, candidate_labels: string[], options?: ZeroShotObjectDetectionPipelineOptions) => Promise>} ZeroShotObjectDetectionPipelineCallback + */ + +/** + * Zero-shot object detection pipeline. This pipeline predicts bounding boxes of + * objects when you provide an image and a set of `candidate_labels`. + * + * **Example:** Zero-shot object detection w/ `Xenova/owlvit-base-patch32`. + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const detector = await pipeline('zero-shot-object-detection', 'Xenova/owlvit-base-patch32'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/astronaut.png'; + * const candidate_labels = ['human face', 'rocket', 'helmet', 'american flag']; + * const output = await detector(url, candidate_labels); + * // [ + * // { + * // score: 0.24392342567443848, + * // label: 'human face', + * // box: { xmin: 180, ymin: 67, xmax: 274, ymax: 175 } + * // }, + * // { + * // score: 0.15129457414150238, + * // label: 'american flag', + * // box: { xmin: 0, ymin: 4, xmax: 106, ymax: 513 } + * // }, + * // { + * // score: 0.13649864494800568, + * // label: 'helmet', + * // box: { xmin: 277, ymin: 337, xmax: 511, ymax: 511 } + * // }, + * // { + * // score: 0.10262022167444229, + * // label: 'rocket', + * // box: { xmin: 352, ymin: -1, xmax: 463, ymax: 287 } + * // } + * // ] + * ``` + * + * **Example:** Zero-shot object detection w/ `Xenova/owlvit-base-patch32` (returning top 4 matches and setting a threshold). + * ```javascript + * import { pipeline } from '@huggingface/transformers'; + * + * const detector = await pipeline('zero-shot-object-detection', 'Xenova/owlvit-base-patch32'); + * const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/beach.png'; + * const candidate_labels = ['hat', 'book', 'sunglasses', 'camera']; + * const output = await detector(url, candidate_labels, { top_k: 4, threshold: 0.05 }); + * // [ + * // { + * // score: 0.1606510728597641, + * // label: 'sunglasses', + * // box: { xmin: 347, ymin: 229, xmax: 429, ymax: 264 } + * // }, + * // { + * // score: 0.08935828506946564, + * // label: 'hat', + * // box: { xmin: 38, ymin: 174, xmax: 258, ymax: 364 } + * // }, + * // { + * // score: 0.08530698716640472, + * // label: 'camera', + * // box: { xmin: 187, ymin: 350, xmax: 260, ymax: 411 } + * // }, + * // { + * // score: 0.08349756896495819, + * // label: 'book', + * // box: { xmin: 261, ymin: 280, xmax: 494, ymax: 425 } + * // } + * // ] + * ``` + */ +export class ZeroShotObjectDetectionPipeline + extends /** @type {new (options: TextImagePipelineConstructorArgs) => ZeroShotObjectDetectionPipelineType} */ ( + Pipeline + ) +{ + async _call(images, candidate_labels, { threshold = 0.1, top_k = null, percentage = false } = {}) { + const isBatched = Array.isArray(images); + const preparedImages = await prepareImages(images); + + // Run tokenization + const text_inputs = this.tokenizer(candidate_labels, { + padding: true, + truncation: true, + }); + + // Run processor + const model_inputs = await this.processor(preparedImages); + + // Since non-maximum suppression is performed for exporting, we need to + // process each image separately. For more information, see: + // https://github.com/huggingface/optimum/blob/e3b7efb1257c011db907ef40ab340e795cc5684c/optimum/exporters/onnx/model_configs.py#L1028-L1032 + const toReturn = []; + for (let i = 0; i < preparedImages.length; ++i) { + const image = preparedImages[i]; + const imageSize = percentage ? null : [[image.height, image.width]]; + const pixel_values = model_inputs.pixel_values[i].unsqueeze_(0); + + // Run model with both text and pixel inputs + const output = await this.model({ ...text_inputs, pixel_values }); + + let result; + if ('post_process_grounded_object_detection' in this.processor) { + // @ts-ignore + const processed = this.processor.post_process_grounded_object_detection(output, text_inputs.input_ids, { + // TODO: support separate threshold values + box_threshold: threshold, + text_threshold: threshold, + target_sizes: imageSize, + })[0]; + result = processed.boxes.map((box, i) => ({ + score: processed.scores[i], + label: processed.labels[i], + box: get_bounding_box(box, !percentage), + })); + } else { + // @ts-ignore + const processed = this.processor.image_processor.post_process_object_detection( + output, + threshold, + imageSize, + true, + )[0]; + result = processed.boxes.map((box, i) => ({ + score: processed.scores[i], + label: candidate_labels[processed.classes[i]], + box: get_bounding_box(box, !percentage), + })); + } + result.sort((a, b) => b.score - a.score); + + if (top_k !== null) { + result = result.slice(0, top_k); + } + toReturn.push(result); + } + + return isBatched ? toReturn : toReturn[0]; + } +} diff --git a/packages/transformers/src/processing_utils.js b/packages/transformers/src/processing_utils.js new file mode 100644 index 0000000..b28be3e --- /dev/null +++ b/packages/transformers/src/processing_utils.js @@ -0,0 +1,162 @@ +/** + * @file Processors are used to prepare inputs (e.g., text, image or audio) for a model. + * + * **Example:** Using a `WhisperProcessor` to prepare an audio input for a model. + * ```javascript + * import { AutoProcessor, read_audio } from '@huggingface/transformers'; + * + * const processor = await AutoProcessor.from_pretrained('openai/whisper-tiny.en'); + * const audio = await read_audio('https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/mlk.flac', 16000); + * const { input_features } = await processor(audio); + * // Tensor { + * // data: Float32Array(240000) [0.4752984642982483, 0.5597258806228638, 0.56434166431427, ...], + * // dims: [1, 80, 3000], + * // type: 'float32', + * // size: 240000, + * // } + * ``` + * + * @module processors + */ +import { PROCESSOR_NAME, CHAT_TEMPLATE_NAME } from './utils/constants.js'; +import { Callable } from './utils/generic.js'; +import { getModelJSON, getModelText } from './utils/hub.js'; + +/** + * @typedef {Object} ProcessorProperties Additional processor-specific properties. + * @typedef {import('./utils/hub.js').PretrainedOptions & ProcessorProperties} PretrainedProcessorOptions + * @typedef {import('./tokenization_utils.js').PreTrainedTokenizer} PreTrainedTokenizer + */ + +/** + * Represents a Processor that extracts features from an input. + */ +export class Processor extends Callable { + static classes = ['image_processor_class', 'tokenizer_class', 'feature_extractor_class']; + static uses_processor_config = false; + static uses_chat_template_file = false; + + /** + * Creates a new Processor with the given components + * @param {Object} config + * @param {Record} components + * @param {string} chat_template + */ + constructor(config, components, chat_template) { + super(); + this.config = config; + this.components = components; + this.chat_template = chat_template; + } + + /** + * @returns {import('./image_processors_utils.js').ImageProcessor|undefined} The image processor of the processor, if it exists. + */ + get image_processor() { + return this.components.image_processor; + } + + /** + * @returns {PreTrainedTokenizer|undefined} The tokenizer of the processor, if it exists. + */ + get tokenizer() { + return this.components.tokenizer; + } + + /** + * @returns {import('./feature_extraction_utils.js').FeatureExtractor|undefined} The feature extractor of the processor, if it exists. + */ + get feature_extractor() { + return this.components.feature_extractor; + } + + /** + * @param {Parameters[0]} messages + * @param {Parameters[1]} options + * @returns {ReturnType} + */ + apply_chat_template(messages, options = {}) { + if (!this.tokenizer) { + throw new Error('Unable to apply chat template without a tokenizer.'); + } + return this.tokenizer.apply_chat_template(messages, { + tokenize: false, // default to false + chat_template: this.chat_template ?? undefined, + ...options, + }); + } + + /** + * @param {Parameters} args + * @returns {ReturnType} + */ + batch_decode(...args) { + if (!this.tokenizer) { + throw new Error('Unable to decode without a tokenizer.'); + } + return this.tokenizer.batch_decode(...args); + } + + /** + * @param {Parameters} args + * @returns {ReturnType} + */ + decode(...args) { + if (!this.tokenizer) { + throw new Error('Unable to decode without a tokenizer.'); + } + return this.tokenizer.decode(...args); + } + + /** + * Calls the feature_extractor function with the given input. + * @param {any} input The input to extract features from. + * @param {...any} args Additional arguments. + * @returns {Promise} A Promise that resolves with the extracted features. + */ + async _call(input, ...args) { + for (const item of [this.image_processor, this.feature_extractor, this.tokenizer]) { + if (item) { + return item(input, ...args); + } + } + throw new Error('No image processor, feature extractor, or tokenizer found.'); + } + + /** + * Instantiate one of the processor classes of the library from a pretrained model. + * + * The processor class to instantiate is selected based on the `image_processor_type` (or `feature_extractor_type`; legacy) + * property of the config object (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible) + * + * @param {string} pretrained_model_name_or_path The name or path of the pretrained model. Can be either: + * - A string, the *model id* of a pretrained processor hosted inside a model repo on huggingface.co. + * Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a + * user or organization name, like `dbmdz/bert-base-german-cased`. + * - A path to a *directory* containing processor files, e.g., `./my_model_directory/`. + * @param {PretrainedProcessorOptions} options Additional options for loading the processor. + * + * @returns {Promise} A new instance of the Processor class. + */ + static async from_pretrained(pretrained_model_name_or_path, options = {}) { + const [config, components, chat_template] = await Promise.all([ + // TODO: + this.uses_processor_config + ? getModelJSON(pretrained_model_name_or_path, PROCESSOR_NAME, true, options) + : {}, + Promise.all( + this.classes + .filter((cls) => cls in this) + .map(async (cls) => { + const component = await this[cls].from_pretrained(pretrained_model_name_or_path, options); + return [cls.replace(/_class$/, ''), component]; + }), + ).then(Object.fromEntries), + this.uses_chat_template_file + ? getModelText(pretrained_model_name_or_path, CHAT_TEMPLATE_NAME, true, options) + : null, + ]); + + return new this(config, components, chat_template); + } +} diff --git a/packages/transformers/src/tokenization_utils.js b/packages/transformers/src/tokenization_utils.js new file mode 100644 index 0000000..29de6f1 --- /dev/null +++ b/packages/transformers/src/tokenization_utils.js @@ -0,0 +1,863 @@ +/** + * @file Tokenization utilities + * + * @module tokenizers + */ + +import { Tokenizer } from '@huggingface/tokenizers'; +import { Template } from '@huggingface/jinja'; +import { Callable } from './utils/generic.js'; + +import { isIntegralNumber, mergeArrays } from './utils/core.js'; +import { getModelJSON } from './utils/hub.js'; +import { max } from './utils/maths.js'; +import { Tensor } from './utils/tensor.js'; +import { logger } from './utils/logger.js'; +import { get_tokenizer_files } from './utils/model_registry/get_tokenizer_files.js'; + +/** + * @typedef {import('./utils/hub.js').PretrainedOptions} PretrainedTokenizerOptions + */ + +/** + * Loads a tokenizer from the specified path. + * @param {string} pretrained_model_name_or_path The path to the tokenizer directory. + * @param {PretrainedTokenizerOptions} options Additional options for loading the tokenizer. + * @returns {Promise} A promise that resolves with information about the loaded tokenizer. + */ +export async function loadTokenizer(pretrained_model_name_or_path, options) { + const tokenizerFiles = await get_tokenizer_files(pretrained_model_name_or_path); + return await Promise.all( + tokenizerFiles.map((file) => getModelJSON(pretrained_model_name_or_path, file, true, options)), + ); +} + +/** + * Helper function to convert a tensor to a list before decoding. + * @param {Tensor} tensor The tensor to convert. + * @returns {number[]} The tensor as a list. + */ +export function prepareTensorForDecode(tensor) { + const dims = tensor.dims; + switch (dims.length) { + case 1: + return tensor.tolist(); + case 2: + if (dims[0] !== 1) { + throw new Error( + 'Unable to decode tensor with `batch size !== 1`. Use `tokenizer.batch_decode(...)` for batched inputs.', + ); + } + return tensor.tolist()[0]; + default: + throw new Error(`Expected tensor to have 1-2 dimensions, got ${dims.length}.`); + } +} + +const SPECIAL_TOKEN_ATTRIBUTES = [ + 'bos_token', + 'eos_token', + 'unk_token', + 'sep_token', + 'pad_token', + 'cls_token', + 'mask_token', + // additional_special_tokens (TODO) +]; + +/** + * @typedef {{ type: 'text', text: string, [key: string]: any }} TextContent + * @property {'text'} type The type of content (must be 'text'). + * @property {string} text The text content. + */ + +/** + * @typedef {{ type: 'image', image?: string | import('./utils/image.js').RawImage, [key: string]: any }} ImageContent + * @property {'image'} type The type of content (must be 'image'). + * @property {string | import('./utils/image.js').RawImage} [image] Optional URL or instance of the image. + * + * Note: This works for SmolVLM. Qwen2VL and Idefics3 have different implementations. + */ + +/** + * @typedef {TextContent | ImageContent | { type: string & {}, [key: string]: any }} MessageContent + * Base type for message content. This is a discriminated union that can be extended with additional content types. + * Example: `@typedef {TextContent | ImageContent | AudioContent} MessageContent` + */ + +/** + * @typedef {Object} Message + * @property {'user' | 'assistant' | 'system' | (string & {})} role The role of the message. + * @property {string | MessageContent[]} content The content of the message. Can be a simple string or an array of content objects. + */ + +/** + * + * Helper function for padding values of an object, which are each arrays. + * NOTE: No additional checks are made here for validity of arguments. + * @param {Record} item The input object. + * @param {number} length The length to pad to. + * @param {(key: string) => any} value_fn Determine the value to fill the array, based on its key. + * @param {string} side Which side to pad the array. + * @private + */ +function padHelper(item, length, value_fn, side) { + for (const key of Object.keys(item)) { + const diff = length - item[key].length; + const value = value_fn(key); + + const padData = new Array(diff).fill(value); + item[key] = side === 'right' ? mergeArrays(item[key], padData) : mergeArrays(padData, item[key]); + } +} + +/** + * Helper function for truncating values of an object, which are each arrays. + * NOTE: No additional checks are made here for validity of arguments. + * @param {Record} item The input object. + * @param {number} length The length to truncate to. + * @private + */ +function truncateHelper(item, length) { + // Setting .length to a lower value truncates the array in-place: + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/length + for (const key of Object.keys(item)) { + item[key].length = length; + } +} + +/** + * Returns the value of the first matching key in the tokenizer config object. + * @param {Object} config The tokenizer config object. + * @param {...string} keys One or more keys to search for in the tokenizer config object. + * @returns {string|null} The value associated with the first matching key, or null if no match is found. + * @throws {Error} If an object is found for a matching key and its __type property is not "AddedToken". + * @private + */ +function getTokenFromConfig(config, ...keys) { + for (const key of keys) { + if (!Object.hasOwn(config, key)) continue; + const item = config[key]; + if (!item) continue; + + if (typeof item === 'object') { + if (item.__type === 'AddedToken') { + return item.content; + } else { + throw Error(`Unknown token: ${item}`); + } + } else { + return item; + } + } + return null; +} + +/** + * + * @param {import('@huggingface/tokenizers').Tokenizer} tokenizer + * @returns {import('@huggingface/tokenizers').AddedToken[]} + * @private + */ +function getSpecialTokens(tokenizer) { + const special = []; + for (const value of tokenizer.get_added_tokens_decoder().values()) { + if (value.special) special.push(value); + } + return special; +} + +/** + * @template {string|string[]} TText + * @typedef {TText extends string ? number[] : number[][]} BatchEncodingArrayItem + */ + +/** + * @template {string|string[]} TText + * @template {boolean} [TReturnTensor=true] + * @typedef {TReturnTensor extends true ? Tensor : BatchEncodingArrayItem} BatchEncodingItem + */ + +/** + * @template TItem + * @typedef {Object} BatchEncoding + * @property {TItem} input_ids List of token ids to be fed to a model. + * @property {TItem} attention_mask List of indices specifying which tokens should be attended to by the model. + * @property {TItem} [token_type_ids] List of token type ids to be fed to a model. + */ + +/** + * @template {string|string[]} TText + * @template {boolean} [TReturnTensor=true] + * @typedef {Object} TokenizerCallOptions + * @property {TText extends string ? string|null : string[]|null} [text_pair=null] Optional second sequence to be encoded. If set, must be the same type as text. + * @property {boolean|'max_length'} [padding=false] Whether to pad the input sequences. + * @property {boolean} [add_special_tokens=true] Whether or not to add the special tokens associated with the corresponding model. + * @property {boolean|null} [truncation=null] Whether to truncate the input sequences. + * @property {number|null} [max_length=null] Maximum length of the returned list and optionally padding length. + * @property {TReturnTensor} [return_tensor=true] Whether to return the results as Tensors or arrays. + * @property {boolean|null} [return_token_type_ids=null] Whether to return the token type ids. + */ + +/** + * @typedef {(text: TText, options?: TokenizerCallOptions) => BatchEncoding>} PreTrainedTokenizerCallback + */ + +/** + * @template {boolean} [TTokenize=true] + * @template {boolean} [TReturnTensor=true] + * @template {boolean} [TReturnDict=true] + * @typedef {Object} ApplyChatTemplateOptions + * @property {string|null} [chat_template=null] A Jinja template to use for this conversion. + * @property {Object[]|null} [tools=null] A list of tools (callable functions) that will be accessible to the model. + * @property {Record[]|null} [documents=null] Documents that will be accessible to the model. + * @property {boolean} [add_generation_prompt=false] Whether to end the prompt with the token(s) that indicate the start of an assistant message. + * @property {TTokenize} [tokenize=true] Whether to tokenize the output. If false, the output will be a string. + * @property {boolean} [padding=false] Whether to pad sequences to the maximum length. Has no effect if tokenize is false. + * @property {boolean} [truncation=false] Whether to truncate sequences to the maximum length. Has no effect if tokenize is false. + * @property {number|null} [max_length=null] Maximum length (in tokens) to use for padding or truncation. Has no effect if tokenize is false. + * @property {TReturnTensor} [return_tensor=true] Whether to return the output as a Tensor or an Array. Has no effect if tokenize is false. + * @property {TReturnDict} [return_dict=true] Whether to return a dictionary with named outputs. Has no effect if tokenize is false. + * @property {Object} [tokenizer_kwargs={}] Additional options to pass to the tokenizer. + */ + +/** + * @template {boolean} [TTokenize=true] + * @template {boolean} [TReturnTensor=true] + * @template {boolean} [TReturnDict=true] + * @typedef {TTokenize extends false ? string : TReturnDict extends false ? BatchEncodingItem : BatchEncoding>} ApplyChatTemplateReturn + */ + +export class PreTrainedTokenizer + extends /** @type {new (tokenizerJSON: Object, tokenizerConfig: Object) => PreTrainedTokenizerCallback} */ ( + Callable + ) +{ + return_token_type_ids = false; + + padding_side = 'right'; + /** + * Create a new PreTrainedTokenizer instance. + * @param {Object} tokenizerJSON The JSON of the tokenizer. + * @param {Object} tokenizerConfig The config of the tokenizer. + */ + constructor(tokenizerJSON, tokenizerConfig) { + super(); + + this._tokenizerJSON = tokenizerJSON; + this._tokenizerConfig = tokenizerConfig; + this._tokenizer = new Tokenizer(tokenizerJSON, tokenizerConfig); + + this.config = tokenizerConfig; + + this.padding_side = tokenizerConfig.padding_side ?? this.padding_side; + + // Set mask token if present (otherwise will be undefined, which is fine) + this.mask_token = getTokenFromConfig(tokenizerConfig, 'mask_token'); + this.mask_token_id = this._tokenizer.token_to_id(this.mask_token); + + this.pad_token = getTokenFromConfig(tokenizerConfig, 'pad_token', 'eos_token'); + this.pad_token_id = this._tokenizer.token_to_id(this.pad_token); + + this.sep_token = getTokenFromConfig(tokenizerConfig, 'sep_token'); + this.sep_token_id = this._tokenizer.token_to_id(this.sep_token); + + this.unk_token = getTokenFromConfig(tokenizerConfig, 'unk_token'); + this.unk_token_id = this._tokenizer.token_to_id(this.unk_token); + + this.bos_token = getTokenFromConfig(tokenizerConfig, 'bos_token'); + this.bos_token_id = this._tokenizer.token_to_id(this.bos_token); + + this.eos_token = getTokenFromConfig(tokenizerConfig, 'eos_token'); + this.eos_token_id = this._tokenizer.token_to_id(this.eos_token); + + this.chat_template = tokenizerConfig.chat_template ?? null; + if (Array.isArray(this.chat_template)) { + // Chat templates are stored as lists of dicts with fixed key names, + // we reconstruct that into a single dict while loading them. + const chat_template = Object.create(null); + for (const { name, template } of this.chat_template) { + if (typeof name !== 'string' || typeof template !== 'string') { + throw new Error('Chat template must be a list of objects with "name" and "template" properties'); + } + chat_template[name] = template; + } + this.chat_template = chat_template; + } + this._compiled_template_cache = new Map(); + + const special_tokens = getSpecialTokens(this._tokenizer); + this.all_special_ids = special_tokens.map((t) => t.id); + this.all_special_tokens = special_tokens.map((t) => t.content); + } + + /** + * Loads a pre-trained tokenizer from the given `pretrained_model_name_or_path`. + * + * @param {string} pretrained_model_name_or_path The path to the pre-trained tokenizer. + * @param {PretrainedTokenizerOptions} options Additional options for loading the tokenizer. + * + * @throws {Error} Throws an error if the tokenizer.json or tokenizer_config.json files are not found in the `pretrained_model_name_or_path`. + * @returns {Promise} A new instance of the `PreTrainedTokenizer` class. + */ + static async from_pretrained( + pretrained_model_name_or_path, + { progress_callback = null, config = null, cache_dir = null, local_files_only = false, revision = 'main' } = {}, + ) { + const info = await loadTokenizer(pretrained_model_name_or_path, { + progress_callback, + config, + cache_dir, + local_files_only, + revision, + }); + + // @ts-ignore + return new this(...info); + } + + get_vocab() { + return this._tokenizer.get_vocab(); + } + + get model_max_length() { + return this._tokenizerConfig.model_max_length ?? Infinity; + } + + get add_eos_token() { + return this._tokenizerConfig.add_eos_token; + } + get add_bos_token() { + return this._tokenizerConfig.add_bos_token; + } + + /** + * Converts a token string (or a sequence of tokens) into a single integer id (or a sequence of ids), using the vocabulary. + * + * @template {string|string[]} T + * @param {T} tokens One or several token(s) to convert to token id(s). + * @returns {T extends string ? number : number[]} The token id or list of token ids. + */ + convert_tokens_to_ids(tokens) { + if (typeof tokens === 'string') { + return /** @type {any} */ (this._tokenizer.token_to_id(tokens)); + } else { + return /** @type {any} */ (tokens.map((token) => this._tokenizer.token_to_id(token))); + } + } + + /** + * Encode/tokenize the given text(s). + * @template {string|string[]} TText + * @template {boolean} [TReturnTensor=true] + * @param {TText} text The text to tokenize. + * @param {TokenizerCallOptions} [options] Additional tokenization options. + * @returns {BatchEncoding>} Object to be passed to the model. + */ + _call( + // Required positional arguments + text, + options = {}, + ) { + const { text_pair = null, add_special_tokens = true, padding = false, return_token_type_ids = null } = options; + let { truncation = null, max_length = null } = options; + const return_tensor = /** @type {TReturnTensor} */ (options.return_tensor ?? true); // Different to HF + + const isBatched = Array.isArray(text); + + let encodedTokens; + + if (isBatched) { + if (text.length === 0) { + throw Error('text array must be non-empty'); + } + + if (text_pair !== null) { + if (!Array.isArray(text_pair)) { + throw Error('text_pair must also be an array'); + } else if (text.length !== text_pair.length) { + throw Error('text and text_pair must have the same length'); + } + + encodedTokens = text.map((t, i) => + this._encode_plus(t, { text_pair: text_pair[i], add_special_tokens, return_token_type_ids }), + ); + } else { + encodedTokens = text.map((x) => this._encode_plus(x, { add_special_tokens, return_token_type_ids })); + } + } else { + if (text === null || text === undefined) { + throw Error('text may not be null or undefined'); + } + + if (Array.isArray(text_pair)) { + throw Error( + 'When specifying `text_pair`, since `text` is a string, `text_pair` must also be a string (i.e., not an array).', + ); + } + + // For single input, we just wrap in an array, and then unwrap later. + encodedTokens = [this._encode_plus(text, { text_pair, add_special_tokens, return_token_type_ids })]; + } + // At this point, `encodedTokens` is batched, of shape [batch_size, tokens]. + // However, array may be jagged. So, we may need pad to max_length. + if (max_length === null) { + max_length = this.model_max_length; + } else if (truncation === null) { + if (padding === true) { + logger.warn( + '`max_length` is ignored when `padding: true` and there is no truncation strategy. ' + + "To pad to max length, use `padding: 'max_length'`.", + ); + max_length = this.model_max_length; + } else if (padding === false) { + logger.warn( + 'Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation: true` to explicitly truncate examples to max length.', + ); + truncation = true; + } + } + + // padding: 'max_length' doesn't require any additional calculation + // but padding: true has to calculate max_length from the sequences + if (padding === true) { + max_length = Math.min(max(encodedTokens.map((x) => x.input_ids.length))[0], max_length ?? Infinity); + } + + // Ensure it is less than model max length + max_length = Math.min(max_length, this.model_max_length ?? Infinity); + + if (padding || truncation) { + // Perform padding and/or truncation + for (let i = 0; i < encodedTokens.length; ++i) { + if (encodedTokens[i].input_ids.length === max_length) { + continue; + } else if (encodedTokens[i].input_ids.length > max_length) { + // possibly truncate + if (truncation) { + truncateHelper(encodedTokens[i], max_length); + } + } else { + // t.length < max_length + // possibly pad + if (padding) { + padHelper( + encodedTokens[i], + max_length, + (key) => (key === 'input_ids' ? this.pad_token_id : 0), + this.padding_side, + ); + } + } + } + } + + const result = {}; + + if (return_tensor) { + if (!(padding && truncation)) { + // Not, guaranteed that all items have same length, so + // we perform additional check + + if ( + encodedTokens.some((x) => { + for (const key of Object.keys(x)) { + if (x[key].length !== encodedTokens[0][key]?.length) { + return true; + } + } + return false; + }) + ) { + throw Error( + 'Unable to create tensor, you should probably activate truncation and/or padding ' + + "with 'padding=true' and 'truncation=true' to have batched tensors with the same length.", + ); + } + } + + // Now we actually convert to tensor + // NOTE: In the same way as the python library, we return a batched tensor, regardless of + // whether we have a single input or multiple inputs. + const dims = [encodedTokens.length, encodedTokens[0].input_ids.length]; + + for (const key of Object.keys(encodedTokens[0])) { + result[key] = new Tensor( + 'int64', + BigInt64Array.from(encodedTokens.flatMap((x) => x[key]).map(BigInt)), + dims, + ); + } + } else { + for (const key of Object.keys(encodedTokens[0])) { + result[key] = encodedTokens.map((x) => x[key]); + } + + // If not returning a tensor, we match the input type + if (!isBatched) { + // Input was not batched, so we unwrap + for (const key of Object.keys(result)) { + result[key] = result[key][0]; + } + } + } + + return /** @type {BatchEncoding>} */ (result); + } + + /** + * Encodes a single text using the preprocessor pipeline of the tokenizer. + * + * @param {string|null} text The text to encode. + * @returns {string[]|null} The encoded tokens. + */ + _encode_text(text) { + if (text === null) return null; + return this._tokenizer.encode(text).tokens; + } + + /** + * Encodes a single text or a pair of texts using the model's tokenizer. + * + * @param {string} text The text to encode. + * @param {Object} options An optional object containing the following properties: + * @param {string|null} [options.text_pair=null] The optional second text to encode. + * @param {boolean} [options.add_special_tokens=true] Whether or not to add the special tokens associated with the corresponding model. + * @param {boolean|null} [options.return_token_type_ids=null] Whether to return token_type_ids. + * @returns {{input_ids: number[], attention_mask: number[], token_type_ids?: number[]}} An object containing the encoded text. + * @private + */ + _encode_plus(text, { text_pair = null, add_special_tokens = true, return_token_type_ids = null } = {}) { + const { ids, attention_mask, token_type_ids } = this._tokenizer.encode(text, { + text_pair, + add_special_tokens, + return_token_type_ids: return_token_type_ids ?? this.return_token_type_ids, + }); + return { + input_ids: ids, + attention_mask, + ...(token_type_ids ? { token_type_ids } : {}), + }; + } + + /** + * Converts a string into a sequence of tokens. + * @param {string} text The sequence to be encoded. + * @param {Object} options An optional object containing the following properties: + * @param {string|null} [options.pair] A second sequence to be encoded with the first. + * @param {boolean} [options.add_special_tokens=false] Whether or not to add the special tokens associated with the corresponding model. + * @returns {string[]} The list of tokens. + */ + tokenize(text, { pair = null, add_special_tokens = false } = {}) { + return this._tokenizer.tokenize(text, { text_pair: pair, add_special_tokens }); + } + + /** + * Encodes a single text or a pair of texts using the model's tokenizer. + * + * @param {string} text The text to encode. + * @param {Object} options An optional object containing the following properties: + * @param {string|null} [options.text_pair=null] The optional second text to encode. + * @param {boolean} [options.add_special_tokens=true] Whether or not to add the special tokens associated with the corresponding model. + * @param {boolean|null} [options.return_token_type_ids=null] Whether to return token_type_ids. + * @returns {number[]} An array of token IDs representing the encoded text(s). + */ + encode(text, { text_pair = null, add_special_tokens = true, return_token_type_ids = null } = {}) { + return this._tokenizer.encode(text, { + text_pair, + add_special_tokens, + return_token_type_ids, + }).ids; + } + + /** + * Decode a batch of tokenized sequences. + * @param {number[][]|Tensor} batch List/Tensor of tokenized input sequences. + * @param {Object} decode_args (Optional) Object with decoding arguments. + * @returns {string[]} List of decoded sequences. + */ + batch_decode(batch, decode_args = {}) { + if (batch instanceof Tensor) { + batch = batch.tolist(); + } + return batch.map((x) => this.decode(x, decode_args)); + } + + /** + * Decodes a sequence of token IDs back to a string. + * + * @param {number[]|bigint[]|Tensor} token_ids List/Tensor of token IDs to decode. + * @param {Object} [decode_args={}] + * @param {boolean} [decode_args.skip_special_tokens=false] If true, special tokens are removed from the output string. + * @param {boolean} [decode_args.clean_up_tokenization_spaces=true] If true, spaces before punctuations and abbreviated forms are removed. + * + * @returns {string} The decoded string. + * @throws {Error} If `token_ids` is not a non-empty array of integers. + */ + decode(token_ids, decode_args = {}) { + if (token_ids instanceof Tensor) { + token_ids = prepareTensorForDecode(token_ids); + } + + if (!Array.isArray(token_ids) || token_ids.length === 0 || !isIntegralNumber(token_ids[0])) { + throw Error('token_ids must be a non-empty array of integers.'); + } + + return this.decode_single(token_ids, decode_args); + } + + /** + * Decode a single list of token ids to a string. + * @param {number[]|bigint[]} token_ids List of token ids to decode + * @param {Object} decode_args Optional arguments for decoding + * @param {boolean} [decode_args.skip_special_tokens=false] Whether to skip special tokens during decoding + * @param {boolean|null} [decode_args.clean_up_tokenization_spaces=null] Whether to clean up tokenization spaces during decoding. + * If null, the value is set to `this.decoder.cleanup` if it exists, falling back to `this.clean_up_tokenization_spaces` if it exists, falling back to `true`. + * @returns {string} The decoded string + */ + decode_single(token_ids, { skip_special_tokens = false, clean_up_tokenization_spaces = null }) { + return this._tokenizer.decode(token_ids, { + skip_special_tokens, + clean_up_tokenization_spaces, + }); + } + + /** + * Retrieve the chat template string used for tokenizing chat messages. This template is used + * internally by the `apply_chat_template` method and can also be used externally to retrieve the model's chat + * template for better generation tracking. + * + * @param {Object} options An optional object containing the following properties: + * @param {string|null} [options.chat_template=null] + * A Jinja template or the name of a template to use for this conversion. + * It is usually not necessary to pass anything to this argument, + * as the model's template will be used by default. + * @param {Object[]} [options.tools=null] + * A list of tools (callable functions) that will be accessible to the model. If the template does not + * support function calling, this argument will have no effect. Each tool should be passed as a JSON Schema, + * giving the name, description and argument types for the tool. See our + * [chat templating guide](https://huggingface.co/docs/transformers/main/en/chat_templating#automated-function-conversion-for-tool-use) + * for more information. + * @returns {string} The chat template string. + */ + get_chat_template({ chat_template = null, tools = null } = {}) { + // First, handle the cases when the model has a dict of multiple templates + if (this.chat_template && typeof this.chat_template === 'object') { + const template_dict = this.chat_template; + + if (chat_template !== null && Object.hasOwn(template_dict, chat_template)) { + // The user can pass the name of a template to the chat template argument instead of an entire template + chat_template = template_dict[chat_template]; + } else if (chat_template === null) { + if (tools !== null && 'tool_use' in template_dict) { + chat_template = template_dict['tool_use']; + } else if ('default' in template_dict) { + chat_template = template_dict['default']; + } else { + throw Error( + `This model has multiple chat templates with no default specified! Please either pass a chat ` + + `template or the name of the template you wish to use to the 'chat_template' argument. Available ` + + `template names are ${Object.keys(template_dict).sort()}.`, + ); + } + } + } else if (chat_template === null) { + // These are the cases when the model has a single template + // priority: `chat_template` argument > `tokenizer.chat_template` + if (this.chat_template) { + chat_template = this.chat_template; + } else { + throw Error( + 'Cannot use apply_chat_template() because tokenizer.chat_template is not set and no template ' + + 'argument was passed! For information about writing templates and setting the ' + + 'tokenizer.chat_template attribute, please see the documentation at ' + + 'https://huggingface.co/docs/transformers/main/en/chat_templating', + ); + } + } + return chat_template; + } + + /** + * Converts a list of message objects with `"role"` and `"content"` keys to a list of token + * ids. This method is intended for use with chat models, and will read the tokenizer's chat_template attribute to + * determine the format and control tokens to use when converting. + * + * See [here](https://huggingface.co/docs/transformers/chat_templating) for more information. + * + * **Example:** Applying a chat template to a conversation. + * + * ```javascript + * import { AutoTokenizer } from "@huggingface/transformers"; + * + * const tokenizer = await AutoTokenizer.from_pretrained("Xenova/mistral-tokenizer-v1"); + * + * const chat = [ + * { "role": "user", "content": "Hello, how are you?" }, + * { "role": "assistant", "content": "I'm doing great. How can I help you today?" }, + * { "role": "user", "content": "I'd like to show off how chat templating works!" }, + * ] + * + * const text = tokenizer.apply_chat_template(chat, { tokenize: false }); + * // "[INST] Hello, how are you? [/INST]I'm doing great. How can I help you today? [INST] I'd like to show off how chat templating works! [/INST]" + * + * const input_ids = tokenizer.apply_chat_template(chat, { tokenize: true, return_tensor: false }); + * // [1, 733, 16289, 28793, 22557, 28725, 910, 460, 368, 28804, 733, 28748, 16289, 28793, 28737, 28742, 28719, 2548, 1598, 28723, 1602, 541, 315, 1316, 368, 3154, 28804, 2, 28705, 733, 16289, 28793, 315, 28742, 28715, 737, 298, 1347, 805, 910, 10706, 5752, 1077, 3791, 28808, 733, 28748, 16289, 28793] + * ``` + * + * @param {Message[]} conversation A list of message objects with `"role"` and `"content"` keys, + * representing the chat history so far. + * @template {boolean} [TTokenize=true] + * @template {boolean} [TReturnTensor=true] + * @template {boolean} [TReturnDict=true] + * @param {Object} [options] An optional object containing the following properties: + * @param {string|null} [options.chat_template=null] A Jinja template to use for this conversion. If + * this is not passed, the model's chat template will be used instead. + * @param {Object[]} [options.tools=null] + * A list of tools (callable functions) that will be accessible to the model. If the template does not + * support function calling, this argument will have no effect. Each tool should be passed as a JSON Schema, + * giving the name, description and argument types for the tool. See our + * [chat templating guide](https://huggingface.co/docs/transformers/main/en/chat_templating#automated-function-conversion-for-tool-use) + * for more information. + * @param {Record[]} [options.documents=null] + * A list of dicts representing documents that will be accessible to the model if it is performing RAG + * (retrieval-augmented generation). If the template does not support RAG, this argument will have no + * effect. We recommend that each document should be a dict containing "title" and "text" keys. Please + * see the RAG section of the [chat templating guide](https://huggingface.co/docs/transformers/main/en/chat_templating#arguments-for-RAG) + * for examples of passing documents with chat templates. + * @param {boolean} [options.add_generation_prompt=false] Whether to end the prompt with the token(s) that indicate + * the start of an assistant message. This is useful when you want to generate a response from the model. + * Note that this argument will be passed to the chat template, and so it must be supported in the + * template for this argument to have any effect. + * @param {TTokenize} [options.tokenize=true] Whether to tokenize the output. If false, the output will be a string. + * @param {boolean} [options.padding=false] Whether to pad sequences to the maximum length. Has no effect if tokenize is false. + * @param {boolean} [options.truncation=false] Whether to truncate sequences to the maximum length. Has no effect if tokenize is false. + * @param {number|null} [options.max_length=null] Maximum length (in tokens) to use for padding or truncation. Has no effect if tokenize is false. + * If not specified, the tokenizer's `max_length` attribute will be used as a default. + * @param {TReturnTensor} [options.return_tensor=true] Whether to return the output as a Tensor or an Array. Has no effect if tokenize is false. + * @param {TReturnDict} [options.return_dict=true] Whether to return a dictionary with named outputs. Has no effect if tokenize is false. + * @param {Object} [options.tokenizer_kwargs={}] Additional options to pass to the tokenizer. + * @returns {ApplyChatTemplateReturn} The tokenized output. + */ + apply_chat_template( + conversation, + options = /** @type {ApplyChatTemplateOptions} */ ({}), + ) { + let { + tools = null, + documents = null, + chat_template = null, + add_generation_prompt = false, + tokenize = /** @type {TTokenize} */ (true), + padding = false, + truncation = false, + max_length = null, + return_tensor = /** @type {TReturnTensor} */ (true), + return_dict = /** @type {TReturnDict} */ (true), + tokenizer_kwargs = {}, + ...kwargs + } = options; + chat_template = this.get_chat_template({ chat_template, tools }); + + if (typeof chat_template !== 'string') { + throw Error(`chat_template must be a string, but got ${typeof chat_template}`); + } + + // Compilation function uses a cache to avoid recompiling the same template + /** @type {import('@huggingface/jinja').Template} */ + let compiledTemplate = this._compiled_template_cache.get(chat_template); + if (compiledTemplate === undefined) { + compiledTemplate = new Template(chat_template); + this._compiled_template_cache.set(chat_template, compiledTemplate); + } + + const special_tokens_map = Object.create(null); + for (const key of SPECIAL_TOKEN_ATTRIBUTES) { + const value = getTokenFromConfig(this.config, key); + if (value) { + special_tokens_map[key] = value; + } + } + + const rendered = compiledTemplate.render({ + messages: conversation, + add_generation_prompt, + tools, + documents, + ...special_tokens_map, + ...kwargs, + }); + + if (tokenize) { + const out = this._call(rendered, { + add_special_tokens: false, + padding, + truncation, + max_length, + return_tensor, + ...tokenizer_kwargs, + }); + return /** @type {ApplyChatTemplateReturn} */ ( + return_dict ? out : out.input_ids + ); + } + + return /** @type {ApplyChatTemplateReturn} */ (rendered); + } +} + +/** + * Helper function to build translation inputs for an `NllbTokenizer` or `M2M100Tokenizer`. + * @param {PreTrainedTokenizer} self The tokenizer instance. + * @param {string|string[]} raw_inputs The text to tokenize. + * @param {Object} tokenizer_options Options to be sent to the tokenizer + * @param {Object} generate_kwargs Generation options. + * @returns {Object} Object to be passed to the model. + */ +export function _build_translation_inputs(self, raw_inputs, tokenizer_options, generate_kwargs) { + if (!('language_codes' in self) || !Array.isArray(self.language_codes)) { + throw new Error( + 'Tokenizer must have `language_codes` attribute set and it should be an array of language ids.', + ); + } + if (!('languageRegex' in self) || !(self.languageRegex instanceof RegExp)) { + throw new Error('Tokenizer must have `languageRegex` attribute set and it should be a regular expression.'); + } + if (!('lang_to_token' in self) || typeof self.lang_to_token !== 'function') { + throw new Error('Tokenizer must have `lang_to_token` attribute set and it should be a function.'); + } + const src_lang_token = generate_kwargs.src_lang; + const tgt_lang_token = generate_kwargs.tgt_lang; + + // Check that the target language is valid: + if (!self.language_codes.includes(tgt_lang_token)) { + throw new Error( + `Target language code "${tgt_lang_token}" is not valid. Must be one of: {${self.language_codes.join(', ')}}`, + ); + } + + // Allow `src_lang` to be optional. If not set, we'll use the tokenizer's default. + if (src_lang_token !== undefined) { + // Check that the source language is valid: + if (!self.language_codes.includes(src_lang_token)) { + throw new Error( + `Source language code "${src_lang_token}" is not valid. Must be one of: {${self.language_codes.join(', ')}}`, + ); + } + + // In the same way as the Python library, we override the post-processor + // to force the source language to be first: + for (const item of self._tokenizer.post_processor.config.single) { + if ('SpecialToken' in item && self.languageRegex.test(item.SpecialToken.id)) { + item.SpecialToken.id = self.lang_to_token(src_lang_token); + break; + } + } + // TODO: Do the same for pair? + } + + // Override the `forced_bos_token_id` to force the correct language + generate_kwargs.forced_bos_token_id = self._tokenizer.token_to_id(self.lang_to_token(tgt_lang_token)); + + return self._call(raw_inputs, tokenizer_options); +} diff --git a/packages/transformers/src/transformers.js b/packages/transformers/src/transformers.js new file mode 100644 index 0000000..ef01569 --- /dev/null +++ b/packages/transformers/src/transformers.js @@ -0,0 +1,71 @@ +/** + * @file Entry point for the Transformers.js library. Only the exports from this file + * are available to the end user, and are grouped as follows: + * + * 1. [Environment variables](./env) + * 2. [Pipelines](./pipelines) + * 3. [Models](./models) + * 4. [Tokenizers](./tokenizers) + * 5. [Processors](./processors) + * 6. [Configs](./configs) + * + * @module transformers + */ + +// Environment variables +export { env, LogLevel } from './env.js'; + +// Pipelines +export * from './pipelines.js'; + +// Models +export * from './models/models.js'; +export * from './models/auto/modeling_auto.js'; + +// Tokenizers +export * from './models/tokenizers.js'; +export * from './models/auto/tokenization_auto.js'; + +// Feature Extractors +export * from './models/feature_extractors.js'; +export * from './models/auto/feature_extraction_auto.js'; + +// Image Processors +export * from './models/image_processors.js'; +export * from './models/auto/image_processing_auto.js'; + +// Processors +export * from './models/processors.js'; +export * from './models/auto/processing_auto.js'; + +// Configs +export { PretrainedConfig, AutoConfig } from './configs.js'; + +// Additional exports +export * from './generation/streamers.js'; +export * from './generation/stopping_criteria.js'; +export * from './generation/logits_process.js'; + +export { load_audio, read_audio, RawAudio } from './utils/audio.js'; +export { load_image, RawImage } from './utils/image.js'; +export { load_video, RawVideo, RawVideoFrame } from './utils/video.js'; +export * from './utils/tensor.js'; +export { softmax, log_softmax, dot, cos_sim } from './utils/maths.js'; +export { random } from './utils/random.js'; + +export { DynamicCache } from './cache_utils.js'; + +// Cache and file management +export { ModelRegistry } from './utils/model_registry/ModelRegistry.js'; + +// Expose common types used across the library for developers to access +/** + * @typedef {import('./utils/hub.js').PretrainedModelOptions} PretrainedModelOptions + * @typedef {import('./processing_utils.js').PretrainedProcessorOptions} PretrainedProcessorOptions + * @typedef {import('./tokenization_utils.js').Message} Message + * @typedef {import('./tokenization_utils.js').PretrainedTokenizerOptions} PretrainedTokenizerOptions + * @typedef {import('./utils/dtypes.js').DataType} DataType + * @typedef {import('./utils/devices.js').DeviceType} DeviceType + * @typedef {import('./utils/core.js').ProgressCallback} ProgressCallback + * @typedef {import('./utils/core.js').ProgressInfo} ProgressInfo + */ diff --git a/packages/transformers/src/utils/audio.js b/packages/transformers/src/utils/audio.js new file mode 100644 index 0000000..c2af807 --- /dev/null +++ b/packages/transformers/src/utils/audio.js @@ -0,0 +1,884 @@ +/** + * @file Helper module for audio processing. + * + * These functions and classes are only used internally, + * meaning an end-user shouldn't need to access anything here. + * + * @module utils/audio + */ + +import { getFile } from './hub.js'; +import { FFT, max } from './maths.js'; +import { calculateReflectOffset } from './core.js'; +import { saveBlob } from './io.js'; +import { Tensor, matmul } from './tensor.js'; +import { logger } from './logger.js'; + +/** + * Helper function to load audio from a path/URL. + * @param {string|URL} url The path/URL to load the audio from. + * @param {number} sampling_rate The sampling rate to use when decoding the audio. + * @returns {Promise} The decoded audio as a `Float32Array`. + */ +export async function load_audio(url, sampling_rate) { + if (typeof AudioContext === 'undefined') { + // Running in node or an environment without AudioContext + throw Error( + 'Unable to load audio from path/URL since `AudioContext` is not available in your environment. ' + + 'Instead, audio data should be passed directly to the pipeline/processor. ' + + 'For more information and some example code, see https://huggingface.co/docs/transformers.js/guides/node-audio-processing.', + ); + } + + const response = await (await getFile(url)).arrayBuffer(); + const audioCTX = new AudioContext({ sampleRate: sampling_rate }); + if (typeof sampling_rate === 'undefined') { + logger.warn(`No sampling rate provided, using default of ${audioCTX.sampleRate}Hz.`); + } + const decoded = await audioCTX.decodeAudioData(response); + + /** @type {Float32Array} */ + let audio; + + // We now replicate HuggingFace's `ffmpeg_read` method: + if (decoded.numberOfChannels === 2) { + // When downmixing a stereo audio file to mono using the -ac 1 option in FFmpeg, + // the audio signal is summed across both channels to create a single mono channel. + // However, if the audio is at full scale (i.e. the highest possible volume level), + // the summing of the two channels can cause the audio signal to clip or distort. + + // To prevent this clipping, FFmpeg applies a scaling factor of 1/sqrt(2) (~ 0.707) + // to the audio signal before summing the two channels. This scaling factor ensures + // that the combined audio signal will not exceed the maximum possible level, even + // if both channels are at full scale. + + // After applying this scaling factor, the audio signal from both channels is summed + // to create a single mono channel. It's worth noting that this scaling factor is + // only applied when downmixing stereo audio to mono using the -ac 1 option in FFmpeg. + // If you're using a different downmixing method, or if you're not downmixing the + // audio at all, this scaling factor may not be needed. + const SCALING_FACTOR = Math.sqrt(2); + + const left = decoded.getChannelData(0); + const right = decoded.getChannelData(1); + + audio = new Float32Array(left.length); + for (let i = 0; i < decoded.length; ++i) { + audio[i] = (SCALING_FACTOR * (left[i] + right[i])) / 2; + } + } else { + // If the audio is not stereo, we can just use the first channel: + audio = decoded.getChannelData(0); + } + + return audio; +} + +/** + * @deprecated Use {@link load_audio} instead. + */ +export const read_audio = load_audio; + +/** + * Helper function to generate windows that are special cases of the generalized cosine window. + * See https://www.mathworks.com/help/signal/ug/generalized-cosine-windows.html for more information. + * @param {number} M Number of points in the output window. If zero or less, an empty array is returned. + * @param {number} a_0 Offset for the generalized cosine window. + * @returns {Float64Array} The generated window. + */ +function generalized_cosine_window(M, a_0) { + if (M < 1) { + return new Float64Array(); + } + if (M === 1) { + return new Float64Array([1]); + } + + const a_1 = 1 - a_0; + const factor = (2 * Math.PI) / (M - 1); + + const cos_vals = new Float64Array(M); + for (let i = 0; i < M; ++i) { + cos_vals[i] = a_0 - a_1 * Math.cos(i * factor); + } + return cos_vals; +} + +/** + * Generates a Hanning window of length M. + * See https://numpy.org/doc/stable/reference/generated/numpy.hanning.html for more information. + * + * @param {number} M The length of the Hanning window to generate. + * @returns {Float64Array} The generated Hanning window. + */ +export function hanning(M) { + return generalized_cosine_window(M, 0.5); +} + +/** + * Generates a Hamming window of length M. + * See https://numpy.org/doc/stable/reference/generated/numpy.hamming.html for more information. + * + * @param {number} M The length of the Hamming window to generate. + * @returns {Float64Array} The generated Hamming window. + */ +export function hamming(M) { + return generalized_cosine_window(M, 0.54); +} + +const HERTZ_TO_MEL_MAPPING = { + htk: (/** @type {number} */ freq) => 2595.0 * Math.log10(1.0 + freq / 700.0), + kaldi: (/** @type {number} */ freq) => 1127.0 * Math.log(1.0 + freq / 700.0), + slaney: (/** @type {number} */ freq, min_log_hertz = 1000.0, min_log_mel = 15.0, logstep = 27.0 / Math.log(6.4)) => + freq >= min_log_hertz ? min_log_mel + Math.log(freq / min_log_hertz) * logstep : (3.0 * freq) / 200.0, +}; + +/** + * @template {Float32Array|Float64Array|number} T + * @param {T} freq + * @param {keyof typeof HERTZ_TO_MEL_MAPPING} [mel_scale] + * @returns {T} + */ +function hertz_to_mel(freq, mel_scale = 'htk') { + const fn = HERTZ_TO_MEL_MAPPING[mel_scale]; + if (!fn) { + throw new Error('mel_scale should be one of "htk", "slaney" or "kaldi".'); + } + + // @ts-expect-error ts(2322) + return typeof freq === 'number' ? fn(freq) : freq.map((x) => fn(x)); +} + +const MEL_TO_HERTZ_MAPPING = { + htk: (/** @type {number} */ mels) => 700.0 * (10.0 ** (mels / 2595.0) - 1.0), + kaldi: (/** @type {number} */ mels) => 700.0 * (Math.exp(mels / 1127.0) - 1.0), + slaney: (/** @type {number} */ mels, min_log_hertz = 1000.0, min_log_mel = 15.0, logstep = Math.log(6.4) / 27.0) => + mels >= min_log_mel ? min_log_hertz * Math.exp(logstep * (mels - min_log_mel)) : (200.0 * mels) / 3.0, +}; + +/** + * @template {Float32Array|Float64Array|number} T + * @param {T} mels + * @param {keyof typeof MEL_TO_HERTZ_MAPPING} [mel_scale] + * @returns {T} + */ +function mel_to_hertz(mels, mel_scale = 'htk') { + const fn = MEL_TO_HERTZ_MAPPING[mel_scale]; + if (!fn) { + throw new Error('mel_scale should be one of "htk", "slaney" or "kaldi".'); + } + + // @ts-expect-error ts(2322) + return typeof mels === 'number' ? fn(mels) : mels.map((x) => fn(x)); +} + +/** + * Creates a triangular filter bank. + * + * Adapted from torchaudio and librosa. + * + * @param {Float64Array} fft_freqs Discrete frequencies of the FFT bins in Hz, of shape `(num_frequency_bins,)`. + * @param {Float64Array} filter_freqs Center frequencies of the triangular filters to create, in Hz, of shape `(num_mel_filters,)`. + * @returns {number[][]} of shape `(num_frequency_bins, num_mel_filters)`. + */ +function _create_triangular_filter_bank(fft_freqs, filter_freqs) { + const filter_diff = Float64Array.from( + { length: filter_freqs.length - 1 }, + (_, i) => filter_freqs[i + 1] - filter_freqs[i], + ); + + const slopes = Array.from( + { + length: fft_freqs.length, + }, + () => new Array(filter_freqs.length), + ); + + for (let j = 0; j < fft_freqs.length; ++j) { + const slope = slopes[j]; + for (let i = 0; i < filter_freqs.length; ++i) { + slope[i] = filter_freqs[i] - fft_freqs[j]; + } + } + + const numFreqs = filter_freqs.length - 2; + const ret = Array.from({ length: numFreqs }, () => new Array(fft_freqs.length)); + + for (let j = 0; j < fft_freqs.length; ++j) { + // 201 + const slope = slopes[j]; + for (let i = 0; i < numFreqs; ++i) { + // 80 + const down = -slope[i] / filter_diff[i]; + const up = slope[i + 2] / filter_diff[i + 1]; + ret[i][j] = Math.max(0, Math.min(down, up)); + } + } + return ret; +} + +/** + * Return evenly spaced numbers over a specified interval. + * @param {number} start The starting value of the sequence. + * @param {number} end The end value of the sequence. + * @param {number} num Number of samples to generate. + * @returns `num` evenly spaced samples, calculated over the interval `[start, stop]`. + */ +function linspace(start, end, num) { + const step = (end - start) / (num - 1); + return Float64Array.from({ length: num }, (_, i) => start + step * i); +} + +/** + * Creates a frequency bin conversion matrix used to obtain a mel spectrogram. This is called a *mel filter bank*, and + * various implementation exist, which differ in the number of filters, the shape of the filters, the way the filters + * are spaced, the bandwidth of the filters, and the manner in which the spectrum is warped. The goal of these + * features is to approximate the non-linear human perception of the variation in pitch with respect to the frequency. + * @param {number} num_frequency_bins Number of frequency bins (should be the same as `n_fft // 2 + 1` + * where `n_fft` is the size of the Fourier Transform used to compute the spectrogram). + * @param {number} num_mel_filters Number of mel filters to generate. + * @param {number} min_frequency Lowest frequency of interest in Hz. + * @param {number} max_frequency Highest frequency of interest in Hz. This should not exceed `sampling_rate / 2`. + * @param {number} sampling_rate Sample rate of the audio waveform. + * @param {string|null} [norm] If `"slaney"`, divide the triangular mel weights by the width of the mel band (area normalization). + * @param {keyof typeof MEL_TO_HERTZ_MAPPING} [mel_scale] The mel frequency scale to use, `"htk"` or `"slaney"`. + * @param {boolean} [triangularize_in_mel_space] If this option is enabled, the triangular filter is applied in mel space rather than frequency space. + * This should be set to `true` in order to get the same results as `torchaudio` when computing mel filters. + * @returns {number[][]} Triangular filter bank matrix, which is a 2D array of shape (`num_frequency_bins`, `num_mel_filters`). + * This is a projection matrix to go from a spectrogram to a mel spectrogram. + */ +export function mel_filter_bank( + num_frequency_bins, + num_mel_filters, + min_frequency, + max_frequency, + sampling_rate, + norm = null, + mel_scale = 'htk', + triangularize_in_mel_space = false, +) { + if (norm !== null && norm !== 'slaney') { + throw new Error('norm must be one of null or "slaney"'); + } + + if (num_frequency_bins < 2) { + throw new Error(`Require num_frequency_bins: ${num_frequency_bins} >= 2`); + } + + if (min_frequency > max_frequency) { + throw new Error(`Require min_frequency: ${min_frequency} <= max_frequency: ${max_frequency}`); + } + + const mel_min = hertz_to_mel(min_frequency, mel_scale); + const mel_max = hertz_to_mel(max_frequency, mel_scale); + const mel_freqs = linspace(mel_min, mel_max, num_mel_filters + 2); + + let filter_freqs = mel_to_hertz(mel_freqs, mel_scale); + let fft_freqs; // frequencies of FFT bins in Hz + + if (triangularize_in_mel_space) { + const fft_bin_width = sampling_rate / ((num_frequency_bins - 1) * 2); + fft_freqs = hertz_to_mel( + Float64Array.from({ length: num_frequency_bins }, (_, i) => i * fft_bin_width), + mel_scale, + ); + filter_freqs = mel_freqs; + } else { + fft_freqs = linspace(0, Math.floor(sampling_rate / 2), num_frequency_bins); + } + + const mel_filters = _create_triangular_filter_bank(fft_freqs, filter_freqs); + + if (norm !== null && norm === 'slaney') { + // Slaney-style mel is scaled to be approx constant energy per channel + for (let i = 0; i < num_mel_filters; ++i) { + const filter = mel_filters[i]; + const enorm = 2.0 / (filter_freqs[i + 2] - filter_freqs[i]); + for (let j = 0; j < num_frequency_bins; ++j) { + // Apply this enorm to all frequency bins + filter[j] *= enorm; + } + } + } + + // TODO warn if there is a zero row + + return mel_filters; +} + +/** + * @template {Float32Array|Float64Array} T + * Pads an array with a reflected version of itself on both ends. + * @param {T} array The array to pad. + * @param {number} left The amount of padding to add to the left. + * @param {number} right The amount of padding to add to the right. + * @returns {T} The padded array. + */ +function padReflect(array, left, right) { + // @ts-ignore + const padded = new array.constructor(array.length + left + right); + const w = array.length - 1; + + for (let i = 0; i < array.length; ++i) { + padded[left + i] = array[i]; + } + + for (let i = 1; i <= left; ++i) { + padded[left - i] = array[calculateReflectOffset(i, w)]; + } + + for (let i = 1; i <= right; ++i) { + padded[w + left + i] = array[calculateReflectOffset(w - i, w)]; + } + + return padded; +} + +/** + * Helper function to compute `amplitude_to_db` and `power_to_db`. + * @template {Float32Array|Float64Array} T + * @param {T} spectrogram + * @param {number} factor + * @param {number} reference + * @param {number} min_value + * @param {number} db_range + * @returns {T} + */ +function _db_conversion_helper(spectrogram, factor, reference, min_value, db_range) { + if (reference <= 0) { + throw new Error('reference must be greater than zero'); + } + + if (min_value <= 0) { + throw new Error('min_value must be greater than zero'); + } + + reference = Math.max(min_value, reference); + + const logReference = Math.log10(reference); + for (let i = 0; i < spectrogram.length; ++i) { + spectrogram[i] = factor * Math.log10(Math.max(min_value, spectrogram[i]) - logReference); + } + + if (db_range !== null) { + if (db_range <= 0) { + throw new Error('db_range must be greater than zero'); + } + const maxValue = max(spectrogram)[0] - db_range; + for (let i = 0; i < spectrogram.length; ++i) { + spectrogram[i] = Math.max(spectrogram[i], maxValue); + } + } + + return spectrogram; +} + +/** + * Converts an amplitude spectrogram to the decibel scale. This computes `20 * log10(spectrogram / reference)`, + * using basic logarithm properties for numerical stability. NOTE: Operates in-place. + * + * The motivation behind applying the log function on the (mel) spectrogram is that humans do not hear loudness on a + * linear scale. Generally to double the perceived volume of a sound we need to put 8 times as much energy into it. + * This means that large variations in energy may not sound all that different if the sound is loud to begin with. + * This compression operation makes the (mel) spectrogram features match more closely what humans actually hear. + * + * @template {Float32Array|Float64Array} T + * @param {T} spectrogram The input amplitude (mel) spectrogram. + * @param {number} [reference=1.0] Sets the input spectrogram value that corresponds to 0 dB. + * For example, use `np.max(spectrogram)` to set the loudest part to 0 dB. Must be greater than zero. + * @param {number} [min_value=1e-5] The spectrogram will be clipped to this minimum value before conversion to decibels, + * to avoid taking `log(0)`. The default of `1e-5` corresponds to a minimum of -100 dB. Must be greater than zero. + * @param {number} [db_range=null] Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the + * difference between the peak value and the smallest value will never be more than 80 dB. Must be greater than zero. + * @returns {T} The modified spectrogram in decibels. + */ +function amplitude_to_db(spectrogram, reference = 1.0, min_value = 1e-5, db_range = null) { + return _db_conversion_helper(spectrogram, 20.0, reference, min_value, db_range); +} + +/** + * Converts a power spectrogram to the decibel scale. This computes `10 * log10(spectrogram / reference)`, + * using basic logarithm properties for numerical stability. NOTE: Operates in-place. + * + * The motivation behind applying the log function on the (mel) spectrogram is that humans do not hear loudness on a + * linear scale. Generally to double the perceived volume of a sound we need to put 8 times as much energy into it. + * This means that large variations in energy may not sound all that different if the sound is loud to begin with. + * This compression operation makes the (mel) spectrogram features match more closely what humans actually hear. + * + * Based on the implementation of `librosa.power_to_db`. + * + * @template {Float32Array|Float64Array} T + * @param {T} spectrogram The input power (mel) spectrogram. Note that a power spectrogram has the amplitudes squared! + * @param {number} [reference=1.0] Sets the input spectrogram value that corresponds to 0 dB. + * For example, use `np.max(spectrogram)` to set the loudest part to 0 dB. Must be greater than zero. + * @param {number} [min_value=1e-10] The spectrogram will be clipped to this minimum value before conversion to decibels, + * to avoid taking `log(0)`. The default of `1e-10` corresponds to a minimum of -100 dB. Must be greater than zero. + * @param {number} [db_range=null] Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the + * difference between the peak value and the smallest value will never be more than 80 dB. Must be greater than zero. + * @returns {T} The modified spectrogram in decibels. + */ +function power_to_db(spectrogram, reference = 1.0, min_value = 1e-10, db_range = null) { + return _db_conversion_helper(spectrogram, 10.0, reference, min_value, db_range); +} + +/** + * Calculates a spectrogram over one waveform using the Short-Time Fourier Transform. + * + * This function can create the following kinds of spectrograms: + * - amplitude spectrogram (`power = 1.0`) + * - power spectrogram (`power = 2.0`) + * - complex-valued spectrogram (`power = None`) + * - log spectrogram (use `log_mel` argument) + * - mel spectrogram (provide `mel_filters`) + * - log-mel spectrogram (provide `mel_filters` and `log_mel`) + * + * In this implementation, the window is assumed to be zero-padded to have the same size as the analysis frame. + * A padded window can be obtained from `window_function()`. The FFT input buffer may be larger than the analysis frame, + * typically the next power of two. + * + * @param {Float32Array|Float64Array} waveform The input waveform of shape `(length,)`. This must be a single real-valued, mono waveform. + * @param {Float32Array|Float64Array} window The windowing function to apply of shape `(frame_length,)`, including zero-padding if necessary. The actual window length may be + * shorter than `frame_length`, but we're assuming the array has already been zero-padded. + * @param {number} frame_length The length of the analysis frames in samples (a.k.a., `fft_length`). + * @param {number} hop_length The stride between successive analysis frames in samples. + * @param {Object} options + * @param {number} [options.fft_length=null] The size of the FFT buffer in samples. This determines how many frequency bins the spectrogram will have. + * For optimal speed, this should be a power of two. If `null`, uses `frame_length`. + * @param {number} [options.power=1.0] If 1.0, returns the amplitude spectrogram. If 2.0, returns the power spectrogram. If `null`, returns complex numbers. + * @param {boolean} [options.center=true] Whether to pad the waveform so that frame `t` is centered around time `t * hop_length`. If `false`, frame + * `t` will start at time `t * hop_length`. + * @param {string} [options.pad_mode="reflect"] Padding mode used when `center` is `true`. Possible values are: `"constant"` (pad with zeros), + * `"edge"` (pad with edge values), `"reflect"` (pads with mirrored values). + * @param {boolean} [options.onesided=true] If `true`, only computes the positive frequencies and returns a spectrogram containing `fft_length // 2 + 1` + * frequency bins. If `false`, also computes the negative frequencies and returns `fft_length` frequency bins. + * @param {number} [options.preemphasis=null] Coefficient for a low-pass filter that applies pre-emphasis before the DFT. + * @param {boolean} [options.preemphasis_htk_flavor=true] Whether to apply the pre-emphasis filter in the HTK flavor. + * @param {number[][]} [options.mel_filters=null] The mel filter bank of shape `(num_freq_bins, num_mel_filters)`. + * If supplied, applies this filter bank to create a mel spectrogram. + * @param {number} [options.mel_floor=1e-10] Minimum value of mel frequency banks. + * @param {string} [options.log_mel=null] How to convert the spectrogram to log scale. Possible options are: + * `null` (don't convert), `"log"` (take the natural logarithm), `"log10"` (take the base-10 logarithm), `"dB"` (convert to decibels), + * `"log10_max_norm"` (take `log10`, then apply `(max(x, maxVal - 8) + 4) / 4` normalization, where `maxVal` is computed from data or given by `max_log_mel`). + * Can only be used when `power` is not `null`. + * @param {number} [options.max_log_mel=null] When `log_mel` is `"log10_max_norm"`, use this fixed value as the max instead of computing from data. + * @param {number} [options.reference=1.0] Sets the input spectrogram value that corresponds to 0 dB. For example, use `max(spectrogram)[0]` to set + * the loudest part to 0 dB. Must be greater than zero. + * @param {number} [options.min_value=1e-10] The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking `log(0)`. + * For a power spectrogram, the default of `1e-10` corresponds to a minimum of -100 dB. For an amplitude spectrogram, the value `1e-5` corresponds to -100 dB. + * Must be greater than zero. + * @param {number} [options.db_range=null] Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the + * peak value and the smallest value will never be more than 80 dB. Must be greater than zero. + * @param {boolean} [options.remove_dc_offset=null] Subtract mean from waveform on each frame, applied before pre-emphasis. This should be set to `true` in + * order to get the same results as `torchaudio.compliance.kaldi.fbank` when computing mel filters. + * @param {number} [options.max_num_frames=null] If provided, limits the number of frames to compute to this value. + * @param {number} [options.min_num_frames=null] If provided, ensures the number of frames to compute is at least this value. + * @param {boolean} [options.do_pad=true] If `true`, pads the output spectrogram to have `max_num_frames` frames. + * @param {boolean} [options.transpose=false] If `true`, the returned spectrogram will have shape `(num_frames, num_frequency_bins/num_mel_filters)`. If `false`, the returned spectrogram will have shape `(num_frequency_bins/num_mel_filters, num_frames)`. + * @param {number} [options.mel_offset=0] Offset to add to the mel spectrogram to avoid taking the log of zero. + * @param {string} [options.mel_floor_mode="clamp"] If `mel_offset` is provided, this option determines how to apply it. If `"clamp"`, the mel spectrogram will be clamped to have a minimum value of `mel_offset`. If `"add"`, `mel_offset` will be added to all values of the mel spectrogram. + * @returns {Promise} Spectrogram of shape `(num_frequency_bins, length)` (regular spectrogram) or shape `(num_mel_filters, length)` (mel spectrogram). + */ +export async function spectrogram( + waveform, + window, + frame_length, + hop_length, + { + fft_length = null, + power = 1.0, + center = true, + pad_mode = 'reflect', + onesided = true, + preemphasis = null, + preemphasis_htk_flavor = true, + mel_filters = null, + mel_floor = 1e-10, + log_mel = null, + max_log_mel = null, + reference = 1.0, + min_value = 1e-10, + db_range = null, + remove_dc_offset = null, + + // Custom parameters for efficiency reasons + min_num_frames = null, + max_num_frames = null, + do_pad = true, + transpose = false, + mel_offset = 0, + mel_floor_mode = 'clamp', + } = {}, +) { + const window_length = window.length; + if (fft_length === null) { + fft_length = frame_length; + } + if (frame_length > fft_length) { + throw Error(`frame_length (${frame_length}) may not be larger than fft_length (${fft_length})`); + } + + if (window_length !== frame_length) { + throw new Error(`Length of the window (${window_length}) must equal frame_length (${frame_length})`); + } + + if (hop_length <= 0) { + throw new Error('hop_length must be greater than zero'); + } + + if (power === null && mel_filters !== null) { + throw new Error( + 'You have provided `mel_filters` but `power` is `None`. Mel spectrogram computation is not yet supported for complex-valued spectrogram. ' + + 'Specify `power` to fix this issue.', + ); + } + + if (!preemphasis_htk_flavor) { + throw new Error('`preemphasis_htk_flavor=false` is not currently supported.'); + } + + if (center) { + const padding = Math.floor(frame_length / 2); + switch (pad_mode) { + case 'reflect': { + waveform = padReflect(waveform, padding, padding); + break; + } + case 'constant': { + // @ts-expect-error ts(2351) + const padded = new waveform.constructor(waveform.length + 2 * padding); + padded.set(waveform, padding); + waveform = padded; + break; + } + case 'semicausal': { + // Prepend padding zeros only (no right padding) + // @ts-expect-error ts(2351) + const padded = new waveform.constructor(waveform.length + padding); + padded.set(waveform, padding); + waveform = padded; + break; + } + default: + throw new Error(`pad_mode="${pad_mode}" not implemented yet.`); + } + } + + // split waveform into frames of frame_length size + let num_frames = Math.floor(1 + Math.floor((waveform.length - frame_length) / hop_length)); + if (min_num_frames !== null && num_frames < min_num_frames) { + num_frames = min_num_frames; + } + const num_frequency_bins = onesided ? Math.floor(fft_length / 2) + 1 : fft_length; + + let d1 = num_frames; + let d1Max = num_frames; + + // If maximum number of frames is provided, we must either pad or truncate + if (max_num_frames !== null) { + if (max_num_frames > num_frames) { + // input is too short, so we pad + if (do_pad) { + d1Max = max_num_frames; + } + } else { + // input is too long, so we truncate + d1Max = d1 = max_num_frames; + } + } + + // Preallocate arrays to store output. + const fft = new FFT(fft_length); + const inputBuffer = new Float64Array(fft_length); + const outputBuffer = new Float64Array(fft.outputBufferSize); + const transposedMagnitudeData = new Float32Array(num_frequency_bins * d1Max); + + for (let i = 0; i < d1; ++i) { + // Populate buffer with waveform data + const offset = i * hop_length; + const buffer_size = Math.min(waveform.length - offset, frame_length); + if (buffer_size !== frame_length) { + // The full buffer is not needed, so we need to reset it (avoid overflow from previous iterations) + // NOTE: We don't need to reset the buffer if it's full since we overwrite the first + // `frame_length` values and the rest (`fft_length - frame_length`) remains zero. + inputBuffer.fill(0, 0, frame_length); + } + + for (let j = 0; j < buffer_size; ++j) { + inputBuffer[j] = waveform[offset + j]; + } + + if (remove_dc_offset) { + let sum = 0; + for (let j = 0; j < buffer_size; ++j) { + sum += inputBuffer[j]; + } + const mean = sum / buffer_size; + for (let j = 0; j < buffer_size; ++j) { + inputBuffer[j] -= mean; + } + } + + if (preemphasis !== null) { + // Done in reverse to avoid copies and destructive modification + for (let j = buffer_size - 1; j >= 1; --j) { + inputBuffer[j] -= preemphasis * inputBuffer[j - 1]; + } + inputBuffer[0] *= 1 - preemphasis; + } + + // Apply window function + for (let j = 0; j < window.length; ++j) { + inputBuffer[j] *= window[j]; + } + + fft.realTransform(outputBuffer, inputBuffer); + + // compute magnitudes + for (let j = 0; j < num_frequency_bins; ++j) { + const j2 = j << 1; + + // NOTE: We transpose the data here to avoid doing it later + transposedMagnitudeData[j * d1Max + i] = outputBuffer[j2] ** 2 + outputBuffer[j2 + 1] ** 2; + } + } + + if (power !== null && power !== 2) { + // slight optimization to not sqrt + const pow = power / 2; // we use 2 since we already squared + for (let i = 0; i < transposedMagnitudeData.length; ++i) { + transposedMagnitudeData[i] **= pow; + } + } + + // TODO: What if `mel_filters` is null? + const num_mel_filters = mel_filters.length; + + // Perform matrix muliplication: + // mel_spec = mel_filters @ magnitudes.T + // - mel_filters.shape=(80, 201) + // - magnitudes.shape=(3000, 201) => magnitudes.T.shape=(201, 3000) + // - mel_spec.shape=(80, 3000) + let mel_spec = await matmul( + // TODO: Make `mel_filters` a Tensor during initialization + new Tensor('float32', mel_filters.flat(), [num_mel_filters, num_frequency_bins]), + new Tensor('float32', transposedMagnitudeData, [num_frequency_bins, d1Max]), + ); + if (transpose) { + mel_spec = mel_spec.transpose(1, 0); + } + + const mel_spec_data = /** @type {Float32Array} */ (mel_spec.data); + if (mel_floor_mode === 'add') { + for (let i = 0; i < mel_spec_data.length; ++i) { + mel_spec_data[i] = mel_offset + mel_spec_data[i] + mel_floor; + } + } else { + for (let i = 0; i < mel_spec_data.length; ++i) { + mel_spec_data[i] = mel_offset + Math.max(mel_floor, mel_spec_data[i]); + } + } + + if (power !== null && log_mel !== null) { + const o = Math.min(mel_spec_data.length, d1 * num_mel_filters); + // NOTE: operates in-place + switch (log_mel) { + case 'log': + for (let i = 0; i < o; ++i) { + mel_spec_data[i] = Math.log(mel_spec_data[i]); + } + break; + case 'log10': + for (let i = 0; i < o; ++i) { + mel_spec_data[i] = Math.log10(mel_spec_data[i]); + } + break; + case 'log10_max_norm': { + for (let i = 0; i < o; ++i) { + mel_spec_data[i] = Math.log10(mel_spec_data[i]); + } + const logMax = max_log_mel ?? max(mel_spec_data)[0]; + const threshold = logMax - 8.0; + for (let i = 0; i < o; ++i) { + mel_spec_data[i] = (Math.max(mel_spec_data[i], threshold) + 4.0) / 4.0; + } + break; + } + case 'dB': + if (power === 1.0) { + amplitude_to_db(mel_spec_data, reference, min_value, db_range); + } else if (power === 2.0) { + power_to_db(mel_spec_data, reference, min_value, db_range); + } else { + throw new Error(`Cannot use log_mel option '${log_mel}' with power ${power}`); + } + break; + default: + throw new Error( + `log_mel must be one of null, 'log', 'log10', 'log10_max_norm', or 'dB'. Got '${log_mel}'`, + ); + } + } + + return mel_spec; +} + +/** + * Returns an array containing the specified window. + * @param {number} window_length The length of the window in samples. + * @param {string} name The name of the window function. + * @param {Object} options Additional options. + * @param {boolean} [options.periodic=true] Whether the window is periodic or symmetric. + * @param {number} [options.frame_length=null] The length of the analysis frames in samples. + * Provide a value for `frame_length` if the window is smaller than the frame length, so that it will be zero-padded. + * @param {boolean} [options.center=true] Whether to center the window inside the FFT buffer. Only used when `frame_length` is provided. + * @returns {Float64Array} The window of shape `(window_length,)` or `(frame_length,)`. + */ +export function window_function(window_length, name, { periodic = true, frame_length = null, center = true } = {}) { + const length = periodic ? window_length + 1 : window_length; + let window; + switch (name) { + case 'boxcar': + window = new Float64Array(length).fill(1.0); + break; + case 'hann': + case 'hann_window': + window = hanning(length); + break; + case 'hamming': + window = hamming(length); + break; + case 'povey': + window = hanning(length).map((x) => Math.pow(x, 0.85)); + break; + default: + throw new Error(`Unknown window type ${name}.`); + } + if (periodic) { + window = window.subarray(0, window_length); + } + if (frame_length === null || window_length === frame_length) { + return window; + } + if (window_length > frame_length) { + throw new Error( + `Length of the window (${window_length}) may not be larger than frame_length (${frame_length})`, + ); + } + + const padded = new Float64Array(frame_length); + const offset = center ? Math.floor((frame_length - window_length) / 2) : 0; + padded.set(window, offset); + return padded; +} + +/** + * Efficiently encode audio data to a WAV file. + * WAV file specs : https://en.wikipedia.org/wiki/WAV#WAV_File_header + * + * Adapted from https://www.npmjs.com/package/audiobuffer-to-wav + * @param {Float32Array[]} chunks The audio samples. + * @param {number} rate The sample rate. + * @returns {Blob} The WAV file as a Blob. + */ +function encodeWAV(chunks, rate) { + const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0); + + const buffer = new ArrayBuffer(44); + const view = new DataView(buffer); + + /* RIFF identifier */ + writeString(view, 0, 'RIFF'); + /* RIFF chunk length */ + view.setUint32(4, 36 + totalLength * 4, true); + /* RIFF type */ + writeString(view, 8, 'WAVE'); + /* format chunk identifier */ + writeString(view, 12, 'fmt '); + /* format chunk length */ + view.setUint32(16, 16, true); + /* sample format (raw) */ + view.setUint16(20, 3, true); + /* channel count */ + view.setUint16(22, 1, true); + /* sample rate */ + view.setUint32(24, rate, true); + /* byte rate (sample rate * block align) */ + view.setUint32(28, rate * 4, true); + /* block align (channel count * bytes per sample) */ + view.setUint16(32, 4, true); + /* bits per sample */ + view.setUint16(34, 32, true); + /* data chunk identifier */ + writeString(view, 36, 'data'); + /* data chunk length */ + view.setUint32(40, totalLength * 4, true); + + return new Blob([buffer, ...chunks.map((chunk) => /** @type {ArrayBuffer} */ (chunk.buffer))], { + type: 'audio/wav', + }); +} + +function writeString(view, offset, string) { + for (let i = 0; i < string.length; ++i) { + view.setUint8(offset + i, string.charCodeAt(i)); + } +} + +export class RawAudio { + /** + * Create a new `RawAudio` object. + * @param {Float32Array|Float32Array[]} audio Audio data, either as a single `Float32Array` chunk or multiple `Float32Array` chunks. + * @param {number} sampling_rate Sampling rate of the audio data + */ + constructor(audio, sampling_rate) { + this.audio = audio; + this.sampling_rate = sampling_rate; + } + + /** + * Get the audio data, accumulating all chunks if necessary. + * @returns {Float32Array} The audio data. + */ + get data() { + if (Array.isArray(this.audio)) { + if (this.audio.length === 0) { + return new Float32Array(0); + } + if (this.audio.length === 1) { + return this.audio[0]; + } + // Concatenate all chunks into a single Float32Array + const totalLength = this.audio.reduce((acc, chunk) => acc + chunk.length, 0); + const result = new Float32Array(totalLength); + let offset = 0; + for (const chunk of this.audio) { + result.set(chunk, offset); + offset += chunk.length; + } + return result; + } else { + return this.audio; + } + } + + /** + * Convert the audio to a blob. + * @returns {Blob} + */ + toBlob() { + let audio = this.audio; + if (audio instanceof Float32Array) { + audio = [audio]; // Ensure audio is an array of chunks + } + return encodeWAV(audio, this.sampling_rate); + } + + /** + * Save the audio to a wav file. + * @param {string} path + * @returns {Promise} + */ + async save(path) { + return saveBlob(path, this.toBlob()); + } +} diff --git a/packages/transformers/src/utils/cache.js b/packages/transformers/src/utils/cache.js new file mode 100644 index 0000000..ca8e279 --- /dev/null +++ b/packages/transformers/src/utils/cache.js @@ -0,0 +1,90 @@ +import { apis, env } from '../env.js'; +import { FileCache } from './cache/FileCache.js'; +import { logger } from './logger.js'; +import { CrossOriginStorage } from './cache/CrossOriginStorageCache.js'; + +/** + * @typedef {Object} CacheInterface + * @property {(request: string) => Promise} match + * Checks if a request is in the cache and returns the cached response if found. + * @property {(request: string, response: Response, progress_callback?: (data: {progress: number, loaded: number, total: number}) => void) => Promise} put + * Adds a response to the cache. + * @property {(request: string) => Promise} [delete] + * Deletes a request from the cache. Returns true if deleted, false otherwise. + */ + +/** + * Retrieves an appropriate caching backend based on the environment configuration. + * Attempts to use custom cache, browser cache, or file system cache in that order of priority. + * @returns {Promise} + * @param file_cache_dir {string|null} Path to a directory in which a downloaded pretrained model configuration should be cached if using the file system cache. + */ +export async function getCache(file_cache_dir = null) { + // First, check if the a caching backend is available + // If no caching mechanism available, will download the file every time + let cache = null; + if (env.useCustomCache) { + // Allow the user to specify a custom cache system. + if (!env.customCache) { + throw Error('`env.useCustomCache=true`, but `env.customCache` is not defined.'); + } + + // Check that the required methods are defined: + if (!env.customCache.match || !env.customCache.put) { + throw new Error( + '`env.customCache` must be an object which implements the `match` and `put` functions of the Web Cache API. ' + + 'For more information, see https://developer.mozilla.org/en-US/docs/Web/API/Cache', + ); + } + cache = env.customCache; + } + + if (!cache && env.experimental_useCrossOriginStorage && CrossOriginStorage.isAvailable()) { + cache = new CrossOriginStorage(); + } + + if (!cache && env.useBrowserCache) { + if (typeof caches === 'undefined') { + throw Error('Browser cache is not available in this environment.'); + } + try { + // In some cases, the browser cache may be visible, but not accessible due to security restrictions. + // For example, when running an application in an iframe, if a user attempts to load the page in + // incognito mode, the following error is thrown: `DOMException: Failed to execute 'open' on 'CacheStorage': + // An attempt was made to break through the security policy of the user agent.` + // So, instead of crashing, we just ignore the error and continue without using the cache. + cache = await caches.open(env.cacheKey); + } catch (e) { + logger.warn('An error occurred while opening the browser cache:', e); + } + } + + if (!cache && env.useFSCache) { + if (!apis.IS_FS_AVAILABLE) { + throw Error('File System Cache is not available in this environment.'); + } + + // If `cache_dir` is not specified, use the default cache directory + cache = new FileCache(file_cache_dir ?? env.cacheDir); + } + + return cache; +} + +/** + * Searches the cache for any of the provided names and returns the first match found. + * @param {CacheInterface} cache The cache to search + * @param {...string} names The names of the items to search for + * @returns {Promise} The item from the cache, or undefined if not found. + */ +export async function tryCache(cache, ...names) { + for (let name of names) { + try { + let result = await cache.match(name); + if (result) return result; + } catch (e) { + continue; + } + } + return undefined; +} diff --git a/packages/transformers/src/utils/cache/CrossOriginStorageCache.js b/packages/transformers/src/utils/cache/CrossOriginStorageCache.js new file mode 100644 index 0000000..280f060 --- /dev/null +++ b/packages/transformers/src/utils/cache/CrossOriginStorageCache.js @@ -0,0 +1,252 @@ +/// + +const HASH_ALGORITHM = 'SHA-256'; + +/** + * Name of the Cache API bucket used to persist the url→hash mapping. + */ +const HASH_CACHE_NAME = 'experimental_transformers-hash-cache'; + +/** + * Builds the hash descriptor object expected by the cross-origin storage API. + * + * @param {string} value Hex-encoded SHA-256 hash. + * @returns {{ algorithm: string, value: string }} + */ +const makeHashDescriptor = (value) => ({ algorithm: HASH_ALGORITHM, value }); + +/** + * A cache implementation backed by the experimental `navigator.crossOriginStorage` API, + * which allows sharing cached files (identified by content hash) across origins. + * + * Implements {@link import('../cache.js').CacheInterface}. + * + * @see https://github.com/explainers-by-googlers/cross-origin-storage + */ +export class CrossOriginStorage { + /** @type {Promise | null} */ + #hashCache = null; + + /** + * Returns (and lazily opens) the hash cache, reusing the same promise across concurrent callers. + * @returns {Promise} + */ + _getHashCache = () => { + this.#hashCache ??= caches.open(HASH_CACHE_NAME); + return this.#hashCache; + }; + + /** + * Returns whether the `navigator.crossOriginStorage` API is available in the current environment. + * @returns {boolean} + */ + static isAvailable = () => typeof navigator !== 'undefined' && 'crossOriginStorage' in navigator; + + /** + * Looks up a cached response for the given URL by resolving its SHA-256 hash and requesting + * the corresponding file handle from cross-origin storage. + * + * Implements `CacheInterface.match`. + * + * @param {string} request The URL of the resource to look up. + * @returns {Promise} The cached `Response`, or `undefined` if not found. + */ + match = async (request) => { + const hashValue = await this._getFileHash(request); + if (!hashValue) { + return undefined; + } + try { + const handle = await navigator.crossOriginStorage.requestFileHandle(makeHashDescriptor(hashValue)); + const blob = await handle.getFile(); + return new Response(blob, { + headers: { + 'Content-Length': String(blob.size), + }, + }); + } catch { + return undefined; + } + }; + + /** + * Stores a response in cross-origin storage, keyed by its SHA-256 hash. + * + * For LFS-backed URLs the hash is resolved cheaply via `_getFileHash` (which checks + * `HASH_CACHE_NAME` first, then falls back to fetching the Git LFS pointer file) + * without reading the response body a second time. + * + * For non-LFS resources the hash is unknown upfront. In that case the body is consumed + * in the background: the stream is read to compute the content hash, the file is written + * into cross-origin storage, and the computed hash is persisted to `HASH_CACHE_NAME` + * so that future `match` calls can resolve the file without a network round-trip. + * + * Implements `CacheInterface.put`. + * + * @param {string} request The URL of the resource (used as the hash-cache key). + * @param {Response} response The response whose body will be written to the cache. + * @returns {Promise} + */ + put = async (request, response) => { + const hashValue = await this._getFileHash(request); + + if (hashValue) { + // Fast path: LFS hash already known. Consume the body and store directly. + const blob = await response.blob(); + await this._storeBlobInCOS(blob, hashValue); + } else { + // Slow path: hash unknown. Process in the background so put() returns promptly. + // The caller already holds a reference to the original response; we receive it + // here only to buffer and hash its body. + this._processAndStore(request, response.body); + } + }; + + /** + * Writes a blob into cross-origin storage using the given pre-computed hex hash string. + * + * @param {Blob} blob + * @param {string} hashHex Hex-encoded SHA-256 hash of `blob`. + * @returns {Promise} + */ + _storeBlobInCOS = async (blob, hashHex) => { + const handle = await navigator.crossOriginStorage.requestFileHandle(makeHashDescriptor(hashHex), { + create: true, + origins: '*', + }); + const writableStream = await handle.createWritable(); + await writableStream.write(blob); + await writableStream.close(); + }; + + /** + * Background task for non-LFS resources: consumes `stream`, computes the SHA-256 hash + * of the resulting blob, stores it in cross-origin storage, and persists the computed + * hash to `HASH_CACHE_NAME` keyed by `request` so future `match` calls can resolve the + * file without a network round-trip. + * + * Called fire-and-forget from `put` — errors are swallowed so failures never surface to + * the caller. + * + * @param {string} request The original resource URL. + * @param {ReadableStream} stream The response body stream to consume. + * @returns {Promise} + */ + _processAndStore = async (request, stream) => { + try { + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + const blob = new Blob(chunks); + const hashHex = await this._getBlobHash(blob); + + await this._storeBlobInCOS(blob, hashHex); + + // Persist the computed hash so future match() calls resolve without the network. + try { + const hashCache = await this._getHashCache(); + await hashCache.put(request, new Response(hashHex)); + } catch { + // Cache API unavailable (e.g. non-secure context): COS entry still written. + } + } catch { + // Non-fatal: background store failure must not affect the caller. + } + }; + + /** + * Deletes the cache entry for the given request. + * + * Removes the hash entry from `HASH_CACHE_NAME`. Note: cross-origin storage itself does not + * expose a delete API, so only the local hash mapping is removed. For non-LFS URLs this + * permanently prevents `match` from resolving the file. For LFS-backed URLs, `match` will + * re-fetch the LFS pointer file on the next call and repopulate the hash cache automatically. + * + * Implements `CacheInterface.delete`. + * + * @param {string} request + * @returns {Promise} Resolves to `true` if the hash entry was deleted, `false` otherwise. + */ + delete = async (request) => { + try { + const hashCache = await this._getHashCache(); + return await hashCache.delete(request); + } catch { + return false; + } + }; + + /** + * Resolves the SHA-256 hash for a given URL. + * + * Returns the cached hash immediately if one has been persisted to `HASH_CACHE_NAME`. + * Otherwise falls back to `_getLfsFileHash` to retrieve the hash from the Hugging Face + * LFS pointer file, persisting the result to `HASH_CACHE_NAME` for future lookups. + * + * Returns `null` if the hash cannot be determined (e.g. non-LFS URL with no cached entry). + * + * @param {string} url The resource URL to resolve a hash for. + * @returns {Promise} The hex-encoded SHA-256 hash, or `null` if unavailable. + */ + _getFileHash = async (url) => { + try { + const hashCache = await this._getHashCache(); + const cached = await hashCache.match(url); + if (cached) { + return cached.text(); + } + + const hash = await this._getLfsFileHash(url); + if (hash) { + await hashCache.put(url, new Response(hash)); + return hash; + } + + return null; + } catch { + return null; + } + }; + + /** + * Attempts to retrieve the SHA-256 hash for a Hugging Face resource URL from its raw + * Git LFS pointer file. + * + * Only applicable to URLs containing `/resolve/` (i.e. Hugging Face resolved file URLs). + * The `/resolve/` segment is rewritten to `/raw/` to fetch the LFS pointer directly. + * Returns `null` for non-LFS URLs or when the network request fails. + * + * @see https://huggingface.co/docs/hub/en/storage-backends#xet + * @param {string} url The resolved Hugging Face URL of the resource. + * @returns {Promise} The hex-encoded SHA-256 hash, or `null` if unavailable. + */ + _getLfsFileHash = async (url) => { + if (!url.includes('/resolve/')) { + return null; + } + + const rawUrl = url.replace('/resolve/', '/raw/'); + + try { + const text = await fetch(rawUrl).then((r) => r.text()); + const match = text.match(/^oid sha256:([0-9a-f]+)$/m); + return match ? match[1] : null; + } catch { + return null; + } + }; + + /** + * Computes the SHA-256 hash of a `Blob`'s contents. + * + * @param {Blob} blob The blob to hash. + * @returns {Promise} The lowercase hex-encoded SHA-256 hash. + */ + _getBlobHash = async (blob) => { + const arrayBuffer = await blob.arrayBuffer(); + const hashBuffer = await crypto.subtle.digest(HASH_ALGORITHM, arrayBuffer); + const hashArray = Array.from(new Uint8Array(hashBuffer)); + return hashArray.map((byte) => byte.toString(16).padStart(2, '0')).join(''); + }; +} diff --git a/packages/transformers/src/utils/cache/FileCache.js b/packages/transformers/src/utils/cache/FileCache.js new file mode 100644 index 0000000..9e81d4a --- /dev/null +++ b/packages/transformers/src/utils/cache/FileCache.js @@ -0,0 +1,128 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { FileResponse } from '../hub/FileResponse.js'; +import { Random } from '../random.js'; +import { apis } from '../../env.js'; + +// Create a dedicated random instance for generating unique temporary file names +const rng = new Random(); + +/** + * File system cache implementation that implements the CacheInterface. + * Provides `match` and `put` methods compatible with the Web Cache API. + */ +export class FileCache { + /** + * Instantiate a `FileCache` object. + * @param {string} path + */ + constructor(path) { + this.path = path; + } + + /** + * Checks whether the given request is in the cache. + * @param {string} request + * @returns {Promise} + */ + async match(request) { + let filePath = path.join(this.path, request); + let file = new FileResponse(filePath); + + if (file.exists) { + return file; + } else { + return undefined; + } + } + + /** + * Adds the given response to the cache. + * @param {string} request + * @param {Response} response + * @param {(data: {progress: number, loaded: number, total: number}) => void} [progress_callback] Optional. + * The function to call with progress updates + * @returns {Promise} + */ + async put(request, response, progress_callback = undefined) { + const filePath = path.join(this.path, request); + + // Include both PID and a random suffix so that concurrent put() call within the same process + // (e.g., multiple pipelines loading the same file in parallel) each get their own temp file + // and don't corrupt each other's writes. + const id = apis.IS_PROCESS_AVAILABLE ? process.pid : Date.now(); + const randomSuffix = rng._int32().toString(36); + const tmpPath = filePath + `.tmp.${id}.${randomSuffix}`; + + try { + const contentLength = response.headers.get('Content-Length'); + const total = parseInt(contentLength ?? '0'); + let loaded = 0; + + await fs.promises.mkdir(path.dirname(filePath), { recursive: true }); + const fileStream = fs.createWriteStream(tmpPath); + const reader = response.body.getReader(); + + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + + await new Promise((resolve, reject) => { + fileStream.write(value, (err) => { + if (err) { + reject(err); + return; + } + resolve(); + }); + }); + + loaded += value.length; + const progress = total ? (loaded / total) * 100 : 0; + + progress_callback?.({ progress, loaded, total }); + } + + await new Promise((resolve, reject) => { + fileStream.close((err) => (err ? reject(err) : resolve())); + }); + + // Atomically move the completed temp file to the final path so that + // concurrent readers (other processes or other in-process calls) + // never observe a partially-written file. + await fs.promises.rename(tmpPath, filePath); + } catch (error) { + // Clean up the temp file if an error occurred during download + try { + await fs.promises.unlink(tmpPath); + } catch {} + throw error; + } + } + + /** + * Deletes the cache entry for the given request. + * @param {string} request + * @returns {Promise} A Promise that resolves to `true` if the cache entry was deleted, `false` otherwise. + */ + async delete(request) { + let filePath = path.join(this.path, request); + + try { + await fs.promises.unlink(filePath); + return true; + } catch (error) { + // File doesn't exist or couldn't be deleted + return false; + } + } + + // TODO add the rest? + // addAll(requests: RequestInfo[]): Promise; + // keys(request?: RequestInfo | URL, options?: CacheQueryOptions): Promise>; + // match(request: RequestInfo | URL, options?: CacheQueryOptions): Promise; + // matchAll(request?: RequestInfo | URL, options?: CacheQueryOptions): Promise>; +} diff --git a/packages/transformers/src/utils/cache/cross-origin-storage.d.ts b/packages/transformers/src/utils/cache/cross-origin-storage.d.ts new file mode 100644 index 0000000..b1b7ca1 --- /dev/null +++ b/packages/transformers/src/utils/cache/cross-origin-storage.d.ts @@ -0,0 +1,47 @@ +/** + * Type definitions for the Cross-Origin Storage API + * Source: https://github.com/WICG/cross-origin-storage/blob/main/cross-origin-storage.d.ts + * @see https://github.com/WICG/cross-origin-storage + */ + +/** + * Represents the dictionary for hash algorithm and value. + */ +interface CrossOriginStorageRequestFileHandleHash { + value: string; + algorithm: string; +} + +/** + * Represents the options for requesting a file handle. + */ +interface CrossOriginStorageRequestFileHandleOptions { + create?: boolean; + /** + * Restricts (or opens up) which origins may later read the stored file. + * `'*'` makes the resource available to all origins; an array of origin + * strings restricts it to that set; omitting the field keeps the + * default (same-site only). Visibility can be upgraded but never + * downgraded. + * @see https://github.com/WICG/cross-origin-storage#example-restricting-resources-to-specific-origins + */ + origins?: '*' | string[]; +} + +/** + * The CrossOriginStorageManager interface. + * [SecureContext] + */ +interface CrossOriginStorageManager { + requestFileHandle( + hash: CrossOriginStorageRequestFileHandleHash, + options?: CrossOriginStorageRequestFileHandleOptions, + ): Promise; +} + +/** + * Augment the standard Navigator interface. + */ +interface Navigator { + readonly crossOriginStorage: CrossOriginStorageManager; +} diff --git a/packages/transformers/src/utils/constants.js b/packages/transformers/src/utils/constants.js new file mode 100644 index 0000000..2acf16c --- /dev/null +++ b/packages/transformers/src/utils/constants.js @@ -0,0 +1,8 @@ +export const GITHUB_ISSUE_URL = 'https://github.com/huggingface/transformers.js/issues/new/choose'; + +export const CONFIG_NAME = 'config.json'; +export const FEATURE_EXTRACTOR_NAME = 'preprocessor_config.json'; +export const IMAGE_PROCESSOR_NAME = FEATURE_EXTRACTOR_NAME; +export const PROCESSOR_NAME = 'processor_config.json'; +export const CHAT_TEMPLATE_NAME = 'chat_template.jinja'; +export const GENERATION_CONFIG_NAME = 'generation_config.json'; diff --git a/packages/transformers/src/utils/core.js b/packages/transformers/src/utils/core.js new file mode 100644 index 0000000..3304d5f --- /dev/null +++ b/packages/transformers/src/utils/core.js @@ -0,0 +1,304 @@ +/** + * @file Core utility functions/classes for Transformers.js. + * + * These are only used internally, meaning an end-user shouldn't + * need to access anything here. + * + * @module utils/core + */ + +import { Callable } from './generic.js'; + +/** + * @typedef {Object} InitiateProgressInfo + * @property {'initiate'} status + * @property {string} name The model id or directory path. + * @property {string} file The name of the file. + */ + +/** + * @typedef {Object} DownloadProgressInfo + * @property {'download'} status + * @property {string} name The model id or directory path. + * @property {string} file The name of the file. + */ + +/** + * @typedef {Object} ProgressStatusInfo + * @property {'progress'} status + * @property {string} name The model id or directory path. + * @property {string} file The name of the file. + * @property {number} progress A number between 0 and 100. + * @property {number} loaded The number of bytes loaded. + * @property {number} total The total number of bytes to be loaded. + */ + +/** + * @typedef {Object} FileLoadingProgress + * @property {number} loaded The number of bytes loaded for this file. + * @property {number} total The total number of bytes for this file. + */ + +/** + * @typedef {Record} FilesLoadingMap + * A mapping of file names to their loading progress. Each key is a file path and each value contains + * the loaded and total bytes for that file. + */ + +/** + * @typedef {Object} TotalProgressInfo + * @property {'progress_total'} status + * @property {string} name The model id or directory path. + * @property {number} progress A number between 0 and 100. + * @property {number} loaded The number of bytes loaded. + * @property {number} total The total number of bytes to be loaded. + * @property {FilesLoadingMap} files A mapping of file names to their loading progress. + */ + +/** + * @typedef {Object} DoneProgressInfo + * @property {'done'} status + * @property {string} name The model id or directory path. + * @property {string} file The name of the file. + */ + +/** + * @typedef {Object} ReadyProgressInfo + * @property {'ready'} status + * @property {string} task The loaded task. + * @property {string} model The loaded model. + */ + +/** + * @typedef {InitiateProgressInfo | DownloadProgressInfo | ProgressStatusInfo | DoneProgressInfo | ReadyProgressInfo | TotalProgressInfo} ProgressInfo + */ + +/** + * A callback function that is called with progress information. + * @callback ProgressCallback + * @param {ProgressInfo} progressInfo + * @returns {void} + */ + +/** + * Helper function to dispatch progress callbacks. + * + * @param {ProgressCallback | null | undefined} progress_callback The progress callback function to dispatch. + * @param {ProgressInfo} data The data to pass to the progress callback function. + * @returns {void} + * @private + */ +export function dispatchCallback(progress_callback, data) { + if (progress_callback) progress_callback(data); +} + +/** + * A callable progress callback that wraps an original callback and emits + * aggregate `progress_total` events. Because it extends `Callable`, instances + * can be passed directly wherever a plain callback function is expected. + * + * Callers can check `callback instanceof DefaultProgressCallback` to avoid + * double-wrapping when both `pipeline()` and `from_pretrained()` would + * otherwise each add their own wrapper. + */ +export class DefaultProgressCallback extends Callable { + /** + * @param {ProgressCallback} callback The original callback. + * @param {FilesLoadingMap} files_loading Mutable map storing per-file progress. + */ + constructor(callback, files_loading) { + super(); + this.callback = callback; + this.files_loading = files_loading; + /** @type {Map>} Pending and completed file loads, used to deduplicate work within a single pipeline() call. */ + this.loads = new Map(); + } + + /** + * @param {ProgressInfo} info + */ + _call(info) { + if (info.status === 'progress') { + this.files_loading[info.file] = { + loaded: info.loaded, + total: info.total, + }; + + const loaded = Object.values(this.files_loading).reduce((acc, curr) => acc + curr.loaded, 0); + const total = Object.values(this.files_loading).reduce((acc, curr) => acc + curr.total, 0); + const progress = total > 0 ? (loaded / total) * 100 : 0; + + this.callback({ + status: 'progress_total', + name: info.name, + progress, + loaded, + total, + files: structuredClone(this.files_loading), + }); + } + this.callback(info); + } +} + +/** + * Reverses the keys and values of an object. + * + * @param {Object} data The object to reverse. + * @returns {Object} The reversed object. + * @see https://ultimatecourses.com/blog/reverse-object-keys-and-values-in-javascript + */ +export function reverseDictionary(data) { + // https://ultimatecourses.com/blog/reverse-object-keys-and-values-in-javascript + return Object.fromEntries(Object.entries(data).map(([key, value]) => [value, key])); +} + +/** + * Escapes regular expression special characters from a string by replacing them with their escaped counterparts. + * + * @param {string} string The string to escape. + * @returns {string} The escaped string. + */ +export function escapeRegExp(string) { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string +} + +/** + * Check if a value is a typed array. + * @param {*} val The value to check. + * @returns {boolean} True if the value is a `TypedArray`, false otherwise. + * + * Adapted from https://stackoverflow.com/a/71091338/13989043 + */ +export function isTypedArray(val) { + return val?.prototype?.__proto__?.constructor?.name === 'TypedArray'; +} + +/** + * Check if a value is an integer. + * @param {*} x The value to check. + * @returns {boolean} True if the value is a string, false otherwise. + */ +export function isIntegralNumber(x) { + return Number.isInteger(x) || typeof x === 'bigint'; +} + +/** + * Determine if a provided width or height is nullish. + * @param {*} x The value to check. + * @returns {boolean} True if the value is `null`, `undefined` or `-1`, false otherwise. + */ +export function isNullishDimension(x) { + return x === null || x === undefined || x === -1; +} + +/** + * Calculates the dimensions of a nested array. + * + * @param {any[]} arr The nested array to calculate dimensions for. + * @returns {number[]} An array containing the dimensions of the input array. + */ +export function calculateDimensions(arr) { + const dimensions = []; + let current = arr; + while (Array.isArray(current)) { + dimensions.push(current.length); + current = current[0]; + } + return dimensions; +} + +/** + * Replicate python's .pop() method for objects. + * @param {Object} obj The object to pop from. + * @param {string} key The key to pop. + * @param {*} defaultValue The default value to return if the key does not exist. + * @returns {*} The value of the popped key. + * @throws {Error} If the key does not exist and no default value is provided. + */ +export function pop(obj, key, defaultValue = undefined) { + const value = obj[key]; + if (value !== undefined) { + delete obj[key]; + return value; + } + if (defaultValue === undefined) { + throw Error(`Key ${key} does not exist in object.`); + } + return defaultValue; +} + +/** + * Efficiently merge arrays, creating a new copy. + * Adapted from https://stackoverflow.com/a/6768642/13989043 + * @param {any[]} arrs Arrays to merge. + * @returns {any[]} The merged array. + */ +export function mergeArrays(...arrs) { + return Array.prototype.concat.apply([], arrs); +} + +/** + * Compute the Cartesian product of given arrays + * @param {...any[]} a Arrays to compute the product + * @returns {any[]} Returns the computed Cartesian product as an array + * @private + */ +export function product(...a) { + // Cartesian product of items + // Adapted from https://stackoverflow.com/a/43053803 + return a.reduce((a, b) => a.flatMap((d) => b.map((e) => [d, e]))); +} + +/** + * Calculates the index offset for a given index and window size. + * @param {number} i The index. + * @param {number} w The window size. + * @returns {number} The index offset. + */ +export function calculateReflectOffset(i, w) { + return Math.abs(((i + w) % (2 * w)) - w); +} + +/** + * + * @param {Object} o + * @param {string[]} props + * @returns {Object} + */ +export function pick(o, props) { + return Object.assign( + {}, + ...props.map((prop) => { + if (o[prop] !== undefined) { + return { [prop]: o[prop] }; + } + }), + ); +} + +/** + * Calculate the length of a string, taking multi-byte characters into account. + * This mimics the behavior of Python's `len` function. + * @param {string} s The string to calculate the length of. + * @returns {number} The length of the string. + */ +export function len(s) { + let length = 0; + for (const c of s) ++length; + return length; +} + +/** + * Count the occurrences of a value in an array or string. + * This mimics the behavior of Python's `count` method. + * @param {any[]|string} arr The array or string to search. + * @param {any} value The value to count. + */ +export function count(arr, value) { + let count = 0; + for (const v of arr) { + if (v === value) ++count; + } + return count; +} diff --git a/packages/transformers/src/utils/devices.js b/packages/transformers/src/utils/devices.js new file mode 100644 index 0000000..613e7e5 --- /dev/null +++ b/packages/transformers/src/utils/devices.js @@ -0,0 +1,44 @@ +import { apis } from '../env.js'; + +/** + * The list of devices supported by Transformers.js + */ +export const DEVICE_TYPES = Object.freeze({ + auto: 'auto', // Auto-detect based on device and environment + gpu: 'gpu', // Auto-detect GPU + cpu: 'cpu', // CPU + wasm: 'wasm', // WebAssembly + webgpu: 'webgpu', // WebGPU + cuda: 'cuda', // CUDA + dml: 'dml', // DirectML + coreml: 'coreml', // CoreML + + webnn: 'webnn', // WebNN (default) + 'webnn-npu': 'webnn-npu', // WebNN NPU + 'webnn-gpu': 'webnn-gpu', // WebNN GPU + 'webnn-cpu': 'webnn-cpu', // WebNN CPU +}); +const DEFAULT_DEVICE = apis.IS_NODE_ENV ? 'cpu' : 'wasm'; + +/** + * @typedef {keyof typeof DEVICE_TYPES} DeviceType + */ + +/** + * Resolves a device configuration value to a concrete device string. + * Handles string, per-file object, or undefined config, with a default fallback. + * @param {string|Record|null|undefined} deviceConfig The device config value. + * @param {string} fileName The model file name to look up if deviceConfig is an object. + * @param {Object} [options] + * @param {(message: string) => void} [options.warn] Optional callback invoked when deviceConfig is a per-file object but fileName is not found. + * @returns {string} The resolved device string. + */ +export function selectDevice(deviceConfig, fileName, { warn } = {}) { + if (!deviceConfig) return DEFAULT_DEVICE; + if (typeof deviceConfig === 'string') return deviceConfig; + if (deviceConfig.hasOwnProperty(fileName)) return deviceConfig[fileName]; + if (warn) { + warn(`device not specified for "${fileName}". Using the default device (${DEFAULT_DEVICE}).`); + } + return DEFAULT_DEVICE; +} diff --git a/packages/transformers/src/utils/dtypes.js b/packages/transformers/src/utils/dtypes.js new file mode 100644 index 0000000..ff026f7 --- /dev/null +++ b/packages/transformers/src/utils/dtypes.js @@ -0,0 +1,146 @@ +/// + +import { apis } from '../env.js'; + +import { DEVICE_TYPES } from './devices.js'; + +// TODO: Use the adapter from `env.backends.onnx.webgpu.adapter` to check for `shader-f16` support, +// when available in https://github.com/microsoft/onnxruntime/pull/19940. +// For more information, see https://github.com/microsoft/onnxruntime/pull/19857#issuecomment-1999984753 + +/** + * Checks if WebGPU fp16 support is available in the current environment. + */ +export const isWebGpuFp16Supported = (function () { + /** @type {boolean} */ + let cachedResult; + + return async function () { + if (cachedResult === undefined) { + if (!apis.IS_WEBGPU_AVAILABLE) { + cachedResult = false; + } else { + try { + const adapter = await navigator.gpu.requestAdapter(); + cachedResult = adapter.features.has('shader-f16'); + } catch (e) { + cachedResult = false; + } + } + } + return cachedResult; + }; +})(); + +export const DATA_TYPES = Object.freeze({ + auto: 'auto', // Auto-detect based on environment + fp32: 'fp32', + fp16: 'fp16', + q8: 'q8', + int8: 'int8', + uint8: 'uint8', + q4: 'q4', + bnb4: 'bnb4', + q4f16: 'q4f16', // fp16 model with 4-bit block weight quantization + q2: 'q2', + q2f16: 'q2f16', // fp16 model with 2-bit block weight quantization + q1: 'q1', + q1f16: 'q1f16', // fp16 model with 1-bit block weight quantization +}); +/** @typedef {keyof typeof DATA_TYPES} DataType */ + +export const DEFAULT_DEVICE_DTYPE = DATA_TYPES.fp32; +export const DEFAULT_DEVICE_DTYPE_MAPPING = Object.freeze({ + // NOTE: If not specified, will default to fp32 + [DEVICE_TYPES.wasm]: DATA_TYPES.q8, +}); + +/** @type {Record, string>} */ +export const DEFAULT_DTYPE_SUFFIX_MAPPING = Object.freeze({ + [DATA_TYPES.fp32]: '', + [DATA_TYPES.fp16]: '_fp16', + [DATA_TYPES.int8]: '_int8', + [DATA_TYPES.uint8]: '_uint8', + [DATA_TYPES.q8]: '_quantized', + [DATA_TYPES.q4]: '_q4', + [DATA_TYPES.q2]: '_q2', + [DATA_TYPES.q1]: '_q1', + [DATA_TYPES.q4f16]: '_q4f16', + [DATA_TYPES.q2f16]: '_q2f16', + [DATA_TYPES.q1f16]: '_q1f16', + [DATA_TYPES.bnb4]: '_bnb4', +}); + +/** + * Resolves a dtype configuration value to a concrete dtype string. + * Handles string, per-file object, and "auto" forms with device-based fallback. + * @param {DataType|Record|null|undefined} dtype The dtype config value. + * @param {string} fileName The model file name to look up if dtype is an object. + * @param {string} selectedDevice The resolved device string for fallback. + * @param {Object} [options] + * @param {DataType|Record|null} [options.configDtype=null] Config dtype used as fallback when dtype is "auto" (supports device_config overlay in session.js). + * @param {(message: string) => void} [options.warn] Optional callback invoked when dtype is a per-file object but fileName is not found. + * @returns {DataType} The resolved dtype string. + */ +export function selectDtype(dtype, fileName, selectedDevice, { configDtype = null, warn } = {}) { + /** @type {string|null|undefined} */ + let resolved; + let needsWarn = false; + if (dtype && typeof dtype !== 'string') { + if (dtype.hasOwnProperty(fileName)) { + resolved = dtype[fileName]; + } else { + resolved = null; + needsWarn = true; + } + } else { + resolved = /** @type {string|null|undefined} */ (dtype); + } + + /** @type {DataType} */ + let result; + + // Handle 'auto': try configDtype fallback + if (resolved === DATA_TYPES.auto) { + if (configDtype) { + const fallback = typeof configDtype === 'string' ? configDtype : configDtype?.[fileName]; + if (fallback && fallback !== DATA_TYPES.auto && DATA_TYPES.hasOwnProperty(fallback)) { + return /** @type {DataType} */ (fallback); + } + } + result = DEFAULT_DEVICE_DTYPE_MAPPING[selectedDevice] ?? DEFAULT_DEVICE_DTYPE; + } else if (resolved && DATA_TYPES.hasOwnProperty(resolved)) { + // Valid known dtype + result = /** @type {DataType} */ (resolved); + } else { + // Fallback to device default + result = DEFAULT_DEVICE_DTYPE_MAPPING[selectedDevice] ?? DEFAULT_DEVICE_DTYPE; + } + + if (needsWarn && warn) { + warn( + `dtype not specified for "${fileName}". Using the default dtype (${result}) for this device (${selectedDevice}).`, + ); + } + return result; +} + +export const DataTypeMap = Object.freeze({ + float32: Float32Array, + // @ts-ignore ts(2552) Limited availability of Float16Array across browsers: + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Float16Array + float16: typeof Float16Array !== 'undefined' ? Float16Array : Uint16Array, + float64: Float64Array, + string: Array, // string[] + int8: Int8Array, + uint8: Uint8Array, + int16: Int16Array, + uint16: Uint16Array, + int32: Int32Array, + uint32: Uint32Array, + int64: BigInt64Array, + uint64: BigUint64Array, + bool: Uint8Array, + uint4: Uint8Array, + int4: Int8Array, +}); diff --git a/packages/transformers/src/utils/generic.js b/packages/transformers/src/utils/generic.js new file mode 100644 index 0000000..3e8fc29 --- /dev/null +++ b/packages/transformers/src/utils/generic.js @@ -0,0 +1,36 @@ +/** + * A base class for creating callable objects. + * See [here](https://stackoverflow.com/q/76073890) for more information. + * + * @type {new () => {(...args: any[]): any, _call(...args: any[]): any}} + */ +export const Callable = /** @type {any} */ ( + class { + /** + * Creates a new instance of the Callable class. + */ + constructor() { + /** + * Creates a closure that delegates to a private method '_call' with the given arguments. + * @type {any} + * @param {...any} args Zero or more arguments to pass to the '_call' method. + * @returns {*} The result of calling the '_call' method. + */ + let closure = function (...args) { + return closure._call(...args); + }; + return Object.setPrototypeOf(closure, new.target.prototype); + } + + /** + * This method should be implemented in subclasses to provide the + * functionality of the callable object. + * + * @param {any[]} args + * @throws {Error} If the subclass does not implement the `_call` method. + */ + _call(...args) { + throw Error('Must implement _call method in subclass'); + } + } +); diff --git a/packages/transformers/src/utils/hub.js b/packages/transformers/src/utils/hub.js new file mode 100755 index 0000000..09f42e1 --- /dev/null +++ b/packages/transformers/src/utils/hub.js @@ -0,0 +1,580 @@ +/** + * @file Utility functions to interact with the Hugging Face Hub (https://huggingface.co/models) + * + * @module utils/hub + */ + +import { apis, env } from '../env.js'; +import { DefaultProgressCallback, dispatchCallback } from './core.js'; +import { FileResponse } from './hub/FileResponse.js'; +import { FileCache } from './cache/FileCache.js'; +import { + handleError, + isValidUrl, + pathJoin, + isValidHfModelId, + makePretrainedOptionsKey, + readResponse, +} from './hub/utils.js'; +import { getCache, tryCache } from './cache.js'; +import { get_file_metadata } from './model_registry/get_file_metadata.js'; +import { logger } from './logger.js'; + +export { MAX_EXTERNAL_DATA_CHUNKS } from './hub/constants.js'; + +/** + * @typedef {boolean|number} ExternalData + * Specifies whether to load the model using the external data format. + * - `false`: Do not use external data format + * - `true`: Use external data format with 1 chunk + * - `number`: Use external data format with the specified number of chunks + */ + +/** + * @typedef {Object} PretrainedOptions Options for loading a pretrained model. + * @property {import('./core.js').ProgressCallback} [progress_callback=null] If specified, this function will be called during model construction, to provide the user with progress updates. + * @property {import('../configs.js').PretrainedConfig} [config=null] Configuration for the model to use instead of an automatically loaded configuration. Configuration can be automatically loaded when: + * - The model is a model provided by the library (loaded with the *model id* string of a pretrained model). + * - The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a configuration JSON file named *config.json* is found in the directory. + * @property {string} [cache_dir=null] Path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used. + * @property {boolean} [local_files_only=false] Whether or not to only look at local files (e.g., not try downloading the model). + * @property {string} [revision='main'] The specific model version to use. It can be a branch name, a tag name, or a commit id, + * since we use a git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git. + * NOTE: This setting is ignored for local requests. + */ + +/** + * @typedef {Object} ModelSpecificPretrainedOptions Options for loading a pretrained model. + * @property {string} [subfolder='onnx'] In case the relevant files are located inside a subfolder of the model repo on huggingface.co, + * you can specify the folder name here. + * @property {string} [model_file_name=null] If specified, load the model with this name (excluding the dtype and .onnx suffixes). Currently only valid for encoder- or decoder-only models. + * @property {import("./devices.js").DeviceType|Record} [device=null] The device to run the model on. If not specified, the device will be chosen from the environment settings. + * @property {import("./dtypes.js").DataType|Record} [dtype=null] The data type to use for the model. If not specified, the data type will be chosen from the environment settings. + * @property {ExternalData|Record} [use_external_data_format=false] Whether to load the model using the external data format (used for models >= 2GB in size). + * @property {import('onnxruntime-common').InferenceSession.SessionOptions} [session_options] (Optional) User-specified session options passed to the runtime. If not provided, suitable defaults will be chosen. + */ + +/** + * @typedef {PretrainedOptions & ModelSpecificPretrainedOptions} PretrainedModelOptions Options for loading a pretrained model. + */ + +/** + * Helper function to get a file, using either the Fetch API or FileSystem API. + * + * @param {URL|string} urlOrPath The URL/path of the file to get. + * @returns {Promise} A promise that resolves to a FileResponse object (if the file is retrieved using the FileSystem API), or a Response object (if the file is retrieved using the Fetch API). + */ +export async function getFile(urlOrPath) { + if (env.useFS && !isValidUrl(urlOrPath, ['http:', 'https:', 'blob:'])) { + return new FileResponse( + urlOrPath instanceof URL + ? urlOrPath.protocol === 'file:' + ? urlOrPath.pathname + : urlOrPath.toString() + : urlOrPath, + ); + } else { + return env.fetch(urlOrPath, { + headers: getFetchHeaders(urlOrPath), + }); + } +} + +/** + * Generates appropriate HTTP headers for fetching resources. + * In Node.js environments, adds User-Agent and Authorization headers when applicable. + * In browser environments, returns minimal headers for security. + * + * @param {URL|string} urlOrPath The URL or path being fetched. + * @returns {Headers} A Headers object with appropriate headers for the request. + */ +export function getFetchHeaders(urlOrPath) { + const isNode = typeof process !== 'undefined' && process?.release?.name === 'node'; + const headers = new Headers(); + + if (isNode) { + const IS_CI = !!process.env?.TESTING_REMOTELY; + const version = env.version; + headers.set('User-Agent', `transformers.js/${version}; is_ci/${IS_CI};`); + + const isHFURL = isValidUrl(urlOrPath, ['http:', 'https:'], ['huggingface.co', 'hf.co']); + if (isHFURL) { + // If an access token is present in the environment variables, + // we add it to the request headers. + // NOTE: We keep `HF_ACCESS_TOKEN` for backwards compatibility (as a fallback). + const token = process.env?.HF_TOKEN ?? process.env?.HF_ACCESS_TOKEN; + if (token) { + headers.set('Authorization', `Bearer ${token}`); + } + } + } else { + // Running in a browser-environment, so we use default headers + // NOTE: We do not allow passing authorization headers in the browser, + // since this would require exposing the token to the client. + } + + return headers; +} + +/** + * Builds the resource paths and URLs for a model file. + * Can be used to get the resource URL or path without loading the file. + * + * @param {string} path_or_repo_id This can be either: + * - a string, the *model id* of a model repo on huggingface.co. + * - a path to a *directory* potentially containing the file. + * @param {string} filename The name of the file to locate. + * @param {PretrainedOptions} [options] An object containing optional parameters. + * @param {import('./cache.js').CacheInterface | null} [cache] The cache instance to use for determining cache keys. + * @returns {{ requestURL: string, localPath: string, remoteURL: string, proposedCacheKey: string, validModelId: boolean }} + * An object containing all the paths and URLs for the resource. + */ +export function buildResourcePaths(path_or_repo_id, filename, options = {}, cache = null) { + const revision = options.revision ?? 'main'; + const requestURL = pathJoin(path_or_repo_id, filename); + + const validModelId = isValidHfModelId(path_or_repo_id); + const localPath = validModelId ? pathJoin(env.localModelPath, requestURL) : requestURL; + const remoteURL = pathJoin( + env.remoteHost, + env.remotePathTemplate + .replaceAll('{model}', path_or_repo_id) + .replaceAll('{revision}', encodeURIComponent(revision)), + filename, + ); + + const proposedCacheKey = + cache instanceof FileCache + ? // Choose cache key for filesystem cache + // When using the main revision (default), we use the request URL as the cache key. + // If a specific revision is requested, we account for this in the cache key. + revision === 'main' + ? requestURL + : pathJoin(path_or_repo_id, revision, filename) + : remoteURL; + + return { + requestURL, + localPath, + remoteURL, + proposedCacheKey, + validModelId, + }; +} + +/** + * Checks if a resource exists in cache. + * + * @param {import('./cache.js').CacheInterface | null} cache The cache instance to check. + * @param {string} localPath The local path to try first. + * @param {string} proposedCacheKey The proposed cache key to try second. + * @returns {Promise} + * The cached response if found, undefined otherwise. + */ +export async function checkCachedResource(cache, localPath, proposedCacheKey) { + if (!cache) { + return undefined; + } + + // A caching system is available, so we try to get the file from it. + // 1. We first try to get from cache using the local path. In some environments (like deno), + // non-URL cache keys are not allowed. In these cases, `response` will be undefined. + // 2. If no response is found, we try to get from cache using the remote URL or file system cache. + return await tryCache(cache, localPath, proposedCacheKey); +} + +/** + * Stores a resource in the cache. + * + * @param {string} path_or_repo_id The path or repo ID of the model. + * @param {string} filename The name of the file to cache. + * @param {import('./cache.js').CacheInterface} cache The cache instance to store in. + * @param {string} cacheKey The cache key to use. + * @param {Response|import('./hub/FileResponse.js').FileResponse} response The response to cache. + * @param {Uint8Array} [result] The result buffer if already read. + * @param {PretrainedOptions} [options] Options containing progress callback and context for progress updates. + * @returns {Promise} + */ +export async function storeCachedResource(path_or_repo_id, filename, cache, cacheKey, response, result, options = {}) { + // Check again whether request is in cache. If not, we add the response to the cache + if ((await cache.match(cacheKey)) !== undefined) { + return; + } + + if (!result) { + // We haven't yet read the response body, so we need to do so now. + // Ensure progress updates include consistent metadata. + const wrapped_progress = options.progress_callback + ? (data) => + dispatchCallback(options.progress_callback, { + status: 'progress', + name: path_or_repo_id, + file: filename, + ...data, + }) + : undefined; + await cache.put(cacheKey, /** @type {Response} */ (response), wrapped_progress); + } else if (typeof response !== 'string') { + // NOTE: We use `new Response(buffer, ...)` instead of `response.clone()` to handle LFS files + // Explicitly set content-length from the buffer size, since the browser Cache API may strip it. + const headers = new Headers(response.headers); + headers.set('content-length', result.byteLength.toString()); + await cache + .put( + cacheKey, + new Response(/** @type {any} */ (result), { + headers, + }), + ) + .catch((err) => { + // Do not crash if unable to add to cache (e.g., QuotaExceededError). + // Rather, log a warning and proceed with execution. + logger.warn(`Unable to add response to browser cache: ${err}.`); + }); + } +} + +/** + * Loads a resource file from local or remote sources. + * + * @param {string} path_or_repo_id This can be either: + * - a string, the *model id* of a model repo on huggingface.co. + * - a path to a *directory* potentially containing the file. + * @param {string} filename The name of the file to locate. + * @param {boolean} [fatal=true] Whether to throw an error if the file is not found. + * @param {PretrainedOptions} [options] An object containing optional parameters. + * @param {boolean} [return_path=false] Whether to return the path of the file instead of the file content. + * @param {import('./cache.js').CacheInterface | null} [cache] The cache instance to use. + * + * @throws Will throw an error if the file is not found and `fatal` is true. + * @returns {Promise} A Promise that resolves with the file content as a Uint8Array if `return_path` is false, or the file path as a string if `return_path` is true. + */ +export async function loadResourceFile( + path_or_repo_id, + filename, + fatal = true, + options = {}, + return_path = false, + cache = null, +) { + const { requestURL, localPath, remoteURL, proposedCacheKey, validModelId } = buildResourcePaths( + path_or_repo_id, + filename, + options, + cache, + ); + + /** @type {string} */ + let cacheKey; + + // Whether to cache the final response in the end. + let toCacheResponse = false; + + /** @type {Response|import('./hub/FileResponse.js').FileResponse|undefined|string} */ + let response; + + // Check cache + response = await checkCachedResource(cache, localPath, proposedCacheKey); + + const cacheHit = response !== undefined; + if (cacheHit) { + cacheKey = proposedCacheKey; + } else { + // Caching not available, or file is not cached, so we perform the request + + if (env.allowLocalModels) { + // Accessing local models is enabled, so we try to get the file locally. + // If request is a valid HTTP URL, we skip the local file check. Otherwise, we try to get the file locally. + const isURL = isValidUrl(requestURL, ['http:', 'https:']); + if (!isURL) { + try { + response = await getFile(localPath); + cacheKey = localPath; // Update the cache key to be the local path + } catch (e) { + // Something went wrong while trying to get the file locally. + // NOTE: error handling is done in the next step (since `response` will be undefined) + logger.warn(`Unable to load from local path "${localPath}": "${e}"`); + } + } else if (options.local_files_only) { + throw new Error(`\`local_files_only=true\`, but attempted to load a remote file from: ${requestURL}.`); + } else if (!env.allowRemoteModels) { + throw new Error( + `\`env.allowRemoteModels=false\`, but attempted to load a remote file from: ${requestURL}.`, + ); + } + } + + if (response === undefined || (typeof response !== 'string' && response.status === 404)) { + // File not found locally. This means either: + // - The user has disabled local file access (`env.allowLocalModels=false`) + // - the path is a valid HTTP url (`response === undefined`) + // - the path is not a valid HTTP url and the file is not present on the file system or local server (`response.status === 404`) + + if (options.local_files_only || !env.allowRemoteModels) { + // User requested local files only, but the file is not found locally. + if (fatal) { + throw Error( + `\`local_files_only=true\` or \`env.allowRemoteModels=false\` and file was not found locally at "${localPath}".`, + ); + } else { + // File not found, but this file is optional. + // TODO in future, cache the response? + return null; + } + } + if (!validModelId) { + // Before making any requests to the remote server, we check if the model ID is valid. + // This prevents unnecessary network requests for invalid model IDs. + throw Error( + `Local file missing at "${localPath}" and download aborted due to invalid model ID "${path_or_repo_id}".`, + ); + } + + // File not found locally, so we try to download it from the remote server + response = await getFile(remoteURL); + + if (response.status !== 200) { + return handleError(response.status, remoteURL, fatal); + } + + // Success! We use the proposed cache key from earlier + cacheKey = proposedCacheKey; + } + + // Only cache the response if: + toCacheResponse = + cache && // 1. A caching system is available + typeof Response !== 'undefined' && // 2. `Response` is defined (i.e., we are in a browser-like environment) + response instanceof Response && // 3. result is a `Response` object (i.e., not a `FileResponse`) + response.status === 200; // 4. request was successful (status code 200) + } + + // Start downloading + dispatchCallback(options.progress_callback, { + status: 'download', + name: path_or_repo_id, + file: filename, + }); + + let result; + if (apis.IS_NODE_ENV && return_path) { + // In Node.js with return_path, we skip the buffer read (ONNX runtime + // loads from disk directly). A completion progress event is emitted + // after the caching block below to ensure progress_total reaches 100%. + } else { + /** @type {Uint8Array} */ + let buffer; + + if (typeof response !== 'string') { + if (!options.progress_callback) { + // If no progress callback is specified, we can use the `.arrayBuffer()` + // method to read the response. + buffer = new Uint8Array(await response.arrayBuffer()); + } else if ( + cacheHit && // The item is being read from the cache + typeof navigator !== 'undefined' && + /firefox/i.test(navigator.userAgent) // We are in Firefox + ) { + // Due to bug in Firefox, we cannot display progress when loading from cache. + // Fortunately, since this should be instantaneous, this should not impact users too much. + buffer = new Uint8Array(await response.arrayBuffer()); + + // For completeness, we still fire the final progress callback + dispatchCallback(options.progress_callback, { + status: 'progress', + name: path_or_repo_id, + file: filename, + progress: 100, + loaded: buffer.length, + total: buffer.length, + }); + } else { + // Get expected file size from response headers or metadata + // This helps with progress tracking when content-length is missing + let expectedSize; + const contentLength = response.headers.get('content-length'); + if (contentLength) { + expectedSize = parseInt(contentLength, 10); + } else { + // Try to get size from metadata (useful when content-length is missing during download) + try { + const metadata = await get_file_metadata(path_or_repo_id, filename, options); + if (metadata.size) { + expectedSize = metadata.size; + } + } catch (e) { + // Ignore metadata fetch errors + } + } + + buffer = await readResponse( + response, + (data) => { + dispatchCallback(options.progress_callback, { + status: 'progress', + name: path_or_repo_id, + file: filename, + ...data, + }); + }, + expectedSize, + ); + } + } + result = buffer; + } + + if ( + // Only cache web responses + // i.e., do not cache FileResponses (prevents duplication) + toCacheResponse && + cacheKey && + typeof response !== 'string' + ) { + await storeCachedResource(path_or_repo_id, filename, cache, cacheKey, response, result, options); + } + + // In Node.js with return_path, the buffer read is skipped so no progress + // events are emitted during loading. Emit a final completion event so + // that aggregate progress_total tracking reaches 100%. This is placed + // after storeCachedResource so it doesn't conflict with caching progress. + if (apis.IS_NODE_ENV && return_path && options.progress_callback && typeof response !== 'string') { + const size = parseInt(response.headers.get('content-length'), 10) || 0; + dispatchCallback(options.progress_callback, { + status: 'progress', + name: path_or_repo_id, + file: filename, + progress: 100, + loaded: size, + total: size, + }); + } + + dispatchCallback(options.progress_callback, { + status: 'done', + name: path_or_repo_id, + file: filename, + }); + + if (result) { + if (!apis.IS_NODE_ENV && return_path) { + throw new Error('Cannot return path in a browser environment.'); + } + return result; + } + if (response instanceof FileResponse) { + return response.filePath; + } + + // Otherwise, return the cached response (most likely a `FileResponse`). + // NOTE: A custom cache may return a Response, or a string (file path) + const cachedResponse = await cache?.match(cacheKey); + if (cachedResponse instanceof FileResponse) { + return cachedResponse.filePath; + } else if (cachedResponse instanceof Response) { + return new Uint8Array(await cachedResponse.arrayBuffer()); + } else if (typeof cachedResponse === 'string') { + return cachedResponse; + } + + throw new Error('Unable to get model file path or buffer.'); +} + +/** @type {Map>} Pending file loads keyed by resource identity. */ +const INFLIGHT_LOADS = new Map(); + +/** + * Retrieves a file from either a remote URL using the Fetch API or from the local file system using the FileSystem API. + * If the filesystem is available and `env.useCache = true`, the file will be downloaded and cached. + * + * @param {string} path_or_repo_id This can be either: + * - a string, the *model id* of a model repo on huggingface.co. + * - a path to a *directory* potentially containing the file. + * @param {string} filename The name of the file to locate in `path_or_repo`. + * @param {boolean} [fatal=true] Whether to throw an error if the file is not found. + * @param {PretrainedOptions} [options] An object containing optional parameters. + * @param {boolean} [return_path=false] Whether to return the path of the file instead of the file content. + * + * @throws Will throw an error if the file is not found and `fatal` is true. + * @returns {Promise} A Promise that resolves with the file content as a Uint8Array if `return_path` is false, or the file path as a string if `return_path` is true. + */ +export async function getModelFile(path_or_repo_id, filename, fatal = true, options = {}, return_path = false) { + if (!env.allowLocalModels) { + // User has disabled local models, so we just make sure other settings are correct. + + if (options.local_files_only) { + throw Error( + 'Invalid configuration detected: local models are disabled (`env.allowLocalModels=false`) but you have requested to only use local models (`local_files_only=true`).', + ); + } else if (!env.allowRemoteModels) { + throw Error( + 'Invalid configuration detected: both local and remote models are disabled. Fix by setting `env.allowLocalModels` or `env.allowRemoteModels` to `true`.', + ); + } + } + + // Deduplicate loads of the same file. When an aggregate progress callback + // is active, scope dedup to its `loads` map so concurrent and sequential + // sibling calls within one pipeline() share a single download (and a + // single `initiate` event). Otherwise fall back to the global in-flight + // map for concurrent dedup, with entries cleared on settle. + const key = makePretrainedOptionsKey(path_or_repo_id, options, filename, fatal, return_path); + const { progress_callback } = options; + const loads = progress_callback instanceof DefaultProgressCallback ? progress_callback.loads : INFLIGHT_LOADS; + let pending = loads.get(key); + if (!pending) { + dispatchCallback(progress_callback, { + status: 'initiate', + name: path_or_repo_id, + file: filename, + }); + pending = getCache(options.cache_dir).then((cache) => + loadResourceFile(path_or_repo_id, filename, fatal, options, return_path, cache), + ); + if (loads === INFLIGHT_LOADS) { + pending = pending.finally(() => INFLIGHT_LOADS.delete(key)); + } + loads.set(key, pending); + } + return await pending; +} + +/** + * Fetches a text file from a given path and file name. + * + * @param {string} modelPath The path to the directory containing the file. + * @param {string} fileName The name of the file to fetch. + * @param {boolean} [fatal=true] Whether to throw an error if the file is not found. + * @param {PretrainedOptions} [options] An object containing optional parameters. + * @returns {Promise} The text content of the file. + * @throws Will throw an error if the file is not found and `fatal` is true. + */ +export async function getModelText(modelPath, fileName, fatal = true, options = {}) { + const buffer = await getModelFile(modelPath, fileName, fatal, options, false); + if (buffer === null) { + return null; + } + + const decoder = new TextDecoder('utf-8'); + return decoder.decode(/** @type {Uint8Array} */ (buffer)); +} + +/** + * Fetches a JSON file from a given path and file name. + * + * @param {string} modelPath The path to the directory containing the file. + * @param {string} fileName The name of the file to fetch. + * @param {boolean} [fatal=true] Whether to throw an error if the file is not found. + * @param {PretrainedOptions} [options] An object containing optional parameters. + * @returns {Promise} The JSON data parsed into a JavaScript object. + * @throws Will throw an error if the file is not found and `fatal` is true. + */ +export async function getModelJSON(modelPath, fileName, fatal = true, options = {}) { + const text = await getModelText(modelPath, fileName, fatal, options); + if (text === null) { + // Return empty object + return {}; + } + + return JSON.parse(text); +} diff --git a/packages/transformers/src/utils/hub/FileResponse.js b/packages/transformers/src/utils/hub/FileResponse.js new file mode 100644 index 0000000..d0102dd --- /dev/null +++ b/packages/transformers/src/utils/hub/FileResponse.js @@ -0,0 +1,121 @@ +import fs from 'node:fs'; + +/** + * Mapping from file extensions to MIME types. + */ +const CONTENT_TYPE_MAP = { + txt: 'text/plain', + html: 'text/html', + css: 'text/css', + js: 'text/javascript', + json: 'application/json', + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + gif: 'image/gif', +}; + +export class FileResponse { + /** + * Creates a new `FileResponse` object. + * @param {string} filePath + */ + constructor(filePath) { + this.filePath = filePath; + this.headers = new Headers(); + + this.exists = fs.existsSync(filePath); + if (this.exists) { + this.status = 200; + this.statusText = 'OK'; + + let stats = fs.statSync(filePath); + this.headers.set('content-length', stats.size.toString()); + + this.updateContentType(); + + const stream = fs.createReadStream(filePath); + this.body = new ReadableStream({ + start(controller) { + stream.on('data', (chunk) => controller.enqueue(chunk)); + stream.on('end', () => controller.close()); + stream.on('error', (err) => controller.error(err)); + }, + cancel() { + stream.destroy(); + }, + }); + } else { + this.status = 404; + this.statusText = 'Not Found'; + this.body = null; + } + } + + /** + * Updates the 'content-type' header property of the response based on the extension of + * the file specified by the filePath property of the current object. + * @returns {void} + */ + updateContentType() { + // Set content-type header based on file extension + const extension = this.filePath.toString().split('.').pop().toLowerCase(); + this.headers.set('content-type', CONTENT_TYPE_MAP[extension] ?? 'application/octet-stream'); + } + + /** + * Clone the current FileResponse object. + * @returns {FileResponse} A new FileResponse object with the same properties as the current object. + */ + clone() { + let response = new FileResponse(this.filePath); + response.exists = this.exists; + response.status = this.status; + response.statusText = this.statusText; + response.headers = new Headers(this.headers); + return response; + } + + /** + * Reads the contents of the file specified by the filePath property and returns a Promise that + * resolves with an ArrayBuffer containing the file's contents. + * @returns {Promise} A Promise that resolves with an ArrayBuffer containing the file's contents. + * @throws {Error} If the file cannot be read. + */ + async arrayBuffer() { + const data = await fs.promises.readFile(this.filePath); + return /** @type {ArrayBuffer} */ (data.buffer); + } + + /** + * Reads the contents of the file specified by the filePath property and returns a Promise that + * resolves with a Blob containing the file's contents. + * @returns {Promise} A Promise that resolves with a Blob containing the file's contents. + * @throws {Error} If the file cannot be read. + */ + async blob() { + const data = await fs.promises.readFile(this.filePath); + return new Blob([/** @type {any} */ (data)], { type: this.headers.get('content-type') }); + } + + /** + * Reads the contents of the file specified by the filePath property and returns a Promise that + * resolves with a string containing the file's contents. + * @returns {Promise} A Promise that resolves with a string containing the file's contents. + * @throws {Error} If the file cannot be read. + */ + async text() { + return await fs.promises.readFile(this.filePath, 'utf8'); + } + + /** + * Reads the contents of the file specified by the filePath property and returns a Promise that + * resolves with a parsed JavaScript object containing the file's contents. + * + * @returns {Promise} A Promise that resolves with a parsed JavaScript object containing the file's contents. + * @throws {Error} If the file cannot be read. + */ + async json() { + return JSON.parse(await this.text()); + } +} diff --git a/packages/transformers/src/utils/hub/constants.js b/packages/transformers/src/utils/hub/constants.js new file mode 100644 index 0000000..b87c7e5 --- /dev/null +++ b/packages/transformers/src/utils/hub/constants.js @@ -0,0 +1,18 @@ +export const ERROR_MAPPING = { + // 4xx errors (https://developer.mozilla.org/en-US/docs/Web/HTTP/Status#client_error_responses) + 400: 'Bad request error occurred while trying to load file', + 401: 'Unauthorized access to file', + 403: 'Forbidden access to file', + 404: 'Could not locate file', + 408: 'Request timeout error occurred while trying to load file', + + // 5xx errors (https://developer.mozilla.org/en-US/docs/Web/HTTP/Status#server_error_responses) + 500: 'Internal server error error occurred while trying to load file', + 502: 'Bad gateway error occurred while trying to load file', + 503: 'Service unavailable error occurred while trying to load file', + 504: 'Gateway timeout error occurred while trying to load file', +}; + +export const MAX_EXTERNAL_DATA_CHUNKS = 100; + +export const REPO_ID_REGEX = /^(\b[\w\-.]+\b\/)?\b[\w\-.]{1,96}\b$/; diff --git a/packages/transformers/src/utils/hub/utils.js b/packages/transformers/src/utils/hub/utils.js new file mode 100644 index 0000000..3cb1431 --- /dev/null +++ b/packages/transformers/src/utils/hub/utils.js @@ -0,0 +1,189 @@ +import { ERROR_MAPPING, REPO_ID_REGEX } from './constants.js'; +import { logger } from '../logger.js'; + +/** + * Joins multiple parts of a path into a single path, while handling leading and trailing slashes. + * + * @param {...string} parts Multiple parts of a path. + * @returns {string} A string representing the joined path. + */ +export function pathJoin(...parts) { + // https://stackoverflow.com/a/55142565 + parts = parts.map((part, index) => { + if (index) { + part = part.replace(new RegExp('^/'), ''); + } + if (index !== parts.length - 1) { + part = part.replace(new RegExp('/$'), ''); + } + return part; + }); + return parts.join('/'); +} + +/** + * Determines whether the given string is a valid URL. + * @param {string|URL} string The string to test for validity as an URL. + * @param {string[]} [protocols=null] A list of valid protocols. If specified, the protocol must be in this list. + * @param {string[]} [validHosts=null] A list of valid hostnames. If specified, the URL's hostname must be in this list. + * @returns {boolean} True if the string is a valid URL, false otherwise. + */ +export function isValidUrl(string, protocols = null, validHosts = null) { + let url; + try { + url = new URL(string); + } catch (_) { + return false; + } + if (protocols && !protocols.includes(url.protocol)) { + return false; + } + if (validHosts && !validHosts.includes(url.hostname)) { + return false; + } + return true; +} + +/** + * Tests whether a string is a valid Hugging Face model ID or not. + * Adapted from https://github.com/huggingface/huggingface_hub/blob/6378820ebb03f071988a96c7f3268f5bdf8f9449/src/huggingface_hub/utils/_validators.py#L119-L170 + * + * @param {string} string The string to test + * @returns {boolean} True if the string is a valid model ID, false otherwise. + */ +export function isValidHfModelId(string) { + if (!REPO_ID_REGEX.test(string)) return false; + if (string.includes('..') || string.includes('--')) return false; + if (string.endsWith('.git') || string.endsWith('.ipynb')) return false; + return true; +} + +/** + * Builds a stable key for memoizing model/revision/cache scoped work. + * + * @param {string} model_id Model ID or local model path. + * @param {Object} [options] Pretrained loading options. + * @param {string} [options.revision='main'] Model revision. + * @param {string|null} [options.cache_dir=null] Custom cache directory. + * @param {boolean} [options.local_files_only=false] Whether to avoid remote lookups. + * @param {...unknown} parts Additional key parts for the specific operation. + * @returns {string} + */ +export function makePretrainedOptionsKey(model_id, options = {}, ...parts) { + return JSON.stringify([ + model_id, + options.revision ?? 'main', + options.cache_dir ?? null, + options.local_files_only ?? false, + ...parts, + ]); +} + +/** + * Helper method to handle fatal errors that occur while trying to load a file from the Hugging Face Hub. + * @param {number} status The HTTP status code of the error. + * @param {string} remoteURL The URL of the file that could not be loaded. + * @param {boolean} fatal Whether to raise an error if the file could not be loaded. + * @returns {null} Returns `null` if `fatal = true`. + * @throws {Error} If `fatal = false`. + */ +export function handleError(status, remoteURL, fatal) { + if (!fatal) { + // File was not loaded correctly, but it is optional. + // TODO in future, cache the response? + return null; + } + + const message = ERROR_MAPPING[status] ?? `Error (${status}) occurred while trying to load file`; + throw Error(`${message}: "${remoteURL}".`); +} + +/** + * Read and track progress when reading a Response object + * + * @param {Response|import('./FileResponse.js').FileResponse} response The Response object to read + * @param {(data: {progress: number, loaded: number, total: number}) => void} progress_callback The function to call with progress updates + * @param {number} [expectedSize] The expected size of the file (used when content-length header is missing) + * @returns {Promise} A Promise that resolves with the Uint8Array buffer + */ +export async function readResponse(response, progress_callback, expectedSize) { + const contentLength = response.headers.get('Content-Length'); + + // Use content-length if available, otherwise fall back to expectedSize (from metadata) + let total = contentLength ? parseInt(contentLength, 10) : (expectedSize ?? 0); + + if (contentLength === null && !expectedSize) { + logger.warn('Unable to determine content-length from response headers. Will expand buffer when needed.'); + } + + let buffer = new Uint8Array(total); + let loaded = 0; + + const reader = response.body.getReader(); + async function read() { + const { done, value } = await reader.read(); + if (done) return; + + const newLoaded = loaded + value.length; + if (newLoaded > total) { + total = newLoaded; + + // Adding the new data will overflow buffer. + // In this case, we extend the buffer + const newBuffer = new Uint8Array(total); + + // copy contents + newBuffer.set(buffer); + + buffer = newBuffer; + } + buffer.set(value, loaded); + loaded = newLoaded; + + const progress = (loaded / total) * 100; + + // Call your function here + progress_callback({ progress, loaded, total }); + + return read(); + } + + // Actually read + await read(); + + return buffer; +} + +/** + * Checks if the given URL is a blob URL (created via URL.createObjectURL). + * Blob URLs should not be cached as they are temporary in-memory references. + * @param {string} url - The URL to check. + * @returns {boolean} True if the URL is a blob URL, false otherwise. + */ +export function isBlobURL(url) { + return isValidUrl(url, ['blob:']); +} + +/** + * Converts any URL to an absolute URL if needed. + * If the URL is already absolute (http://, https://, or blob:), returns it unchanged (handled by new URL(...)). + * Otherwise, resolves it relative to the current page location (browser) or module location (Node/Bun/Deno). + * @param {string} url - The URL to convert (can be relative or absolute). + * @returns {string} The absolute URL. + */ +export function toAbsoluteURL(url) { + let baseURL; + + if (typeof location !== 'undefined' && location.href) { + // Browser environment: use location.href + baseURL = location.href; + } else if (typeof import.meta !== 'undefined' && import.meta.url) { + // Node.js/Bun/Deno module environment: use import.meta.url + baseURL = import.meta.url; + } else { + // Fallback: if no base is available, return the URL unchanged + return url; + } + + return new URL(url, baseURL).href; +} diff --git a/packages/transformers/src/utils/image.js b/packages/transformers/src/utils/image.js new file mode 100644 index 0000000..d7d8c7f --- /dev/null +++ b/packages/transformers/src/utils/image.js @@ -0,0 +1,818 @@ +/** + * @file Helper module for image processing. + * + * These functions and classes are only used internally, + * meaning an end-user shouldn't need to access anything here. + * + * @module utils/image + */ + +import { isNullishDimension } from './core.js'; +import { getFile } from './hub.js'; +import { apis } from '../env.js'; +import { Tensor } from './tensor.js'; +import { saveBlob } from './io.js'; + +// Will be empty (or not used) if running in browser or web-worker +import sharp from 'sharp'; +import { logger } from './logger.js'; + +let createCanvasFunction; +let ImageDataClass; +let loadImageFunction; +if (apis.IS_WEB_ENV) { + // Running in browser or web-worker + createCanvasFunction = (/** @type {number} */ width, /** @type {number} */ height) => { + if (!self.OffscreenCanvas) { + throw new Error('OffscreenCanvas not supported by this environment.'); + } + return new self.OffscreenCanvas(width, height); + }; + loadImageFunction = self.createImageBitmap; + ImageDataClass = self.ImageData; +} else if (sharp) { + // Running in Node.js, electron, or other non-browser environment + + loadImageFunction = async (/**@type {sharp.Sharp}*/ img) => { + const metadata = await img.metadata(); + const rawChannels = metadata.channels; + + const { data, info } = await img.rotate().raw().toBuffer({ resolveWithObject: true }); + + const newImage = new RawImage(new Uint8ClampedArray(data), info.width, info.height, info.channels); + if (rawChannels !== undefined && rawChannels !== info.channels) { + // Make sure the new image has the same number of channels as the input image. + // This is necessary for grayscale images. + newImage.convert(rawChannels); + } + return newImage; + }; +} else { + throw new Error('Unable to load image processing library.'); +} + +// Defined here: https://github.com/python-pillow/Pillow/blob/a405e8406b83f8bfb8916e93971edc7407b8b1ff/src/libImaging/Imaging.h#L262-L268 +const RESAMPLING_MAPPING = { + 0: 'nearest', + 1: 'lanczos', + 2: 'bilinear', + 3: 'bicubic', + 4: 'box', + 5: 'hamming', +}; + +/** + * Mapping from file extensions to MIME types. + */ +const CONTENT_TYPE_MAP = new Map([ + ['png', 'image/png'], + ['jpg', 'image/jpeg'], + ['jpeg', 'image/jpeg'], + ['gif', 'image/gif'], +]); + +export class RawImage { + /** + * Create a new `RawImage` object. + * @param {Uint8ClampedArray|Uint8Array} data The pixel data. + * @param {number} width The width of the image. + * @param {number} height The height of the image. + * @param {1|2|3|4} channels The number of channels. + */ + constructor(data, width, height, channels) { + this.data = data; + this.width = width; + this.height = height; + this.channels = channels; + } + + /** + * Returns the size of the image (width, height). + * @returns {[number, number]} The size of the image (width, height). + */ + get size() { + return [this.width, this.height]; + } + + /** + * Helper method for reading an image from a variety of input types. + * @param {RawImage|string|URL|Blob|HTMLCanvasElement|OffscreenCanvas} input + * @returns {Promise} The image object. + * + * **Example:** Read image from a URL. + * ```javascript + * let image = await RawImage.read('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg'); + * // RawImage { + * // "data": Uint8ClampedArray [ 25, 25, 25, 19, 19, 19, ... ], + * // "width": 800, + * // "height": 533, + * // "channels": 3 + * // } + * ``` + */ + static async read(input) { + if (input instanceof RawImage) { + return input; + } else if (typeof input === 'string' || input instanceof URL) { + return await this.fromURL(input); + } else if (input instanceof Blob) { + return await this.fromBlob(input); + } else if ( + (typeof HTMLCanvasElement !== 'undefined' && input instanceof HTMLCanvasElement) || + (typeof OffscreenCanvas !== 'undefined' && input instanceof OffscreenCanvas) + ) { + return this.fromCanvas(input); + } else { + throw new Error(`Unsupported input type: ${typeof input}`); + } + } + + /** + * Read an image from a canvas. + * @param {HTMLCanvasElement|OffscreenCanvas} canvas The canvas to read the image from. + * @returns {RawImage} The image object. + */ + static fromCanvas(canvas) { + if (!apis.IS_WEB_ENV) { + throw new Error('fromCanvas() is only supported in browser environments.'); + } + + const ctx = /** @type {CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D} */ ( + canvas.getContext('2d') + ); + const data = ctx.getImageData(0, 0, canvas.width, canvas.height).data; + return new RawImage(data, canvas.width, canvas.height, 4); + } + + /** + * Read an image from a URL or file path. + * @param {string|URL} url The URL or file path to read the image from. + * @returns {Promise} The image object. + */ + static async fromURL(url) { + const response = await getFile(url); + if (response.status !== 200) { + throw new Error(`Unable to read image from "${url}" (${response.status} ${response.statusText})`); + } + const blob = await response.blob(); + return this.fromBlob(blob); + } + + /** + * Helper method to create a new Image from a blob. + * @param {Blob} blob The blob to read the image from. + * @returns {Promise} The image object. + */ + static async fromBlob(blob) { + if (apis.IS_WEB_ENV) { + // Running in environment with canvas + const img = await loadImageFunction(blob); + + const ctx = createCanvasFunction(img.width, img.height).getContext('2d'); + + // Draw image to context + ctx.drawImage(img, 0, 0); + + return new this(ctx.getImageData(0, 0, img.width, img.height).data, img.width, img.height, 4); + } else { + // Use sharp.js to read (and possible resize) the image. + const img = sharp(await blob.arrayBuffer()); + + return await loadImageFunction(img); + } + } + + /** + * Helper method to create a new Image from a tensor + * @param {Tensor} tensor + */ + static fromTensor(tensor, channel_format = 'CHW') { + if (tensor.dims.length !== 3) { + throw new Error(`Tensor should have 3 dimensions, but has ${tensor.dims.length} dimensions.`); + } + + if (channel_format === 'CHW') { + tensor = tensor.transpose(1, 2, 0); + } else if (channel_format === 'HWC') { + // Do nothing + } else { + throw new Error(`Unsupported channel format: ${channel_format}`); + } + if (!(tensor.data instanceof Uint8ClampedArray || tensor.data instanceof Uint8Array)) { + throw new Error(`Unsupported tensor type: ${tensor.type}`); + } + switch (tensor.dims[2]) { + case 1: + case 2: + case 3: + case 4: + return new RawImage(tensor.data, tensor.dims[1], tensor.dims[0], tensor.dims[2]); + default: + throw new Error(`Unsupported number of channels: ${tensor.dims[2]}`); + } + } + + /** + * Convert the image to grayscale format. + * @returns {RawImage} `this` to support chaining. + */ + grayscale() { + if (this.channels === 1) { + return this; + } + + const newData = new Uint8ClampedArray(this.width * this.height * 1); + switch (this.channels) { + case 3: // rgb to grayscale + case 4: // rgba to grayscale + for (let i = 0, offset = 0; i < this.data.length; i += this.channels) { + const red = this.data[i]; + const green = this.data[i + 1]; + const blue = this.data[i + 2]; + + newData[offset++] = Math.round(0.2989 * red + 0.587 * green + 0.114 * blue); + } + break; + default: + throw new Error(`Conversion failed due to unsupported number of channels: ${this.channels}`); + } + return this._update(newData, this.width, this.height, 1); + } + + /** + * Convert the image to RGB format. + * @returns {RawImage} `this` to support chaining. + */ + rgb() { + if (this.channels === 3) { + return this; + } + + const newData = new Uint8ClampedArray(this.width * this.height * 3); + + switch (this.channels) { + case 1: // grayscale to rgb + for (let i = 0, offset = 0; i < this.data.length; ++i) { + newData[offset++] = this.data[i]; + newData[offset++] = this.data[i]; + newData[offset++] = this.data[i]; + } + break; + case 4: // rgba to rgb + for (let i = 0, offset = 0; i < this.data.length; i += 4) { + newData[offset++] = this.data[i]; + newData[offset++] = this.data[i + 1]; + newData[offset++] = this.data[i + 2]; + } + break; + default: + throw new Error(`Conversion failed due to unsupported number of channels: ${this.channels}`); + } + return this._update(newData, this.width, this.height, 3); + } + + /** + * Convert the image to RGBA format. + * @returns {RawImage} `this` to support chaining. + */ + rgba() { + if (this.channels === 4) { + return this; + } + + const newData = new Uint8ClampedArray(this.width * this.height * 4); + + switch (this.channels) { + case 1: // grayscale to rgba + for (let i = 0, offset = 0; i < this.data.length; ++i) { + newData[offset++] = this.data[i]; + newData[offset++] = this.data[i]; + newData[offset++] = this.data[i]; + newData[offset++] = 255; + } + break; + case 3: // rgb to rgba + for (let i = 0, offset = 0; i < this.data.length; i += 3) { + newData[offset++] = this.data[i]; + newData[offset++] = this.data[i + 1]; + newData[offset++] = this.data[i + 2]; + newData[offset++] = 255; + } + break; + default: + throw new Error(`Conversion failed due to unsupported number of channels: ${this.channels}`); + } + + return this._update(newData, this.width, this.height, 4); + } + + /** + * Apply an alpha mask to the image. Operates in place. + * @param {RawImage} mask The mask to apply. It should have a single channel. + * @returns {RawImage} The masked image. + * @throws {Error} If the mask is not the same size as the image. + * @throws {Error} If the image does not have 4 channels. + * @throws {Error} If the mask is not a single channel. + */ + putAlpha(mask) { + if (mask.width !== this.width || mask.height !== this.height) { + throw new Error( + `Expected mask size to be ${this.width}x${this.height}, but got ${mask.width}x${mask.height}`, + ); + } + if (mask.channels !== 1) { + throw new Error(`Expected mask to have 1 channel, but got ${mask.channels}`); + } + + const this_data = this.data; + const mask_data = mask.data; + const num_pixels = this.width * this.height; + if (this.channels === 3) { + // Convert to RGBA and simultaneously apply mask to alpha channel + const newData = new Uint8ClampedArray(num_pixels * 4); + for (let i = 0, in_offset = 0, out_offset = 0; i < num_pixels; ++i) { + newData[out_offset++] = this_data[in_offset++]; + newData[out_offset++] = this_data[in_offset++]; + newData[out_offset++] = this_data[in_offset++]; + newData[out_offset++] = mask_data[i]; + } + return this._update(newData, this.width, this.height, 4); + } else if (this.channels === 4) { + // Apply mask to alpha channel in place + for (let i = 0; i < num_pixels; ++i) { + this_data[4 * i + 3] = mask_data[i]; + } + return this; + } + throw new Error(`Expected image to have 3 or 4 channels, but got ${this.channels}`); + } + + /** + * Resize the image to the given dimensions. This method uses the canvas API to perform the resizing. + * @param {number} width The width of the new image. `null` or `-1` will preserve the aspect ratio. + * @param {number} height The height of the new image. `null` or `-1` will preserve the aspect ratio. + * @param {Object} options Additional options for resizing. + * @param {0|1|2|3|4|5|string} [options.resample] The resampling method to use. + * @returns {Promise} `this` to support chaining. + */ + async resize(width, height, { resample = 2 } = {}) { + // Do nothing if the image already has the desired size + if (this.width === width && this.height === height) { + return this; + } + + // Ensure resample method is a string + let resampleMethod = RESAMPLING_MAPPING[resample] ?? resample; + + // Calculate width / height to maintain aspect ratio, in the event that + // the user passed a null value in. + // This allows users to pass in something like `resize(320, null)` to + // resize to 320 width, but maintain aspect ratio. + const nullish_width = isNullishDimension(width); + const nullish_height = isNullishDimension(height); + if (nullish_width && nullish_height) { + return this; + } else if (nullish_width) { + width = (height / this.height) * this.width; + } else if (nullish_height) { + height = (width / this.width) * this.height; + } + + if (apis.IS_WEB_ENV) { + // TODO use `resample` in browser environment + + // Store number of channels before resizing + const numChannels = this.channels; + + // Create canvas object for this image + const canvas = this.toCanvas(); + + // Actually perform resizing using the canvas API + const ctx = createCanvasFunction(width, height).getContext('2d'); + + // Draw image to context, resizing in the process + ctx.drawImage(canvas, 0, 0, width, height); + + // Create image from the resized data + const resizedImage = new RawImage(ctx.getImageData(0, 0, width, height).data, width, height, 4); + + // Convert back so that image has the same number of channels as before + return resizedImage.convert(numChannels); + } else { + // Create sharp image from raw data, and resize + let img = this.toSharp(); + + switch (resampleMethod) { + case 'box': + case 'hamming': + if (resampleMethod === 'box' || resampleMethod === 'hamming') { + logger.warn( + `Resampling method ${resampleMethod} is not yet supported. Using bilinear instead.`, + ); + resampleMethod = 'bilinear'; + } + + case 'nearest': + case 'bilinear': + case 'bicubic': + // Perform resizing using affine transform. + // This matches how the python Pillow library does it. + img = img.affine([width / this.width, 0, 0, height / this.height], { + interpolator: resampleMethod, + }); + break; + + case 'lanczos': + // https://github.com/python-pillow/Pillow/discussions/5519 + // https://github.com/lovell/sharp/blob/main/docs/api-resize.md + img = img.resize({ + width, + height, + fit: 'fill', + kernel: 'lanczos3', // PIL Lanczos uses a kernel size of 3 + }); + break; + + default: + throw new Error(`Resampling method ${resampleMethod} is not supported.`); + } + + return await loadImageFunction(img); + } + } + + async pad([left, right, top, bottom]) { + left = Math.max(left, 0); + right = Math.max(right, 0); + top = Math.max(top, 0); + bottom = Math.max(bottom, 0); + + if (left === 0 && right === 0 && top === 0 && bottom === 0) { + // No padding needed + return this; + } + + if (apis.IS_WEB_ENV) { + // Store number of channels before padding + const numChannels = this.channels; + + // Create canvas object for this image + const canvas = this.toCanvas(); + + const newWidth = this.width + left + right; + const newHeight = this.height + top + bottom; + + // Create a new canvas of the desired size. + const ctx = createCanvasFunction(newWidth, newHeight).getContext('2d'); + + // Draw image to context, padding in the process + ctx.drawImage(canvas, 0, 0, this.width, this.height, left, top, this.width, this.height); + + // Create image from the padded data + const paddedImage = new RawImage(ctx.getImageData(0, 0, newWidth, newHeight).data, newWidth, newHeight, 4); + + // Convert back so that image has the same number of channels as before + return paddedImage.convert(numChannels); + } else { + const img = this.toSharp().extend({ left, right, top, bottom }); + return await loadImageFunction(img); + } + } + + async crop([x_min, y_min, x_max, y_max]) { + // Ensure crop bounds are within the image + x_min = Math.max(x_min, 0); + y_min = Math.max(y_min, 0); + x_max = Math.min(x_max, this.width - 1); + y_max = Math.min(y_max, this.height - 1); + + // Do nothing if the crop is the entire image + if (x_min === 0 && y_min === 0 && x_max === this.width - 1 && y_max === this.height - 1) { + return this; + } + + const crop_width = x_max - x_min + 1; + const crop_height = y_max - y_min + 1; + + if (apis.IS_WEB_ENV) { + // Store number of channels before resizing + const numChannels = this.channels; + + // Create canvas object for this image + const canvas = this.toCanvas(); + + // Create a new canvas of the desired size. This is needed since if the + // image is too small, we need to pad it with black pixels. + const ctx = createCanvasFunction(crop_width, crop_height).getContext('2d'); + + // Draw image to context, cropping in the process + ctx.drawImage(canvas, x_min, y_min, crop_width, crop_height, 0, 0, crop_width, crop_height); + + // Create image from the resized data + const resizedImage = new RawImage( + ctx.getImageData(0, 0, crop_width, crop_height).data, + crop_width, + crop_height, + 4, + ); + + // Convert back so that image has the same number of channels as before + return resizedImage.convert(numChannels); + } else { + // Create sharp image from raw data + const img = this.toSharp().extract({ + left: x_min, + top: y_min, + width: crop_width, + height: crop_height, + }); + + return await loadImageFunction(img); + } + } + + async center_crop(crop_width, crop_height) { + // If the image is already the desired size, return it + if (this.width === crop_width && this.height === crop_height) { + return this; + } + + // Determine bounds of the image in the new canvas + const width_offset = (this.width - crop_width) / 2; + const height_offset = (this.height - crop_height) / 2; + + if (apis.IS_WEB_ENV) { + // Store number of channels before resizing + const numChannels = this.channels; + + // Create canvas object for this image + const canvas = this.toCanvas(); + + // Create a new canvas of the desired size. This is needed since if the + // image is too small, we need to pad it with black pixels. + const ctx = createCanvasFunction(crop_width, crop_height).getContext('2d'); + + let sourceX = 0; + let sourceY = 0; + let destX = 0; + let destY = 0; + + if (width_offset >= 0) { + sourceX = width_offset; + } else { + destX = -width_offset; + } + + if (height_offset >= 0) { + sourceY = height_offset; + } else { + destY = -height_offset; + } + + // Draw image to context, cropping in the process + ctx.drawImage(canvas, sourceX, sourceY, crop_width, crop_height, destX, destY, crop_width, crop_height); + + // Create image from the resized data + const resizedImage = new RawImage( + ctx.getImageData(0, 0, crop_width, crop_height).data, + crop_width, + crop_height, + 4, + ); + + // Convert back so that image has the same number of channels as before + return resizedImage.convert(numChannels); + } else { + // Create sharp image from raw data + let img = this.toSharp(); + + if (width_offset >= 0 && height_offset >= 0) { + // Cropped image lies entirely within the original image + img = img.extract({ + left: Math.floor(width_offset), + top: Math.floor(height_offset), + width: crop_width, + height: crop_height, + }); + } else if (width_offset <= 0 && height_offset <= 0) { + // Cropped image lies entirely outside the original image, + // so we add padding + const top = Math.floor(-height_offset); + const left = Math.floor(-width_offset); + img = img.extend({ + top: top, + left: left, + + // Ensures the resulting image has the desired dimensions + right: crop_width - this.width - left, + bottom: crop_height - this.height - top, + }); + } else { + // Cropped image lies partially outside the original image. + // We first pad, then crop. + + let y_padding = [0, 0]; + let y_extract = 0; + if (height_offset < 0) { + y_padding[0] = Math.floor(-height_offset); + y_padding[1] = crop_height - this.height - y_padding[0]; + } else { + y_extract = Math.floor(height_offset); + } + + let x_padding = [0, 0]; + let x_extract = 0; + if (width_offset < 0) { + x_padding[0] = Math.floor(-width_offset); + x_padding[1] = crop_width - this.width - x_padding[0]; + } else { + x_extract = Math.floor(width_offset); + } + + img = img + .extend({ + top: y_padding[0], + bottom: y_padding[1], + left: x_padding[0], + right: x_padding[1], + }) + .extract({ + left: x_extract, + top: y_extract, + width: crop_width, + height: crop_height, + }); + } + + return await loadImageFunction(img); + } + } + + async toBlob(type = 'image/png', quality = 1) { + if (!apis.IS_WEB_ENV) { + throw new Error('toBlob() is only supported in browser environments.'); + } + + const canvas = this.toCanvas(); + return await canvas.convertToBlob({ type, quality }); + } + + toTensor(channel_format = 'CHW') { + let tensor = new Tensor('uint8', new Uint8Array(this.data), [this.height, this.width, this.channels]); + + if (channel_format === 'HWC') { + // Do nothing + } else if (channel_format === 'CHW') { + // hwc -> chw + tensor = tensor.permute(2, 0, 1); + } else { + throw new Error(`Unsupported channel format: ${channel_format}`); + } + return tensor; + } + + toCanvas() { + if (!apis.IS_WEB_ENV) { + throw new Error('toCanvas() is only supported in browser environments.'); + } + + // Clone, and convert data to RGBA before drawing to canvas. + // This is because the canvas API only supports RGBA + const cloned = this.clone().rgba(); + + // Create canvas object for the cloned image + const clonedCanvas = createCanvasFunction(cloned.width, cloned.height); + + // Draw image to context + const data = new ImageDataClass(cloned.data, cloned.width, cloned.height); + clonedCanvas.getContext('2d').putImageData(data, 0, 0); + + return clonedCanvas; + } + + /** + * Split this image into individual bands. This method returns an array of individual image bands from an image. + * For example, splitting an "RGB" image creates three new images each containing a copy of one of the original bands (red, green, blue). + * + * Inspired by PIL's `Image.split()` [function](https://pillow.readthedocs.io/en/latest/reference/Image.html#PIL.Image.Image.split). + * @returns {RawImage[]} An array containing bands. + */ + split() { + const { data, width, height, channels } = this; + + /** @type {typeof Uint8Array | typeof Uint8ClampedArray} */ + const data_type = /** @type {any} */ (data.constructor); + const per_channel_length = data.length / channels; + + // Pre-allocate buffers for each channel + const split_data = Array.from({ length: channels }, () => new data_type(per_channel_length)); + + // Write pixel data + for (let i = 0; i < per_channel_length; ++i) { + const data_offset = channels * i; + for (let j = 0; j < channels; ++j) { + split_data[j][i] = data[data_offset + j]; + } + } + return split_data.map((data) => new RawImage(data, width, height, 1)); + } + + /** + * Helper method to update the image data. + * @param {Uint8ClampedArray} data The new image data. + * @param {number} width The new width of the image. + * @param {number} height The new height of the image. + * @param {1|2|3|4|null} [channels] The new number of channels of the image. + * @private + */ + _update(data, width, height, channels = null) { + this.data = data; + this.width = width; + this.height = height; + if (channels !== null) { + this.channels = channels; + } + return this; + } + + /** + * Clone the image + * @returns {RawImage} The cloned image + */ + clone() { + return new RawImage(this.data.slice(), this.width, this.height, this.channels); + } + + /** + * Helper method for converting image to have a certain number of channels + * @param {number} numChannels The number of channels. Must be 1, 3, or 4. + * @returns {RawImage} `this` to support chaining. + */ + convert(numChannels) { + if (this.channels === numChannels) return this; // Already correct number of channels + + switch (numChannels) { + case 1: + this.grayscale(); + break; + case 3: + this.rgb(); + break; + case 4: + this.rgba(); + break; + default: + throw new Error(`Conversion failed due to unsupported number of channels: ${this.channels}`); + } + return this; + } + + /** + * Save the image to the given path. + * @param {string} path The path to save the image to. + * @returns {Promise} + */ + async save(path) { + if (apis.IS_WEB_ENV) { + if (apis.IS_WEBWORKER_ENV) { + throw new Error('Unable to save an image from a Web Worker.'); + } + + const extension = path.split('.').pop().toLowerCase(); + const mime = CONTENT_TYPE_MAP.get(extension) ?? 'image/png'; + + // Convert image to Blob + const blob = await this.toBlob(mime); + + return saveBlob(path, blob); + } else if (apis.IS_FS_AVAILABLE) { + const img = this.toSharp(); + await img.toFile(path); + } else { + throw new Error('Unable to save the image because filesystem is disabled in this environment.'); + } + } + + /** + * Convert the image to a Sharp instance. + * @returns {import('sharp').Sharp} The Sharp instance. + */ + toSharp() { + if (apis.IS_WEB_ENV) { + throw new Error('toSharp() is only supported in server-side environments.'); + } + + return sharp(this.data, { + raw: { + width: this.width, + height: this.height, + channels: this.channels, + }, + }); + } +} + +/** + * Helper function to load an image from a URL, path, etc. + */ +export const load_image = RawImage.read.bind(RawImage); diff --git a/packages/transformers/src/utils/io.js b/packages/transformers/src/utils/io.js new file mode 100644 index 0000000..4aa8683 --- /dev/null +++ b/packages/transformers/src/utils/io.js @@ -0,0 +1,49 @@ +import fs from 'node:fs'; +import { Readable } from 'node:stream'; +import { pipeline as pipe } from 'node:stream/promises'; + +import { apis } from '../env.js'; + +/** + * Save blob file. + * @param {string} path The path to save the blob to + * @param {Blob} blob The blob to save + * @returns {Promise} A promise that resolves when the blob has been saved + */ +export async function saveBlob(path, blob) { + if (apis.IS_BROWSER_ENV) { + if (apis.IS_WEBWORKER_ENV) { + throw new Error('Unable to save a file from a Web Worker.'); + } + // Convert the canvas content to a data URL + const dataURL = URL.createObjectURL(blob); + + // Create an anchor element with the data URL as the href attribute + const downloadLink = document.createElement('a'); + downloadLink.href = dataURL; + + // Set the download attribute to specify the desired filename for the downloaded image + downloadLink.download = path; + + // Trigger the download + downloadLink.click(); + + // Clean up: remove the anchor element from the DOM + downloadLink.remove(); + + // Revoke the Object URL to free up memory + URL.revokeObjectURL(dataURL); + } else if (apis.IS_FS_AVAILABLE) { + // Convert Blob to a Node.js Readable Stream + const webStream = blob.stream(); + const nodeStream = Readable.fromWeb(webStream); + + // Create the file write stream + const fileStream = fs.createWriteStream(path); + + // Pipe the readable stream to the file write stream + await pipe(nodeStream, fileStream); + } else { + throw new Error('Unable to save because filesystem is disabled in this environment.'); + } +} diff --git a/packages/transformers/src/utils/logger.js b/packages/transformers/src/utils/logger.js new file mode 100644 index 0000000..ec59404 --- /dev/null +++ b/packages/transformers/src/utils/logger.js @@ -0,0 +1,67 @@ +/** + * @file Logger utility for Transformers.js with configurable log levels. + * + * @module utils/logger + */ + +import { env, LogLevel } from '../env.js'; + +/** + * Logger that respects the configured log level in env.logLevel. + * + * @example + * import { logger } from './utils/logger.js'; + * logger.info('Model loaded successfully'); + * logger.warn('Deprecated method used'); + * logger.error('Failed to load model'); + * logger.debug('Token count:', tokens.length); + */ +export const logger = { + /** + * Log an error message. Only suppressed when logLevel is NONE. + * @param {...any} args - Arguments to log + */ + error(...args) { + if (env.logLevel <= LogLevel.ERROR) { + console.error(...args); + } + }, + + /** + * Log a warning message. Shown when logLevel <= WARNING. + * @param {...any} args - Arguments to log + */ + warn(...args) { + if (env.logLevel <= LogLevel.WARNING) { + console.warn(...args); + } + }, + + /** + * Log an info message. Shown when logLevel <= INFO. + * @param {...any} args - Arguments to log + */ + info(...args) { + if (env.logLevel <= LogLevel.INFO) { + console.log(...args); + } + }, + + /** + * Log a debug message. Only shown when logLevel is DEBUG. + * @param {...any} args - Arguments to log + */ + debug(...args) { + if (env.logLevel <= LogLevel.DEBUG) { + console.log(...args); + } + }, + + /** + * Log a message (alias for info). Shown when logLevel <= INFO. + * @param {...any} args - Arguments to log + */ + log(...args) { + this.info(...args); + }, +}; diff --git a/packages/transformers/src/utils/lru_cache.js b/packages/transformers/src/utils/lru_cache.js new file mode 100644 index 0000000..2636c56 --- /dev/null +++ b/packages/transformers/src/utils/lru_cache.js @@ -0,0 +1,67 @@ +/** + * A simple Least Recently Used (LRU) cache implementation in JavaScript. + * This cache stores key-value pairs and evicts the least recently used item + * when the capacity is exceeded. + */ +export class LRUCache { + /** @type {number} */ + #capacity; + + /** @type {Map} */ + #cache; + + /** + * Creates an LRUCache instance. + * @param {number} capacity The maximum number of items the cache can hold. + */ + constructor(capacity) { + this.#capacity = capacity; + this.#cache = new Map(); + } + + /** + * Retrieves the value associated with the given key and marks the key as recently used. + * @param {any} key The key to retrieve. + * @returns {any} The value associated with the key, or undefined if the key does not exist. + */ + get(key) { + if (!this.#cache.has(key)) return undefined; + const value = this.#cache.get(key); + this.#cache.delete(key); + this.#cache.set(key, value); + return value; + } + + /** + * Inserts or updates the key-value pair in the cache. + * If the key already exists, it is updated and marked as recently used. + * If the cache exceeds its capacity, the least recently used item is evicted. + * @param {any} key The key to add or update. + * @param {any} value The value to associate with the key. + */ + put(key, value) { + if (this.#cache.has(key)) { + this.#cache.delete(key); + } + this.#cache.set(key, value); + if (this.#cache.size > this.#capacity) { + this.#cache.delete(this.#cache.keys().next().value); + } + } + + /** + * Removes the entry for the given key from the cache. + * @param {any} key The key to delete. + * @returns {boolean} `true` if the entry existed and was removed, `false` otherwise. + */ + delete(key) { + return this.#cache.delete(key); + } + + /** + * Clears the cache. + */ + clear() { + this.#cache.clear(); + } +} diff --git a/packages/transformers/src/utils/maths.js b/packages/transformers/src/utils/maths.js new file mode 100644 index 0000000..8a46edb --- /dev/null +++ b/packages/transformers/src/utils/maths.js @@ -0,0 +1,1104 @@ +/** + * @file Helper module for mathematical processing. + * + * These functions and classes are only used internally, + * meaning an end-user shouldn't need to access anything here. + * + * @module utils/maths + */ + +/** + * @typedef {Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | Float16Array | Float32Array | Float64Array} TypedArray + * @typedef {BigInt64Array | BigUint64Array} BigTypedArray + * @typedef {TypedArray | BigTypedArray} AnyTypedArray + */ + +/** + * @param {TypedArray} input + */ +export function interpolate_data( + input, + [in_channels, in_height, in_width], + [out_height, out_width], + mode = 'bilinear', + align_corners = false, +) { + // TODO use mode and align_corners + + // Output image dimensions + const x_scale = out_width / in_width; + const y_scale = out_height / in_height; + + // Output image + // @ts-ignore + const out_img = new input.constructor(out_height * out_width * in_channels); + + // Pre-calculate strides + const inStride = in_height * in_width; + const outStride = out_height * out_width; + + for (let i = 0; i < out_height; ++i) { + for (let j = 0; j < out_width; ++j) { + // Calculate output offset + const outOffset = i * out_width + j; + + // Calculate input pixel coordinates + const x = (j + 0.5) / x_scale - 0.5; + const y = (i + 0.5) / y_scale - 0.5; + + // Calculate the four nearest input pixels + // We also check if the input pixel coordinates are within the image bounds + let x1 = Math.floor(x); + let y1 = Math.floor(y); + const x2 = Math.min(x1 + 1, in_width - 1); + const y2 = Math.min(y1 + 1, in_height - 1); + + x1 = Math.max(x1, 0); + y1 = Math.max(y1, 0); + + // Calculate the fractional distances between the input pixel and the four nearest pixels + const s = x - x1; + const t = y - y1; + + // Perform bilinear interpolation + const w1 = (1 - s) * (1 - t); + const w2 = s * (1 - t); + const w3 = (1 - s) * t; + const w4 = s * t; + + // Calculate the four nearest input pixel indices + const yStride = y1 * in_width; + const xStride = y2 * in_width; + const idx1 = yStride + x1; + const idx2 = yStride + x2; + const idx3 = xStride + x1; + const idx4 = xStride + x2; + + for (let k = 0; k < in_channels; ++k) { + // Calculate channel offset + const cOffset = k * inStride; + + out_img[k * outStride + outOffset] = + w1 * input[cOffset + idx1] + + w2 * input[cOffset + idx2] + + w3 * input[cOffset + idx3] + + w4 * input[cOffset + idx4]; + } + } + } + + return out_img; +} + +/** + * Helper method to permute a `AnyTypedArray` directly + * @template {AnyTypedArray} T + * @param {T} array + * @param {number[]} dims + * @param {number[]} axes + * @returns {[T, number[]]} The permuted array and the new shape. + */ +export function permute_data(array, dims, axes) { + // Calculate the new shape of the permuted array + // and the stride of the original array + const shape = new Array(axes.length); + const stride = new Array(axes.length); + + for (let i = axes.length - 1, s = 1; i >= 0; --i) { + stride[i] = s; + shape[i] = dims[axes[i]]; + s *= shape[i]; + } + + // Precompute inverse mapping of stride + const invStride = axes.map((_, i) => stride[axes.indexOf(i)]); + + // Create the permuted array with the new shape + // @ts-ignore + const permutedData = new array.constructor(array.length); + + // Permute the original array to the new array + for (let i = 0; i < array.length; ++i) { + let newIndex = 0; + for (let j = dims.length - 1, k = i; j >= 0; --j) { + newIndex += (k % dims[j]) * invStride[j]; + k = Math.floor(k / dims[j]); + } + permutedData[newIndex] = array[i]; + } + + return [permutedData, shape]; +} + +/** + * Compute the softmax of an array of numbers. + * @template {TypedArray|number[]} T + * @param {T} arr The array of numbers to compute the softmax of. + * @returns {T} The softmax array. + */ +export function softmax(arr) { + // Compute the maximum value in the array + const maxVal = max(arr)[0]; + + // Compute the exponentials of the array values + const exps = arr.map((x) => Math.exp(x - maxVal)); + + // Compute the sum of the exponentials + // @ts-ignore + const sumExps = exps.reduce((acc, val) => acc + val, 0); + + // Compute the softmax values + const softmaxArr = exps.map((x) => x / sumExps); + + return /** @type {T} */ (softmaxArr); +} + +/** + * Calculates the logarithm of the softmax function for the input array. + * @template {TypedArray|number[]} T + * @param {T} arr The input array to calculate the log_softmax function for. + * @returns {T} The resulting log_softmax array. + */ +export function log_softmax(arr) { + // Compute the maximum value in the array + const maxVal = max(arr)[0]; + + // Compute the sum of the exponentials + let sumExps = 0; + for (let i = 0; i < arr.length; ++i) { + sumExps += Math.exp(arr[i] - maxVal); + } + + // Compute the log of the sum + const logSum = Math.log(sumExps); + + // Compute the softmax values + const logSoftmaxArr = arr.map((x) => x - maxVal - logSum); + + return /** @type {T} */ (logSoftmaxArr); +} + +/** + * Calculates the dot product of two arrays. + * @param {number[]} arr1 The first array. + * @param {number[]} arr2 The second array. + * @returns {number} The dot product of arr1 and arr2. + */ +export function dot(arr1, arr2) { + let result = 0; + for (let i = 0; i < arr1.length; ++i) { + result += arr1[i] * arr2[i]; + } + return result; +} + +/** + * Computes the cosine similarity between two arrays. + * + * @param {number[]} arr1 The first array. + * @param {number[]} arr2 The second array. + * @returns {number} The cosine similarity between the two arrays. + */ +export function cos_sim(arr1, arr2) { + // Calculate dot product of the two arrays + const dotProduct = dot(arr1, arr2); + + // Calculate the magnitude of the first array + const magnitudeA = magnitude(arr1); + + // Calculate the magnitude of the second array + const magnitudeB = magnitude(arr2); + + // Calculate the cosine similarity + const cosineSimilarity = dotProduct / (magnitudeA * magnitudeB); + + return cosineSimilarity; +} + +/** + * Calculates the magnitude of a given array. + * @param {number[]} arr The array to calculate the magnitude of. + * @returns {number} The magnitude of the array. + */ +function magnitude(arr) { + return Math.sqrt(arr.reduce((acc, val) => acc + val * val, 0)); +} + +/** + * Returns the value and index of the minimum element in an array. + * @template {number[]|bigint[]|AnyTypedArray} T + * @param {T} arr array of numbers. + * @returns {T extends bigint[]|BigTypedArray ? [bigint, number] : [number, number]} the value and index of the minimum element, of the form: [valueOfMin, indexOfMin] + * @throws {Error} If array is empty. + */ +export function min(arr) { + if (arr.length === 0) throw Error('Array must not be empty'); + let min = arr[0]; + let indexOfMin = 0; + for (let i = 1; i < arr.length; ++i) { + if (arr[i] < min) { + min = arr[i]; + indexOfMin = i; + } + } + return /** @type {T extends bigint[]|BigTypedArray ? [bigint, number] : [number, number]} */ ([min, indexOfMin]); +} + +/** + * Returns the value and index of the maximum element in an array. + * @template {number[]|bigint[]|AnyTypedArray} T + * @param {T} arr array of numbers. + * @returns {T extends bigint[]|BigTypedArray ? [bigint, number] : [number, number]} the value and index of the maximum element, of the form: [valueOfMax, indexOfMax] + * @throws {Error} If array is empty. + */ +export function max(arr) { + if (arr.length === 0) throw Error('Array must not be empty'); + let max = arr[0]; + let indexOfMax = 0; + for (let i = 1; i < arr.length; ++i) { + if (arr[i] > max) { + max = arr[i]; + indexOfMax = i; + } + } + return /** @type {T extends bigint[]|BigTypedArray ? [bigint, number] : [number, number]} */ ([max, indexOfMax]); +} + +function isPowerOfTwo(number) { + // Check if the number is greater than 0 and has only one bit set to 1 + return number > 0 && (number & (number - 1)) === 0; +} + +/** + * Implementation of Radix-4 FFT. + * + * P2FFT class provides functionality for performing Fast Fourier Transform on arrays + * which are a power of two in length. + * Code adapted from https://www.npmjs.com/package/fft.js + */ +class P2FFT { + /** + * @param {number} size The size of the input array. Must be a power of two larger than 1. + * @throws {Error} FFT size must be a power of two larger than 1. + */ + constructor(size) { + this.size = size | 0; // convert to a 32-bit signed integer + if (this.size <= 1 || !isPowerOfTwo(this.size)) + throw new Error('FFT size must be a power of two larger than 1'); + + this._csize = size << 1; + + this.table = new Float64Array(this.size * 2); + for (let i = 0; i < this.table.length; i += 2) { + const angle = (Math.PI * i) / this.size; + this.table[i] = Math.cos(angle); + this.table[i + 1] = -Math.sin(angle); + } + + // Find size's power of two + let power = 0; + for (let t = 1; this.size > t; t <<= 1) ++power; + + // Calculate initial step's width: + // * If we are full radix-4, it is 2x smaller to give inital len=8 + // * Otherwise it is the same as `power` to give len=4 + this._width = power % 2 === 0 ? power - 1 : power; + + // Pre-compute bit-reversal patterns + this._bitrev = new Int32Array(1 << this._width); + for (let j = 0; j < this._bitrev.length; ++j) { + this._bitrev[j] = 0; + for (let shift = 0; shift < this._width; shift += 2) { + const revShift = this._width - shift - 2; + this._bitrev[j] |= ((j >>> shift) & 3) << revShift; + } + } + } + + /** + * Create a complex number array with size `2 * size` + * + * @returns {Float64Array} A complex number array with size `2 * size` + */ + createComplexArray() { + return new Float64Array(this._csize); + } + + /** + * Converts a complex number representation stored in a Float64Array to an array of real numbers. + * + * @param {Float64Array} complex The complex number representation to be converted. + * @param {number[]} [storage] An optional array to store the result in. + * @returns {number[]} An array of real numbers representing the input complex number representation. + */ + fromComplexArray(complex, storage) { + const res = storage || new Array(complex.length >>> 1); + for (let i = 0; i < complex.length; i += 2) res[i >>> 1] = complex[i]; + return res; + } + + /** + * Convert a real-valued input array to a complex-valued output array. + * @param {Float64Array} input The real-valued input array. + * @param {Float64Array} [storage] Optional buffer to store the output array. + * @returns {Float64Array} The complex-valued output array. + */ + toComplexArray(input, storage) { + const res = storage || this.createComplexArray(); + for (let i = 0; i < res.length; i += 2) { + res[i] = input[i >>> 1]; + res[i + 1] = 0; + } + return res; + } + + /** + * Performs a Fast Fourier Transform (FFT) on the given input data and stores the result in the output buffer. + * + * @param {Float64Array} out The output buffer to store the result. + * @param {Float64Array} data The input data to transform. + * + * @throws {Error} Input and output buffers must be different. + * + * @returns {void} + */ + transform(out, data) { + if (out === data) throw new Error('Input and output buffers must be different'); + + this._transform4(out, data, 1 /* DONE */); + } + + /** + * Performs a real-valued forward FFT on the given input buffer and stores the result in the given output buffer. + * The input buffer must contain real values only, while the output buffer will contain complex values. The input and + * output buffers must be different. + * + * @param {Float64Array} out The output buffer. + * @param {Float64Array} data The input buffer containing real values. + * + * @throws {Error} If the input and output buffers are the same. + */ + realTransform(out, data) { + if (out === data) throw new Error('Input and output buffers must be different'); + + this._realTransform4(out, data, 1 /* DONE */); + } + + /** + * Performs an inverse FFT transformation on the given `data` array, and stores the result in `out`. + * The `out` array must be a different buffer than the `data` array. The `out` array will contain the + * result of the transformation. The `data` array will not be modified. + * + * @param {Float64Array} out The output buffer for the transformed data. + * @param {Float64Array} data The input data to transform. + * @throws {Error} If `out` and `data` refer to the same buffer. + * @returns {void} + */ + inverseTransform(out, data) { + if (out === data) throw new Error('Input and output buffers must be different'); + + this._transform4(out, data, -1 /* DONE */); + for (let i = 0; i < out.length; ++i) out[i] /= this.size; + } + + /** + * Performs a radix-4 implementation of a discrete Fourier transform on a given set of data. + * + * @param {Float64Array} out The output buffer for the transformed data. + * @param {Float64Array} data The input buffer of data to be transformed. + * @param {number} inv A scaling factor to apply to the transform. + * @returns {void} + */ + _transform4(out, data, inv) { + // radix-4 implementation + + const size = this._csize; + + // Initial step (permute and transform) + const width = this._width; + let step = 1 << width; + let len = (size / step) << 1; + + let outOff; + let t; + const bitrev = this._bitrev; + if (len === 4) { + for (outOff = 0, t = 0; outOff < size; outOff += len, ++t) { + const off = bitrev[t]; + this._singleTransform2(data, out, outOff, off, step); + } + } else { + // len === 8 + for (outOff = 0, t = 0; outOff < size; outOff += len, ++t) { + const off = bitrev[t]; + this._singleTransform4(data, out, outOff, off, step, inv); + } + } + + // Loop through steps in decreasing order + const table = this.table; + for (step >>= 2; step >= 2; step >>= 2) { + len = (size / step) << 1; + const quarterLen = len >>> 2; + + // Loop through offsets in the data + for (outOff = 0; outOff < size; outOff += len) { + // Full case + const limit = outOff + quarterLen - 1; + for (let i = outOff, k = 0; i < limit; i += 2, k += step) { + const A = i; + const B = A + quarterLen; + const C = B + quarterLen; + const D = C + quarterLen; + + // Original values + const Ar = out[A]; + const Ai = out[A + 1]; + const Br = out[B]; + const Bi = out[B + 1]; + const Cr = out[C]; + const Ci = out[C + 1]; + const Dr = out[D]; + const Di = out[D + 1]; + + const tableBr = table[k]; + const tableBi = inv * table[k + 1]; + const MBr = Br * tableBr - Bi * tableBi; + const MBi = Br * tableBi + Bi * tableBr; + + const tableCr = table[2 * k]; + const tableCi = inv * table[2 * k + 1]; + const MCr = Cr * tableCr - Ci * tableCi; + const MCi = Cr * tableCi + Ci * tableCr; + + const tableDr = table[3 * k]; + const tableDi = inv * table[3 * k + 1]; + const MDr = Dr * tableDr - Di * tableDi; + const MDi = Dr * tableDi + Di * tableDr; + + // Pre-Final values + const T0r = Ar + MCr; + const T0i = Ai + MCi; + const T1r = Ar - MCr; + const T1i = Ai - MCi; + const T2r = MBr + MDr; + const T2i = MBi + MDi; + const T3r = inv * (MBr - MDr); + const T3i = inv * (MBi - MDi); + + // Final values + out[A] = T0r + T2r; + out[A + 1] = T0i + T2i; + out[B] = T1r + T3i; + out[B + 1] = T1i - T3r; + out[C] = T0r - T2r; + out[C + 1] = T0i - T2i; + out[D] = T1r - T3i; + out[D + 1] = T1i + T3r; + } + } + } + } + + /** + * Performs a radix-2 implementation of a discrete Fourier transform on a given set of data. + * + * @param {Float64Array} data The input buffer of data to be transformed. + * @param {Float64Array} out The output buffer for the transformed data. + * @param {number} outOff The offset at which to write the output data. + * @param {number} off The offset at which to begin reading the input data. + * @param {number} step The step size for indexing the input data. + * @returns {void} + */ + _singleTransform2(data, out, outOff, off, step) { + // radix-2 implementation + // NOTE: Only called for len=4 + + const evenR = data[off]; + const evenI = data[off + 1]; + const oddR = data[off + step]; + const oddI = data[off + step + 1]; + + out[outOff] = evenR + oddR; + out[outOff + 1] = evenI + oddI; + out[outOff + 2] = evenR - oddR; + out[outOff + 3] = evenI - oddI; + } + + /** + * Performs radix-4 transformation on input data of length 8 + * + * @param {Float64Array} data Input data array of length 8 + * @param {Float64Array} out Output data array of length 8 + * @param {number} outOff Index of output array to start writing from + * @param {number} off Index of input array to start reading from + * @param {number} step Step size between elements in input array + * @param {number} inv Scaling factor for inverse transform + * + * @returns {void} + */ + _singleTransform4(data, out, outOff, off, step, inv) { + // radix-4 + // NOTE: Only called for len=8 + const step2 = step * 2; + const step3 = step * 3; + + // Original values + const Ar = data[off]; + const Ai = data[off + 1]; + const Br = data[off + step]; + const Bi = data[off + step + 1]; + const Cr = data[off + step2]; + const Ci = data[off + step2 + 1]; + const Dr = data[off + step3]; + const Di = data[off + step3 + 1]; + + // Pre-Final values + const T0r = Ar + Cr; + const T0i = Ai + Ci; + const T1r = Ar - Cr; + const T1i = Ai - Ci; + const T2r = Br + Dr; + const T2i = Bi + Di; + const T3r = inv * (Br - Dr); + const T3i = inv * (Bi - Di); + + // Final values + out[outOff] = T0r + T2r; + out[outOff + 1] = T0i + T2i; + out[outOff + 2] = T1r + T3i; + out[outOff + 3] = T1i - T3r; + out[outOff + 4] = T0r - T2r; + out[outOff + 5] = T0i - T2i; + out[outOff + 6] = T1r - T3i; + out[outOff + 7] = T1i + T3r; + } + + /** + * Real input radix-4 implementation + * @param {Float64Array} out Output array for the transformed data + * @param {Float64Array} data Input array of real data to be transformed + * @param {number} inv The scale factor used to normalize the inverse transform + */ + _realTransform4(out, data, inv) { + // Real input radix-4 implementation + const size = this._csize; + + // Initial step (permute and transform) + const width = this._width; + let step = 1 << width; + let len = (size / step) << 1; + + let outOff; + let t; + const bitrev = this._bitrev; + if (len === 4) { + for (outOff = 0, t = 0; outOff < size; outOff += len, ++t) { + const off = bitrev[t]; + this._singleRealTransform2(data, out, outOff, off >>> 1, step >>> 1); + } + } else { + // len === 8 + for (outOff = 0, t = 0; outOff < size; outOff += len, ++t) { + const off = bitrev[t]; + this._singleRealTransform4(data, out, outOff, off >>> 1, step >>> 1, inv); + } + } + + // Loop through steps in decreasing order + const table = this.table; + for (step >>= 2; step >= 2; step >>= 2) { + len = (size / step) << 1; + const halfLen = len >>> 1; + const quarterLen = halfLen >>> 1; + const hquarterLen = quarterLen >>> 1; + + // Loop through offsets in the data + for (outOff = 0; outOff < size; outOff += len) { + for (let i = 0, k = 0; i <= hquarterLen; i += 2, k += step) { + const A = outOff + i; + const B = A + quarterLen; + const C = B + quarterLen; + const D = C + quarterLen; + + // Original values + const Ar = out[A]; + const Ai = out[A + 1]; + const Br = out[B]; + const Bi = out[B + 1]; + const Cr = out[C]; + const Ci = out[C + 1]; + const Dr = out[D]; + const Di = out[D + 1]; + + // Middle values + const MAr = Ar; + const MAi = Ai; + + const tableBr = table[k]; + const tableBi = inv * table[k + 1]; + const MBr = Br * tableBr - Bi * tableBi; + const MBi = Br * tableBi + Bi * tableBr; + + const tableCr = table[2 * k]; + const tableCi = inv * table[2 * k + 1]; + const MCr = Cr * tableCr - Ci * tableCi; + const MCi = Cr * tableCi + Ci * tableCr; + + const tableDr = table[3 * k]; + const tableDi = inv * table[3 * k + 1]; + const MDr = Dr * tableDr - Di * tableDi; + const MDi = Dr * tableDi + Di * tableDr; + + // Pre-Final values + const T0r = MAr + MCr; + const T0i = MAi + MCi; + const T1r = MAr - MCr; + const T1i = MAi - MCi; + const T2r = MBr + MDr; + const T2i = MBi + MDi; + const T3r = inv * (MBr - MDr); + const T3i = inv * (MBi - MDi); + + // Final values + out[A] = T0r + T2r; + out[A + 1] = T0i + T2i; + out[B] = T1r + T3i; + out[B + 1] = T1i - T3r; + + // Output final middle point + if (i === 0) { + out[C] = T0r - T2r; + out[C + 1] = T0i - T2i; + continue; + } + + // Do not overwrite ourselves + if (i === hquarterLen) continue; + + const SA = outOff + quarterLen - i; + const SB = outOff + halfLen - i; + + out[SA] = T1r - inv * T3i; + out[SA + 1] = -T1i - inv * T3r; + out[SB] = T0r - inv * T2r; + out[SB + 1] = -T0i + inv * T2i; + } + } + } + + // Complete the spectrum by adding its mirrored negative frequency components. + const half = size >>> 1; + for (let i = 2; i < half; i += 2) { + out[size - i] = out[i]; + out[size - i + 1] = -out[i + 1]; + } + } + + /** + * Performs a single real input radix-2 transformation on the provided data + * + * @param {Float64Array} data The input data array + * @param {Float64Array} out The output data array + * @param {number} outOff The output offset + * @param {number} off The input offset + * @param {number} step The step + * + * @returns {void} + */ + _singleRealTransform2(data, out, outOff, off, step) { + // radix-2 implementation + // NOTE: Only called for len=4 + + const evenR = data[off]; + const oddR = data[off + step]; + + out[outOff] = evenR + oddR; + out[outOff + 1] = 0; + out[outOff + 2] = evenR - oddR; + out[outOff + 3] = 0; + } + + /** + * Computes a single real-valued transform using radix-4 algorithm. + * This method is only called for len=8. + * + * @param {Float64Array} data The input data array. + * @param {Float64Array} out The output data array. + * @param {number} outOff The offset into the output array. + * @param {number} off The offset into the input array. + * @param {number} step The step size for the input array. + * @param {number} inv The value of inverse. + */ + _singleRealTransform4(data, out, outOff, off, step, inv) { + // radix-4 + // NOTE: Only called for len=8 + const step2 = step * 2; + const step3 = step * 3; + + // Original values + const Ar = data[off]; + const Br = data[off + step]; + const Cr = data[off + step2]; + const Dr = data[off + step3]; + + // Pre-Final values + const T0r = Ar + Cr; + const T1r = Ar - Cr; + const T2r = Br + Dr; + const T3r = inv * (Br - Dr); + + // Final values + out[outOff] = T0r + T2r; + out[outOff + 1] = 0; + out[outOff + 2] = T1r; + out[outOff + 3] = -T3r; + out[outOff + 4] = T0r - T2r; + out[outOff + 5] = 0; + out[outOff + 6] = T1r; + out[outOff + 7] = T3r; + } +} + +/** + * NP2FFT class provides functionality for performing Fast Fourier Transform on arrays + * which are not a power of two in length. In such cases, the chirp-z transform is used. + * + * For more information, see: https://math.stackexchange.com/questions/77118/non-power-of-2-ffts/77156#77156 + */ +class NP2FFT { + /** + * Constructs a new NP2FFT object. + * @param {number} fft_length The length of the FFT + */ + constructor(fft_length) { + // Helper variables + const a = 2 * (fft_length - 1); + const b = 2 * (2 * fft_length - 1); + const nextP2 = 2 ** Math.ceil(Math.log2(b)); + this.bufferSize = nextP2; + this._a = a; + + // Define buffers + // Compute chirp for transform + const chirp = new Float64Array(b); + const ichirp = new Float64Array(nextP2); + this._chirpBuffer = new Float64Array(nextP2); + this._buffer1 = new Float64Array(nextP2); + this._buffer2 = new Float64Array(nextP2); + this._outBuffer1 = new Float64Array(nextP2); + this._outBuffer2 = new Float64Array(nextP2); + + // Compute complex exponentiation + const theta = (-2 * Math.PI) / fft_length; + const baseR = Math.cos(theta); + const baseI = Math.sin(theta); + + // Precompute helper for chirp-z transform + for (let i = 0; i < b >> 1; ++i) { + // Compute complex power: + const e = (i + 1 - fft_length) ** 2 / 2.0; + + // Compute the modulus and argument of the result + const result_mod = Math.sqrt(baseR ** 2 + baseI ** 2) ** e; + const result_arg = e * Math.atan2(baseI, baseR); + + // Convert the result back to rectangular form + // and assign to chirp and ichirp + const i2 = 2 * i; + chirp[i2] = result_mod * Math.cos(result_arg); + chirp[i2 + 1] = result_mod * Math.sin(result_arg); + + // conjugate + ichirp[i2] = chirp[i2]; + ichirp[i2 + 1] = -chirp[i2 + 1]; + } + this._slicedChirpBuffer = chirp.subarray(a, b); + + // create object to perform Fast Fourier Transforms + // with `nextP2` complex numbers + this._f = new P2FFT(nextP2 >> 1); + this._f.transform(this._chirpBuffer, ichirp); + } + + _transform(output, input, real) { + const ib1 = this._buffer1; + const ib2 = this._buffer2; + const ob2 = this._outBuffer1; + const ob3 = this._outBuffer2; + const cb = this._chirpBuffer; + const sb = this._slicedChirpBuffer; + const a = this._a; + + if (real) { + // Real multiplication + for (let j = 0; j < sb.length; j += 2) { + const j2 = j + 1; + const j3 = j >> 1; + + const a_real = input[j3]; + ib1[j] = a_real * sb[j]; + ib1[j2] = a_real * sb[j2]; + } + } else { + // Complex multiplication + for (let j = 0; j < sb.length; j += 2) { + const j2 = j + 1; + ib1[j] = input[j] * sb[j] - input[j2] * sb[j2]; + ib1[j2] = input[j] * sb[j2] + input[j2] * sb[j]; + } + } + this._f.transform(ob2, ib1); + + for (let j = 0; j < cb.length; j += 2) { + const j2 = j + 1; + + ib2[j] = ob2[j] * cb[j] - ob2[j2] * cb[j2]; + ib2[j2] = ob2[j] * cb[j2] + ob2[j2] * cb[j]; + } + this._f.inverseTransform(ob3, ib2); + + for (let j = 0; j < ob3.length; j += 2) { + const a_real = ob3[j + a]; + const a_imag = ob3[j + a + 1]; + const b_real = sb[j]; + const b_imag = sb[j + 1]; + + output[j] = a_real * b_real - a_imag * b_imag; + output[j + 1] = a_real * b_imag + a_imag * b_real; + } + } + + transform(output, input) { + this._transform(output, input, false); + } + + realTransform(output, input) { + this._transform(output, input, true); + } +} + +export class FFT { + constructor(fft_length) { + this.fft_length = fft_length; + this.isPowerOfTwo = isPowerOfTwo(fft_length); + if (this.isPowerOfTwo) { + this.fft = new P2FFT(fft_length); + this.outputBufferSize = 2 * fft_length; + } else { + this.fft = new NP2FFT(fft_length); + this.outputBufferSize = this.fft.bufferSize; + } + } + + realTransform(out, input) { + this.fft.realTransform(out, input); + } + + transform(out, input) { + this.fft.transform(out, input); + } +} + +/** + * Performs median filter on the provided data. Padding is done by mirroring the data. + * @param {AnyTypedArray} data The input array + * @param {number} windowSize The window size + */ +export function medianFilter(data, windowSize) { + if (windowSize % 2 === 0 || windowSize <= 0) { + throw new Error('Window size must be a positive odd number'); + } + + // @ts-ignore + const outputArray = new data.constructor(data.length); + + // @ts-ignore + const buffer = new data.constructor(windowSize); // Reusable array for storing values + + const halfWindowSize = Math.floor(windowSize / 2); + + for (let i = 0; i < data.length; ++i) { + let valuesIndex = 0; + + for (let j = -halfWindowSize; j <= halfWindowSize; ++j) { + let index = i + j; + if (index < 0) { + index = Math.abs(index); + } else if (index >= data.length) { + index = 2 * (data.length - 1) - index; + } + + buffer[valuesIndex++] = data[index]; + } + + buffer.sort(); + outputArray[i] = buffer[halfWindowSize]; + } + + return outputArray; +} + +/** + * Helper function to round a number to a given number of decimals + * @param {number} num The number to round + * @param {number} decimals The number of decimals + * @returns {number} The rounded number + */ +export function round(num, decimals) { + const pow = Math.pow(10, decimals); + return Math.round(num * pow) / pow; +} + +/** + * Helper function to round a number to the nearest integer, with ties rounded to the nearest even number. + * Also known as "bankers' rounding". This is the default rounding mode in python. For example: + * 1.5 rounds to 2 and 2.5 rounds to 2. + * + * @param {number} x The number to round + * @returns {number} The rounded number + */ +export function bankers_round(x) { + const r = Math.round(x); + const br = Math.abs(x) % 1 === 0.5 ? (r % 2 === 0 ? r : r - 1) : r; + return br; +} + +/** + * Measures similarity between two temporal sequences (e.g., input audio and output tokens + * to generate token-level timestamps). + * @param {number[][]} matrix + * @returns {number[][]} + */ +export function dynamic_time_warping(matrix) { + const output_length = matrix.length; + const input_length = matrix[0].length; + + const outputShape = [output_length + 1, input_length + 1]; + + const cost = Array.from({ length: outputShape[0] }, () => Array(outputShape[1]).fill(Infinity)); + cost[0][0] = 0; + + const trace = Array.from({ length: outputShape[0] }, () => Array(outputShape[1]).fill(-1)); + + for (let j = 1; j < outputShape[1]; ++j) { + for (let i = 1; i < outputShape[0]; ++i) { + const c0 = cost[i - 1][j - 1]; + const c1 = cost[i - 1][j]; + const c2 = cost[i][j - 1]; + + let c, t; + if (c0 < c1 && c0 < c2) { + c = c0; + t = 0; + } else if (c1 < c0 && c1 < c2) { + c = c1; + t = 1; + } else { + c = c2; + t = 2; + } + cost[i][j] = matrix[i - 1][j - 1] + c; + trace[i][j] = t; + } + } + + for (let i = 0; i < outputShape[1]; ++i) { + // trace[0, :] = 2 + trace[0][i] = 2; + } + for (let i = 0; i < outputShape[0]; ++i) { + // trace[:, 0] = 1 + trace[i][0] = 1; + } + + // backtrace + let i = output_length; + let j = input_length; + let text_indices = []; + let time_indices = []; + while (i > 0 || j > 0) { + text_indices.push(i - 1); + time_indices.push(j - 1); + + switch (trace[i][j]) { + case 0: + --i; + --j; + break; + case 1: + --i; + break; + case 2: + --j; + break; + default: + throw new Error( + `Internal error in dynamic time warping. Unexpected trace[${i}, ${j}]. Please file a bug report.`, + ); + } + } + + text_indices.reverse(); + time_indices.reverse(); + + return [text_indices, time_indices]; +} + +/** + * Efficiently converts a Uint16Array of float16 values to a Float32Array. + * This implementation uses a lazily initialized lookup table (LUT) for fast conversion. + */ +export const uint16_to_float32 = (function () { + let float16LUT = null; // The Lookup Table + + return function (/** @type {Uint16Array} */ u16Array) { + if (!float16LUT) { + // Lazily initialize LUT + float16LUT = new Float32Array(65536); + const buffer = new ArrayBuffer(4); + const u32 = new Uint32Array(buffer); + const f32 = new Float32Array(buffer); + + for (let i = 0; i < float16LUT.length; ++i) { + let outBits = 0; + const sign = (i & 0x8000) << 16; + const exp = (i & 0x7c00) >> 10; + let mantissa = i & 0x03ff; + + if (exp === 0x1f) { + // Infinity or NaN + outBits = sign | 0x7f800000 | (mantissa << 13); + } else if (exp === 0) { + // Zero or Subnormal + if (mantissa === 0) { + outBits = sign; + } else { + let renormExp = 113; + while ((mantissa & 0x0400) === 0) { + mantissa <<= 1; + --renormExp; + } + mantissa &= ~0x0400; + outBits = sign | (renormExp << 23) | (mantissa << 13); + } + } else { + // Normal + outBits = sign | ((exp + 112) << 23) | (mantissa << 13); + } + + u32[0] = outBits; + float16LUT[i] = f32[0]; + } + } + + const length = u16Array.length; + const lut = float16LUT; + const out = new Float32Array(length); + for (let i = 0; i < length; ++i) { + out[i] = lut[u16Array[i]]; + } + + return out; + }; +})(); diff --git a/packages/transformers/src/utils/memoize_promise.js b/packages/transformers/src/utils/memoize_promise.js new file mode 100644 index 0000000..500a098 --- /dev/null +++ b/packages/transformers/src/utils/memoize_promise.js @@ -0,0 +1,45 @@ +/** + * @file Utility for memoizing promises by key. + * + * Ensures that a given async operation is only initiated once per key. + * Subsequent calls with the same key return the same pending or resolved promise. + * Rejected promises are evicted from the cache so callers can retry. + * + * The cache is bounded by `MAX_CACHE_SIZE`. When the limit is exceeded, the + * least-recently-used entry is evicted. + */ + +import { LRUCache } from './lru_cache.js'; + +const MAX_CACHE_SIZE = 100; + +/** @type {LRUCache} */ +const cache = new LRUCache(MAX_CACHE_SIZE); + +/** + * Returns the cached promise for `key`, or calls `factory` to create one and caches it. + * Subsequent calls with the same key return the same promise whether it is still + * pending or already resolved, so the factory is never invoked more than once per key. + * If the promise rejects, the entry is removed from the cache so the operation can be retried. + * + * @template T + * @param {string} key A unique identifier for this async operation. + * @param {() => Promise} factory A function that returns the promise to memoize. + * Only called when no entry exists for `key`. + * @returns {Promise} + */ +export function memoizePromise(key, factory) { + const cached = cache.get(key); + if (cached !== undefined) { + return cached; + } + const promise = factory().then( + (value) => value, + (err) => { + cache.delete(key); + return Promise.reject(err); + }, + ); + cache.put(key, promise); + return promise; +} diff --git a/packages/transformers/src/utils/model-loader.js b/packages/transformers/src/utils/model-loader.js new file mode 100644 index 0000000..599aefe --- /dev/null +++ b/packages/transformers/src/utils/model-loader.js @@ -0,0 +1,111 @@ +import { getModelFile, MAX_EXTERNAL_DATA_CHUNKS } from './hub.js'; +import { apis } from '../env.js'; + +/** + * Resolves an `use_external_data_format` config value to the number of data chunks for a given file. + * @param {import('./hub.js').ExternalData|Record|null|undefined} config The external data format configuration. + * @param {string} fullName The full ONNX file name (e.g., "model_quantized.onnx"). + * @param {string} fileName The base file name (e.g., "model"). + * @returns {number} The number of external data chunks (0 if none). + */ +export function resolveExternalDataFormat(config, fullName, fileName) { + if (!config) return 0; + if (typeof config === 'object' && config !== null) { + if (config.hasOwnProperty(fullName)) return +config[fullName]; + if (config.hasOwnProperty(fileName)) return +config[fileName]; + return 0; + } + return +config; // (false=0, true=1, number remains the same) +} + +/** + * Generates the file names for external data chunks. + * @param {string} fullName The full ONNX file name (e.g., "model_quantized.onnx"). + * @param {number} numChunks The number of external data chunks. + * @returns {string[]} Array of external data file names. + */ +export function getExternalDataChunkNames(fullName, numChunks) { + const names = []; + for (let i = 0; i < numChunks; ++i) { + names.push(`${fullName}_data${i === 0 ? '' : '_' + i}`); + } + return names; +} + +/** + * Loads the core model file. + * + * @param {string} pretrained_model_name_or_path The path to the directory containing the model file. + * @param {string} fileName The base name of the model file (without suffix or extension). + * @param {import('./hub.js').PretrainedModelOptions} options Additional options for loading the model. + * @param {string} suffix The suffix to append to the file name (e.g., '_q4', '_quantized'). + * @returns {Promise} A Promise that resolves to the model file buffer or path. + */ +export async function getCoreModelFile(pretrained_model_name_or_path, fileName, options, suffix) { + const baseName = `${fileName}${suffix}.onnx`; + const fullPath = `${options.subfolder ?? ''}/${baseName}`; + + return await getModelFile(pretrained_model_name_or_path, fullPath, true, options, apis.IS_NODE_ENV); +} + +/** + * Loads external data files for a model. + * + * @param {string} pretrained_model_name_or_path The path to the directory containing the model files. + * @param {string} fileName The base name of the model file (without suffix or extension). + * @param {string} suffix The suffix to append to the file name (e.g., '_q4'). + * @param {import('./hub.js').PretrainedModelOptions} options Additional options for loading the model. + * @param {import('./hub.js').ExternalData|Record|undefined} use_external_data_format External data format configuration. + * @param {any} [session_options] Optional session options that may contain externalData configuration. + * @returns {Promise>} A Promise that resolves to an array of external data files. + */ +export async function getModelDataFiles( + pretrained_model_name_or_path, + fileName, + suffix, + options, + use_external_data_format, + session_options = {}, +) { + const baseName = `${fileName}${suffix}.onnx`; + const return_path = apis.IS_NODE_ENV; + + /** @type {Promise[]} */ + let externalDataPromises = []; + + const num_chunks = resolveExternalDataFormat(use_external_data_format, baseName, fileName); + if (num_chunks > 0) { + if (num_chunks > MAX_EXTERNAL_DATA_CHUNKS) { + throw new Error( + `The number of external data chunks (${num_chunks}) exceeds the maximum allowed value (${MAX_EXTERNAL_DATA_CHUNKS}).`, + ); + } + const chunkNames = getExternalDataChunkNames(baseName, num_chunks); + for (const path of chunkNames) { + const fullPath = `${options.subfolder ?? ''}/${path}`; + externalDataPromises.push( + new Promise(async (resolve, reject) => { + const data = await getModelFile( + pretrained_model_name_or_path, + fullPath, + true, + options, + return_path, + ); + resolve(data instanceof Uint8Array ? { path, data } : path); + }), + ); + } + } else if (session_options.externalData !== undefined) { + externalDataPromises = session_options.externalData.map(async (ext) => { + // if the external data is a string, fetch the file and replace the string with its content + if (typeof ext.data === 'string') { + const ext_buffer = await getModelFile(pretrained_model_name_or_path, ext.data, true, options); + return { ...ext, data: ext_buffer }; + } + return ext; + }); + } + + return Promise.all(externalDataPromises); +} diff --git a/packages/transformers/src/utils/model_registry/ModelRegistry.js b/packages/transformers/src/utils/model_registry/ModelRegistry.js new file mode 100644 index 0000000..1179a92 --- /dev/null +++ b/packages/transformers/src/utils/model_registry/ModelRegistry.js @@ -0,0 +1,382 @@ +/** + * @file Model registry for cache and file operations + * + * Provides static methods for: + * - Discovering which files a model needs + * - Detecting available quantization levels (dtypes) + * - Getting file metadata + * - Checking cache status + * + * **Example:** Get all files needed for a model + * ```javascript + * const files = await ModelRegistry.get_files( + * "onnx-community/all-MiniLM-L6-v2-ONNX", + * { dtype: "fp16" }, + * ); + * console.log(files); // [ 'config.json', 'onnx/model_fp16.onnx', 'onnx/model_fp16.onnx_data', 'tokenizer.json', 'tokenizer_config.json' ] + * ``` + * + * **Example:** Get all files needed for a specific pipeline task + * ```javascript + * const files = await ModelRegistry.get_pipeline_files( + * "text-generation", + * "onnx-community/Qwen3-0.6B-ONNX", + * { dtype: "q4" }, + * ); + * console.log(files); // [ 'config.json', 'onnx/model_q4.onnx', 'generation_config.json', 'tokenizer.json', 'tokenizer_config.json' ] + * ``` + * + * **Example:** Get specific component files + * ```javascript + * const modelFiles = await ModelRegistry.get_model_files("onnx-community/all-MiniLM-L6-v2-ONNX", { dtype: "q4" }); + * const tokenizerFiles = await ModelRegistry.get_tokenizer_files("onnx-community/all-MiniLM-L6-v2-ONNX"); + * const processorFiles = await ModelRegistry.get_processor_files("onnx-community/all-MiniLM-L6-v2-ONNX"); + * console.log(modelFiles); // [ 'config.json', 'onnx/model_q4.onnx', 'onnx/model_q4.onnx_data' ] + * console.log(tokenizerFiles); // [ 'tokenizer.json', 'tokenizer_config.json' ] + * console.log(processorFiles); // [ ] + * ``` + * + * **Example:** Detect available quantization levels for a model + * ```javascript + * const dtypes = await ModelRegistry.get_available_dtypes("onnx-community/all-MiniLM-L6-v2-ONNX"); + * console.log(dtypes); // [ 'fp32', 'fp16', 'int8', 'uint8', 'q8', 'q4' ] + * + * // Use the result to pick the best available dtype + * const preferredDtype = dtypes.includes("q4") ? "q4" : "fp32"; + * const files = await ModelRegistry.get_files("onnx-community/all-MiniLM-L6-v2-ONNX", { dtype: preferredDtype }); + * ``` + * + * **Example:** Check file metadata without downloading + * ```javascript + * const metadata = await ModelRegistry.get_file_metadata( + * "onnx-community/Qwen3-0.6B-ONNX", + * "config.json" + * ); + * console.log(metadata); // { exists: true, size: 912, contentType: 'application/json', fromCache: true } + * ``` + * + * **Example:** Model cache management + * ```javascript + * const modelId = "onnx-community/Qwen3-0.6B-ONNX"; + * const options = { dtype: "q4" }; + * + * // Quickly check if the model is cached (probably false) + * let cached = await ModelRegistry.is_cached(modelId, options); + * console.log(cached); // false + * + * // Get per-file cache detail + * let cacheStatus = await ModelRegistry.is_cached_files(modelId, options); + * console.log(cacheStatus); + * // { + * // allCached: false, + * // files: [ { file: 'config.json', cached: true }, { file: 'onnx/model_q4.onnx', cached: false }, { file: 'generation_config.json', cached: false }, { file: 'tokenizer.json', cached: false }, { file: 'tokenizer_config.json', cached: false } ] + * // } + * + * // Download the model by instantiating a pipeline + * const generator = await pipeline("text-generation", modelId, options); + * const output = await generator( + * [{ role: "user", content: "What is the capital of France?" }], + * { max_new_tokens: 256, do_sample: false }, + * ); + * console.log(output[0].generated_text.at(-1).content); // ...\n\nThe capital of France is **Paris**. + * + * // Check if the model is cached (should be true now) + * cached = await ModelRegistry.is_cached(modelId, options); + * console.log(cached); // true + * + * // Clear the cache + * const clearResult = await ModelRegistry.clear_cache(modelId, options); + * console.log(clearResult); + * // { + * // filesDeleted: 5, + * // filesCached: 5, + * // files: [ { file: 'config.json', deleted: true, wasCached: true }, { file: 'onnx/model_q4.onnx', deleted: true, wasCached: true }, { file: 'generation_config.json', deleted: true, wasCached: true }, { file: 'tokenizer.json', deleted: true, wasCached: true }, { file: 'tokenizer_config.json', deleted: true, wasCached: true } ] + * // } + * + * // Check if the model is cached (should be false again) + * cached = await ModelRegistry.is_cached(modelId, options); + * console.log(cached); // false + * ``` + * + * @module utils/model_registry + */ + +import { get_files } from './get_files.js'; +import { get_pipeline_files } from './get_pipeline_files.js'; +import { get_model_files } from './get_model_files.js'; +import { get_tokenizer_files } from './get_tokenizer_files.js'; +import { get_processor_files } from './get_processor_files.js'; +import { is_cached, is_cached_files, is_pipeline_cached, is_pipeline_cached_files } from './is_cached.js'; +import { get_file_metadata } from './get_file_metadata.js'; +import { clear_cache, clear_pipeline_cache } from './clear_cache.js'; +import { get_available_dtypes } from './get_available_dtypes.js'; + +/** + * Static class for cache and file management operations. + * @hideconstructor + */ +export class ModelRegistry { + /** + * Get all files (model, tokenizer, processor) needed for a model. + * + * @param {string} modelId - The model id (e.g., "onnx-community/bert-base-uncased-ONNX") + * @param {Object} [options] - Optional parameters + * @param {import('../../configs.js').PretrainedConfig} [options.config=null] - Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype=null] - Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device=null] - Override device + * @param {string} [options.model_file_name=null] - Override the model file name (excluding .onnx suffix) + * @param {boolean} [options.include_tokenizer=true] - Whether to check for tokenizer files + * @param {boolean} [options.include_processor=true] - Whether to check for processor files + * @returns {Promise} Array of file paths + * + * @example + * const files = await ModelRegistry.get_files('onnx-community/gpt2-ONNX'); + * console.log(files); // ['config.json', 'tokenizer.json', 'onnx/model_q4.onnx', ...] + */ + static async get_files(modelId, options = {}) { + return get_files(modelId, options); + } + + /** + * Get all files needed for a specific pipeline task. + * Automatically determines which components are needed based on the task. + * + * @param {string} task - The pipeline task (e.g., "text-generation", "background-removal") + * @param {string} modelId - The model id (e.g., "onnx-community/bert-base-uncased-ONNX") + * @param {Object} [options] - Optional parameters + * @param {import('../../configs.js').PretrainedConfig} [options.config=null] - Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype=null] - Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device=null] - Override device + * @param {string} [options.model_file_name=null] - Override the model file name (excluding .onnx suffix) + * @returns {Promise} Array of file paths + * + * @example + * const files = await ModelRegistry.get_pipeline_files('text-generation', 'onnx-community/gpt2-ONNX'); + * console.log(files); // ['config.json', 'tokenizer.json', 'onnx/model_q4.onnx', ...] + */ + static async get_pipeline_files(task, modelId, options = {}) { + return get_pipeline_files(task, modelId, options); + } + + /** + * Get model files needed for a specific model. + * + * @param {string} modelId - The model id + * @param {Object} [options] - Optional parameters + * @param {import('../../configs.js').PretrainedConfig} [options.config=null] - Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype=null] - Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device=null] - Override device + * @param {string} [options.model_file_name=null] - Override the model file name (excluding .onnx suffix) + * @returns {Promise} Array of model file paths + * + * @example + * const files = await ModelRegistry.get_model_files('onnx-community/bert-base-uncased-ONNX'); + * console.log(files); // ['config.json', 'onnx/model_q4.onnx', 'generation_config.json'] + */ + static async get_model_files(modelId, options = {}) { + return get_model_files(modelId, options); + } + + /** + * Get tokenizer files needed for a specific model. + * + * @param {string} modelId - The model id + * @returns {Promise} Array of tokenizer file paths + * + * @example + * const files = await ModelRegistry.get_tokenizer_files('onnx-community/gpt2-ONNX'); + * console.log(files); // ['tokenizer.json', 'tokenizer_config.json'] + */ + static async get_tokenizer_files(modelId) { + return get_tokenizer_files(modelId); + } + + /** + * Get processor files needed for a specific model. + * + * @param {string} modelId - The model id + * @returns {Promise} Array of processor file paths + * + * @example + * const files = await ModelRegistry.get_processor_files('onnx-community/vit-base-patch16-224-ONNX'); + * console.log(files); // ['preprocessor_config.json'] + */ + static async get_processor_files(modelId) { + return get_processor_files(modelId); + } + + /** + * Detects which quantization levels (dtypes) are available for a model + * by checking which ONNX files exist on the hub or locally. + * + * A dtype is considered available if all required model session files + * exist for that dtype. + * + * @param {string} modelId - The model id (e.g., "onnx-community/all-MiniLM-L6-v2-ONNX") + * @param {Object} [options] - Optional parameters + * @param {import('../../configs.js').PretrainedConfig} [options.config=null] - Pre-loaded config + * @param {string} [options.model_file_name=null] - Override the model file name (excluding .onnx suffix) + * @param {string} [options.revision='main'] - Model revision + * @param {string} [options.cache_dir=null] - Custom cache directory + * @param {boolean} [options.local_files_only=false] - Only check local files + * @returns {Promise} Array of available dtype strings (e.g., ['fp32', 'fp16', 'q4', 'q8']) + * + * @example + * const dtypes = await ModelRegistry.get_available_dtypes('onnx-community/all-MiniLM-L6-v2-ONNX'); + * console.log(dtypes); // ['fp32', 'fp16', 'int8', 'uint8', 'q8', 'q4'] + */ + static async get_available_dtypes(modelId, options = {}) { + return get_available_dtypes(modelId, options); + } + + /** + * Quickly checks if a model is fully cached by verifying `config.json` is present, + * then confirming all required files are cached. + * Returns a plain boolean — use `is_cached_files` if you need per-file detail. + * + * @param {string} modelId - The model id + * @param {Object} [options] - Optional parameters + * @param {string} [options.cache_dir] - Custom cache directory + * @param {string} [options.revision] - Model revision (default: 'main') + * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype=null] - Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device=null] - Override device + * @returns {Promise} Whether all required files are cached + * + * @example + * const cached = await ModelRegistry.is_cached('onnx-community/bert-base-uncased-ONNX'); + * console.log(cached); // true or false + */ + static async is_cached(modelId, options = {}) { + return is_cached(modelId, options); + } + + /** + * Checks if all files for a given model are already cached, with per-file detail. + * Automatically determines which files are needed using get_files(). + * + * @param {string} modelId - The model id + * @param {Object} [options] - Optional parameters + * @param {string} [options.cache_dir] - Custom cache directory + * @param {string} [options.revision] - Model revision (default: 'main') + * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype=null] - Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device=null] - Override device + * @returns {Promise} Object with allCached boolean and files array with cache status + * + * @example + * const status = await ModelRegistry.is_cached_files('onnx-community/bert-base-uncased-ONNX'); + * console.log(status.allCached); // true or false + * console.log(status.files); // [{ file: 'config.json', cached: true }, ...] + */ + static async is_cached_files(modelId, options = {}) { + return is_cached_files(modelId, options); + } + + /** + * Quickly checks if all files for a specific pipeline task are cached by verifying + * `config.json` is present, then confirming all required files are cached. + * Returns a plain boolean — use `is_pipeline_cached_files` if you need per-file detail. + * + * @param {string} task - The pipeline task (e.g., "text-generation", "background-removal") + * @param {string} modelId - The model id + * @param {Object} [options] - Optional parameters + * @param {string} [options.cache_dir] - Custom cache directory + * @param {string} [options.revision] - Model revision (default: 'main') + * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype=null] - Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device=null] - Override device + * @returns {Promise} Whether all required files are cached + * + * @example + * const cached = await ModelRegistry.is_pipeline_cached('text-generation', 'onnx-community/gpt2-ONNX'); + * console.log(cached); // true or false + */ + static async is_pipeline_cached(task, modelId, options = {}) { + return is_pipeline_cached(task, modelId, options); + } + + /** + * Checks if all files for a specific pipeline task are already cached, with per-file detail. + * Automatically determines which components are needed based on the task. + * + * @param {string} task - The pipeline task (e.g., "text-generation", "background-removal") + * @param {string} modelId - The model id + * @param {Object} [options] - Optional parameters + * @param {string} [options.cache_dir] - Custom cache directory + * @param {string} [options.revision] - Model revision (default: 'main') + * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype=null] - Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device=null] - Override device + * @returns {Promise} Object with allCached boolean and files array with cache status + * + * @example + * const status = await ModelRegistry.is_pipeline_cached_files('text-generation', 'onnx-community/gpt2-ONNX'); + * console.log(status.allCached); // true or false + * console.log(status.files); // [{ file: 'config.json', cached: true }, ...] + */ + static async is_pipeline_cached_files(task, modelId, options = {}) { + return is_pipeline_cached_files(task, modelId, options); + } + + /** + * Get metadata for a specific file without downloading it. + * + * @param {string} path_or_repo_id - Model id or path + * @param {string} filename - The file name + * @param {import('../hub.js').PretrainedOptions} [options] - Optional parameters + * @returns {Promise<{exists: boolean, size?: number, contentType?: string, fromCache?: boolean}>} File metadata + * + * @example + * const metadata = await ModelRegistry.get_file_metadata('onnx-community/gpt2-ONNX', 'config.json'); + * console.log(metadata.exists, metadata.size); // true, 665 + */ + static async get_file_metadata(path_or_repo_id, filename, options = {}) { + return get_file_metadata(path_or_repo_id, filename, options); + } + + /** + * Clears all cached files for a given model. + * Automatically determines which files are needed and removes them from the cache. + * + * @param {string} modelId - The model id (e.g., "onnx-community/gpt2-ONNX") + * @param {Object} [options] - Optional parameters + * @param {string} [options.cache_dir] - Custom cache directory + * @param {string} [options.revision] - Model revision (default: 'main') + * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype] - Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device] - Override device + * @param {boolean} [options.include_tokenizer=true] - Whether to clear tokenizer files + * @param {boolean} [options.include_processor=true] - Whether to clear processor files + * @returns {Promise} Object with deletion statistics and file status + * + * @example + * const result = await ModelRegistry.clear_cache('onnx-community/bert-base-uncased-ONNX'); + * console.log(`Deleted ${result.filesDeleted} of ${result.filesCached} cached files`); + */ + static async clear_cache(modelId, options = {}) { + return clear_cache(modelId, options); + } + + /** + * Clears all cached files for a specific pipeline task. + * Automatically determines which components are needed based on the task. + * + * @param {string} task - The pipeline task (e.g., "text-generation", "image-classification") + * @param {string} modelId - The model id (e.g., "onnx-community/gpt2-ONNX") + * @param {Object} [options] - Optional parameters + * @param {string} [options.cache_dir] - Custom cache directory + * @param {string} [options.revision] - Model revision (default: 'main') + * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype] - Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device] - Override device + * @returns {Promise} Object with deletion statistics and file status + * + * @example + * const result = await ModelRegistry.clear_pipeline_cache('text-generation', 'onnx-community/gpt2-ONNX'); + * console.log(`Deleted ${result.filesDeleted} of ${result.filesCached} cached files`); + */ + static async clear_pipeline_cache(task, modelId, options = {}) { + return clear_pipeline_cache(task, modelId, options); + } +} diff --git a/packages/transformers/src/utils/model_registry/clear_cache.js b/packages/transformers/src/utils/model_registry/clear_cache.js new file mode 100644 index 0000000..fe4850e --- /dev/null +++ b/packages/transformers/src/utils/model_registry/clear_cache.js @@ -0,0 +1,128 @@ +/** + * @file Cache clearing utilities for model files + * + * Provides functions to clear cached model files from the cache system. + */ + +import { getCache } from '../cache.js'; +import { buildResourcePaths, checkCachedResource } from '../hub.js'; +import { get_files } from './get_files.js'; +import { get_pipeline_files } from './get_pipeline_files.js'; + +/** + * @typedef {Object} FileClearStatus + * @property {string} file - The file path + * @property {boolean} deleted - Whether the file was successfully deleted + * @property {boolean} wasCached - Whether the file was cached before deletion + */ + +/** + * @typedef {Object} CacheClearResult + * @property {number} filesDeleted - Number of files successfully deleted + * @property {number} filesCached - Number of files that were in cache + * @property {FileClearStatus[]} files - Array of files with their deletion status + */ + +/** + * Internal helper to clear cached files. + * + * + * @private + * @param {string} modelId - The model id + * @param {string[]} files - List of file paths to clear + * @param {Object} options - Options including cache_dir + * @returns {Promise} + */ +async function clear_files_from_cache(modelId, files, options = {}) { + const cache = await getCache(options?.cache_dir); + + if (!cache) { + return { + filesDeleted: 0, + filesCached: 0, + files: files.map((filename) => ({ file: filename, deleted: false, wasCached: false })), + }; + } + + if (!cache.delete) { + throw new Error('Cache does not support delete operation'); + } + + const results = await Promise.all( + files.map(async (filename) => { + const { localPath, proposedCacheKey } = buildResourcePaths(modelId, filename, options, cache); + + const cached = await checkCachedResource(cache, localPath, proposedCacheKey); + const wasCached = !!cached; + + let deleted = false; + if (wasCached) { + // Try proposedCacheKey first (remote URL for browser Cache API, request path for FileCache), + // then fall back to localPath in case the entry was cached under the local key instead. + const deletedWithProposed = await cache.delete(proposedCacheKey); + const deletedWithLocal = + !deletedWithProposed && proposedCacheKey !== localPath ? await cache.delete(localPath) : false; + + deleted = deletedWithProposed || deletedWithLocal; + } + + return { file: filename, deleted, wasCached }; + }), + ); + + return { + filesDeleted: results.filter((r) => r.deleted).length, + filesCached: results.filter((r) => r.wasCached).length, + files: results, + }; +} + +/** + * Clears all cached files for a given model. + * Automatically determines which files are needed using get_files(). + * + * @param {string} modelId - The model id (e.g., "Xenova/gpt2") + * @param {Object} [options] - Optional parameters + * @param {string} [options.cache_dir] - Custom cache directory + * @param {string} [options.revision] - Model revision (default: 'main') + * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype] - Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device] - Override device + * @param {boolean} [options.include_tokenizer=true] - Whether to clear tokenizer files + * @param {boolean} [options.include_processor=true] - Whether to clear processor files + * @returns {Promise} Object with deletion statistics and file status + */ +export async function clear_cache(modelId, options = {}) { + if (!modelId) { + throw new Error('modelId is required'); + } + + const files = await get_files(modelId, options); + return await clear_files_from_cache(modelId, files, options); +} + +/** + * Clears all cached files for a specific pipeline task. + * Automatically determines which components are needed based on the task. + * + * @param {string} task - The pipeline task (e.g., "text-generation", "image-classification") + * @param {string} modelId - The model id (e.g., "Xenova/gpt2") + * @param {Object} [options] - Optional parameters + * @param {string} [options.cache_dir] - Custom cache directory + * @param {string} [options.revision] - Model revision (default: 'main') + * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype] - Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device] - Override device + * @returns {Promise} Object with deletion statistics and file status + */ +export async function clear_pipeline_cache(task, modelId, options = {}) { + if (!task) { + throw new Error('task is required'); + } + if (!modelId) { + throw new Error('modelId is required'); + } + + const files = await get_pipeline_files(task, modelId, options); + return await clear_files_from_cache(modelId, files, options); +} diff --git a/packages/transformers/src/utils/model_registry/get_available_dtypes.js b/packages/transformers/src/utils/model_registry/get_available_dtypes.js new file mode 100644 index 0000000..fad61da --- /dev/null +++ b/packages/transformers/src/utils/model_registry/get_available_dtypes.js @@ -0,0 +1,68 @@ +import { getSessionsConfig } from '../../models/session_config.js'; +import { DEFAULT_DTYPE_SUFFIX_MAPPING } from '../dtypes.js'; +import { get_file_metadata } from './get_file_metadata.js'; +import { get_config } from './get_model_files.js'; +import { resolve_model_type } from './resolve_model_type.js'; + +/** + * @typedef {import('../../configs.js').PretrainedConfig} PretrainedConfig + */ + +/** + * The dtypes to probe for availability (excludes 'auto' which is not a concrete dtype). + * @type {string[]} + */ +const CONCRETE_DTYPES = Object.keys(DEFAULT_DTYPE_SUFFIX_MAPPING); + +/** + * Detects which quantization levels (dtypes) are available for a model + * by checking which ONNX files exist on the hub or locally. + * + * A dtype is considered available if *all* required model session files + * exist for that dtype. For example, a Seq2Seq model needs both an encoder + * and decoder file — the dtype is only listed if both are present. + * + * @param {string} modelId The model id (e.g., "onnx-community/all-MiniLM-L6-v2-ONNX") + * @param {Object} [options] Optional parameters + * @param {PretrainedConfig} [options.config=null] Pre-loaded model config (optional, will be fetched if not provided) + * @param {string} [options.model_file_name=null] Override the model file name (excluding .onnx suffix) + * @param {string} [options.revision='main'] Model revision + * @param {string} [options.cache_dir=null] Custom cache directory + * @param {boolean} [options.local_files_only=false] Only check local files + * @returns {Promise} Array of available dtype strings (e.g., ['fp32', 'fp16', 'q4', 'q8']) + */ +export async function get_available_dtypes( + modelId, + { config = null, model_file_name = null, revision = 'main', cache_dir = null, local_files_only = false } = {}, +) { + config = await get_config(modelId, { config, cache_dir, local_files_only, revision }); + + const subfolder = 'onnx'; + + const modelType = resolve_model_type(config); + + const { sessions } = getSessionsConfig(modelType, config, { model_file_name }); + + // Get all base names for model session files + const baseNames = Object.values(sessions); + + // For each dtype, check if all session files exist + const metadataOptions = { revision, cache_dir, local_files_only }; + + // Probe all (dtype, baseName) combinations in parallel + const probeResults = await Promise.all( + CONCRETE_DTYPES.map(async (dtype) => { + const suffix = DEFAULT_DTYPE_SUFFIX_MAPPING[dtype] ?? ''; + const allExist = await Promise.all( + baseNames.map(async (baseName) => { + const filename = `${subfolder}/${baseName}${suffix}.onnx`; + const metadata = await get_file_metadata(modelId, filename, metadataOptions); + return metadata.exists; + }), + ); + return { dtype, available: allExist.every(Boolean) }; + }), + ); + + return probeResults.filter((r) => r.available).map((r) => r.dtype); +} diff --git a/packages/transformers/src/utils/model_registry/get_file_metadata.js b/packages/transformers/src/utils/model_registry/get_file_metadata.js new file mode 100644 index 0000000..2ee8a50 --- /dev/null +++ b/packages/transformers/src/utils/model_registry/get_file_metadata.js @@ -0,0 +1,156 @@ +/** + * @file File metadata utilities for cache-aware operations + */ + +import { env } from '../../env.js'; +import { getCache } from '../cache.js'; +import { buildResourcePaths, checkCachedResource, getFetchHeaders, getFile } from '../hub.js'; +import { isValidUrl, makePretrainedOptionsKey } from '../hub/utils.js'; +import { logger } from '../logger.js'; +import { memoizePromise } from '../memoize_promise.js'; + +/** + * @typedef {import('../hub.js').PretrainedOptions} PretrainedOptions + */ + +/** + * Makes a Range request to get file metadata without downloading full content. + * + * We use a Range request (first byte only) instead of HEAD because: + * 1. Ensures we get the uncompressed file size, which is critical for accurate progress tracking + * 2. When compression is used, HEAD request content-length reflects compressed size + * 3. But during actual download, fetch API decompresses transparently and reports uncompressed bytes + * 4. This mismatch causes progress bar inconsistencies when tracking multiple files + * 5. Range requests typically aren't compressed, and content-range header shows true uncompressed size + * + * @param {URL|string} urlOrPath The URL/path of the file. + * @returns {Promise} A promise that resolves to a Response object or null if not supported. + * @private + */ +async function fetch_file_head(urlOrPath) { + // Range requests only make sense for HTTP URLs + if (!isValidUrl(urlOrPath, ['http:', 'https:'])) { + return null; + } + + const headers = getFetchHeaders(urlOrPath); + headers.set('Range', 'bytes=0-0'); + return env.fetch(urlOrPath, { method: 'GET', headers, cache: 'no-store' }); +} + +/** + * Gets file metadata (size, content-type, etc.) without downloading the full content. + * Uses Range requests for remote files to be efficient. + * Can also be used as a lightweight file existence check by checking the `.exists` property. + * + * @param {string} path_or_repo_id This can be either: + * - a string, the *model id* of a model repo on huggingface.co. + * - a path to a *directory* potentially containing the file. + * @param {string} filename The name of the file to check. + * @param {PretrainedOptions} [options] An object containing optional parameters. + * @returns {Promise<{exists: boolean, size?: number, contentType?: string, fromCache?: boolean}>} A Promise that resolves to file metadata. + */ +export function get_file_metadata(path_or_repo_id, filename, options = {}) { + const key = makePretrainedOptionsKey(path_or_repo_id, options, filename); + return memoizePromise(key, () => _get_file_metadata(path_or_repo_id, filename, options)); +} + +async function _get_file_metadata(path_or_repo_id, filename, options) { + /** @type {import('../cache.js').CacheInterface | null} */ + const cache = await getCache(options?.cache_dir); + const { localPath, remoteURL, proposedCacheKey, validModelId } = buildResourcePaths( + path_or_repo_id, + filename, + options, + cache, + ); + + // Check cache first - if cached, we can get metadata from the cached response + const cachedResponse = await checkCachedResource(cache, localPath, proposedCacheKey); + if (cachedResponse !== undefined && typeof cachedResponse !== 'string') { + const size = cachedResponse.headers.get('content-length'); + const contentType = cachedResponse.headers.get('content-type'); + return { + exists: true, + size: size ? parseInt(size, 10) : undefined, + contentType: contentType || undefined, + fromCache: true, + }; + } + + // Check local file system + if (env.allowLocalModels) { + const isURL = isValidUrl(localPath, ['http:', 'https:']); + if (!isURL) { + try { + const response = await getFile(localPath); + if (typeof response !== 'string' && response.status !== 404) { + const size = response.headers.get('content-length'); + const contentType = response.headers.get('content-type'); + + return { + exists: true, + size: size ? parseInt(size, 10) : undefined, + contentType: contentType || undefined, + fromCache: false, + }; + } + } catch (e) { + // File doesn't exist locally, continue to remote check + } + } + } + + // Check remote if allowed - use Range request for efficiency + if (env.allowRemoteModels && !options.local_files_only && validModelId) { + try { + // Make a Range request to get metadata without downloading full content + const rangeResponse = await fetch_file_head(remoteURL); + + if (rangeResponse && rangeResponse.status >= 200 && rangeResponse.status < 300) { + let size; + const contentType = rangeResponse.headers.get('content-type'); + + // If server supports Range requests, we get a 206 Partial Content response + // with a content-range header showing the total uncompressed file size + if (rangeResponse.status === 206) { + const contentRange = rangeResponse.headers.get('content-range'); + if (contentRange) { + // Format: "bytes 0-0/12345" where 12345 is the total uncompressed size + const match = contentRange.match(/bytes \d+-\d+\/(\d+)/); + if (match) { + size = parseInt(match[1], 10); + } + } + } else if (rangeResponse.status === 200) { + // Server doesn't support Range requests and returned the full file (200 OK) + // Cancel the response body immediately to avoid downloading the entire file + try { + await rangeResponse.body?.cancel(); + } catch (cancelError) { + // Ignore cancellation errors - body might already be consumed or not cancellable + } + } + + // Fallback to content-length if content-range is not available (200 OK response) + // This can happen if server doesn't support Range requests + if (size === undefined) { + const contentLength = rangeResponse.headers.get('content-length'); + size = contentLength ? parseInt(contentLength, 10) : undefined; + } + + return { + exists: true, + size, + contentType: contentType || undefined, + fromCache: false, + }; + } + } catch (e) { + // Range request failed most likely because of a network error, timeout, etc. + logger.warn(`Unable to fetch file metadata for "${remoteURL}": ${e}`); + } + } + + return { exists: false, fromCache: false }; +} diff --git a/packages/transformers/src/utils/model_registry/get_files.js b/packages/transformers/src/utils/model_registry/get_files.js new file mode 100644 index 0000000..eeda289 --- /dev/null +++ b/packages/transformers/src/utils/model_registry/get_files.js @@ -0,0 +1,42 @@ +import { get_tokenizer_files } from './get_tokenizer_files.js'; +import { get_model_files } from './get_model_files.js'; +import { get_processor_files } from './get_processor_files.js'; + +/** + * Returns the list of files that will be loaded for a model based on its configuration. + * Automatically detects which files are needed (tokenizer, processor, model files). + * + * @param {string} modelId The model id (e.g., "Xenova/llama-2-7b") + * @param {Object} [options] Optional parameters + * @param {import('../../configs.js').PretrainedConfig} [options.config=null] Pre-loaded model config (optional, will be fetched if not provided) + * @param {import('../dtypes.js').DataType|Record} [options.dtype=null] Override dtype (use this if passing dtype to pipeline) + * @param {import('../devices.js').DeviceType|Record} [options.device=null] Override device (use this if passing device to pipeline) + * @param {string|null} [options.model_file_name=null|null] Override the model file name (excluding .onnx suffix) + * @param {boolean} [options.include_tokenizer=true] Whether to check for tokenizer files (set to false for vision-only models) + * @param {boolean} [options.include_processor=true] Whether to check for processor files + * @returns {Promise} Array of file paths that will be loaded + */ +export async function get_files( + modelId, + { + config = null, + dtype = null, + device = null, + model_file_name = null, + include_tokenizer = true, + include_processor = true, + } = {}, +) { + const files = await get_model_files(modelId, { config, dtype, device, model_file_name }); + + if (include_tokenizer) { + const tokenizerFiles = await get_tokenizer_files(modelId); + files.push(...tokenizerFiles); + } + if (include_processor) { + const processorFiles = await get_processor_files(modelId); + files.push(...processorFiles); + } + + return files; +} diff --git a/packages/transformers/src/utils/model_registry/get_model_files.js b/packages/transformers/src/utils/model_registry/get_model_files.js new file mode 100644 index 0000000..4676424 --- /dev/null +++ b/packages/transformers/src/utils/model_registry/get_model_files.js @@ -0,0 +1,115 @@ +import { DEFAULT_DTYPE_SUFFIX_MAPPING, selectDtype } from '../dtypes.js'; +import { selectDevice } from '../devices.js'; +import { resolveExternalDataFormat, getExternalDataChunkNames } from '../model-loader.js'; +import { getSessionsConfig } from '../../models/session_config.js'; +import { AutoConfig } from '../../configs.js'; +import { makePretrainedOptionsKey } from '../hub/utils.js'; +import { memoizePromise } from '../memoize_promise.js'; +import { resolve_model_type } from './resolve_model_type.js'; + +/** + * @typedef {import('../../configs.js').PretrainedConfig} PretrainedConfig + */ + +/** + * Returns a memoized AutoConfig for the given model ID and options. + * If the same model ID and options have been requested before — even while + * the first request is still in-flight — the cached promise is returned + * so that config.json is only fetched once. + * When a pre-loaded `config` object is supplied the result is not memoized, + * since the caller already has the config and no network operation is performed. + * + * @param {string} modelId The model id (e.g., "onnx-community/granite-4.0-350m-ONNX-web") + * @param {Object} [options] + * @param {PretrainedConfig|null} [options.config=null] Pre-loaded config; skips fetching if provided. + * @param {string|null} [options.cache_dir=null] Custom local cache directory. + * @param {boolean} [options.local_files_only=false] Never hit the network if true. + * @param {string} [options.revision='main'] Git branch, tag, or commit SHA. + * @returns {Promise} + */ +export function get_config( + modelId, + { config = null, cache_dir = null, local_files_only = false, revision = 'main' } = {}, +) { + // When a pre-loaded config is provided, skip memoization — no fetch occurs + // and there is no meaningful key to deduplicate on. + if (config !== null) { + return AutoConfig.from_pretrained(modelId, { config, cache_dir, local_files_only, revision }); + } + const key = makePretrainedOptionsKey(modelId, { cache_dir, local_files_only, revision }); + return memoizePromise(key, () => + AutoConfig.from_pretrained(modelId, { config, cache_dir, local_files_only, revision }), + ); +} + +/** + * Returns the list of files that will be loaded for a model based on its configuration. + * + * This function reads configuration from the model's config.json on the hub. + * If dtype/device are not specified in the config, you can provide them to match + * what the pipeline will actually use. + * + * @param {string} modelId The model id (e.g., "onnx-community/granite-4.0-350m-ONNX-web") + * @param {Object} [options] Optional parameters + * @param {import('../../configs.js').PretrainedConfig} [options.config=null] Pre-loaded model config (optional, will be fetched if not provided) + * @param {import('../dtypes.js').DataType|Record} [options.dtype=null] Override dtype (use this if passing dtype to pipeline) + * @param {import('../devices.js').DeviceType|Record} [options.device=null] Override device (use this if passing device to pipeline) + * @param {string} [options.model_file_name=null] Override the model file name (excluding .onnx suffix). + * @returns {Promise} Array of file paths that will be loaded + */ +export async function get_model_files( + modelId, + { config = null, dtype: overrideDtype = null, device: overrideDevice = null, model_file_name = null } = {}, +) { + config = await get_config(modelId, { config }); + + const files = [ + // Add config.json (always loaded) + 'config.json', + ]; + const custom_config = config['transformers.js_config'] ?? {}; + + const use_external_data_format = custom_config.use_external_data_format; + const subfolder = 'onnx'; // Always 'onnx' as per the default in from_pretrained + + const rawDevice = overrideDevice ?? custom_config.device; + let dtype = overrideDtype ?? custom_config.dtype; + + // Infer model type from config + const modelType = resolve_model_type(config); + + const add_model_file = (fileName, baseName = null) => { + baseName = baseName ?? fileName; + const selectedDevice = selectDevice(rawDevice, fileName); + const selectedDtype = selectDtype(dtype, fileName, selectedDevice); + + const suffix = DEFAULT_DTYPE_SUFFIX_MAPPING[selectedDtype] ?? ''; + const fullName = `${baseName}${suffix}.onnx`; + const fullPath = subfolder ? `${subfolder}/${fullName}` : fullName; + files.push(fullPath); + + // Check for external data files + const num_chunks = resolveExternalDataFormat(use_external_data_format, fullName, fileName); + for (const dataFileName of getExternalDataChunkNames(fullName, num_chunks)) { + const dataFilePath = subfolder ? `${subfolder}/${dataFileName}` : dataFileName; + files.push(dataFilePath); + } + }; + + // Get session configuration from the shared source of truth + const { sessions, optional_configs } = getSessionsConfig(modelType, config, { model_file_name }); + + // Add model files based on sessions + for (const [sessionKey, baseName] of Object.entries(sessions)) { + add_model_file(sessionKey, baseName); + } + + // Add optional config files + if (optional_configs) { + for (const configFile of Object.values(optional_configs)) { + files.push(configFile); + } + } + + return files; +} diff --git a/packages/transformers/src/utils/model_registry/get_pipeline_files.js b/packages/transformers/src/utils/model_registry/get_pipeline_files.js new file mode 100644 index 0000000..c6d4e3a --- /dev/null +++ b/packages/transformers/src/utils/model_registry/get_pipeline_files.js @@ -0,0 +1,62 @@ +import { get_files } from './get_files.js'; +import { get_config } from './get_model_files.js'; +import { resolve_model_type } from './resolve_model_type.js'; +import { getTextOnlySessions } from '../../models/session_config.js'; +import { SUPPORTED_TASKS, TASK_ALIASES } from '../../pipelines/index.js'; + +/** + * Get all files needed for a specific pipeline task. + * Automatically detects which components (tokenizer, processor) are needed by checking + * whether the model has the corresponding files (tokenizer_config.json, preprocessor_config.json). + * + * @param {string} task - The pipeline task (e.g., "text-generation", "image-classification") + * @param {string} modelId - The model id (e.g., "Xenova/bert-base-uncased") + * @param {Object} [options] - Optional parameters + * @param {import('../../configs.js').PretrainedConfig} [options.config=null] - Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype=null] - Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device=null] - Override device + * @param {string} [options.model_file_name=null] - Override the model file name (excluding .onnx suffix) + * @returns {Promise} Array of file paths that will be loaded + * @throws {Error} If the task is not supported + */ +export async function get_pipeline_files(task, modelId, options = {}) { + // Apply task aliases + task = TASK_ALIASES[task] ?? task; + + // Validate that the task is supported + const taskConfig = SUPPORTED_TASKS[task]; + if (!taskConfig) { + throw new Error( + `Unsupported pipeline task: ${task}. Must be one of [${Object.keys(SUPPORTED_TASKS).join(', ')}]`, + ); + } + + // Use the task type to determine which components to auto-detect: + // - 'text' tasks: always check tokenizer, skip processor (text models rarely have one) + // - 'audio'/'image' tasks: skip tokenizer, always check processor + // - 'multimodal' tasks: check both + const { type } = taskConfig; + const include_tokenizer = type !== 'audio' && type !== 'image'; + const include_processor = type !== 'text'; + + const files = await get_files(modelId, { + ...options, + include_tokenizer, + include_processor, + }); + + // When loading multimodal models via the text-generation pipeline, + // only load the sessions needed for text generation (embed_tokens, decoder_model_merged) + if (task === 'text-generation') { + const config = await get_config(modelId, options); + const modelType = resolve_model_type(config); + const textOnlySessions = getTextOnlySessions(modelType); + + if (textOnlySessions) { + const allowedPrefixes = Object.values(textOnlySessions).map((s) => `onnx/${s}`); + return files.filter((f) => !f.startsWith('onnx/') || allowedPrefixes.some((p) => f.startsWith(p))); + } + } + + return files; +} diff --git a/packages/transformers/src/utils/model_registry/get_processor_files.js b/packages/transformers/src/utils/model_registry/get_processor_files.js new file mode 100644 index 0000000..894748d --- /dev/null +++ b/packages/transformers/src/utils/model_registry/get_processor_files.js @@ -0,0 +1,20 @@ +import { IMAGE_PROCESSOR_NAME } from '../constants.js'; +import { get_file_metadata } from './get_file_metadata.js'; + +/** + * Returns the list of processor files that will be loaded for a model. + * Auto-detects if the model has a processor by checking if preprocessor_config.json exists. + * + * @param {string} modelId The model id (e.g., "Xenova/detr-resnet-50") + * @returns {Promise} Array of processor file names (empty if no processor) + */ +export async function get_processor_files(modelId) { + if (!modelId) { + throw new Error('modelId is required'); + } + + // Check if preprocessor_config.json exists + const metadata = await get_file_metadata(modelId, IMAGE_PROCESSOR_NAME, {}); + + return metadata.exists ? [IMAGE_PROCESSOR_NAME] : []; +} diff --git a/packages/transformers/src/utils/model_registry/get_tokenizer_files.js b/packages/transformers/src/utils/model_registry/get_tokenizer_files.js new file mode 100644 index 0000000..1024f99 --- /dev/null +++ b/packages/transformers/src/utils/model_registry/get_tokenizer_files.js @@ -0,0 +1,21 @@ +import { get_file_metadata } from './get_file_metadata.js'; + +/** + * Returns the list of files that will be loaded for a tokenizer. + * Automatically detects whether the model has tokenizer files. + * + * @param {string} modelId The model id to check for tokenizer files + * @returns {Promise} An array of file names that will be loaded + */ +export async function get_tokenizer_files(modelId) { + if (!modelId) { + throw new Error('modelId is required for get_tokenizer_files'); + } + + const metadata = await get_file_metadata(modelId, 'tokenizer_config.json', {}); + if (metadata.exists) { + return ['tokenizer.json', 'tokenizer_config.json']; + } + + return []; +} diff --git a/packages/transformers/src/utils/model_registry/is_cached.js b/packages/transformers/src/utils/model_registry/is_cached.js new file mode 100644 index 0000000..9c65301 --- /dev/null +++ b/packages/transformers/src/utils/model_registry/is_cached.js @@ -0,0 +1,169 @@ +import { getCache } from '../cache.js'; +import { buildResourcePaths, checkCachedResource } from '../hub.js'; +import { get_files } from './get_files.js'; +import { get_pipeline_files } from './get_pipeline_files.js'; + +/** + * @typedef {Object} FileCacheStatus + * @property {string} file - The file path + * @property {boolean} cached - Whether the file is cached + */ + +/** + * @typedef {Object} CacheCheckResult + * @property {boolean} allCached - Whether all files are cached + * @property {FileCacheStatus[]} files - Array of files with their cache status + */ + +/** + * Internal helper to check cache status for a list of files + * @private + * @param {string} modelId - The model id + * @param {string[]} files - List of file paths to check + * @param {Object} options - Options including cache_dir + * @returns {Promise} + */ +async function check_files_cache(modelId, files, options = {}) { + const cache = await getCache(options?.cache_dir); + + if (!cache) { + const fileStatuses = files.map((filename) => ({ file: filename, cached: false })); + // No cache available, all files considered not cached + return { allCached: false, files: fileStatuses }; + } + + const fileStatuses = await Promise.all( + files.map(async (filename) => { + const { localPath, proposedCacheKey } = buildResourcePaths(modelId, filename, options, cache); + const cached = await checkCachedResource(cache, localPath, proposedCacheKey); + return { file: filename, cached: !!cached }; + }), + ); + + return { allCached: fileStatuses.every((f) => f.cached), files: fileStatuses }; +} + +/** + * Internal helper to check whether a single file is cached. + * @private + * @param {string} modelId - The model id + * @param {string} filename - The file path to check + * @param {Object} options - Options including cache_dir + * @returns {Promise} + */ +async function is_file_cached(modelId, filename, options = {}) { + const cache = await getCache(options?.cache_dir); + if (!cache) return false; + const { localPath, proposedCacheKey } = buildResourcePaths(modelId, filename, options, cache); + return !!(await checkCachedResource(cache, localPath, proposedCacheKey)); +} + +/** + * Quickly checks if a model is cached by verifying that `config.json` is present, + * then confirming all required files are cached. + * Returns a plain boolean — use `is_cached_files` if you need per-file detail. + * + * @param {string} modelId The model id (e.g., "Xenova/gpt2") + * @param {Object} [options] Optional parameters + * @param {string} [options.cache_dir] Custom cache directory + * @param {string} [options.revision] Model revision (default: 'main') + * @param {import('../../configs.js').PretrainedConfig} [options.config] Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype] Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device] Override device + * @returns {Promise} Whether all required files are cached + */ +export async function is_cached(modelId, options = {}) { + if (!modelId) { + throw new Error('modelId is required'); + } + + // Fast early-exit: if config.json is missing we can expect the rest not to be cached too. + if (!(await is_file_cached(modelId, 'config.json', options))) { + return false; + } + + const files = await get_files(modelId, options); + const result = await check_files_cache(modelId, files, options); + return result.allCached; +} + +/** + * Checks if all files for a given model are already cached, with per-file detail. + * Automatically determines which files are needed using get_files(). + * + * @param {string} modelId The model id (e.g., "Xenova/gpt2") + * @param {Object} [options] Optional parameters + * @param {string} [options.cache_dir] Custom cache directory + * @param {string} [options.revision] Model revision (default: 'main') + * @param {import('../../configs.js').PretrainedConfig} [options.config] Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype] Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device] Override device + * @returns {Promise} Object with allCached boolean and files array with cache status + */ +export async function is_cached_files(modelId, options = {}) { + if (!modelId) { + throw new Error('modelId is required'); + } + + const files = await get_files(modelId, options); + return await check_files_cache(modelId, files, options); +} + +/** + * Quickly checks if all files for a specific pipeline task are cached by verifying + * that `config.json` is present, then confirming all required files are cached. + * Returns a plain boolean — use `is_pipeline_cached_files` if you need per-file detail. + * + * @param {string} task - The pipeline task (e.g., "text-generation", "image-classification") + * @param {string} modelId - The model id (e.g., "Xenova/gpt2") + * @param {Object} [options] - Optional parameters + * @param {string} [options.cache_dir] - Custom cache directory + * @param {string} [options.revision] - Model revision (default: 'main') + * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype] - Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device] - Override device + * @returns {Promise} Whether all required files are cached + */ +export async function is_pipeline_cached(task, modelId, options = {}) { + if (!task) { + throw new Error('task is required'); + } + if (!modelId) { + throw new Error('modelId is required'); + } + + // Fast early-exit: if config.json is missing we can expect the rest not to be cached too. + if (!(await is_file_cached(modelId, 'config.json', options))) { + return false; + } + + const files = await get_pipeline_files(task, modelId, options); + const result = await check_files_cache(modelId, files, options); + return result.allCached; +} + +/** + * Checks if all files for a specific pipeline task are already cached, with per-file detail. + * Automatically determines which components are needed based on the task. + * + * @param {string} task - The pipeline task (e.g., "text-generation", "image-classification") + * @param {string} modelId - The model id (e.g., "Xenova/gpt2") + * @param {Object} [options] - Optional parameters + * @param {string} [options.cache_dir] - Custom cache directory + * @param {string} [options.revision] - Model revision (default: 'main') + * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config + * @param {import('../dtypes.js').DataType|Record} [options.dtype] - Override dtype + * @param {import('../devices.js').DeviceType|Record} [options.device] - Override device + * @returns {Promise} Object with allCached boolean and files array with cache status + */ +export async function is_pipeline_cached_files(task, modelId, options = {}) { + if (!task) { + throw new Error('task is required'); + } + if (!modelId) { + throw new Error('modelId is required'); + } + + const files = await get_pipeline_files(task, modelId, options); + return await check_files_cache(modelId, files, options); +} diff --git a/packages/transformers/src/utils/model_registry/resolve_model_type.js b/packages/transformers/src/utils/model_registry/resolve_model_type.js new file mode 100644 index 0000000..43d0640 --- /dev/null +++ b/packages/transformers/src/utils/model_registry/resolve_model_type.js @@ -0,0 +1,67 @@ +import { MODEL_MAPPING_NAMES, MODEL_TYPE_MAPPING } from '../../models/modeling_utils.js'; +import { MODEL_TYPES } from '../../models/session_config.js'; +import { GITHUB_ISSUE_URL } from '../constants.js'; +import { logger } from '../logger.js'; + +/** + * @typedef {import('../../configs.js').PretrainedConfig} PretrainedConfig + */ + +/** + * Resolves the model type (e.g., EncoderOnly, DecoderOnly, Seq2Seq, …) from a + * model config by checking architectures and model_type against the known + * MODEL_TYPE_MAPPING. + * + * Resolution order: + * 1. `config.architectures` entries looked up in MODEL_TYPE_MAPPING + * 2. `config.model_type` looked up directly in MODEL_TYPE_MAPPING + * 3. `config.model_type` looked up via MODEL_MAPPING_NAMES → architecture → MODEL_TYPE_MAPPING + * 4. Fallback to `MODEL_TYPES.EncoderOnly` + * + * @param {PretrainedConfig} config The model config object. + * @param {{ warn?: boolean }} [options] Set `warn` to false to suppress the + * fallback warning (defaults to true). + * @returns {number} One of the MODEL_TYPES enum values. + */ +export function resolve_model_type(config, { warn = true } = {}) { + // @ts-ignore - architectures is set via Object.assign in PretrainedConfig constructor + const architectures = /** @type {string[]} */ (config.architectures || []); + + // 1. Try architectures against MODEL_TYPE_MAPPING + for (const arch of architectures) { + const mappedType = MODEL_TYPE_MAPPING.get(arch); + if (mappedType !== undefined) { + return mappedType; + } + } + + // 2. Try config.model_type directly + if (config.model_type) { + const mappedType = MODEL_TYPE_MAPPING.get(config.model_type); + if (mappedType !== undefined) { + return mappedType; + } + + // 3. Try MODEL_MAPPING_NAMES as a last resort + for (const mapping of Object.values(MODEL_MAPPING_NAMES)) { + if (mapping.has(config.model_type)) { + const resolved = MODEL_TYPE_MAPPING.get(mapping.get(config.model_type)); + if (resolved !== undefined) { + return resolved; + } + } + } + } + + // 4. Fallback + if (warn) { + const archList = architectures.length > 0 ? architectures.join(', ') : '(none)'; + logger.warn( + `[resolve_model_type] Architecture(s) not found in MODEL_TYPE_MAPPING: [${archList}] ` + + `for model type '${config.model_type}'. Falling back to EncoderOnly (single model.onnx file). ` + + `If you encounter issues, please report at: ${GITHUB_ISSUE_URL}`, + ); + } + + return MODEL_TYPES.EncoderOnly; +} diff --git a/packages/transformers/src/utils/random.js b/packages/transformers/src/utils/random.js new file mode 100644 index 0000000..76319be --- /dev/null +++ b/packages/transformers/src/utils/random.js @@ -0,0 +1,225 @@ +/** + * Let there be order amidst the chaos. + * + * This file implements Mersenne Twister 19937, matching Python's `random` module exactly for reproducibility. + * + * ```javascript + * import { random } from '@huggingface/transformers'; + * + * random.seed(42); + * random.random(); // 0.6394267984578837 (matches Python) + * random.gauss(0, 1); // normal-distributed value + * random.choices(['a','b'], [3, 1]); // weighted pick + * + * const arr = [1, 2, 3, 4, 5]; + * random.shuffle(arr); // in-place Fisher-Yates shuffle + * + * // Use a separate instance to avoid affecting the global state: + * const rng = new random.Random(42); + * rng.random(); // 0.6394267984578837 (same seed, independent state) + * ``` + * + * **Note on Reproducibility:** + * Similarly to the [Python random](https://docs.python.org/3/library/random.html#notes-on-reproducibility) + * module, it is useful to be able to reproduce the sequences given by a pseudo-random number generator. + * By reusing a seed value, the same sequence should be reproducible from run to run as long as multiple + * threads or asynchronous operations are not running concurrently. + * + * @module utils/random + */ + +import { apis } from '../env.js'; + +/** + * Mersenne Twister 19937 PRNG, matching Python's `random.Random` class exactly. + * + * Each instance has its own independent state, so seeding one instance does not + * affect any other instance or the global helper functions. + * + * @example + * const rng1 = new Random(42); + * const rng2 = new Random(42); + * rng1.random() === rng2.random(); // true (same seed, independent state) + */ +export class Random { + constructor(seed) { + this._mt = new Uint32Array(624); + this._idx = 625; + this._gauss_next = null; + this._random_fn = this.random.bind(this); + this.seed(seed); + } + + /** + * Seeds this instance's PRNG. + * + * When called with a number, initializes the state deterministically from that value. + * When called with no arguments (or `undefined`/`null`), seeds from OS entropy + * via `crypto.getRandomValues`, matching Python's `random.seed()` behaviour. + * + * @param {number} [n] The seed value. Omit to seed from OS entropy. + */ + seed(n) { + if (n === undefined || n === null) { + if (apis.IS_CRYPTO_AVAILABLE) { + const buf = new Uint32Array(1); + crypto.getRandomValues(buf); + n = buf[0]; + } else { + n = Date.now() >>> 0; + } + } + const mt = this._mt; + const u = (a, b) => Math.imul(a, b) >>> 0, + key = []; + for (let v = n || 0; v > 0; v = Math.floor(v / 0x100000000)) key.push(v & 0xffffffff); + if (!key.length) key.push(0); + mt[0] = 19650218; + for (let k = 1; k < 624; ++k) mt[k] = (u(1812433253, mt[k - 1] ^ (mt[k - 1] >>> 30)) + k) >>> 0; + let i = 1, + j = 0; + for (let k = Math.max(624, key.length); k > 0; --k, ++i, ++j) { + if (i >= 624) { + mt[0] = mt[623]; + i = 1; + } + if (j >= key.length) j = 0; + mt[i] = ((mt[i] ^ u(mt[i - 1] ^ (mt[i - 1] >>> 30), 1664525)) + key[j] + j) >>> 0; + } + for (let k = 623; k > 0; --k, ++i) { + if (i >= 624) { + mt[0] = mt[623]; + i = 1; + } + mt[i] = ((mt[i] ^ u(mt[i - 1] ^ (mt[i - 1] >>> 30), 1566083941)) - i) >>> 0; + } + mt[0] = 0x80000000; + this._idx = 624; + this._gauss_next = null; + } + + /** + * Generates a random unsigned 32-bit integer. + * + * Performs the "twist" step when the state buffer is exhausted, + * then applies the standard MT19937 tempering transform. + * + * @returns {number} A random integer in the range [0, 2^32 - 1]. + */ + _int32() { + const mt = this._mt; + if (this._idx >= 624) { + for (let k = 0; k < 624; ++k) { + // twist + const y = (mt[k] & 0x80000000) | (mt[(k + 1) % 624] & 0x7fffffff); + mt[k] = (mt[(k + 397) % 624] ^ (y >>> 1) ^ (y & 1 ? 0x9908b0df : 0)) >>> 0; + } + this._idx = 0; + } + let y = mt[this._idx++]; + y ^= y >>> 11; + y ^= (y << 7) & 0x9d2c5680; + y ^= (y << 15) & 0xefc60000; + y ^= y >>> 18; + return y >>> 0; + } + + /** + * Generates a random floating-point number in the half-open interval [0, 1). + * + * Combines two 32-bit integers (using 53 bits of precision) to produce + * a uniformly distributed double, matching Python's `random.random()`. + * + * @returns {number} A random float in [0, 1). + */ + random() { + return ((this._int32() >>> 5) * 67108864.0 + (this._int32() >>> 6)) / 9007199254740992.0; + } + + /** + * Generates a random number from a Gaussian (normal) distribution. + * + * Uses the Box-Muller transform with a cached spare value, + * matching Python's `random.gauss()` output for the same seed. + * + * @param {number} [mu=0] The mean of the distribution. + * @param {number} [sigma=1] The standard deviation of the distribution. + * @returns {number} A normally distributed random value. + */ + gauss(mu = 0, sigma = 1) { + let z = this._gauss_next; + this._gauss_next = null; + if (z === null) { + const x2pi = this.random() * 2 * Math.PI, + g2rad = Math.sqrt(-2 * Math.log(1 - this.random())); + z = Math.cos(x2pi) * g2rad; + this._gauss_next = Math.sin(x2pi) * g2rad; + } + return mu + z * sigma; + } + + /** + * Shuffles an array in-place using the Fisher-Yates algorithm. + * + * Uses rejection sampling via `getrandbits`-style bit masking to ensure + * a uniform distribution, matching Python's `random.shuffle()`. + * + * @param {any[]} arr The array to shuffle in-place. + */ + shuffle(arr) { + for (let i = arr.length - 1; i > 0; --i) { + const k = 32 - Math.clz32(i + 1); + let r = this._int32() >>> (32 - k); + while (r > i) r = this._int32() >>> (32 - k); + const t = arr[i]; + arr[i] = arr[r]; + arr[r] = t; + } + } + + /** + * Selects a single element from a weighted population. + * + * Matches Python's `random.choices(population, weights=weights, k=1)[0]` + * + * @param {any[]} population The array of items to choose from. + * @param {number[]} weights An array of non-negative weights, one per population element. + * @returns {*} A single randomly selected element from the population. + */ + choices(population, weights) { + return population[_weightedIndexWith(this._random_fn, weights)]; + } +} + +/** + * Returns a random index into `weights`, where each index's probability + * is proportional to its weight. Uses a linear scan: O(n) time, O(1) memory. + * + * @param {() => number} randomFn A function returning a uniform random float in [0, 1). + * @param {ArrayLike} weights Non-negative weights. + * @returns {number} A randomly selected index in `[0, weights.length)`. + */ +function _weightedIndexWith(randomFn, weights) { + let sum = 0; + for (let i = 0; i < weights.length; ++i) sum += weights[i]; + let x = randomFn() * sum; + for (let i = 0; i < weights.length; ++i) { + x -= weights[i]; + if (x < 0) return i; + } + return weights.length - 1; // floating-point guard +} + +// Global default instance: mirrors the module-level functions in Python's `random` module. +const _default = new Random(); +export const random = Object.freeze({ + Random, + seed: _default.seed.bind(_default), + random: _default.random.bind(_default), + gauss: _default.gauss.bind(_default), + shuffle: _default.shuffle.bind(_default), + choices: _default.choices.bind(_default), +}); + +// Private helper function, used by LogitsSampler, but not exported as part of the public API. +export const _weightedIndex = (weights) => _weightedIndexWith(random.random, weights); diff --git a/packages/transformers/src/utils/tensor.js b/packages/transformers/src/utils/tensor.js new file mode 100644 index 0000000..85da403 --- /dev/null +++ b/packages/transformers/src/utils/tensor.js @@ -0,0 +1,1673 @@ +/** + * @file Helper module for `Tensor` processing. + * + * These functions and classes are only used internally, + * meaning an end-user shouldn't need to access anything here. + * + * @module utils/tensor + */ + +import { interpolate_data, max, min, permute_data, uint16_to_float32 } from './maths.js'; + +import { Tensor as ONNXTensor, isONNXTensor } from '../backends/onnx.js'; + +import { TensorOpRegistry } from '../ops/registry.js'; + +import { DataTypeMap } from './dtypes.js'; + +import { random } from './random.js'; + +/** + * @typedef {keyof typeof DataTypeMap} DataType + * @typedef {import('./maths.js').AnyTypedArray | any[]} DataArray + */ + +export class Tensor { + /** + * Dimensions of the tensor. + * @type {number[]} + */ + get dims() { + // @ts-ignore + return this.ort_tensor.dims; + } + set dims(value) { + // FIXME: ONNXTensor declares dims as readonly so one needs to use the constructor() if dims change. + // @ts-ignore + this.ort_tensor.dims = value; + } + + /** + * Type of the tensor. + * @type {DataType} + */ + get type() { + return this.ort_tensor.type; + } + + /** + * The data stored in the tensor. + * @type {DataArray} + */ + get data() { + return this.ort_tensor.data; + } + + /** + * The number of elements in the tensor. + * @type {number} + */ + get size() { + return this.ort_tensor.size; + } + + /** + * The location of the tensor data. + * @type {string} + */ + get location() { + return this.ort_tensor.location; + } + + ort_tensor; + + /** + * Create a new Tensor or copy an existing Tensor. + * @param {[DataType, DataArray, number[]]|[ONNXTensor]} args + */ + constructor(...args) { + if (isONNXTensor(args[0])) { + this.ort_tensor = /** @type {ONNXTensor} */ (args[0]); + } else { + // Create new tensor + this.ort_tensor = new ONNXTensor( + /** @type {DataType} */ (args[0]), + // @ts-expect-error ts(2769) Type 'number' is not assignable to type 'bigint'. + /** @type {Exclude} */ (args[1]), + args[2], + ); + } + + return new Proxy(this, { + get: (obj, key) => { + if (typeof key === 'string') { + let index = Number(key); + if (Number.isInteger(index)) { + // key is an integer (i.e., index) + return obj._getitem(index); + } + } + // @ts-ignore + return obj[key]; + }, + set: (obj, key, value) => { + // TODO allow setting of data + + // @ts-ignore + return (obj[key] = value); + }, + }); + } + + dispose() { + this.ort_tensor.dispose(); + // this.ort_tensor = undefined; + } + + /** + * Returns an iterator object for iterating over the tensor data in row-major order. + * If the tensor has more than one dimension, the iterator will yield subarrays. + * @returns {Iterator} An iterator object for iterating over the tensor data in row-major order. + */ + *[Symbol.iterator]() { + const [iterLength, ...iterDims] = this.dims; + + if (iterDims.length > 0) { + const iterSize = iterDims.reduce((a, b) => a * b); + for (let i = 0; i < iterLength; ++i) { + yield this._subarray(i, iterSize, iterDims); + } + } else { + yield* this.data; + } + } + + /** + * Index into a Tensor object. + * @param {number} index The index to access. + * @returns {Tensor} The data at the specified index. + */ + _getitem(index) { + const [iterLength, ...iterDims] = this.dims; + + index = safeIndex(index, iterLength); + + if (iterDims.length > 0) { + const iterSize = iterDims.reduce((a, b) => a * b); + return this._subarray(index, iterSize, iterDims); + } else { + return new Tensor(this.type, [this.data[index]], iterDims); + } + } + + /** + * @param {number|bigint} item The item to search for in the tensor + * @returns {number} The index of the first occurrence of item in the tensor data. + */ + indexOf(item) { + const this_data = this.data; + for (let index = 0; index < this_data.length; ++index) { + // Note: == instead of === so we can match Ints with BigInts + if (this_data[index] == item) { + return index; + } + } + return -1; + } + + /** + * @param {number} index + * @param {number} iterSize + * @param {any} iterDims + * @returns {Tensor} + */ + _subarray(index, iterSize, iterDims) { + const o1 = index * iterSize; + const o2 = (index + 1) * iterSize; + + // We use subarray if available (typed array), otherwise we use slice (normal array) + const data = 'subarray' in this.data ? this.data.subarray(o1, o2) : this.data.slice(o1, o2); + return new Tensor(this.type, data, iterDims); + } + + /** + * Returns the value of this tensor as a standard JavaScript Number. This only works + * for tensors with one element. For other cases, see `Tensor.tolist()`. + * @returns {number|bigint} The value of this tensor as a standard JavaScript Number. + * @throws {Error} If the tensor has more than one element. + */ + item() { + const this_data = this.data; + if (this_data.length !== 1) { + throw new Error(`a Tensor with ${this_data.length} elements cannot be converted to Scalar`); + } + return this_data[0]; + } + + /** + * Convert tensor data to a n-dimensional JS list + * @returns {any[]} + */ + tolist() { + return reshape(this.data, this.dims); + } + + /** + * Return a new Tensor with the sigmoid function applied to each element. + * @returns {Tensor} The tensor with the sigmoid function applied. + */ + sigmoid() { + return this.clone().sigmoid_(); + } + + /** + * Applies the sigmoid function to the tensor in place. + * @returns {Tensor} Returns `this`. + */ + sigmoid_() { + const this_data = this.data; + for (let i = 0; i < this_data.length; ++i) { + this_data[i] = 1 / (1 + Math.exp(-this_data[i])); + } + return this; + } + + /** + * Return a new Tensor with a callback function applied to each element. + * @param {Function} callback - The function to apply to each element. It should take three arguments: + * the current element, its index, and the tensor's data array. + * @returns {Tensor} A new Tensor with the callback function applied to each element. + */ + map(callback) { + return this.clone().map_(callback); + } + + /** + * Apply a callback function to each element of the tensor in place. + * @param {Function} callback - The function to apply to each element. It should take three arguments: + * the current element, its index, and the tensor's data array. + * @returns {Tensor} Returns `this`. + */ + map_(callback) { + const this_data = this.data; + for (let i = 0; i < this_data.length; ++i) { + this_data[i] = callback(this_data[i], i, this_data); + } + return this; + } + + /** + * Return a new Tensor with every element multiplied by a constant. + * @param {number} val The value to multiply by. + * @returns {Tensor} The new tensor. + */ + mul(val) { + return this.clone().mul_(val); + } + + /** + * Multiply the tensor by a constant in place. + * @param {number} val The value to multiply by. + * @returns {Tensor} Returns `this`. + */ + mul_(val) { + const this_data = this.data; + for (let i = 0; i < this_data.length; ++i) { + this_data[i] *= val; + } + return this; + } + + /** + * Return a new Tensor with every element divided by a constant. + * @param {number} val The value to divide by. + * @returns {Tensor} The new tensor. + */ + div(val) { + return this.clone().div_(val); + } + + /** + * Divide the tensor by a constant in place. + * @param {number} val The value to divide by. + * @returns {Tensor} Returns `this`. + */ + div_(val) { + const this_data = this.data; + for (let i = 0; i < this_data.length; ++i) { + this_data[i] /= val; + } + return this; + } + + /** + * Return a new Tensor with every element added by a constant. + * @param {number} val The value to add by. + * @returns {Tensor} The new tensor. + */ + add(val) { + return this.clone().add_(val); + } + + /** + * Add the tensor by a constant in place. + * @param {number} val The value to add by. + * @returns {Tensor} Returns `this`. + */ + add_(val) { + const this_data = this.data; + for (let i = 0; i < this_data.length; ++i) { + this_data[i] += val; + } + return this; + } + + /** + * Return a new Tensor with every element subtracted by a constant. + * @param {number} val The value to subtract by. + * @returns {Tensor} The new tensor. + */ + sub(val) { + return this.clone().sub_(val); + } + + /** + * Subtract the tensor by a constant in place. + * @param {number} val The value to subtract by. + * @returns {Tensor} Returns `this`. + */ + sub_(val) { + const this_data = this.data; + for (let i = 0; i < this_data.length; ++i) { + this_data[i] -= val; + } + return this; + } + + /** + * Creates a deep copy of the current Tensor. + * @returns {Tensor} A new Tensor with the same type, data, and dimensions as the original. + */ + clone() { + return new Tensor(this.type, this.data.slice(), this.dims.slice()); + } + + /** + * Performs a slice operation on the Tensor along specified dimensions. + * + * Consider a Tensor that has a dimension of [4, 7]: + * ``` + * [ 1, 2, 3, 4, 5, 6, 7] + * [ 8, 9, 10, 11, 12, 13, 14] + * [15, 16, 17, 18, 19, 20, 21] + * [22, 23, 24, 25, 26, 27, 28] + * ``` + * We can slice against the two dims of row and column, for instance in this + * case we can start at the second element, and return to the second last, + * like this: + * ``` + * tensor.slice([1, -1], [1, -1]); + * ``` + * which would return: + * ``` + * [ 9, 10, 11, 12, 13 ] + * [ 16, 17, 18, 19, 20 ] + * ``` + * + * @param {...(number|number[]|null)} slices The slice specifications for each dimension. + * - If a number is given, then a single element is selected. + * - If an array of two numbers is given, then a range of elements [start, end (exclusive)] is selected. + * - If null is given, then the entire dimension is selected. + * @returns {Tensor} A new Tensor containing the selected elements. + * @throws {Error} If the slice input is invalid. + */ + slice(...slices) { + // This allows for slicing with ranges and numbers + const newTensorDims = []; + const newOffsets = []; + + // slices is an array of numbers or arrays of numbers + // e.g., slices = [0, [1, 3], null, [0, 3]] + for (let sliceIndex = 0; sliceIndex < this.dims.length; ++sliceIndex) { + let slice = slices[sliceIndex]; + + if (slice === null || slice === undefined) { + // null or undefined means take the whole dimension + newOffsets.push([0, this.dims[sliceIndex]]); + newTensorDims.push(this.dims[sliceIndex]); + } else if (typeof slice === 'number') { + slice = safeIndex(slice, this.dims[sliceIndex], sliceIndex); + + // A number means take a single element + newOffsets.push([slice, slice + 1]); + } else if (Array.isArray(slice) && slice.length === 2) { + // An array of length 2 means take a range of elements + let [start, end] = slice; + start = start === null ? 0 : safeIndex(start, this.dims[sliceIndex], sliceIndex, false); + end = end === null ? this.dims[sliceIndex] : safeIndex(end, this.dims[sliceIndex], sliceIndex, false); + + if (start > end) { + throw new Error(`Invalid slice: ${slice}`); + } + + const offsets = [Math.max(start, 0), Math.min(end, this.dims[sliceIndex])]; + + newOffsets.push(offsets); + newTensorDims.push(offsets[1] - offsets[0]); + } else { + throw new Error(`Invalid slice: ${slice}`); + } + } + + const newDims = newOffsets.map(([start, end]) => end - start); + const newBufferSize = newDims.reduce((a, b) => a * b); + + const this_data = this.data; + // Allocate memory + // @ts-ignore + const data = new this_data.constructor(newBufferSize); + + // Precompute strides + const stride = this.stride(); + + // Detect if the slice is contiguous + let isContiguous = true; + for (let i = 1; i < newDims.length; ++i) { + if (newOffsets[i][0] !== 0 || newOffsets[i][1] !== this.dims[i]) { + isContiguous = false; + break; + } + } + + if (isContiguous) { + // Perform bulk copy for contiguous slices to improve performance + const start = newOffsets[0][0] * stride[0]; + const end = newOffsets[0][1] * stride[0]; + + if (ArrayBuffer.isView(this_data)) { + // If this.data is a TypedArray, use subarray + // @ts-ignore + data.set(this_data.subarray(start, end)); + } else if (Array.isArray(this_data)) { + // If this.data is a plain array, use slice + const slicedData = this_data.slice(start, end); + for (let i = 0; i < slicedData.length; ++i) { + data[i] = slicedData[i]; + } + } else { + throw new Error('Unsupported data type for slicing'); + } + } else { + // Fallback to manual copying for non-contiguous slices + for (let i = 0; i < newBufferSize; ++i) { + let originalIndex = 0; + for (let j = newDims.length - 1, num = i; j >= 0; --j) { + const size = newDims[j]; + originalIndex += ((num % size) + newOffsets[j][0]) * stride[j]; + num = Math.floor(num / size); + } + data[i] = this_data[originalIndex]; + } + } + + return new Tensor(this.type, data, newTensorDims); + } + + /** + * Return a permuted version of this Tensor, according to the provided dimensions. + * @param {...number} dims Dimensions to permute. + * @returns {Tensor} The permuted tensor. + */ + permute(...dims) { + return permute(this, dims); + } + + // TODO: implement transpose. For now (backwards compatibility), it's just an alias for permute() + /** @type {Tensor['permute']} */ + transpose(...dims) { + return this.permute(...dims); + } + + /** + * Returns the sum of each row of the input tensor in the given dimension dim. + * + * @param {number|null} [dim=null] The dimension or dimensions to reduce. If `null`, all dimensions are reduced. + * @param {boolean} keepdim Whether the output tensor has `dim` retained or not. + * @returns The summed tensor + */ + sum(dim = null, keepdim = false) { + return this.norm(1, dim, keepdim); + } + + /** + * Returns the matrix norm or vector norm of a given tensor. + * @param {number|string} [p='fro'] The order of norm + * @param {number|null} [dim=null] Specifies which dimension of the tensor to calculate the norm across. + * If dim is None, the norm will be calculated across all dimensions of input. + * @param {boolean} [keepdim=false] Whether the output tensors have dim retained or not. + * @returns {Tensor} The norm of the tensor. + */ + norm(p = 'fro', dim = null, keepdim = false) { + if (p === 'fro') { + // NOTE: Since we only support integer dims, Frobenius norm produces the same result as p=2. + p = 2; + } else if (typeof p === 'string') { + throw Error(`Unsupported norm: ${p}`); + } + + const this_data = this.data; + const is_bigint = this_data instanceof BigInt64Array || this_data instanceof BigUint64Array; + + if (is_bigint && p !== 1) { + throw Error(`Expected a floating point tensor as input. Got ${this.type}`); + } + + let fn, zero; + if (is_bigint) { + fn = (/** @type {bigint} */ a, /** @type {bigint} */ b) => a + b; + zero = 0n; + } else { + fn = (/** @type {number} */ a, /** @type {number} */ b) => a + b ** p; + zero = 0; + } + + if (dim === null) { + // @ts-ignore + let val = this_data.reduce(fn, zero); + if (p !== 1) { + val = val ** (1 / p); + } + return new Tensor(this.type, [val], []); + } + + const [type, result, resultDims] = reduce_helper(fn, this, dim, keepdim); + + if (p !== 1) { + for (let i = 0; i < result.length; ++i) { + result[i] = result[i] ** (1 / p); + } + } + return new Tensor(type, result, resultDims); + } + + /** + * Performs `L_p` normalization of inputs over specified dimension. Operates in place. + * @param {number} [p=2] The exponent value in the norm formulation + * @param {number} [dim=1] The dimension to reduce + * @returns {Tensor} `this` for operation chaining. + */ + normalize_(p = 2.0, dim = 1) { + dim = safeIndex(dim, this.dims.length); + + const norm = this.norm(p, dim, true); + + const this_data = this.data; + const norm_data = norm.data; + for (let i = 0; i < this_data.length; ++i) { + // Calculate the index in the resulting array + let resultIndex = 0; + + for (let j = this.dims.length - 1, num = i, resultMultiplier = 1; j >= 0; --j) { + const size = this.dims[j]; + if (j !== dim) { + const index = num % size; + resultIndex += index * resultMultiplier; + resultMultiplier *= this.dims[j]; + } + num = Math.floor(num / size); + } + + // Divide by normalized value + this_data[i] /= norm_data[resultIndex]; + } + + return this; + } + + /** + * Performs `L_p` normalization of inputs over specified dimension. + * @param {number} [p=2] The exponent value in the norm formulation + * @param {number} [dim=1] The dimension to reduce + * @returns {Tensor} The normalized tensor. + */ + normalize(p = 2.0, dim = 1) { + return this.clone().normalize_(p, dim); + } + + /** + * Compute and return the stride of this tensor. + * Stride is the jump necessary to go from one element to the next one in the specified dimension dim. + * @returns {number[]} The stride of this tensor. + */ + stride() { + return dimsToStride(this.dims); + } + + /** + * Returns a tensor with all specified dimensions of input of size 1 removed. + * + * NOTE: The returned tensor shares the storage with the input tensor, so changing the contents of one will change the contents of the other. + * If you would like a copy, use `tensor.clone()` before squeezing. + * + * @param {number|number[]|null} [dim=null] If given, the input will be squeezed only in the specified dimensions. + * @returns {Tensor} The squeezed tensor + */ + squeeze(dim = null) { + return new Tensor(this.type, this.data, calc_squeeze_dims(this.dims, dim)); + } + + /** + * In-place version of @see {@link Tensor.squeeze} + */ + squeeze_(dim = null) { + this.dims = calc_squeeze_dims(this.dims, dim); + return this; + } + + /** + * Returns a new tensor with a dimension of size one inserted at the specified position. + * + * NOTE: The returned tensor shares the same underlying data with this tensor. + * + * @param {number} dim The index at which to insert the singleton dimension + * @returns {Tensor} The unsqueezed tensor + */ + unsqueeze(dim) { + return new Tensor(this.type, this.data, calc_unsqueeze_dims(this.dims, dim)); + } + + /** + * In-place version of @see {@link Tensor.unsqueeze} + * @type {Tensor['unsqueeze']} + */ + unsqueeze_(dim) { + this.dims = calc_unsqueeze_dims(this.dims, dim); + return this; + } + + /** + * In-place version of @see {@link Tensor.flatten} + */ + flatten_(start_dim = 0, end_dim = -1) { + // TODO validate inputs + end_dim = (end_dim + this.dims.length) % this.dims.length; + + let dimsToKeepBefore = this.dims.slice(0, start_dim); + let dimsToFlatten = this.dims.slice(start_dim, end_dim + 1); + let dimsToKeepAfter = this.dims.slice(end_dim + 1); + + this.dims = [...dimsToKeepBefore, dimsToFlatten.reduce((a, b) => a * b, 1), ...dimsToKeepAfter]; + return this; + } + + /** + * Flattens input by reshaping it into a one-dimensional tensor. + * If `start_dim` or `end_dim` are passed, only dimensions starting with `start_dim` + * and ending with `end_dim` are flattened. The order of elements in input is unchanged. + * @param {number} start_dim the first dim to flatten + * @param {number} end_dim the last dim to flatten + * @returns {Tensor} The flattened tensor. + */ + flatten(start_dim = 0, end_dim = -1) { + return this.clone().flatten_(start_dim, end_dim); + } + + /** + * Returns a new tensor with the same data as the `self` tensor but of a different `shape`. + * @param {...number} dims the desired size + * @returns {Tensor} The tensor with the same data but different shape + */ + view(...dims) { + // TODO: validate dims + let inferredIndex = -1; + for (let i = 0; i < dims.length; ++i) { + if (dims[i] === -1) { + if (inferredIndex !== -1) { + throw new Error('Only one dimension can be inferred'); + } + inferredIndex = i; + } + } + + const this_data = this.data; + if (inferredIndex !== -1) { + // Some dimension must be inferred + const productOther = dims.reduce((product, curr, index) => { + return index !== inferredIndex ? product * curr : product; + }, 1); + + dims[inferredIndex] = this_data.length / productOther; + } + return new Tensor(this.type, this_data, dims); // NOTE: uses same underlying storage + } + + neg_() { + const this_data = this.data; + for (let i = 0; i < this_data.length; ++i) { + this_data[i] = -this_data[i]; + } + return this; + } + neg() { + return this.clone().neg_(); + } + + /** + * Computes input > val element-wise. + * @param {number} val The value to compare with. + * @returns {Tensor} A boolean tensor that is `true` where input is greater than other and `false` elsewhere. + */ + gt(val) { + const mask = new Uint8Array(this.data.length); + const this_data = this.data; + for (let i = 0; i < this_data.length; ++i) { + mask[i] = this_data[i] > val ? 1 : 0; + } + return new Tensor('bool', mask, this.dims); + } + + /** + * Computes input < val element-wise. + * @param {number} val The value to compare with. + * @returns {Tensor} A boolean tensor that is `true` where input is less than other and `false` elsewhere. + */ + lt(val) { + const mask = new Uint8Array(this.data.length); + const this_data = this.data; + for (let i = 0; i < this_data.length; ++i) { + mask[i] = this_data[i] < val ? 1 : 0; + } + return new Tensor('bool', mask, this.dims); + } + + /** + * In-place version of @see {@link Tensor.clamp} + * @type {Tensor['clamp']} + */ + clamp_(min, max) { + const this_data = this.data; + for (let i = 0; i < this_data.length; ++i) { + this_data[i] = Math.min(Math.max(this_data[i], min), max); + } + return this; + } + + /** + * Clamps all elements in input into the range [ min, max ] + * @param {number} min lower-bound of the range to be clamped to + * @param {number} max upper-bound of the range to be clamped to + * @returns {Tensor} the output tensor. + */ + clamp(min, max) { + return this.clone().clamp_(min, max); + } + + /** + * In-place version of @see {@link Tensor.round} + */ + round_() { + const this_data = this.data; + for (let i = 0; i < this_data.length; ++i) { + this_data[i] = Math.round(this_data[i]); + } + return this; + } + + /** + * Rounds elements of input to the nearest integer. + * @returns {Tensor} the output tensor. + */ + round() { + return this.clone().round_(); + } + + mean(dim = null, keepdim = false) { + return mean(this, dim, keepdim); + } + + min(dim = null, keepdim = false) { + if (dim === null) { + // None to reduce over all dimensions. + const val = min(this.data)[0]; + return new Tensor( + this.type, + [val], + [ + /* scalar */ + ], + ); + } + const [type, result, resultDims] = reduce_helper((a, b) => Math.min(a, b), this, dim, keepdim, Infinity); + return new Tensor(type, result, resultDims); + } + + max(dim = null, keepdim = false) { + if (dim === null) { + // None to reduce over all dimensions. + const val = max(this.data)[0]; + return new Tensor( + this.type, + [val], + [ + /* scalar */ + ], + ); + } + const [type, result, resultDims] = reduce_helper((a, b) => Math.max(a, b), this, dim, keepdim, -Infinity); + return new Tensor(type, result, resultDims); + } + + argmin(dim = null, keepdim = false) { + if (dim !== null) { + throw new Error('`dim !== null` not yet implemented.'); + } + const index = min(this.data)[1]; + return new Tensor('int64', [BigInt(index)], []); + } + argmax(dim = null, keepdim = false) { + if (dim !== null) { + throw new Error('`dim !== null` not yet implemented.'); + } + const index = max(this.data)[1]; + return new Tensor('int64', [BigInt(index)], []); + } + + /** + * Repeats this tensor along the specified dimensions. + * @param {...number} repeats The number of times to repeat this tensor along each dimension. + * @returns {Tensor} The repeated tensor. + * @throws {Error} If the number of repeats is less than the number of dimensions. + */ + repeat(...repeats) { + if (repeats.length < this.dims.length) { + throw new Error( + `Number of dimensions of repeat dims (${repeats.length}) cannot be smaller than number of dimensions of tensor (${this.dims.length})`, + ); + } + + // Optimization: if all repeats are 1, just clone with potentially expanded dims + if (repeats.every((r) => r === 1)) { + if (repeats.length === this.dims.length) { + return this.clone(); + } + // Need to expand dimensions by prepending 1s + const numPrependedDims = repeats.length - this.dims.length; + const newDims = Array(numPrependedDims).fill(1).concat(this.dims); + return new Tensor(this.type, this.data.slice(), newDims); + } + + // If repeats has more dimensions than the tensor, prepend 1s to tensor dims + const numPrependedDims = repeats.length - this.dims.length; + const expandedDims = Array(numPrependedDims).fill(1).concat(this.dims); + + // Calculate the output dimensions + const outputDims = expandedDims.map((dim, i) => dim * repeats[i]); + + // Calculate total output size + const outputSize = outputDims.reduce((a, b) => a * b, 1); + + // Allocate output data + const this_data = this.data; + // @ts-ignore + const data = new this_data.constructor(outputSize); + + // Calculate strides for the expanded input tensor + const inputStrides = dimsToStride(expandedDims); + const outputStrides = dimsToStride(outputDims); + + // Fill the output tensor + for (let i = 0; i < outputSize; ++i) { + // Convert flat index to multi-dimensional indices in output + let remaining = i; + let inputIndex = 0; + + for (let d = 0; d < outputDims.length; ++d) { + const outputCoord = Math.floor(remaining / outputStrides[d]); + remaining = remaining % outputStrides[d]; + + // Map output coordinate to input coordinate using modulo + const inputCoord = outputCoord % expandedDims[d]; + inputIndex += inputCoord * inputStrides[d]; + } + + data[i] = this_data[inputIndex]; + } + + return new Tensor(this.type, data, outputDims); + } + + /** + * Constructs a tensor by repeating the elements of input. The dims argument specifies the number of repetitions in each dimension. + * @param {...number} dims The number of repetitions per dimension. + * @returns {Tensor} The tiled tensor. + */ + tile(...dims) { + // If fewer repeats than dims, prepend 1s to repeats + if (dims.length < this.dims.length) { + const numPrependedRepeats = this.dims.length - dims.length; + dims = Array(numPrependedRepeats).fill(1).concat(dims); + } + return this.repeat(...dims); + } + + /** + * Performs Tensor dtype conversion. + * @param {DataType} type The desired data type. + * @returns {Tensor} The converted tensor. + */ + to(type) { + // If the self Tensor already has the correct dtype, then self is returned. + if (this.type === type) return this; + + // Otherwise, the returned tensor is a copy of self with the desired dtype. + if (!DataTypeMap.hasOwnProperty(type)) { + throw new Error(`Unsupported type: ${type}`); + } + + // Handle special cases where a mapping function is needed (e.g., where one type is a bigint and the other is a number) + let map_fn; + const is_source_bigint = ['int64', 'uint64'].includes(this.type); + const is_dest_bigint = ['int64', 'uint64'].includes(type); + if (is_source_bigint && !is_dest_bigint) { + // TypeError: Cannot convert a BigInt value to a number + map_fn = Number; + } else if (!is_source_bigint && is_dest_bigint) { + // TypeError: Cannot convert [x] to a BigInt + if (['float16', 'float32', 'float64'].includes(this.type)) { + map_fn = (x) => BigInt(Math.floor(x)); + } else { + map_fn = BigInt; + } + } else if (this.type === 'float16' && type == 'float32' && this.data instanceof Uint16Array) { + // Certain runtimes do not support Float16Array, so the values are stored in Uint16Array + return new Tensor(type, uint16_to_float32(this.data), this.dims); + } + + return new Tensor( + type, + // @ts-ignore - TypeScript can't verify .from() across union of TypedArray constructors + DataTypeMap[type].from(this.data, map_fn), + this.dims, + ); + } +} + +/** + * This creates a nested array of a given type and depth (see examples). + * + * @example + * NestArray; // string[] + * @example + * NestArray; // number[][] + * @example + * NestArray; // string[][][] etc. + * @template T + * @template {number} Depth + * @template {never[]} [Acc=[]] + * @typedef {Acc['length'] extends Depth ? T : NestArray} NestArray + */ + +/** + * Reshapes a 1-dimensional array into an n-dimensional array, according to the provided dimensions. + * + * @example + * reshape([10 ], [1 ]); // Type: number[] Value: [10] + * reshape([1, 2, 3, 4 ], [2, 2 ]); // Type: number[][] Value: [[1, 2], [3, 4]] + * reshape([1, 2, 3, 4, 5, 6, 7, 8], [2, 2, 2]); // Type: number[][][] Value: [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] + * reshape([1, 2, 3, 4, 5, 6, 7, 8], [4, 2 ]); // Type: number[][] Value: [[1, 2], [3, 4], [5, 6], [7, 8]] + * @param {T[]|DataArray} data The input array to reshape. + * @param {DIM} dimensions The target shape/dimensions. + * @template T + * @template {[number]|number[]} DIM + * @returns {NestArray} The reshaped array. + */ +function reshape(data, dimensions) { + const totalElements = data.length; + const dimensionSize = dimensions.length === 0 ? 1 : dimensions.reduce((a, b) => a * b); + + if (totalElements !== dimensionSize) { + throw Error(`cannot reshape array of size ${totalElements} into shape (${dimensions})`); + } + + /** @type {any} */ + let reshapedArray = data; + + for (let i = dimensions.length - 1; i >= 0; i--) { + reshapedArray = reshapedArray.reduce( + (acc, val) => { + let lastArray = acc[acc.length - 1]; + + if (lastArray.length < dimensions[i]) { + lastArray.push(val); + } else { + acc.push([val]); + } + + return acc; + }, + [[]], + ); + } + + return reshapedArray[0]; +} + +/** + * Permutes a tensor according to the provided axes. + * @param {any} tensor The input tensor to permute. + * @param {number[]} axes The axes to permute the tensor along. + * @returns {Tensor} The permuted tensor. + */ +export function permute(tensor, axes) { + const [permutedData, shape] = permute_data(tensor.data, tensor.dims, axes); + return new Tensor(tensor.type, permutedData, shape); +} + +/** + * Interpolates an Tensor to the given size. + * @param {Tensor} input The input tensor to interpolate. Data must be channel-first (i.e., [c, h, w]) + * @param {number[]} size The output size of the image + * @param {string} mode The interpolation mode + * @param {boolean} align_corners Whether to align corners. + * @returns {Tensor} The interpolated tensor. + */ +export function interpolate(input, [out_height, out_width], mode = 'bilinear', align_corners = false) { + // Input image dimensions + const in_channels = input.dims.at(-3) ?? 1; + const in_height = input.dims.at(-2); + const in_width = input.dims.at(-1); + + const output = interpolate_data( + /** @type {import('./maths.js').TypedArray}*/ (input.data), + [in_channels, in_height, in_width], + [out_height, out_width], + mode, + align_corners, + ); + return new Tensor(input.type, output, [in_channels, out_height, out_width]); +} + +/** + * Down/up samples the input. + * Inspired by https://pytorch.org/docs/stable/generated/torch.nn.functional.interpolate.html. + * @param {Tensor} input the input tensor + * @param {Object} options the options for the interpolation + * @param {[number, number]|[number, number, number]|[number, number, number, number]} [options.size=null] output spatial size. + * @param {"nearest"|"bilinear"|"bicubic"} [options.mode='bilinear'] algorithm used for upsampling + * @returns {Promise} The interpolated tensor. + */ +export async function interpolate_4d(input, { size = null, mode = 'bilinear' } = {}) { + // Error checking + if (input.dims.length !== 4) { + throw new Error('`interpolate_4d` currently only supports 4D input.'); + } + if (!size) { + // TODO: support scale_factor + throw new Error('`interpolate_4d` requires a `size` argument.'); + } + + // Fill in missing dimensions + let targetDims; + if (size.length === 2) { + targetDims = [...input.dims.slice(0, 2), ...size]; + } else if (size.length === 3) { + targetDims = [input.dims[0], ...size]; + } else if (size.length === 4) { + targetDims = size; + } else { + throw new Error('`size` must be of length 2, 3, or 4.'); + } + + let op; + if (mode === 'nearest') { + op = await TensorOpRegistry.nearest_interpolate_4d; + } else if (mode === 'bilinear') { + op = await TensorOpRegistry.bilinear_interpolate_4d; + } else if (mode === 'bicubic') { + op = await TensorOpRegistry.bicubic_interpolate_4d; + } else { + throw new Error(`Unsupported mode: ${mode}`); + } + + const sizeTensor = new Tensor('int64', new BigInt64Array(targetDims.map(BigInt)), [targetDims.length]); + return await op({ x: input, s: sizeTensor }); +} + +/** + * Matrix product of two tensors. + * Inspired by https://pytorch.org/docs/stable/generated/torch.matmul.html + * @param {Tensor} a the first tensor to be multiplied + * @param {Tensor} b the second tensor to be multiplied + * @returns {Promise} The matrix product of the two tensors. + */ +export async function matmul(a, b) { + const op = await TensorOpRegistry.matmul; + return await op({ a, b }); +} + +/** + * Computes the one dimensional Fourier transform of real-valued input. + * Inspired by https://pytorch.org/docs/stable/generated/torch.fft.rfft.html + * @param {Tensor} x the real input tensor + * @param {Tensor} a The dimension along which to take the one dimensional real FFT. + * @returns {Promise} the output tensor. + */ +export async function rfft(x, a) { + const op = await TensorOpRegistry.rfft; + return await op({ x, a }); +} + +/** + * Returns the k largest elements of the given input tensor. + * Inspired by https://pytorch.org/docs/stable/generated/torch.topk.html + * @param {Tensor} x the input tensor + * @param {number} [k] the k in "top-k" + * @returns {Promise<[Tensor, Tensor]>} the output tuple of (Tensor, LongTensor) of top-k elements and their indices. + */ +export async function topk(x, k) { + const op = await TensorOpRegistry.top_k; + + if (k == null) { + k = x.dims.at(-1); + } else { + k = Math.min(k, x.dims.at(-1)); + } + return await op({ + x, + k: new Tensor('int64', [BigInt(k)], [1]), + }); +} + +const arrayToIndexTensor = (array) => new Tensor('int64', array, [array.length]); +/** + * Slice a multidimensional float32 tensor. + * @param {Tensor} data: Tensor of data to extract slices from + * @param {number[]} starts: 1-D array of starting indices of corresponding axis in axes + * @param {number[]} ends: 1-D array of ending indices (exclusive) of corresponding axis in axes + * @param {number[]} axes: 1-D array of axes that starts and ends apply to + * @param {number[]} [steps]: 1-D array of slice step of corresponding axis in axes. + * @returns {Promise} Sliced data tensor. + */ +export async function slice(data, starts, ends, axes, steps) { + const op = await TensorOpRegistry.slice; + return await op({ + x: data, + s: arrayToIndexTensor(starts), + e: arrayToIndexTensor(ends), + a: arrayToIndexTensor(axes), + t: arrayToIndexTensor(steps ?? new Array(axes.length).fill(1)), + }); +} + +/** + * Perform mean pooling of the last hidden state followed by a normalization step. + * @param {Tensor} last_hidden_state Tensor of shape [batchSize, seqLength, embedDim] + * @param {Tensor} attention_mask Tensor of shape [batchSize, seqLength] + * @returns {Tensor} Returns a new Tensor of shape [batchSize, embedDim]. + */ +export function mean_pooling(last_hidden_state, attention_mask) { + // last_hidden_state: [batchSize, seqLength, embedDim] + // attention_mask: [batchSize, seqLength] + const lastHiddenStateData = last_hidden_state.data; + const attentionMaskData = attention_mask.data; + + const shape = [last_hidden_state.dims[0], last_hidden_state.dims[2]]; + + // @ts-ignore + const returnedData = new lastHiddenStateData.constructor(shape[0] * shape[1]); + const [batchSize, seqLength, embedDim] = last_hidden_state.dims; + + let outIndex = 0; + for (let i = 0; i < batchSize; ++i) { + const offset = i * embedDim * seqLength; + + for (let k = 0; k < embedDim; ++k) { + let sum = 0; + let count = 0; + + const attnMaskOffset = i * seqLength; + const offset2 = offset + k; + // Pool over all words in sequence + for (let j = 0; j < seqLength; ++j) { + // index into attention mask + const attn = Number(attentionMaskData[attnMaskOffset + j]); + + count += attn; + sum += lastHiddenStateData[offset2 + j * embedDim] * attn; + } + + const avg = sum / count; + returnedData[outIndex++] = avg; + } + } + + return new Tensor(last_hidden_state.type, returnedData, shape); +} + +/** + * Apply Layer Normalization for last certain number of dimensions. + * @param {Tensor} input The input tensor + * @param {number[]} normalized_shape input shape from an expected input of size + * @param {Object} options The options for the layer normalization + * @param {number} [options.eps=1e-5] A value added to the denominator for numerical stability. + * @returns {Tensor} The normalized tensor. + */ +export function layer_norm(input, normalized_shape, { eps = 1e-5 } = {}) { + if (input.dims.length !== 2) { + throw new Error('`layer_norm` currently only supports 2D input.'); + } + + const [batchSize, featureDim] = input.dims; + + if (normalized_shape.length !== 1 && normalized_shape[0] !== featureDim) { + throw new Error('`normalized_shape` must be a 1D array with shape `[input.dims[1]]`.'); + } + + const [std, mean] = std_mean(input, 1, 0, true); + const stdData = /** @type {Float32Array} */ (std.data); + const meanData = /** @type {Float32Array} */ (mean.data); + + const inputData = /** @type {Float32Array} */ (input.data); + + // @ts-ignore + const returnedData = new inputData.constructor(inputData.length); + + for (let i = 0; i < batchSize; ++i) { + const offset = i * featureDim; + for (let j = 0; j < featureDim; ++j) { + const offset2 = offset + j; + returnedData[offset2] = (inputData[offset2] - meanData[i]) / (stdData[i] + eps); + } + } + return new Tensor(input.type, returnedData, input.dims); +} + +/** + * Helper function to calculate new dimensions when performing a squeeze operation. + * @param {number[]} dims The dimensions of the tensor. + * @param {number|number[]|null} dim The dimension(s) to squeeze. + * @returns {number[]} The new dimensions. + * @private + */ +function calc_squeeze_dims(dims, dim) { + dims = dims.slice(); + if (dim === null) { + dims = dims.filter((d) => d !== 1); + } else if (typeof dim === 'number') { + if (dims[dim] === 1) { + dims.splice(dim, 1); + } + } else if (Array.isArray(dim)) { + dims = dims.filter((x, i) => { + return x !== 1 || !dim.includes(i); + }); + } + return dims; +} + +/** + * Helper function to calculate new dimensions when performing an unsqueeze operation. + * @param {number[]} dims The dimensions of the tensor. + * @param {number} dim The dimension to unsqueeze. + * @returns {number[]} The new dimensions. + * @private + */ +function calc_unsqueeze_dims(dims, dim) { + // Dimension out of range (e.g., "expected to be in range of [-4, 3], but got 4") + // + 1 since we allow inserting at the end (i.e. dim = -1) + dim = safeIndex(dim, dims.length + 1); + dims = dims.slice(); + // Insert 1 into specified dimension + dims.splice(dim, 0, 1); + return dims; +} + +/** + * Safely calculate the index for an array of a given size, allowing negative indexing. + * @param {number} index The index that will be used. + * @param {number} size The size of the array. + * @param {number|null} [dimension=null] The dimension that the index is for (optional). + * @returns {number} The index, guaranteed to be non-negative and less than `arrayLength`. + * + * @throws {Error} If the index is out of range. + * @private + */ +function safeIndex(index, size, dimension = null, boundsCheck = true) { + if (index < -size || index >= size) { + if (boundsCheck) { + throw new Error( + `IndexError: index ${index} is out of bounds for dimension${dimension === null ? '' : ' ' + dimension} with size ${size}`, + ); + } else { + return index < -size ? 0 : size; + } + } + + if (index < 0) { + // Negative indexing, ensuring positive index + index = ((index % size) + size) % size; + } + return index; +} + +/** + * Concatenates an array of tensors along a specified dimension. + * @param {Tensor[]} tensors The array of tensors to concatenate. + * @param {number} dim The dimension to concatenate along. + * @returns {Tensor} The concatenated tensor. + */ +export function cat(tensors, dim = 0) { + dim = safeIndex(dim, tensors[0].dims.length); + + // TODO do validation of shapes + + const resultDims = tensors[0].dims.slice(); + resultDims[dim] = tensors.reduce((a, b) => a + b.dims[dim], 0); + + // Create a new array to store the accumulated values + const resultSize = resultDims.reduce((a, b) => a * b, 1); + // @ts-ignore + const result = new tensors[0].data.constructor(resultSize); + + // Create output tensor of same type as first + const resultType = tensors[0].type; + + if (dim === 0) { + // Handle special case for performance reasons + + let offset = 0; + for (const tensor of tensors) { + const tensorData = tensor.data; + result.set(tensorData, offset); + offset += tensorData.length; + } + } else { + let currentDim = 0; + + for (let t = 0; t < tensors.length; ++t) { + const { data, dims } = tensors[t]; + + // Iterate over the data array + for (let i = 0; i < data.length; ++i) { + // Calculate the index in the resulting array + let resultIndex = 0; + + for (let j = dims.length - 1, num = i, resultMultiplier = 1; j >= 0; --j) { + const size = dims[j]; + let index = num % size; + if (j === dim) { + index += currentDim; + } + resultIndex += index * resultMultiplier; + resultMultiplier *= resultDims[j]; + num = Math.floor(num / size); + } + // Accumulate the value at the current index + result[resultIndex] = data[i]; + } + + currentDim += dims[dim]; + } + } + return new Tensor(resultType, result, resultDims); +} + +/** + * Stack an array of tensors along a specified dimension. + * @param {Tensor[]} tensors The array of tensors to stack. + * @param {number} dim The dimension to stack along. + * @returns {Tensor} The stacked tensor. + */ +export function stack(tensors, dim = 0) { + // TODO do validation of shapes + // NOTE: stack expects each tensor to be equal size + return cat( + tensors.map((t) => t.unsqueeze(dim)), + dim, + ); +} + +/** + * @param {(previousValue: any, currentValue: any, currentIndex?: number, resultIndex?: number) => any} callbackfn + * @param {Tensor} input the input tensor. + * @param {number} dim the dimension to reduce. + * @param {boolean} keepdim whether the output tensor has dim retained or not. + * @param {any} [initialValue=null] the initial value to start the reduction with. + * @returns {[DataType, any, number[]]} The reduced tensor data. + */ +function reduce_helper(callbackfn, input, dim, keepdim = false, initialValue = null) { + const inputData = input.data; + const inputDims = input.dims; + + // Negative indexing + dim = safeIndex(dim, inputDims.length); + + // Calculate the shape of the resulting array after summation + const resultDims = inputDims.slice(); // Copy the original dimensions + resultDims[dim] = 1; // Remove the specified axis + + // Create a new array to store the accumulated values + // @ts-ignore + const result = new inputData.constructor(inputData.length / inputDims[dim]); + if (initialValue !== null) { + result.fill(initialValue); + } + + // Iterate over the data array + for (let i = 0; i < inputData.length; ++i) { + // Calculate the index in the resulting array + let resultIndex = 0; + + for (let j = inputDims.length - 1, num = i, resultMultiplier = 1; j >= 0; --j) { + const size = inputDims[j]; + if (j !== dim) { + const index = num % size; + resultIndex += index * resultMultiplier; + resultMultiplier *= resultDims[j]; + } + num = Math.floor(num / size); + } + + // Accumulate the value at the current index + result[resultIndex] = callbackfn(result[resultIndex], inputData[i], i, resultIndex); + } + + if (!keepdim) resultDims.splice(dim, 1); + + return [input.type, result, resultDims]; +} + +/** + * Calculates the standard deviation and mean over the dimensions specified by dim. dim can be a single dimension or `null` to reduce over all dimensions. + * @param {Tensor} input the input tenso + * @param {number|null} dim the dimension to reduce. If None, all dimensions are reduced. + * @param {number} correction difference between the sample size and sample degrees of freedom. Defaults to Bessel's correction, correction=1. + * @param {boolean} keepdim whether the output tensor has dim retained or not. + * @returns {Tensor[]} A tuple of (std, mean) tensors. + */ +export function std_mean(input, dim = null, correction = 1, keepdim = false) { + const inputData = /** @type {Float32Array} */ (input.data); + const inputDims = input.dims; + + if (dim === null) { + // None to reduce over all dimensions. + const sum = inputData.reduce((a, b) => a + b, 0); + const mean = sum / inputData.length; + const std = Math.sqrt(inputData.reduce((a, b) => a + (b - mean) ** 2, 0) / (inputData.length - correction)); + + const meanTensor = new Tensor( + input.type, + [mean], + [ + /* scalar */ + ], + ); + const stdTensor = new Tensor( + input.type, + [std], + [ + /* scalar */ + ], + ); + + return [stdTensor, meanTensor]; + } + dim = safeIndex(dim, inputDims.length); + const meanTensor = mean(input, dim, keepdim); + const meanTensorData = meanTensor.data; + + // Compute squared sum + const [type, result, resultDims] = reduce_helper( + (a, b, i, j) => a + (b - meanTensorData[j]) ** 2, + input, + dim, + keepdim, + ); + + // Square root of the squared sum + for (let i = 0; i < result.length; ++i) { + result[i] = Math.sqrt(result[i] / (inputDims[dim] - correction)); + } + + const stdTensor = new Tensor(type, result, resultDims); + + return [stdTensor, meanTensor]; +} + +/** + * Returns the mean value of each row of the input tensor in the given dimension dim. + * @param {Tensor} input the input tensor. + * @param {number|null} dim the dimension to reduce. + * @param {boolean} keepdim whether the output tensor has dim retained or not. + * @returns {Tensor} A new tensor with means taken along the specified dimension. + */ +export function mean(input, dim = null, keepdim = false) { + const inputDims = input.dims; + const inputData = /** @type {Float32Array} */ (input.data); + + if (dim === null) { + // None to reduce over all dimensions. + const val = inputData.reduce((a, b) => a + b, 0); + return new Tensor( + input.type, + [val / inputData.length], + [ + /* scalar */ + ], + ); + } + dim = safeIndex(dim, inputDims.length); + + // Compute sum + const [type, result, resultDims] = reduce_helper((a, b) => a + b, input, dim, keepdim); + + // Divide by number of elements in the dimension + if (inputDims[dim] !== 1) { + for (let i = 0; i < result.length; ++i) { + result[i] /= inputDims[dim]; + } + } + + return new Tensor(type, result, resultDims); +} + +function dimsToStride(dims) { + const stride = new Array(dims.length); + for (let i = dims.length - 1, s2 = 1; i >= 0; --i) { + stride[i] = s2; + s2 *= dims[i]; + } + return stride; +} + +function fullHelper(size, fill_value, dtype, cls) { + const numElements = size.reduce((a, b) => a * b, 1); + return new Tensor(dtype, new cls(numElements).fill(fill_value), size); +} + +/** + * Creates a tensor of size size filled with fill_value. The tensor's dtype is inferred from fill_value. + * @param {number[]} size A sequence of integers defining the shape of the output tensor. + * @param {number|bigint|boolean} fill_value The value to fill the output tensor with. + * @returns {Tensor} The filled tensor. + */ +export function full(size, fill_value) { + let dtype; + let typedArrayCls; + if (typeof fill_value === 'number') { + dtype = 'float32'; + typedArrayCls = Float32Array; + } else if (typeof fill_value === 'bigint') { + dtype = 'int64'; + typedArrayCls = BigInt64Array; + } else if (typeof fill_value === 'boolean') { + dtype = 'bool'; + typedArrayCls = Uint8Array; + } else { + // TODO: support other dtypes + throw new Error(`Unsupported data type: ${typeof fill_value}`); + } + return fullHelper(size, fill_value, dtype, typedArrayCls); +} + +export function full_like(tensor, fill_value) { + return full(tensor.dims, fill_value); +} + +/** + * Returns a tensor filled with the scalar value 1, with the shape defined by the variable argument size. + * @param {number[]} size A sequence of integers defining the shape of the output tensor. + * @returns {Tensor} The ones tensor. + */ +export function ones(size) { + return fullHelper(size, 1n, 'int64', BigInt64Array); +} + +/** + * Returns a tensor filled with the scalar value 1, with the same size as input. + * @param {Tensor} tensor The size of input will determine size of the output tensor. + * @returns {Tensor} The ones tensor. + */ +export function ones_like(tensor) { + return ones(tensor.dims); +} + +/** + * Returns a tensor filled with the scalar value 0, with the shape defined by the variable argument size. + * @param {number[]} size A sequence of integers defining the shape of the output tensor. + * @returns {Tensor} The zeros tensor. + */ +export function zeros(size) { + return fullHelper(size, 0n, 'int64', BigInt64Array); +} + +/** + * Returns a tensor filled with the scalar value 0, with the same size as input. + * @param {Tensor} tensor The size of input will determine size of the output tensor. + * @returns {Tensor} The zeros tensor. + */ +export function zeros_like(tensor) { + return zeros(tensor.dims); +} + +/** + * Returns a tensor filled with random numbers from a uniform distribution on the interval [0, 1) + * @param {number[]} size A sequence of integers defining the shape of the output tensor. + * @returns {Tensor} The random tensor. + */ +export function rand(size) { + const length = size.reduce((a, b) => a * b, 1); + return new Tensor( + 'float32', + Float32Array.from({ length }, () => random.random()), + size, + ); +} + +/** + * Returns a tensor filled with random numbers from a normal distribution with mean 0 and variance 1 (also called the standard normal distribution). + * @param {number[]} size A sequence of integers defining the shape of the output tensor. + * @returns {Tensor} The random tensor. + */ +export function randn(size) { + const length = size.reduce((a, b) => a * b, 1); + return new Tensor( + 'float32', + Float32Array.from({ length }, () => random.gauss()), + size, + ); +} + +/** + * Quantizes the embeddings tensor to binary or unsigned binary precision. + * @param {Tensor} tensor The tensor to quantize. + * @param {'binary'|'ubinary'} precision The precision to use for quantization. + * @returns {Tensor} The quantized tensor. + */ +export function quantize_embeddings(tensor, precision) { + if (tensor.dims.length !== 2) { + throw new Error('The tensor must have 2 dimensions'); + } + if (tensor.dims.at(-1) % 8 !== 0) { + throw new Error('The last dimension of the tensor must be a multiple of 8'); + } + if (!['binary', 'ubinary'].includes(precision)) { + throw new Error("The precision must be either 'binary' or 'ubinary'"); + } + + const signed = precision === 'binary'; + const dtype = signed ? 'int8' : 'uint8'; + + // Create a typed array to store the packed bits + const cls = signed ? Int8Array : Uint8Array; + const inputData = tensor.data; + const outputData = new cls(inputData.length / 8); + + // Iterate over each number in the array + for (let i = 0; i < inputData.length; ++i) { + // Determine if the number is greater than 0 + const bit = inputData[i] > 0 ? 1 : 0; + + // Calculate the index in the typed array and the position within the byte + const arrayIndex = Math.floor(i / 8); + const bitPosition = i % 8; + + // Pack the bit into the typed array + outputData[arrayIndex] |= bit << (7 - bitPosition); + if (signed && bitPosition === 0) { + outputData[arrayIndex] -= 128; + } + } + + return new Tensor(dtype, outputData, [tensor.dims[0], tensor.dims[1] / 8]); +} diff --git a/packages/transformers/src/utils/video.js b/packages/transformers/src/utils/video.js new file mode 100644 index 0000000..e81e1f9 --- /dev/null +++ b/packages/transformers/src/utils/video.js @@ -0,0 +1,126 @@ +import { RawImage } from './image.js'; +import { env, apis } from '../env.js'; + +export class RawVideoFrame { + /** + * @param {RawImage} image + * @param {number} timestamp + */ + constructor(image, timestamp) { + this.image = image; + this.timestamp = timestamp; + } +} + +export class RawVideo { + /** + * @param {RawVideoFrame[]|RawImage[]} frames + * @param {number} duration + */ + constructor(frames, duration) { + if (frames.length > 0 && frames[0] instanceof RawImage) { + // Assume uniform timestamps + frames = frames.map((image, i) => new RawVideoFrame(image, ((i + 1) / (frames.length + 1)) * duration)); + } + this.frames = /** @type {RawVideoFrame[]} */ (frames); + this.duration = duration; + } + + get width() { + return this.frames[0].image.width; + } + get height() { + return this.frames[0].image.height; + } + + get fps() { + return this.frames.length / this.duration; + } +} + +/** + * Loads a video. + * + * @param {string|Blob|HTMLVideoElement} src The video to process. + * @param {Object} [options] Optional parameters. + * @param {number} [options.num_frames=null] The number of frames to sample uniformly. + * @param {number} [options.fps=null] The number of frames to sample per second. + * + * @returns {Promise} The loaded video. + */ +export async function load_video(src, { num_frames = null, fps = null } = {}) { + if (!apis.IS_BROWSER_ENV) { + throw new Error('`load_video` is currently only supported in browser environments.'); + } + + // TODO: Support efficiently loading all frames using the WebCodecs API. + // Specfically, https://developer.mozilla.org/en-US/docs/Web/API/VideoDecoder + if (num_frames == null && fps == null) { + throw new Error('Either num_frames or fps must be provided.'); + } + + const frames = []; + + const video = document.createElement('video'); + video.crossOrigin = 'anonymous'; + video.muted = true; // mute to allow autoplay and seeking + + if (typeof src === 'string') { + video.src = src; + } else if (src instanceof Blob) { + video.src = URL.createObjectURL(src); + } else if (src instanceof HTMLVideoElement) { + video.src = src.src; + } else { + throw new Error('Invalid URL or video element provided.'); + } + // Wait for metadata to load to obtain duration + await new Promise((resolve) => (video.onloadedmetadata = resolve)); + + if (video.seekable.start(0) === video.seekable.end(0)) { + // Fallback: Download entire video if not seekable + const response = await env.fetch(video.src); + const blob = await response.blob(); + video.src = URL.createObjectURL(blob); + await new Promise((resolve) => (video.onloadedmetadata = resolve)); + } + + const duration = video.duration; + + let count, step; + if (num_frames != null) { + count = num_frames; + step = num_frames === 1 ? 0 : duration / (num_frames - 1); + } else { + step = 1 / fps; + count = Math.floor(duration / step); + } + + // Build an array of sample times based on num_frames or fps + let sampleTimes = []; + for (let i = 0; i < count; ++i) { + sampleTimes.push(num_frames === 1 ? duration / 2 : i * step); + } + + const canvas = document.createElement('canvas'); + canvas.width = video.videoWidth; + canvas.height = video.videoHeight; + const ctx = canvas.getContext('2d', { willReadFrequently: true }); + for (const t of sampleTimes) { + video.currentTime = t; + await new Promise((resolve) => { + video.onseeked = resolve; + }); + ctx.drawImage(video, 0, 0, canvas.width, canvas.height); + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const frameData = new RawImage(imageData.data, canvas.width, canvas.height, 4); + + const frame = new RawVideoFrame(frameData, t); + frames.push(frame); + } + + // Clean up video element. + video.remove(); + + return new RawVideo(frames, duration); +} diff --git a/packages/transformers/tests/asset_cache.js b/packages/transformers/tests/asset_cache.js new file mode 100644 index 0000000..768eaa0 --- /dev/null +++ b/packages/transformers/tests/asset_cache.js @@ -0,0 +1,74 @@ +import { RawImage } from "../src/transformers.js"; + +const BASE_URL = "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/"; +const TEST_IMAGES = Object.freeze({ + white_image: BASE_URL + "white-image.png", + blue_image: BASE_URL + "blue-image.png", + pattern_3x3: BASE_URL + "pattern_3x3.png", + pattern_3x5: BASE_URL + "pattern_3x5.png", + checkerboard_8x8: BASE_URL + "checkerboard_8x8.png", + checkerboard_64x32: BASE_URL + "checkerboard_64x32.png", + gradient_1280x640: BASE_URL + "gradient_1280x640.png", + receipt: BASE_URL + "receipt.png", + tiger: BASE_URL + "tiger.jpg", + paper: BASE_URL + "nougat_paper.png", + cats: BASE_URL + "cats.jpg", + + // grayscale image + skateboard: "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/ml-web-games/skateboard.png", + + vitmatte_image: BASE_URL + "vitmatte_image.png", + vitmatte_trimap: BASE_URL + "vitmatte_trimap.png", + + beetle: BASE_URL + "beetle.png", + book_cover: BASE_URL + "book-cover.png", + corgi: BASE_URL + "corgi.jpg", + man_on_car: BASE_URL + "young-man-standing-and-leaning-on-car.jpg", + portrait_of_woman: BASE_URL + "portrait-of-woman_small.jpg", +}); + +const TEST_AUDIOS = { + mlk: BASE_URL + "mlk.npy", + whisper_1: BASE_URL + "whisper-test-audio_1.pcm", + whisper_2: BASE_URL + "whisper-test-audio_2.pcm", + whisper_3: BASE_URL + "whisper-test-audio_3.pcm", +}; + +/** @type {Map} */ +const IMAGE_CACHE = new Map(); +const load_image = async (url) => { + const cached = IMAGE_CACHE.get(url); + if (cached) { + return cached; + } + const image = await RawImage.fromURL(url); + IMAGE_CACHE.set(url, image); + return image; +}; + +/** @type {Map} */ +const AUDIO_CACHE = new Map(); +const load_audio = async (url) => { + const cached = AUDIO_CACHE.get(url); + if (cached) { + return cached; + } + const buffer = await (await fetch(url)).arrayBuffer(); + const audio = url.endsWith(".npy") ? Float32Array.from(new Float64Array(buffer)) : new Float32Array(buffer); + AUDIO_CACHE.set(url, audio); + return audio; +}; + +/** + * Load a cached image. + * @param {keyof typeof TEST_IMAGES} name The name of the image to load. + * @returns {Promise} The loaded image. + */ +export const load_cached_image = (name) => load_image(TEST_IMAGES[name]); + +/** + * Load a cached audio. + * @param {keyof typeof TEST_AUDIOS} name The name of the audio to load. + * @returns {Promise} The loaded audio. + */ +export const load_cached_audio = (name) => load_audio(TEST_AUDIOS[name]); diff --git a/packages/transformers/tests/bundles.test.js b/packages/transformers/tests/bundles.test.js new file mode 100644 index 0000000..1bc0411 --- /dev/null +++ b/packages/transformers/tests/bundles.test.js @@ -0,0 +1,37 @@ +import { spawnSync } from "child_process"; + +const MODULE_NAME = "@huggingface/transformers"; + +const CODE_BODY = ` +const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; +const generator = await pipeline("text-generation", model_id, { dtype: "fp32" }); +const result = await generator("hello", { max_new_tokens: 3, return_full_text: false }); +process.stdout.write(result[0].generated_text); +`; + +const TARGET_OUTPUT = "erdingsdelete mely"; + +const wrap_async_iife = (code) => `(async function() { ${code} })();`; + +const check = (code, module = false) => { + const args = ["-e", code]; + if (module) args.push("--input-type=module"); + const { status, stdout, stderr } = spawnSync("node", args); + expect(stderr.toString()).toEqual(""); // No warnings or errors are printed + expect(stdout.toString()).toEqual(TARGET_OUTPUT); // The output should match + expect(status).toEqual(0); // The process should exit cleanly +}; + +describe("Testing the bundle", () => { + it("ECMAScript Module (ESM)", () => { + check(`import { pipeline } from "${MODULE_NAME}";${CODE_BODY}`, true); + }); + + it("CommonJS (CJS) with require", () => { + check(`const { pipeline } = require("${MODULE_NAME}");${wrap_async_iife(CODE_BODY)}`); + }); + + it("CommonJS (CJS) with dynamic import", () => { + check(`${wrap_async_iife(`const { pipeline } = await import("${MODULE_NAME}");${CODE_BODY}`)}`); + }); +}); diff --git a/packages/transformers/tests/configs.test.js b/packages/transformers/tests/configs.test.js new file mode 100644 index 0000000..f66a8a8 --- /dev/null +++ b/packages/transformers/tests/configs.test.js @@ -0,0 +1,23 @@ +import { AutoConfig, env } from "../src/transformers.js"; +import { getFile } from "../src/utils/hub.js"; + +// Initialise the testing environment +env.allowLocalModels = false; +env.useFSCache = false; + +const TEST_DATA = { + "Xenova/bert-base-uncased": { + model_type: "bert", + }, +}; + +describe("Configs", () => { + for (const [model_id, minimal_config] of Object.entries(TEST_DATA)) { + it(model_id, async () => { + const config = await AutoConfig.from_pretrained(model_id); + for (const [key, value] of Object.entries(minimal_config)) { + expect(config[key]).toEqual(value); + } + }); + } +}); diff --git a/packages/transformers/tests/exports.test.js b/packages/transformers/tests/exports.test.js new file mode 100644 index 0000000..3a4a5a1 --- /dev/null +++ b/packages/transformers/tests/exports.test.js @@ -0,0 +1,41 @@ +import { readdirSync } from "node:fs"; +import { dirname, join, relative } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import * as transformers from "../src/transformers.js"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const MODELS_DIR = join(__dirname, "..", "src", "models"); + +function findModelingFiles(dir) { + return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const path = join(dir, entry.name); + + if (entry.isDirectory()) { + return findModelingFiles(path); + } + return entry.isFile() && entry.name.startsWith("modeling_") && entry.name.endsWith(".js") ? [path] : []; + }); +} + +function isPublicModelingFile(file) { + return !file.endsWith(join("models", "modeling_utils.js")); +} + +describe("Public exports", () => { + it("exports every public modeling_* symbol from the root entry point", async () => { + const missing = []; + + for (const file of findModelingFiles(MODELS_DIR).filter(isPublicModelingFile).sort()) { + const moduleExports = await import(pathToFileURL(file).href); + + for (const exportName of Object.keys(moduleExports)) { + if (!Object.hasOwn(transformers, exportName)) { + missing.push(`${relative(MODELS_DIR, file)}: ${exportName}`); + } + } + } + + expect(missing).toEqual([]); + }); +}); diff --git a/packages/transformers/tests/feature_extractors.test.js b/packages/transformers/tests/feature_extractors.test.js new file mode 100644 index 0000000..89ac48f --- /dev/null +++ b/packages/transformers/tests/feature_extractors.test.js @@ -0,0 +1,5 @@ +import { init } from "./init.js"; +import { collect_and_execute_tests } from "./test_utils.js"; + +init(); +await collect_and_execute_tests("Feature extractors", "feature_extraction"); diff --git a/packages/transformers/tests/image_processors.test.js b/packages/transformers/tests/image_processors.test.js new file mode 100644 index 0000000..07e8677 --- /dev/null +++ b/packages/transformers/tests/image_processors.test.js @@ -0,0 +1,5 @@ +import { init } from "./init.js"; +import { collect_and_execute_tests } from "./test_utils.js"; + +init(); +await collect_and_execute_tests("Image processors", "image_processing"); diff --git a/packages/transformers/tests/init.js b/packages/transformers/tests/init.js new file mode 100644 index 0000000..9da694d --- /dev/null +++ b/packages/transformers/tests/init.js @@ -0,0 +1,125 @@ +// Helper functions used when initialising the testing environment. + +// Import Node typing utilities +import * as types from "node:util/types"; + +// Import onnxruntime-node's default backend +import { onnxruntimeBackend } from "onnxruntime-node/dist/backend"; +import * as ONNX_COMMON from "onnxruntime-common"; + +/** + * A workaround to define a new backend for onnxruntime, which + * will not throw an error when running tests with jest. + * For more information, see: https://github.com/jestjs/jest/issues/11864#issuecomment-1261468011 + */ +export function init() { + // In rare cases (specifically when running unit tests with GitHub actions), possibly due to + // a large number of concurrent executions, onnxruntime might fallback to use the WASM backend. + // In this case, we set the number of threads to 1 to avoid errors like: + // - `TypeError: The worker script or module filename must be an absolute path or a relative path starting with './' or '../'. Received "blob:nodedata:..."` + ONNX_COMMON.env.wasm.numThreads = 1; + + let registerBackend = ONNX_COMMON.registerBackend; + + // Define the constructors to monkey-patch + const TYPED_ARRAYS_CONSTRUCTOR_NAMES = ["Int8Array", "Int16Array", "Int32Array", "BigInt64Array", "Uint8Array", "Uint8ClampedArray", "Uint16Array", "Uint32Array", "BigUint64Array", "Float16Array", "Float32Array", "Float64Array"]; + + // Keep a reference to the original initialization method + const originalMethod = onnxruntimeBackend.init; + + // Monkey-patch the initialization function + onnxruntimeBackend.init = function (...args) { + // There is probably a better way to do this + Array.isArray = (x) => typeof x === "object" && x !== null && typeof x.length === "number" && x?.constructor.toString() === Array.toString(); + + // For each typed array constructor + for (const ctorName of TYPED_ARRAYS_CONSTRUCTOR_NAMES) { + // Get the constructor from the current context + const ctor = globalThis[ctorName]; + if (ctor === undefined) continue; // If unavailable, skip the patching + + // Get the corresponding test function from the `util` module + const value = types[`is${ctorName}`].bind(types); + + // Monkey-patch the constructor so "x instanceof ctor" returns "types[`is${ctorName}`](x)" + Object.defineProperty(ctor, Symbol.hasInstance, { + value, + writable: true, // writable=true is necessary to overwrite the default implementation (and allow subsequent overwrites) + configurable: false, + enumerable: false, + }); + } + + // Call the original method + return originalMethod.apply(this, args); + }; + + // Register the backend with the highest priority, so it is used instead of the default one + registerBackend("test", onnxruntimeBackend, Number.POSITIVE_INFINITY); +} + +export const MAX_TOKENIZER_LOAD_TIME = 32_000; // 32 seconds +export const MAX_FEATURE_EXTRACTOR_LOAD_TIME = 10_000; // 10 seconds +export const MAX_PROCESSOR_LOAD_TIME = 10_000; // 10 seconds +export const MAX_MODEL_LOAD_TIME = 15_000; // 15 seconds +export const MAX_TEST_EXECUTION_TIME = 60_000; // 60 seconds +export const MAX_MODEL_DISPOSE_TIME = 1_000; // 1 second + +export const MAX_TEST_TIME = MAX_MODEL_LOAD_TIME + MAX_TEST_EXECUTION_TIME + MAX_MODEL_DISPOSE_TIME; + +export const DEFAULT_MODEL_OPTIONS = Object.freeze({ + dtype: "fp32", +}); + +expect.extend({ + toBeCloseToNested(received, expected, numDigits = 2) { + const compare = (received, expected, path = "") => { + if (typeof received === "number" && typeof expected === "number" && (!Number.isInteger(received) || !Number.isInteger(expected))) { + const pass = Math.abs(received - expected) < Math.pow(10, -numDigits); + return { + pass, + message: () => (pass ? `✓ At path '${path}': expected ${received} not to be close to ${expected} with tolerance of ${numDigits} decimal places` : `✗ At path '${path}': expected ${received} to be close to ${expected} with tolerance of ${numDigits} decimal places`), + }; + } else if (Array.isArray(received) && Array.isArray(expected)) { + if (received.length !== expected.length) { + return { + pass: false, + message: () => `✗ At path '${path}': array lengths differ. Received length ${received.length}, expected length ${expected.length}`, + }; + } + for (let i = 0; i < received.length; i++) { + const result = compare(received[i], expected[i], `${path}[${i}]`); + if (!result.pass) return result; + } + } else if (typeof received === "object" && typeof expected === "object" && received !== null && expected !== null) { + const receivedKeys = Object.keys(received); + const expectedKeys = Object.keys(expected); + if (receivedKeys.length !== expectedKeys.length) { + return { + pass: false, + message: () => `✗ At path '${path}': object keys length differ. Received keys: ${JSON.stringify(receivedKeys)}, expected keys: ${JSON.stringify(expectedKeys)}`, + }; + } + for (const key of receivedKeys) { + if (!expected.hasOwnProperty(key)) { + return { + pass: false, + message: () => `✗ At path '${path}': key '${key}' found in received but not in expected`, + }; + } + const result = compare(received[key], expected[key], `${path}.${key}`); + if (!result.pass) return result; + } + } else { + const pass = received === expected; + return { + pass, + message: () => (pass ? `✓ At path '${path}': expected ${JSON.stringify(received)} not to equal ${JSON.stringify(expected)}` : `✗ At path '${path}': expected ${JSON.stringify(received)} to equal ${JSON.stringify(expected)}`), + }; + } + return { pass: true }; + }; + + return compare(received, expected); + }, +}); diff --git a/packages/transformers/tests/models.test.js b/packages/transformers/tests/models.test.js new file mode 100644 index 0000000..98f1d40 --- /dev/null +++ b/packages/transformers/tests/models.test.js @@ -0,0 +1,81 @@ +/* + * Test that models loaded outside of the `pipeline` function work correctly (e.g., `AutoModel.from_pretrained(...)`); + */ + +import { AutoTokenizer, AutoProcessor, BertForMaskedLM, GPT2LMHeadModel, T5ForConditionalGeneration, BertTokenizer, GPT2Tokenizer, T5Tokenizer, LlamaTokenizer, LlamaForCausalLM, WhisperForConditionalGeneration, WhisperProcessor, AutoModelForMaskedLM, AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoModelForSpeechSeq2Seq } from "../src/transformers.js"; +import { init, MAX_TEST_EXECUTION_TIME, DEFAULT_MODEL_OPTIONS } from "./init.js"; +import { collect_and_execute_tests } from "./test_utils.js"; + +// Initialise the testing environment +init(); + +describe("Loading different architecture types", () => { + // List all models which will be tested + const models_to_test = [ + // [name, [AutoModelClass, ModelClass], [AutoProcessorClass, ProcessorClass], [modelOptions?], [modality?]] + ["hf-internal-testing/tiny-random-BertForMaskedLM", [AutoModelForMaskedLM, BertForMaskedLM], [AutoTokenizer, BertTokenizer]], // Encoder-only + ["hf-internal-testing/tiny-random-GPT2LMHeadModel", [AutoModelForCausalLM, GPT2LMHeadModel], [AutoTokenizer, GPT2Tokenizer]], // Decoder-only + ["hf-internal-testing/tiny-random-T5ForConditionalGeneration", [AutoModelForSeq2SeqLM, T5ForConditionalGeneration], [AutoTokenizer, T5Tokenizer]], // Encoder-decoder + ["onnx-internal-testing/tiny-random-LlamaForCausalLM-ONNX_external", [AutoModelForCausalLM, LlamaForCausalLM], [AutoTokenizer, LlamaTokenizer]], // Decoder-only w/ external data + ["onnx-internal-testing/tiny-random-WhisperForConditionalGeneration-ONNX_external", [AutoModelForSpeechSeq2Seq, WhisperForConditionalGeneration], [AutoProcessor, WhisperProcessor], {}], // Encoder-decoder-only w/ external data + ]; + + const texts = ["Once upon a time", "I like to eat apples"]; + + for (const [model_id, models, processors, modelOptions] of models_to_test) { + // Test that both the auto model and the specific model work + for (let i = 0; i < processors.length; ++i) { + const processorClassToTest = processors[i]; + const modelClassToTest = models[i]; + + it( + `${model_id} (${modelClassToTest.name})`, + async () => { + // Load model and processor + const processor = await processorClassToTest.from_pretrained(model_id); + const model = await modelClassToTest.from_pretrained(model_id, modelOptions ?? DEFAULT_MODEL_OPTIONS); + + const tests = [ + texts[0], // single + texts, // batched + ]; + + const { model_type } = model.config; + const tokenizer = model_type === "whisper" ? processor.tokenizer : processor; + const feature_extractor = model_type === "whisper" ? processor.feature_extractor : null; + + for (const test of tests) { + const inputs = await tokenizer(test, { truncation: true, padding: true }); + if (model.config.is_encoder_decoder) { + inputs.decoder_input_ids = inputs.input_ids; + } + if (feature_extractor) { + Object.assign(inputs, await feature_extractor(new Float32Array(16000))); + } + + const output = await model(inputs); + + if (output.logits) { + // Ensure correct shapes + const input_ids = inputs.input_ids ?? inputs.decoder_input_ids; + const expected_shape = [...input_ids.dims, model.config.vocab_size]; + const actual_shape = output.logits.dims; + expect(expected_shape).toEqual(actual_shape); + } else if (output.last_hidden_state) { + const expected_shape = [...inputs.input_ids.dims, model.config.d_model]; + const actual_shape = output.last_hidden_state.dims; + expect(expected_shape).toEqual(actual_shape); + } else { + console.warn("Unexpected output", output); + throw new Error("Unexpected output"); + } + } + await model.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + } + } +}); + +await collect_and_execute_tests("Model-specific tests", "modeling"); diff --git a/packages/transformers/tests/models/afmoe/test_modeling_afmoe.js b/packages/transformers/tests/models/afmoe/test_modeling_afmoe.js new file mode 100644 index 0000000..0899043 --- /dev/null +++ b/packages/transformers/tests/models/afmoe/test_modeling_afmoe.js @@ -0,0 +1,51 @@ +import { PreTrainedTokenizer, AfmoeForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("AfmoeForCausalLM", () => { + const model_id = "onnx-internal-testing/tiny-random-AfmoeForCausalLM"; + /** @type {AfmoeForCausalLM} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await AfmoeForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[15339n, 73860n, 53854n, 50501n, 12449n, 94537n, 88764n, 50217n, 62648n, 62648n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [100256n, 15339n, 73860n, 53854n, 50501n, 12449n, 94537n, 88764n, 50217n, 62648n], + [15339n, 1917n, 74552n, 87404n, 39726n, 71306n, 16944n, 450n, 68783n, 86239n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/albert/test_tokenization_albert.js b/packages/transformers/tests/models/albert/test_tokenization_albert.js new file mode 100644 index 0000000..6d712d3 --- /dev/null +++ b/packages/transformers/tests/models/albert/test_tokenization_albert.js @@ -0,0 +1,183 @@ +import { AlbertTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, BERT_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = AlbertTokenizer; +export const TEST_CONFIG = { + // - uses `StripAccents` normalizer + "Xenova/albert-base-v2": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581how", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [2, 184, 50, 42, 845, 60, 3], + decoded: "[CLS] how are you doing?[SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581you", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [2, 42, 378, 22, 195, 677, 48, 3], + decoded: "[CLS] you should've done this[SEP]", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u25810", "12", "345", "67", "89", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [2, 713, 918, 21997, 4167, 3877, 713, 137, 172, 203, 268, 331, 400, 453, 469, 561, 332, 808, 6150, 3], + decoded: "[CLS] 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000[SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581the", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u25812016", "."], + ids: [2, 14, 237, 23, 785, 19, 690, 9, 3], + decoded: "[CLS] the company was founded in 2016.[SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581a", "\u2581", "'", "ll", "\u2581", "!!", "to", "?'", "d", '"', "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [2, 21, 13, 22, 211, 13, 19015, 262, 5663, 43, 7, 43, 16, 15, 92, 22, 38, 9, 3], + decoded: "[CLS] a 'll!!to?'d\"d of, can't.[SEP]", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581def", "\u2581main", "(", ")", ":", "\u2581pass"], + ids: [2, 6312, 407, 5, 6, 45, 1477, 3], + decoded: "[CLS] def main(): pass[SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581ob", "j", ".", "to", "string", "(", ")", ";", "\u2581to", "string", "(", ")", ";"], + ids: [2, 408, 21, 800, 5122, 728, 9, 262, 11130, 5, 6, 73, 20, 11130, 5, 6, 73, 3], + decoded: "[CLS] let a = obj.tostring(); tostring();[SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581this", "\u2581is", "\u2581a", "\u2581test", "."], + ids: [2, 48, 25, 21, 1289, 9, 3], + decoded: "[CLS] this is a test.[SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581unwanted", ",", "running"], + ids: [2, 21095, 15, 11325, 3], + decoded: "[CLS] unwanted,running[SEP]", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u25811", "\u0000", "2", "\u25813"], + ids: [2, 137, 1, 135, 203, 3], + decoded: "[CLS] 12 3[SEP]", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [2, 10975, 126, 3], + decoded: "[CLS] hello world[SEP]", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [2, 10975, 126, 3], + decoded: "[CLS] hello world[SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f\u6d3b\u7684\u771f\u8c1b\u662f"], + ids: [2, 13, 1, 3], + decoded: "[CLS] [SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581leading", "\u2581space"], + ids: [2, 1005, 726, 3], + decoded: "[CLS] leading space[SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trailing", "\u2581space"], + ids: [2, 14323, 726, 3], + decoded: "[CLS] trailing space[SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581hi", "\u2581hello"], + ids: [2, 4148, 10975, 3], + decoded: "[CLS] hi hello[SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581r", "2", "\u2581#3", "\u2581", "\u20ac", "4", "\u2581", "\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581", "\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [2, 1289, 3742, 761, 135, 11489, 13, 12, 300, 13, 11, 264, 13, 1, 379, 13, 1, 465, 13, 1, 457, 13, 1, 518, 1289, 3], + decoded: "[CLS] test $1 r2 #3 \u20ac4 \u00a35 6 7 8 9 test[SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581i", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$1", ".", "00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [2, 31, 2448, 40, 4037, 26, 3742, 9, 2032, 35, 14, 1718, 9, 3], + decoded: "[CLS] i bought an apple for $1.00 at the store.[SEP]", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", ".", ".", "."], + ids: [2, 42, 9, 9, 9, 3], + decoded: "[CLS] you...[SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", ".", ".", "."], + ids: [2, 42, 9, 9, 9, 3], + decoded: "[CLS] you...[SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", ".", ".", ".", "\u2581you", ".", ".", "."], + ids: [2, 42, 9, 9, 9, 42, 9, 9, 9, 3], + decoded: "[CLS] you... you...[SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "~", "\u2581edge", "\u2581", "~", "\u2581case"], + ids: [2, 5455, 13, 1, 1407, 13, 1, 610, 3], + decoded: "[CLS] weird edge case[SEP]", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581this", "\u2581is", "\u2581a", "\u2581test", "\u2581", "."], + ids: [2, 48, 25, 21, 1289, 13, 9, 3], + decoded: "[CLS] this is a test.[SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581", "\u2764", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [2, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 3], + decoded: "[CLS] [SEP]", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\u2581", "\ud83d\udc71\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\u2581", "\ud83e\uddd9\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581", "\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68\ud83c\udffb", "\u2581", "\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68\ud83c\udffc"], + ids: [2, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 3], + decoded: "[CLS] [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["\u2581", "ah", "\u535a\u63a8", "zz"], + ids: [2, 13, 1307, 1, 5092, 3], + decoded: "[CLS] ahzz[SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["\u2581hello"], + ids: [2, 10975, 3], + decoded: "[CLS] hello[SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u2581hello", "!", "how", "\u2581are", "\u2581you", "?"], + ids: [2, 10975, 187, 1544, 50, 42, 60, 3], + decoded: "[CLS] hello!how are you?[SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u2581hall", "o", "!", "how", "\u2581are", "\u2581you", "?"], + ids: [2, 554, 111, 187, 1544, 50, 42, 60, 3], + decoded: "[CLS] hallo!how are you?[SEP]", + }, + }, +}; diff --git a/packages/transformers/tests/models/arcee/test_modeling_arcee.js b/packages/transformers/tests/models/arcee/test_modeling_arcee.js new file mode 100644 index 0000000..a852aeb --- /dev/null +++ b/packages/transformers/tests/models/arcee/test_modeling_arcee.js @@ -0,0 +1,51 @@ +import { PreTrainedTokenizer, ArceeForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("ArceeForCausalLM", () => { + const model_id = "onnx-internal-testing/tiny-random-ArceeForCausalLM"; + /** @type {ArceeForCausalLM} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await ArceeForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[1n, 22172n, 1316n, 11038n, 25378n, 11619n, 7959n, 15231n, 15231n, 23659n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [2n, 1n, 22172n, 5706n, 3803n, 11619n, 28763n, 4015n, 18904n, 7959n], + [1n, 22172n, 3186n, 1316n, 11038n, 22918n, 9469n, 25671n, 22918n, 2687n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/audio_spectrogram_transformer/test_feature_extraction_audio_spectrogram_transformer.js b/packages/transformers/tests/models/audio_spectrogram_transformer/test_feature_extraction_audio_spectrogram_transformer.js new file mode 100644 index 0000000..0fced5b --- /dev/null +++ b/packages/transformers/tests/models/audio_spectrogram_transformer/test_feature_extraction_audio_spectrogram_transformer.js @@ -0,0 +1,53 @@ +import { AutoFeatureExtractor, ASTFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_audio } from "../../asset_cache.js"; +import { MAX_FEATURE_EXTRACTOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // ASTFeatureExtractor + describe("ASTFeatureExtractor", () => { + const model_id = "Xenova/ast-finetuned-audioset-10-10-0.4593"; + + /** @type {ASTFeatureExtractor} */ + let feature_extractor; + beforeAll(async () => { + feature_extractor = await AutoFeatureExtractor.from_pretrained(model_id); + }, MAX_FEATURE_EXTRACTOR_LOAD_TIME); + + it( + "truncation", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_values } = await feature_extractor(audio); + expect(input_values.dims).toEqual([1, 1024, 128]); + + expect(input_values.mean().item()).toBeCloseTo(-0.04054912979309085); + expect(input_values.data[0]).toBeCloseTo(-0.5662586092948914); + expect(input_values.data[1]).toBeCloseTo(-1.0300861597061157); + expect(input_values.data[129]).toBeCloseTo(-1.084834098815918); + expect(input_values.data[1025]).toBeCloseTo(-1.1204065084457397); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "padding", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_values } = await feature_extractor(audio.slice(0, 1000)); + expect(input_values.dims).toEqual([1, 1024, 128]); // [1, 4, 128] -> (padded to) -> [1, 1024, 128] + + expect(input_values.mean().item()).toBeCloseTo(0.4647964835166931); + expect(input_values.data[0]).toBeCloseTo(-0.5662586092948914); + expect(input_values.data[1]).toBeCloseTo(-1.0300861597061157); + expect(input_values.data[129]).toBeCloseTo(-1.084834098815918); + + // padded values + expect(input_values.data[1025]).toBeCloseTo(0.46703237295150757); + expect(input_values.data[2049]).toBeCloseTo(0.46703237295150757); + expect(input_values.data[10000]).toBeCloseTo(0.46703237295150757); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/beit/test_image_processing_beit.js b/packages/transformers/tests/models/beit/test_image_processing_beit.js new file mode 100644 index 0000000..065f6cb --- /dev/null +++ b/packages/transformers/tests/models/beit/test_image_processing_beit.js @@ -0,0 +1,31 @@ +import { AutoImageProcessor, BeitFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("BeitFeatureExtractor", () => { + const model_id = "Xenova/beit-base-patch16-224-pt22k-ft22k"; + + /** @type {BeitFeatureExtractor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "default", + async () => { + const image = await load_cached_image("tiger"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 224, 224]); + expect(pixel_values.mean().item()).toBeCloseTo(-0.22706867939852762, 6); + + expect(original_sizes).toEqual([[408, 612]]); + expect(reshaped_input_sizes).toEqual([[224, 224]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/bert/test_modeling_bert.js b/packages/transformers/tests/models/bert/test_modeling_bert.js new file mode 100644 index 0000000..510d2ba --- /dev/null +++ b/packages/transformers/tests/models/bert/test_modeling_bert.js @@ -0,0 +1,211 @@ +import { BertTokenizer, BertModel, BertForMaskedLM, BertForSequenceClassification, BertForTokenClassification, BertForQuestionAnswering } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("BertModel", () => { + const model_id = "hf-internal-testing/tiny-random-BertModel"; + + /** @type {BertModel} */ + let model; + /** @type {BertTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await BertModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await BertTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const { last_hidden_state } = await model(inputs); + expect(last_hidden_state.dims).toEqual([1, 7, 32]); + expect(last_hidden_state.mean().item()).toBeCloseTo(0.0, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const { last_hidden_state } = await model(inputs); + expect(last_hidden_state.dims).toEqual([2, 12, 32]); + expect(last_hidden_state.mean().item()).toBeCloseTo(1.4901161193847656e-8, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("BertForMaskedLM", () => { + const model_id = "hf-internal-testing/tiny-random-BertForMaskedLM"; + + const texts = ["The goal of life is [MASK].", "Paris is the [MASK] of France."]; + + /** @type {BertForMaskedLM} */ + let model; + /** @type {BertTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await BertForMaskedLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await BertTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer(texts[0]); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 19, 1124]); + expect(logits.mean().item()).toBeCloseTo(0.0016587056452408433, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(texts, { padding: true }); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([2, 22, 1124]); + expect(logits.mean().item()).toBeCloseTo(0.0017160633578896523, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("BertForSequenceClassification", () => { + const model_id = "hf-internal-testing/tiny-random-BertForSequenceClassification"; + + /** @type {BertForSequenceClassification} */ + let model; + /** @type {BertTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await BertForSequenceClassification.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await BertTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const { logits } = await model(inputs); + const target = [[0.00043986947275698185, -0.030218850821256638]]; + expect(logits.dims).toEqual([1, 2]); + expect(logits.tolist()).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const { logits } = await model(inputs); + const target = [ + [0.00043986947275698185, -0.030218850821256638], + [0.0003853091038763523, -0.03022204339504242], + ]; + expect(logits.dims).toEqual([2, 2]); + expect(logits.tolist()).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("BertForTokenClassification", () => { + const model_id = "hf-internal-testing/tiny-random-BertForTokenClassification"; + + /** @type {BertForTokenClassification} */ + let model; + /** @type {BertTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await BertForTokenClassification.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await BertTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 7, 2]); + expect(logits.mean().item()).toBeCloseTo(0.07089076191186905, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([2, 12, 2]); + expect(logits.mean().item()).toBeCloseTo(0.04702216014266014, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("BertForQuestionAnswering", () => { + const model_id = "hf-internal-testing/tiny-random-BertForQuestionAnswering"; + + /** @type {BertForQuestionAnswering} */ + let model; + /** @type {BertTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await BertForQuestionAnswering.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await BertTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const { start_logits, end_logits } = await model(inputs); + expect(start_logits.dims).toEqual([1, 7]); + expect(start_logits.mean().item()).toBeCloseTo(0.12772157788276672, 5); + expect(end_logits.dims).toEqual([1, 7]); + expect(end_logits.mean().item()).toBeCloseTo(0.11811424791812897, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const { start_logits, end_logits } = await model(inputs); + expect(start_logits.dims).toEqual([2, 12]); + expect(start_logits.mean().item()).toBeCloseTo(0.12843115627765656, 5); + expect(end_logits.dims).toEqual([2, 12]); + expect(end_logits.mean().item()).toBeCloseTo(0.11745202541351318, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/bert/test_tokenization_bert.js b/packages/transformers/tests/models/bert/test_tokenization_bert.js new file mode 100644 index 0000000..f9a6af6 --- /dev/null +++ b/packages/transformers/tests/models/bert/test_tokenization_bert.js @@ -0,0 +1,1371 @@ +import { BertTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, BERT_TEST_STRINGS, NORMALIZATION_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = BertTokenizer; +export const TEST_CONFIG = { + "Xenova/bert-base-uncased": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["how", "are", "you", "doing", "?"], + ids: [101, 2129, 2024, 2017, 2725, 1029, 102], + decoded: "[CLS] how are you doing? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["you", "should", "'", "ve", "done", "this"], + ids: [101, 2017, 2323, 1005, 2310, 2589, 2023, 102], + decoded: "[CLS] you should've done this [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["the", "company", "was", "founded", "in", "2016", "."], + ids: [101, 1996, 2194, 2001, 2631, 1999, 2355, 1012, 102], + decoded: "[CLS] the company was founded in 2016. [SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [101, 1037, 1005, 2222, 999, 999, 2000, 1029, 1005, 1040, 1005, 1005, 1040, 1997, 1010, 2064, 1005, 1056, 1012, 102], + decoded: "[CLS] a'll!! to?'d'' d of, can't. [SEP]", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "main", "(", ")", ":", "pass"], + ids: [101, 13366, 2364, 1006, 1007, 1024, 3413, 102], + decoded: "[CLS] def main ( ) : pass [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##st", "##ring", "(", ")", ";", "to", "##st", "##ring", "(", ")", ";"], + ids: [101, 2292, 1037, 1027, 27885, 3501, 1012, 2000, 3367, 4892, 1006, 1007, 1025, 2000, 3367, 4892, 1006, 1007, 1025, 102], + decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["this", "is", "a", "test", "."], + ids: [101, 2023, 2003, 1037, 3231, 1012, 102], + decoded: "[CLS] this is a test. [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["unwanted", ",", "running"], + ids: [101, 18162, 1010, 2770, 102], + decoded: "[CLS] unwanted, running [SEP]", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["123"], + ids: [101, 13138, 102], + decoded: "[CLS] 123 [SEP]", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["hello", "world"], + ids: [101, 7592, 2088, 102], + decoded: "[CLS] hello world [SEP]", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "world"], + ids: [101, 7592, 2088, 102], + decoded: "[CLS] hello world [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "[UNK]", "\u7684", "\u771f", "[UNK]", "[UNK]"], + ids: [101, 1910, 100, 1916, 1921, 100, 100, 102], + decoded: "[CLS] \u751f [UNK] \u7684 \u771f [UNK] [UNK] [SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["leading", "space"], + ids: [101, 2877, 2686, 102], + decoded: "[CLS] leading space [SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "space"], + ids: [101, 12542, 2686, 102], + decoded: "[CLS] trailing space [SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["hi", "hello"], + ids: [101, 7632, 7592, 102], + decoded: "[CLS] hi hello [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r", "##2", "#", "3", "\u20ac", "##4", "\u00a35", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [101, 3231, 1002, 1015, 1054, 2475, 1001, 1017, 1574, 2549, 27813, 1071, 2575, 100, 1576, 2620, 1575, 2683, 3231, 102], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["i", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 1045, 4149, 2019, 6207, 2005, 1002, 1015, 1012, 4002, 2012, 1996, 3573, 1012, 102], + decoded: "[CLS] i bought an apple for $ 1. 00 at the store. [SEP]", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u2026"], + ids: [101, 2017, 1529, 102], + decoded: "[CLS] you \u2026 [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u2026"], + ids: [101, 2017, 1529, 102], + decoded: "[CLS] you \u2026 [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u2026", "you", "\u2026"], + ids: [101, 2017, 1529, 2017, 1529, 102], + decoded: "[CLS] you \u2026 you \u2026 [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "\uff5e", "edge", "\uff5e", "case"], + ids: [101, 6881, 1995, 3341, 1995, 2553, 102], + decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "."], + ids: [101, 100, 100, 100, 100, 100, 1012, 102], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK]. [SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [101, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [101, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "\u535a", "[UNK]", "z", "##z"], + ids: [101, 6289, 1786, 100, 1062, 2480, 102], + decoded: "[CLS] ah \u535a [UNK] zz [SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["hello"], + ids: [101, 7592, 102], + decoded: "[CLS] hello [SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["hello", "!", "how", "are", "you", "?"], + ids: [101, 7592, 999, 2129, 2024, 2017, 1029, 102], + decoded: "[CLS] hello! how are you? [SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["hall", "##o", "!", "how", "are", "you", "?"], + ids: [101, 2534, 2080, 999, 2129, 2024, 2017, 1029, 102], + decoded: "[CLS] hallo! how are you? [SEP]", + }, + ONLY_WHITESPACE: { + text: BASE_TEST_STRINGS.ONLY_WHITESPACE, + tokens: [], + ids: [101, 102], + decoded: "[CLS] [SEP]", + }, + + TEXT_PAIR: { + text: "hello", + text_pair: "world", + tokens: ["hello", "world"], + ids: [101, 7592, 102, 2088, 102], + decoded: "[CLS] hello [SEP] world [SEP]", + }, + }, + "Xenova/bert-base-cased": { + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "o", "##b", "##j", ".", "to", "##S", "##tring", "(", ")", ";", "to", "##S", "##tring", "(", ")", ";"], + ids: [101, 1519, 170, 134, 184, 1830, 3361, 119, 1106, 1708, 28108, 113, 114, 132, 1106, 1708, 28108, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "##wan", "##t\u00e9", "##d", ",", "running"], + ids: [101, 7414, 5491, 14608, 1181, 117, 1919, 102], + decoded: "[CLS] UNwant\u00e9d, running [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "[UNK]", "[UNK]", "\u771f", "[UNK]", "[UNK]"], + ids: [101, 1056, 100, 100, 1061, 100, 100, 102], + decoded: "[CLS] \u751f [UNK] [UNK] \u771f [UNK] [UNK] [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "R", "##2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [101, 2774, 109, 122, 155, 1477, 108, 124, 836, 1527, 202, 1571, 203, 1545, 100, 838, 1604, 837, 1580, 2774, 102], + decoded: "[CLS] test $ 1 R2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 146, 3306, 1126, 12075, 1111, 109, 122, 119, 3135, 1120, 1103, 2984, 119, 102], + decoded: "[CLS] I bought an apple for $ 1. 00 at the store. [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "[UNK]", "edge", "[UNK]", "case"], + ids: [101, 6994, 100, 2652, 100, 1692, 102], + decoded: "[CLS] weird [UNK] edge [UNK] case [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "[UNK]", "[UNK]", "z", "##z"], + ids: [101, 18257, 100, 100, 195, 1584, 102], + decoded: "[CLS] ah [UNK] [UNK] zz [SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "##\u00e9", "##llo"], + ids: [101, 145, 2744, 6643, 102], + decoded: "[CLS] H\u00e9llo [SEP]", + }, + }, + + "Xenova/bert-base-multilingual-cased-ner-hrl": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "are", "you", "doing", "?"], + ids: [101, 14962, 10301, 13028, 30918, 136, 102], + decoded: "[CLS] How are you doing? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "should", "'", "ve", "done", "this"], + ids: [101, 11065, 14819, 112, 10323, 20378, 10531, 102], + decoded: "[CLS] You should've done this [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "company", "was", "founded", "in", "2016", "."], + ids: [101, 10117, 12100, 10134, 14078, 10106, 10255, 119, 102], + decoded: "[CLS] The company was founded in 2016. [SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [101, 138, 112, 22469, 106, 106, 10114, 136, 112, 172, 112, 112, 172, 10108, 117, 10944, 112, 188, 119, 102], + decoded: "[CLS] A'll!! to?'d'' d of, can't. [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##S", "##trin", "##g", "(", ")", ";", "to", "##S", "##trin", "##g", "(", ")", ";"], + ids: [101, 13595, 169, 134, 17339, 10418, 119, 10114, 10731, 109163, 10240, 113, 114, 132, 10114, 10731, 109163, 10240, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "is", "a", "test", "."], + ids: [101, 10747, 10124, 169, 15839, 119, 102], + decoded: "[CLS] This is a test. [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "##want", "##\u00e9d", ",", "running"], + ids: [101, 26578, 104216, 84193, 117, 18020, 102], + decoded: "[CLS] UNwant\u00e9d, running [SEP]", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "World"], + ids: [101, 31178, 10315, 102], + decoded: "[CLS] Hello World [SEP]", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hell", "##o", "world"], + ids: [101, 61694, 10133, 11356, 102], + decoded: "[CLS] hello world [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [101, 5600, 4978, 5718, 5769, 7378, 4380, 102], + decoded: "[CLS] \u751f \u6d3b \u7684 \u771f \u8c1b \u662f [SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trail", "##ing", "space"], + ids: [101, 56559, 10230, 16199, 102], + decoded: "[CLS] trailing space [SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "Hello"], + ids: [101, 20065, 31178, 102], + decoded: "[CLS] Hi Hello [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "R2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "[UNK]", "test"], + ids: [101, 15839, 109, 122, 94000, 108, 124, 1775, 11011, 201, 11166, 202, 11211, 100, 1776, 11396, 100, 15839, 102], + decoded: "[CLS] test $ 1 R2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 [UNK] test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "bought", "an", "app", "##le", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 146, 28870, 10151, 72894, 10284, 10142, 109, 122, 119, 11025, 10160, 10105, 13708, 119, 102], + decoded: "[CLS] I bought an apple for $ 1. 00 at the store. [SEP]", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "[UNK]"], + ids: [101, 13028, 100, 102], + decoded: "[CLS] you [UNK] [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "[UNK]"], + ids: [101, 13028, 100, 102], + decoded: "[CLS] you [UNK] [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "[UNK]", "you", "[UNK]"], + ids: [101, 13028, 100, 13028, 100, 102], + decoded: "[CLS] you [UNK] you [UNK] [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["wei", "##rd", "\uff5e", "edge", "\uff5e", "case"], + ids: [101, 86981, 12023, 10096, 30599, 10096, 13474, 102], + decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "\u535a", "\u63a8", "z", "##z"], + ids: [101, 69863, 2684, 4163, 194, 10305, 102], + decoded: "[CLS] ah \u535a \u63a8 zz [SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "##\u00e9l", "##lo"], + ids: [101, 145, 24817, 10715, 102], + decoded: "[CLS] H\u00e9llo [SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["He", "##LL", "##o", "!", "how", "Are", "yo", "##U", "?"], + ids: [101, 10357, 82834, 10133, 106, 14796, 13491, 13672, 12022, 136, 102], + decoded: "[CLS] HeLLo! how Are yoU? [SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["H", "##\u00e4", "##LL", "##o", "!", "how", "Are", "yo", "##U", "?"], + ids: [101, 145, 11013, 82834, 10133, 106, 14796, 13491, 13672, 12022, 136, 102], + decoded: "[CLS] H\u00e4LLo! how Are yoU? [SEP]", + }, + }, + "Xenova/paraphrase-multilingual-MiniLM-L12-v2": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [0, 11249, 621, 398, 20594, 32, 2], + decoded: " How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [0, 2583, 5608, 25, 272, 16940, 903, 2], + decoded: " You should've done this", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u25812016."], + ids: [0, 581, 14380, 509, 14037, 297, 23, 6360, 2], + decoded: " The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581'", "ll", "\u2581!!", "to", "?", "'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [0, 62, 242, 1181, 6506, 188, 32, 25, 71, 4765, 71, 111, 4, 831, 25, 18, 5, 2], + decoded: " A 'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581de", "f", "\u2581main", "(", "):", "\u2581pass"], + ids: [0, 8, 420, 5201, 132, 2077, 27875, 2], + decoded: " def main(): pass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581ob", "j", ".", "to", "Str", "ing", "(", ");", "\u2581to", "Str", "ing", "(", ");"], + ids: [0, 2633, 10, 2203, 995, 170, 5, 188, 71713, 214, 132, 3142, 47, 71713, 214, 132, 3142, 2], + decoded: " let a = obj.toString(); toString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "."], + ids: [0, 3293, 83, 10, 3034, 5, 2], + decoded: " This is a test.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "wan", "t\u00e9", "d", ",", "run", "ning"], + ids: [0, 8274, 3206, 2312, 71, 4, 16428, 592, 2], + decoded: " UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u25811", "\u0000", "2", "\u25813"], + ids: [0, 106, 3, 304, 138, 2], + decoded: " 12 3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581Hello", "\u2581World"], + ids: [0, 35378, 6661, 2], + decoded: " Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hell", "o", "\u2581world"], + ids: [0, 33600, 31, 8999, 2], + decoded: " hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f\u6d3b\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [0, 6, 62668, 5364, 245875, 354, 2], + decoded: " \u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581leading", "\u2581space"], + ids: [0, 105207, 32628, 2], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trail", "ing", "\u2581space"], + ids: [0, 141037, 214, 32628, 2], + decoded: " trailing space", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581Hi", "\u2581Hello"], + ids: [0, 2673, 35378, 2], + decoded: " Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [0, 3034, 38629, 627, 304, 111378, 2505, 617, 11762, 758, 6, 32389, 910, 6, 3, 966, 87316, 1019, 6, 247425, 1126, 3034, 2], + decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1.00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [0, 87, 123997, 142, 108787, 100, 3650, 146533, 99, 70, 4343, 5, 2], + decoded: " I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "..."], + ids: [0, 398, 27, 2], + decoded: " you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "..."], + ids: [0, 398, 27, 2], + decoded: " you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "..."], + ids: [0, 398, 27, 398, 27, 2], + decoded: " you... you...", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], + ids: [0, 179459, 6, 6087, 121303, 6, 6087, 7225, 2], + decoded: " weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581", "."], + ids: [0, 3293, 83, 10, 3034, 6, 5, 2], + decoded: " This is a test.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [0, 6, 115114, 6, 118280, 6, 243385, 6, 84464, 6, 232773, 6, 243816, 6, 113612, 6, 82803, 6, 222326, 6, 201344, 6, 239569, 6, 243544, 6, 191876, 6, 243404, 49933, 15755, 6, 244233, 6, 244162, 6, 244181, 6, 243892, 6, 245820, 6, 161546, 6, 204811, 6, 3, 6, 238992, 6, 167474, 6, 120242, 6, 245561, 6, 244864, 6, 246144, 6, 244459, 6, 244703, 6, 246887, 6, 144400, 6, 246511, 6, 142325, 6, 244230, 6, 245559, 6, 243374, 6, 245200, 2], + decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], + ids: [0, 6, 167474, 6, 243544, 6, 246984, 15755, 6, 247201, 79500, 6, 248325, 6, 228250, 15755, 6, 3, 79500, 6, 228250, 6, 244314, 79500, 6, 246529, 6, 3, 6, 247443, 6, 3, 6, 244785, 49933, 6, 244960, 6, 244314, 6, 244785, 6, 244785, 6, 245719, 6, 246167, 6, 3, 79500, 6, 247443, 6, 3, 79500, 6, 3, 6, 244314, 79500, 49933, 15755, 6, 244960, 6, 244314, 239719, 2], + decoded: " \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75 \u2642\ufe0f \ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb \ud83c\udf3e \ud83e\udd1d \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83d\udc67 \ud83d\udc66 \ud83c\udffb \ud83e\udd1d \ud83c\udffb \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["\u2581ah", "\u535a", "\u63a8", "zz"], + ids: [0, 1263, 11173, 10238, 13894, 2], + decoded: " ah\u535a\u63a8zz", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["\u2581H\u00e9", "llo"], + ids: [0, 88064, 9284, 2], + decoded: " H\u00e9llo", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u2581He", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?"], + ids: [0, 1529, 23708, 31, 38, 47251, 15901, 3005, 1062, 32, 2], + decoded: " HeLLo!how Are yoU?", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u2581H\u00e4", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?"], + ids: [0, 28863, 23708, 31, 38, 47251, 15901, 3005, 1062, 32, 2], + decoded: " H\u00e4LLo!how Are yoU?", + }, + }, + "Xenova/bert-base-multilingual-uncased-sentiment": { + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "tos", "##tri", "##ng", "(", ")", ";", "tos", "##tri", "##ng", "(", ")", ";"], + ids: [101, 12421, 143, 134, 15547, 10428, 119, 53564, 27711, 10422, 113, 114, 132, 53564, 27711, 10422, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["un", "##wan", "##ted", ",", "running"], + ids: [101, 10119, 15134, 11894, 117, 16484, 102], + decoded: "[CLS] unwanted, running [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [101, 14084, 109, 122, 85583, 108, 124, 1329, 11124, 175, 11301, 177, 11325, 100, 1332, 11544, 1330, 11518, 14084, 102], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", + }, + }, + "Xenova/multilingual-e5-small": { + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trail", "ing", "\u2581space", "\u2581"], + ids: [0, 141037, 214, 32628, 6, 2], + decoded: " trailing space ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [0, 398, 27, 6, 2], + decoded: " you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [0, 398, 27, 6, 2], + decoded: " you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "...", "\u2581"], + ids: [0, 398, 27, 398, 27, 6, 2], + decoded: " you... you... ", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u2581He", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?", "\u2581"], + ids: [0, 1529, 23708, 31, 38, 47251, 15901, 3005, 1062, 32, 6, 2], + decoded: " HeLLo!how Are yoU? ", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u2581H\u00e4", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?", "\u2581"], + ids: [0, 28863, 23708, 31, 38, 47251, 15901, 3005, 1062, 32, 6, 2], + decoded: " H\u00e4LLo!how Are yoU? ", + }, + }, + "Xenova/bge-small-zh-v1.5": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["[UNK]", "are", "you", "doi", "##ng", "?"], + ids: [101, 100, 8995, 8357, 9962, 8291, 136, 102], + decoded: "[CLS] [UNK] are you doing? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["[UNK]", "sh", "##ould", "'", "ve", "don", "##e", "this"], + ids: [101, 100, 11167, 11734, 112, 12810, 9524, 8154, 8554, 102], + decoded: "[CLS] [UNK] should've done this [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["[UNK]", "company", "was", "f", "##ound", "##ed", "in", "2016", "."], + ids: [101, 100, 10007, 9947, 148, 11477, 8303, 8217, 8112, 119, 102], + decoded: "[CLS] [UNK] company was founded in 2016. [SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["[UNK]", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [101, 100, 112, 10856, 106, 106, 8228, 136, 112, 146, 112, 112, 146, 8205, 117, 9109, 112, 162, 119, 102], + decoded: "[CLS] [UNK]'ll!! to?'d'' d of, can't. [SEP]", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["de", "##f", "main", "(", ")", ":", "pass"], + ids: [101, 8363, 8189, 9139, 113, 114, 131, 9703, 102], + decoded: "[CLS] def main ( ) : pass [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "[UNK]", "(", ")", ";", "[UNK]", "(", ")", ";"], + ids: [101, 9946, 143, 134, 12639, 8334, 119, 100, 113, 114, 132, 100, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. [UNK] ( ) ; [UNK] ( ) ; [SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["[UNK]", "is", "a", "test", "."], + ids: [101, 100, 8310, 143, 10060, 119, 102], + decoded: "[CLS] [UNK] is a test. [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["[UNK]", ",", "running"], + ids: [101, 100, 117, 11620, 102], + decoded: "[CLS] [UNK], running [SEP]", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["[UNK]", "[UNK]"], + ids: [101, 100, 100, 102], + decoded: "[CLS] [UNK] [UNK] [SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["le", "##ad", "##ing", "space"], + ids: [101, 8983, 8695, 8221, 9634, 102], + decoded: "[CLS] leading space [SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["t", "##rail", "##ing", "space"], + ids: [101, 162, 12783, 8221, 9634, 102], + decoded: "[CLS] trailing space [SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["[UNK]", "[UNK]"], + ids: [101, 100, 100, 102], + decoded: "[CLS] [UNK] [UNK] [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "[UNK]", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "[UNK]", "[UNK]", "test"], + ids: [101, 10060, 109, 122, 100, 108, 124, 359, 8159, 173, 8157, 175, 8158, 100, 100, 100, 10060, 102], + decoded: "[CLS] test $ 1 [UNK] # 3 \u20ac4 \u00a35 \u00a56 [UNK] [UNK] [UNK] test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["[UNK]", "bo", "##ugh", "##t", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 100, 11059, 12667, 8165, 9064, 8350, 8330, 109, 122, 119, 8136, 8243, 8174, 8719, 119, 102], + decoded: "[CLS] [UNK] bought an apple for $ 1. 00 at the store. [SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\ud83d\ude02", "\ud83d\udc4d", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "\ud83d\udd25", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "\ud83d\ude0e", "[UNK]", "[UNK]", "[UNK]", "\u2728", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [101, 8104, 8102, 100, 100, 100, 100, 100, 100, 8103, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 8105, 100, 100, 100, 501, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], + decoded: "[CLS] \ud83d\ude02 \ud83d\udc4d [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] \ud83d\udd25 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] \ud83d\ude0e [UNK] [UNK] [UNK] \u2728 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2728", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [101, 501, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], + decoded: "[CLS] \u2728 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["[UNK]"], + ids: [101, 100, 102], + decoded: "[CLS] [UNK] [SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["[UNK]", "!", "how", "[UNK]", "[UNK]", "?"], + ids: [101, 100, 106, 9510, 100, 100, 136, 102], + decoded: "[CLS] [UNK]! how [UNK] [UNK]? [SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["[UNK]", "!", "how", "[UNK]", "[UNK]", "?"], + ids: [101, 100, 106, 9510, 100, 100, 136, 102], + decoded: "[CLS] [UNK]! how [UNK] [UNK]? [SEP]", + }, + }, + "Xenova/bge-base-zh-v1.5": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["how", "are", "you", "doi", "##ng", "?"], + ids: [101, 9510, 8995, 8357, 9962, 8291, 136, 102], + decoded: "[CLS] how are you doing? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["you", "sh", "##ould", "'", "ve", "don", "##e", "this"], + ids: [101, 8357, 11167, 11734, 112, 12810, 9524, 8154, 8554, 102], + decoded: "[CLS] you should've done this [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["the", "company", "was", "f", "##ound", "##ed", "in", "2016", "."], + ids: [101, 8174, 10007, 9947, 148, 11477, 8303, 8217, 8112, 119, 102], + decoded: "[CLS] the company was founded in 2016. [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["u", "##n", "##wan", "##ted", ",", "running"], + ids: [101, 163, 8171, 9951, 9255, 117, 11620, 102], + decoded: "[CLS] unwanted, running [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "[UNK]", "[UNK]", "test"], + ids: [101, 10060, 109, 122, 11345, 108, 124, 359, 8159, 173, 8157, 175, 8158, 100, 100, 100, 10060, 102], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] [UNK] [UNK] test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["i", "bo", "##ugh", "##t", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 151, 11059, 12667, 8165, 9064, 8350, 8330, 109, 122, 119, 8136, 8243, 8174, 8719, 119, 102], + decoded: "[CLS] i bought an apple for $ 1. 00 at the store. [SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\ud83d\ude02", "\ud83d\udc4d", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "\ud83d\udd25", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "\u2764", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "\ud83d\ude0e", "[UNK]", "[UNK]", "[UNK]", "\u2728", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [101, 8104, 8102, 100, 100, 100, 100, 100, 100, 8103, 100, 100, 100, 100, 100, 506, 100, 100, 100, 100, 100, 8105, 100, 100, 100, 501, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], + decoded: "[CLS] \ud83d\ude02 \ud83d\udc4d [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] \ud83d\udd25 [UNK] [UNK] [UNK] [UNK] [UNK] \u2764 [UNK] [UNK] [UNK] [UNK] [UNK] \ud83d\ude0e [UNK] [UNK] [UNK] \u2728 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + }, + "Xenova/indobert-base-p1": { + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["you", "sho", "##uld", "'", "ve", "don", "##e", "this"], + ids: [2, 3299, 9596, 15370, 30463, 28239, 4081, 30357, 5379, 3], + decoded: "[CLS] you should've done this [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["the", "company", "was", "found", "##ed", "in", "2016", "."], + ids: [2, 1002, 9105, 2738, 11009, 133, 48, 1538, 30470, 3], + decoded: "[CLS] the company was founded in 2016. [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "tos", "##trin", "##g", "(", ")", ";", "tos", "##trin", "##g", "(", ")", ";"], + ids: [2, 4734, 253, 30475, 559, 30372, 30470, 20498, 12448, 30365, 30464, 30465, 30473, 20498, 12448, 30365, 30464, 30465, 30473, 3], + decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["un", "##wan", "##te", "##d", ",", "running"], + ids: [2, 78, 1322, 3298, 30364, 30468, 22715, 3], + decoded: "[CLS] unwanted, running [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [2, 1, 1, 1, 1, 1, 1, 3], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["lead", "##ing", "space"], + ids: [2, 9196, 55, 14561, 3], + decoded: "[CLS] leading space [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r", "##2", "#", "3", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "test"], + ids: [2, 4243, 30460, 111, 56, 30378, 30459, 283, 1, 1, 1, 1, 1, 1, 4243, 3], + decoded: "[CLS] test $ 1 r2 # 3 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["i", "bo", "##ught", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [2, 89, 1880, 25009, 223, 7761, 1548, 30460, 111, 30470, 4230, 117, 1002, 8052, 30470, 3], + decoded: "[CLS] i bought an apple for $ 1. 00 at the store. [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["wei", "##rd", "[UNK]", "edge", "[UNK]", "case"], + ids: [2, 27753, 12548, 1, 21418, 1, 13687, 3], + decoded: "[CLS] weird [UNK] edge [UNK] case [SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["hallo", "!", "how", "are", "you", "?"], + ids: [2, 19598, 30457, 11088, 5811, 3299, 30477, 3], + decoded: "[CLS] hallo! how are you? [SEP]", + }, + }, + "Xenova/spanbert-large-cased": { + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "o", "##b", "##j", ".", "to", "##st", "##ring", "(", ")", ";", "to", "##st", "##ring", "(", ")", ";"], + ids: [101, 1519, 170, 134, 184, 1830, 3361, 119, 1106, 2050, 3384, 113, 114, 132, 1106, 2050, 3384, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r", "##2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [101, 2774, 109, 122, 187, 1477, 108, 124, 836, 1527, 202, 1571, 203, 1545, 100, 838, 1604, 837, 1580, 2774, 102], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", + }, + }, + "Xenova/UMLSBert_ENG": { + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "obj", ".", "tos", "##tring", "(", ")", ";", "tos", "##tring", "(", ")", ";"], + ids: [2, 8894, 42, 32, 2473, 17, 22660, 23640, 11, 12, 30, 22660, 23640, 11, 12, 30, 3], + decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["hel", "##lo", "world"], + ids: [2, 3018, 5368, 4517, 3], + decoded: "[CLS] hello world [SEP]", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hel", "##lo", "world"], + ids: [2, 3018, 5368, 4517, 3], + decoded: "[CLS] hello world [SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["hi", "hel", "##lo"], + ids: [2, 11245, 3018, 5368, 3], + decoded: "[CLS] hi hello [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "\u20a3", "##7", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [2, 2313, 7, 20, 9663, 6, 22, 528, 1017, 74, 1009, 76, 1018, 524, 1019, 531, 1011, 529, 1038, 2313, 3], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "##ir", "##d", "\uff5e", "edge", "\uff5e", "case"], + ids: [2, 1802, 1753, 1022, 943, 9676, 943, 2632, 3], + decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["hel", "##lo"], + ids: [2, 3018, 5368, 3], + decoded: "[CLS] hello [SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["hel", "##lo", "!", "how", "are", "you", "?"], + ids: [2, 3018, 5368, 5, 2135, 1810, 17915, 34, 3], + decoded: "[CLS] hello! how are you? [SEP]", + }, + }, + "Xenova/SapBERT-from-PubMedBERT-fulltext": { + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "\u6d3b", "\u7684", "[UNK]", "[UNK]", "\u662f"], + ids: [2, 799, 776, 811, 1, 1, 731, 3], + decoded: "[CLS] \u751f \u6d3b \u7684 [UNK] [UNK] \u662f [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "[UNK]", "test"], + ids: [2, 2648, 8, 21, 7261, 7, 23, 281, 1006, 76, 1015, 78, 1016, 1, 282, 1025, 1, 2648, 3], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 [UNK] test [SEP]", + }, + }, + "Xenova/rubert-base-cased": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "are", "you", "do", "##ing", "?"], + ids: [101, 15474, 10813, 13540, 10661, 7729, 166, 102], + decoded: "[CLS] How are you doing? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "sh", "##oul", "##d", "'", "ve", "don", "##e", "this"], + ids: [101, 11577, 45942, 76143, 239, 118, 10835, 17450, 241, 11043, 102], + decoded: "[CLS] You should've done this [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "comp", "##any", "was", "f", "##ound", "##ed", "in", "2016", "."], + ids: [101, 6821, 71382, 17927, 10646, 242, 71129, 7491, 10618, 8273, 132, 102], + decoded: "[CLS] The company was founded in 2016. [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##St", "##ring", "(", ")", ";", "to", "##St", "##ring", "(", ")", ";"], + ids: [101, 14107, 232, 162, 17851, 251, 132, 10626, 21568, 13647, 120, 122, 158, 10626, 21568, 13647, 120, 122, 158, 102], + decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "##wan", "##t", "##\u00e9d", ",", "run", "##ning"], + ids: [101, 27090, 14906, 271, 84705, 128, 14607, 11781, 102], + decoded: "[CLS] UNwant\u00e9d, running [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "[UNK]", "\u662f"], + ids: [101, 6104, 5480, 6222, 6273, 100, 4877, 102], + decoded: "[CLS] \u751f \u6d3b \u7684 \u771f [UNK] \u662f [SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["le", "##ading", "sp", "##ace"], + ids: [101, 10653, 73130, 33162, 13967, 102], + decoded: "[CLS] leading space [SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "##ili", "##ng", "sp", "##ace"], + ids: [101, 11776, 14296, 10888, 33162, 13967, 102], + decoded: "[CLS] trailing space [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "bo", "##ught", "an", "app", "##le", "for", "$", "1", ".", "00", "at", "the", "st", "##ore", "."], + ids: [101, 186, 21018, 53718, 10663, 73406, 7159, 10654, 112, 138, 132, 11537, 10672, 10617, 28668, 13536, 132, 102], + decoded: "[CLS] I bought an apple for $ 1. 00 at the store. [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "##ird", "\uff5e", "ed", "##ge", "\uff5e", "cas", "##e"], + ids: [101, 12463, 36865, 10608, 11051, 11037, 10608, 15501, 241, 102], + decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["a", "##h", "\u535a", "\u63a8", "z", "##z"], + ids: [101, 232, 247, 3166, 4657, 282, 283, 102], + decoded: "[CLS] ah \u535a \u63a8 zz [SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["He", "##LL", "##o", "!", "ho", "##w", "Are", "yo", "##U", "?"], + ids: [101, 10869, 83346, 261, 106, 13685, 277, 14003, 14184, 211, 166, 102], + decoded: "[CLS] HeLLo! how Are yoU? [SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["H", "##\u00e4", "##LL", "##o", "!", "ho", "##w", "Are", "yo", "##U", "?"], + ids: [101, 184, 384, 83346, 261, 106, 13685, 277, 14003, 14184, 211, 166, 102], + decoded: "[CLS] H\u00e4LLo! how Are yoU? [SEP]", + }, + }, + "Xenova/kobert": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "?"], + ids: [2, 0, 0, 0, 0, 258, 3], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK]? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["[UNK]", "[UNK]", "'", "[UNK]", "[UNK]", "[UNK]"], + ids: [2, 0, 0, 15, 0, 0, 0, 3], + decoded: "[CLS] [UNK] [UNK]'[UNK] [UNK] [UNK] [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "[UNK]", "[UNK]", "[UNK]", "in", "[UNK]", "."], + ids: [2, 355, 0, 0, 0, 409, 0, 54, 3], + decoded: "[CLS] The [UNK] [UNK] [UNK] in [UNK]. [SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "'", "[UNK]", "!", "!", "[UNK]", "?", "'", "d", "'", "'", "d", "[UNK]", ",", "[UNK]", "'", "t", "."], + ids: [2, 264, 15, 0, 5, 5, 0, 258, 15, 388, 15, 15, 388, 0, 46, 0, 15, 442, 54, 3], + decoded: "[CLS] A'[UNK]!! [UNK]?'d'' d [UNK], [UNK]'t. [SEP]", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["[UNK]", "[UNK]", "(", ")", ":", "[UNK]"], + ids: [2, 0, 0, 18, 40, 249, 0, 3], + decoded: "[CLS] [UNK] [UNK] ( ) : [UNK] [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["[UNK]", "a", "=", "[UNK]", ".", "[UNK]", "(", ")", ";", "[UNK]", "(", ")", ";"], + ids: [2, 0, 367, 254, 0, 54, 0, 18, 40, 252, 0, 18, 40, 252, 3], + decoded: "[CLS] [UNK] a = [UNK]. [UNK] ( ) ; [UNK] ( ) ; [SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["[UNK]", "is", "a", "[UNK]", "."], + ids: [2, 0, 412, 367, 0, 54, 3], + decoded: "[CLS] [UNK] is a [UNK]. [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["[UNK]", ",", "[UNK]"], + ids: [2, 0, 46, 0, 3], + decoded: "[CLS] [UNK], [UNK] [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [2, 5298, 0, 0, 0, 0, 0, 3], + decoded: "[CLS] \u751f [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["[UNK]", "$", "1", "[UNK]", "#", "3", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [2, 0, 10, 93, 0, 9, 142, 0, 0, 0, 0, 0, 0, 0, 3], + decoded: "[CLS] [UNK] $ 1 [UNK] # 3 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "[UNK]", "an", "[UNK]", "[UNK]", "$", "1", ".", "00", "at", "[UNK]", "[UNK]", "."], + ids: [2, 296, 0, 374, 0, 0, 10, 93, 54, 79, 377, 0, 0, 54, 3], + decoded: "[CLS] I [UNK] an [UNK] [UNK] $ 1. 00 at [UNK] [UNK]. [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [2, 0, 0, 0, 0, 3], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [2, 0, 0, 0, 0, 0, 3], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "\u2581", "."], + ids: [2, 0, 0, 0, 0, 517, 54, 3], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] \u2581. [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [2, 0, 0, 0, 0, 3], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["[UNK]", "!", "[UNK]", "[UNK]", "[UNK]", "?"], + ids: [2, 0, 5, 0, 0, 0, 258, 3], + decoded: "[CLS] [UNK]! [UNK] [UNK] [UNK]? [SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["[UNK]", "!", "[UNK]", "[UNK]", "[UNK]", "?"], + ids: [2, 0, 5, 0, 0, 0, 258, 3], + decoded: "[CLS] [UNK]! [UNK] [UNK] [UNK]? [SEP]", + }, + }, + "Xenova/scibert_scivocab_uncased": { + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "obj", ".", "to", "##string", "(", ")", ";", "to", "##string", "(", ")", ";"], + ids: [102, 1293, 106, 275, 2324, 205, 147, 20301, 145, 546, 1814, 147, 20301, 145, 546, 1814, 103], + decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["hi", "hell", "##o"], + ids: [102, 5305, 29423, 30112, 103], + decoded: "[CLS] hi hello [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r", "##2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "[UNK]", "[UNK]", "test"], + ids: [102, 856, 3250, 158, 182, 30132, 3000, 239, 20801, 30140, 11221, 30139, 20704, 30142, 101, 101, 101, 856, 103], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] [UNK] [UNK] test [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "[UNK]", "[UNK]", "zz"], + ids: [102, 7839, 101, 101, 23591, 103], + decoded: "[CLS] ah [UNK] [UNK] zz [SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["hell", "##o"], + ids: [102, 29423, 30112, 103], + decoded: "[CLS] hello [SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["hell", "##o", "!", "how", "are", "you", "?"], + ids: [102, 29423, 30112, 3190, 539, 220, 3034, 3912, 103], + decoded: "[CLS] hello! how are you? [SEP]", + }, + }, + "Xenova/LaBSE": { + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "obj", ".", "to", "##String", "(", ")", ";", "to", "##String", "(", ")", ";"], + ids: [101, 17214, 170, 134, 228877, 119, 14986, 368304, 113, 114, 132, 14986, 368304, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "R2", "#", "3", "\u20ac", "##4", "\u00a35", "\u00a5", "##6", "\u20a3", "##7", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [101, 17678, 109, 122, 51222, 108, 124, 3030, 16006, 279082, 205, 16151, 3023, 16187, 3037, 16175, 3033, 16236, 17678, 102], + decoded: "[CLS] test $ 1 R2 # 3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test [SEP]", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581", "##This", "\u2581", "##is", "\u2581", "##a", "\u2581", "##test", "\u2581", "."], + ids: [101, 3283, 342068, 3283, 15319, 3283, 14983, 3283, 50149, 3283, 119, 102], + decoded: "[CLS] \u2581This \u2581is \u2581a \u2581test \u2581. [SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\ud83d\ude02", "\ud83d\udc4d", "\ud83e\udd23", "\ud83d\ude0d", "\ud83d\ude2d", "\ud83c\udf89", "\ud83d\ude4f", "\ud83d\ude0a", "\ud83d\udd25", "\ud83d\ude01", "\ud83d\ude05", "\ud83e\udd17", "\ud83d\ude06", "\ud83d\udc4f", "\u2764\ufe0f", "\ud83d\udc9c", "\ud83d\udc9a", "\ud83d\udc97", "\ud83d\udc99", "\ud83d\udda4", "\ud83d\ude0e", "\ud83d\udc4c", "\ud83e\udd73", "\ud83d\udcaa", "\u2728", "\ud83d\udc49", "\ud83d\udc40", "\ud83d\udcaf", "\ud83c\udf88", "\ud83d\ude48", "\ud83d\ude4c", "\ud83d\udc80", "\ud83d\udc47", "\ud83d\udc4b", "\u2705", "\ud83c\udf81", "\ud83c\udf1e", "\ud83c\udf38", "\ud83d\udcb0"], + ids: [101, 14820, 14617, 14933, 14831, 14863, 14496, 14893, 14828, 14775, 14819, 14823, 14926, 14824, 14619, 91822, 14687, 14685, 14682, 14684, 14810, 14832, 14616, 14956, 14701, 3496, 14613, 14606, 14706, 14495, 14887, 14891, 14660, 14611, 14615, 3465, 14488, 14416, 14430, 14707, 102], + decoded: "[CLS] \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0 [SEP]", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2728", "\ud83e\udd17", "\ud83d\udc41\ufe0f", "\ud83d\udc71", "##\ud83c\udffb", "[UNK]", "[UNK]", "\ud83d\udc68", "##\ud83c\udffb", "##\ud83c\udf3e", "[UNK]", "\ud83d\udc69", "##\u2764", "##\ud83d\udc8b", "##\ud83d\udc68", "\ud83d\udc69", "##\ud83d\udc69", "##\ud83d\udc67", "##\ud83d\udc66", "[UNK]", "\ud83c\udff4", "\ud83d\udc68", "##\ud83c\udffb", "##\u2764", "##\ufe0f", "##\ud83d\udc8b", "##\ud83d\udc68", "##\ud83c\udffc"], + ids: [101, 3496, 14926, 350545, 14648, 130826, 100, 100, 14639, 130826, 498832, 100, 14640, 488649, 499065, 499034, 14640, 499035, 499033, 499032, 100, 14555, 14639, 130826, 488649, 44450, 499065, 499034, 421916, 102], + decoded: "[CLS] \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb [UNK] [UNK] \ud83d\udc68\ud83c\udffb\ud83c\udf3e [UNK] \ud83d\udc69\u2764\ud83d\udc8b\ud83d\udc68 \ud83d\udc69\ud83d\udc69\ud83d\udc67\ud83d\udc66 [UNK] \ud83c\udff4 \ud83d\udc68\ud83c\udffb\u2764\ufe0f\ud83d\udc8b\ud83d\udc68\ud83c\udffc [SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "\u535a", "\u63a8", "zz"], + ids: [101, 15524, 4573, 6405, 441764, 102], + decoded: "[CLS] ah \u535a \u63a8 zz [SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H\u00e9", "##llo"], + ids: [101, 220855, 23025, 102], + decoded: "[CLS] H\u00e9llo [SEP]", + }, + }, + "Xenova/herbert-large-cased": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["Ho", "w", "are", "you", "do", "ing", "?"], + ids: [0, 5213, 1019, 25720, 20254, 2065, 5129, 1550, 2], + decoded: "How are you doing? ", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "sho", "uld", "'", "ve", "d", "one", "this"], + ids: [0, 32795, 14924, 48273, 1571, 6647, 72, 2290, 48846, 2], + decoded: "You should've done this ", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "co", "mpany", "was", "fo", "un", "de", "d", "in", "20", "16", "."], + ids: [0, 7117, 2406, 41449, 9873, 3435, 2195, 2101, 1038, 2651, 5646, 2555, 1899, 2], + decoded: "The company was founded in 2016. ", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [0, 1012, 1571, 9396, 1725, 1725, 2063, 1550, 1571, 1038, 1571, 1571, 1038, 6595, 1947, 26794, 1571, 1026, 1899, 2], + decoded: "A'll!! to?'d'' d of, can't. ", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["de", "f", "main", "(", ")", ":", "pa", "ss"], + ids: [0, 2101, 1050, 41851, 1341, 1940, 1335, 2083, 5357, 2], + decoded: "def main ( ) : pass ", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "j", ".", "to", "S", "tr", "ing", "(", ")", ";", "to", "S", "tr", "ing", "(", ")", ";"], + ids: [0, 11324, 1011, 1789, 2033, 1013, 1899, 2146, 55, 2518, 5129, 1341, 1940, 1195, 2146, 55, 2518, 5129, 1341, 1940, 1195, 2], + decoded: "let a = obj. toString ( ) ; toString ( ) ; ", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["T", "his", "is", "a", "test", "."], + ids: [0, 56, 22855, 6869, 1011, 14825, 1899, 2], + decoded: "This is a test. ", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "wan", "t", "\u00e9", "d", ",", "run", "ning"], + ids: [0, 23029, 2688, 88, 163, 1038, 1947, 4980, 17843, 2], + decoded: "UNwant\u00e9d, running ", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["123"], + ids: [0, 19049, 2], + decoded: "123 ", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hel", "lo", "World"], + ids: [0, 12156, 6170, 21207, 2], + decoded: "Hello World ", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hel", "lo", "world"], + ids: [0, 11526, 6170, 38188, 2], + decoded: "hello world ", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["", "", "", "", "", "\u662f"], + ids: [0, 3, 3, 3, 3, 3, 1776, 2], + decoded: "\u662f ", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["le", "ad", "ing", "space"], + ids: [0, 2018, 2035, 5129, 46489, 2], + decoded: "leading space ", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "i", "ling", "space"], + ids: [0, 2201, 77, 16342, 46489, 2], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["H", "i", "Hel", "lo"], + ids: [0, 44, 1009, 12156, 6170, 2], + decoded: "Hi Hello ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "R", "2", "#", "3", "\u20ac", "4", "\u00a3", "5", "", "6", "", "7", "", "8", "", "9", "test"], + ids: [0, 14825, 1927, 1029, 54, 1025, 1393, 1034, 706, 1018, 100, 1008, 3, 1036, 3, 1030, 3, 1064, 3, 1017, 14825, 2], + decoded: "test $ 1 R2 # 3 \u20ac4 \u00a35 6 7 8 9 test ", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "bou", "ght", "an", "ap", "ple", "for", "$", "1", ".", "00", "at", "the", "st", "ore", "."], + ids: [0, 1056, 13016, 15272, 2879, 10309, 20861, 15181, 1927, 1029, 1899, 2291, 4772, 6854, 1989, 24005, 1899, 2], + decoded: "I bought an apple for $ 1. 00 at the store. ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u2026"], + ids: [0, 20254, 1826, 2], + decoded: "you \u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u2026"], + ids: [0, 20254, 1826, 2], + decoded: "you \u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u2026", "you", "\u2026"], + ids: [0, 20254, 1826, 20254, 1826, 2], + decoded: "you \u2026 you \u2026 ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ir", "d", "", "e", "dge", "", "ca", "se"], + ids: [0, 2149, 17435, 1038, 3, 73, 25801, 3, 3833, 4417, 2], + decoded: "weird edge case ", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["", "T", "his", "", "is", "", "a", "", "test", "", "."], + ids: [0, 3, 56, 22855, 3, 6869, 3, 1011, 3, 14825, 3, 1899, 2], + decoded: "This is a test . ", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["a", "h", "", "", "zz"], + ids: [0, 69, 1021, 3, 3, 49185, 2], + decoded: "ah zz ", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "\u00e9", "l", "lo"], + ids: [0, 44, 163, 80, 6170, 2], + decoded: "H\u00e9llo ", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["He", "L", "L", "o", "!", "ho", "w", "Ar", "e", "yo", "U", "?"], + ids: [0, 4596, 48, 48, 1007, 1725, 3145, 1019, 2921, 1015, 13908, 1041, 1550, 2], + decoded: "HeLLo! how Are yoU? ", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["H", "\u00e4", "L", "L", "o", "!", "ho", "w", "Ar", "e", "yo", "U", "?"], + ids: [0, 44, 158, 48, 48, 1007, 1725, 3145, 1019, 2921, 1015, 13908, 1041, 1550, 2], + decoded: "H\u00e4LLo! how Are yoU? ", + }, + }, + "Xenova/ernie-gram-zh": { + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r2", "#", "3", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "test"], + ids: [1, 6943, 18005, 208, 6847, 9474, 284, 18017, 18017, 18017, 18017, 18017, 18017, 6943, 2], + decoded: "[CLS] test $ 1 r2 # 3 [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] test [SEP]", + }, + }, + // `model.type` field missing in tokenizer.json + "google-bert/bert-base-cased": { + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "[UNK]", "[UNK]", "z", "##z"], + ids: [101, 18257, 100, 100, 195, 1584, 102], + decoded: "[CLS] ah [UNK] [UNK] zz [SEP]", + }, + }, + // NFD normalizer + "onnx-community/language_detection-ONNX": { + DEFAULT_EXAMPLE: { + text: NORMALIZATION_TEST_STRINGS.DEFAULT_EXAMPLE, + tokens: ["ame", "##lie", "|", "ame", "##lie"], + ids: [1, 21947, 31933, 70, 21947, 31933, 2], + decoded: "[CLS] amelie | amelie [SEP]", + }, + CANONICAL_EQUIVALENCE_NORMALIZATION: { + text: NORMALIZATION_TEST_STRINGS.CANONICAL_EQUIVALENCE_NORMALIZATION, + tokens: ["n", "|", "n"], + ids: [1, 56, 70, 56, 2], + decoded: "[CLS] n | n [SEP]", + }, + COMPATIBILITY_NORMALIZATION: { + text: NORMALIZATION_TEST_STRINGS.COMPATIBILITY_NORMALIZATION, + tokens: ["[UNK]", "|", "ff"], + ids: [1, 0, 70, 40133, 2], + decoded: "[CLS] [UNK] | ff [SEP]", + }, + COMBINED_EXAMPLE: { + text: NORMALIZATION_TEST_STRINGS.COMBINED_EXAMPLE, + tokens: ["ſ", "|", "ſ", "|", "ſ", "|", "s", "|", "s"], + ids: [1, 121, 70, 121, 70, 121, 70, 61, 70, 61, 2], + decoded: "[CLS] ſ | ſ | ſ | s | s [SEP]", + }, + }, +}; diff --git a/packages/transformers/tests/models/bit/test_image_processing_bit.js b/packages/transformers/tests/models/bit/test_image_processing_bit.js new file mode 100644 index 0000000..1c4fdcc --- /dev/null +++ b/packages/transformers/tests/models/bit/test_image_processing_bit.js @@ -0,0 +1,31 @@ +import { AutoImageProcessor, BitImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("BitImageProcessor", () => { + const model_id = "Xenova/dinov2-small-imagenet1k-1-layer"; + + /** @type {BitImageProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "default", + async () => { + const image = await load_cached_image("tiger"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 224, 224]); + expect(pixel_values.mean().item()).toBeCloseTo(0.06262318789958954, 3); + + expect(original_sizes).toEqual([[408, 612]]); + expect(reshaped_input_sizes).toEqual([[224, 224]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/blenderbot_small/test_tokenization_blenderbot_small.js b/packages/transformers/tests/models/blenderbot_small/test_tokenization_blenderbot_small.js new file mode 100644 index 0000000..3904224 --- /dev/null +++ b/packages/transformers/tests/models/blenderbot_small/test_tokenization_blenderbot_small.js @@ -0,0 +1,166 @@ +import { BlenderbotSmallTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, BLENDERBOT_SMALL_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = BlenderbotSmallTokenizer; + +// NOTE: `.tokenize()` is disabled for BlenderbotSmallTokenizer +export const TEST_CONFIG = { + "Xenova/blenderbot_small-90M": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + // "tokens": ["how", "are", "you", "doing", "?"], + ids: [102, 46, 15, 267, 20], + decoded: "how are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + // "tokens": ["you", "should", "'", "ve", "done", "this"], + ids: [15, 197, 8, 117, 369, 36], + decoded: "you should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + // "tokens": ["0@@", "1@@", "2@@", "3@@", "4@@", "5@@", "6@@", "7@@", "89", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], + ids: [1988, 2388, 735, 801, 827, 948, 981, 1110, 4814, 520, 143, 176, 216, 260, 253, 345, 374, 420, 475, 316, 773, 6217], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + // "tokens": ["the", "company", "was", "founded", "in", "2016", "."], + ids: [7, 293, 18, 912, 13, 845, 5], + decoded: "the company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + // "tokens": ["a", "__newln__", "'", "ll", "!", "!@@", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [12, 4, 8, 97, 37, 3, 11, 20, 8, 85, 8, 8, 85, 10, 6, 62, 8, 30, 5], + decoded: "a __newln__'ll! __unk__ to?'d'' d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + // "tokens": ["def", "main", "(", ")@@", ":", "__newln__", "pass"], + ids: [21996, 550, 40, 3, 106, 4, 1314], + decoded: "def main ( __unk__ : __newln__ pass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + // "tokens": ["let", "a", "=", "ob@@", "j", ".@@", "to@@", "string", "(", ")@@", ";", "__newln__", "to@@", "string", "(", ")@@", ";"], + ids: [131, 12, 1381, 2808, 755, 3, 752, 4529, 40, 3, 118, 4, 752, 4529, 40, 3, 118], + decoded: "let a = obj __unk__ tostring ( __unk__ ; __newln__ tostring ( __unk__ ;", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + // "tokens": ["this", "__newln__", "is", "__newln__", "a", "__newln__", "test", "."], + ids: [36, 4, 24, 4, 12, 4, 1248, 5], + decoded: "this __newln__ is __newln__ a __newln__ test.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + // "tokens": ["un@@", "wan@@", "t@@", "\u00e9@@", "d", ",@@", "running"], + ids: [204, 4151, 291, 1677, 85, 3, 785], + decoded: "unwant\u00e9d __unk__ running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + // "tokens": ["1@@", "\u0000@@", "2@@", "\ufffd@@", "3"], + ids: [2388, 3, 735, 3, 216], + decoded: "1__unk__ 2__unk__ 3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + // "tokens": ["hello", "world"], + ids: [880, 159], + decoded: "hello world", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + // "tokens": ["hello", "world"], + ids: [880, 159], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + // "tokens": ["\u751f@@", "\u6d3b@@", "\u7684@@", "\u771f@@", "\u8c1b@@", "\u662f"], + ids: [30488, 32756, 29891, 30813, 3, 34037], + decoded: "\u751f\u6d3b\u7684\u771f__unk__ \u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + // "tokens": ["leading", "space"], + ids: [1164, 833], + decoded: "leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + // "tokens": ["trailing", "space"], + ids: [12499, 833], + decoded: "trailing space", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + // "tokens": ["hi", "hello"], + ids: [792, 880], + decoded: "hi hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + // "tokens": ["test", "$@@", "1", "r@@", "2", "#@@", "3", "\u20ac@@", "4", "\u00a3@@", "5", "\u00a5@@", "6", "\u20a3@@", "7", "\u20b9@@", "8", "\u20b1@@", "9", "test"], + ids: [1248, 3, 143, 510, 176, 3, 216, 3, 260, 3, 253, 3, 345, 3, 374, 3, 420, 3, 475, 1248], + decoded: "test __unk__ 1 r2 __unk__ 3 __unk__ 4 __unk__ 5 __unk__ 6 __unk__ 7 __unk__ 8 __unk__ 9 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + // "tokens": ["i", "bought", "an", "apple", "for", "$@@", "1", ".@@", "00", "at", "the", "store", "."], + ids: [14, 1890, 50, 4758, 26, 3, 143, 3, 1966, 32, 7, 1640, 5], + decoded: "i bought an apple for __unk__ 1 __unk__ 00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + // "tokens": ["you@@", "\u2026"], + ids: [7984, 1244], + decoded: "you\u2026", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + // "tokens": ["you@@", "\u2026"], + ids: [7984, 1244], + decoded: "you\u2026", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + // "tokens": ["you@@", "\u2026", "you@@", "\u2026"], + ids: [7984, 1244, 7984, 1244], + decoded: "you\u2026 you\u2026", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + // "tokens": ["weird", "\uff5e", "edge", "\uff5e", "case"], + ids: [2614, 30831, 1649, 30831, 543], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + // "tokens": ["\u2581@@", "this", "\u2581@@", "is", "\u2581@@", "a", "\u2581@@", "test", "\u2581", "."], + ids: [3, 36, 3, 24, 3, 12, 3, 1248, 50106, 5], + decoded: "__unk__ this __unk__ is __unk__ a __unk__ test \u2581.", + }, + SPECIAL_TOKENS: { + text: BLENDERBOT_SMALL_TEST_STRINGS.SPECIAL_TOKENS, + // "tokens": ["__start__", "hello", "world", "__end__"], + ids: [1, 880, 159, 2], + decoded: "__start__ hello world __end__", + }, + WHITESPACE_1: { + text: BLENDERBOT_SMALL_TEST_STRINGS.WHITESPACE_1, + // "tokens": ["__start__", "hey", "__end__"], + ids: [1, 226, 2], + decoded: "__start__ hey __end__", + }, + WHITESPACE_2: { + text: BLENDERBOT_SMALL_TEST_STRINGS.WHITESPACE_2, + // "tokens": ["__start__", "hey", "__end__"], + ids: [1, 226, 2], + decoded: "__start__ hey __end__", + }, + }, +}; diff --git a/packages/transformers/tests/models/bloom/test_modeling_bloom.js b/packages/transformers/tests/models/bloom/test_modeling_bloom.js new file mode 100644 index 0000000..5eec189 --- /dev/null +++ b/packages/transformers/tests/models/bloom/test_modeling_bloom.js @@ -0,0 +1,50 @@ +import { BloomTokenizer, BloomForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("BloomForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-BloomForCausalLM"; + /** @type {BloomForCausalLM} */ + let model; + /** @type {BloomTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await BloomForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await BloomTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[198n, 803n, 82n, 82n, 82n, 82n, 82n, 82n, 82n, 82n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [3n, 3n, 198n, 803n, 82n, 82n, 82n, 82n, 82n, 82n], + [198n, 803n, 82n, 209n, 753n, 753n, 753n, 753n, 753n, 753n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/bloom/test_tokenization_bloom.js b/packages/transformers/tests/models/bloom/test_tokenization_bloom.js new file mode 100644 index 0000000..5c81d28 --- /dev/null +++ b/packages/transformers/tests/models/bloom/test_tokenization_bloom.js @@ -0,0 +1,194 @@ +import { BloomTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, BLOOM_TEST_STRINGS, SENTENCEPIECE_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = BloomTokenizer; +export const TEST_CONFIG = { + "Xenova/bloom-560m": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [7572, 1306, 1152, 12491, 34], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [5448, 3403, 7300, 11541, 1119], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0123", "456789", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [166660, 145647, 931, 404, 415, 735, 934, 973, 1231, 1392, 1445, 1575, 1581, 4334, 19526], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], + ids: [2175, 16333, 1620, 88289, 361, 5854, 17], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?", "'d", "''", "d", "\u0120of", ",", "\u0120can't", "."], + ids: [36, 189, 8722, 49825, 1025, 34, 10628, 2328, 71, 461, 15, 11229, 17], + decoded: "A\n'll !!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "()", ":", "\u010a\u0109", "pass"], + ids: [7564, 4291, 883, 29, 1582, 12608], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "toString", "()", ";", "\u010a", "toString", "()", ";"], + ids: [2963, 267, 564, 17949, 17, 27392, 883, 30, 189, 27392, 883, 30], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [6168, 603, 290, 189, 68, 189, 9234, 17], + decoded: "This\n\nis\na\ntest.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], + ids: [5777, 75642, 2454, 15, 101897], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [20, 179, 21, 23181, 22], + decoded: "1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [59414, 12155], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [101579, 8876], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [71167, 4137, 1927, 239, 644], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [250, 36128, 12978], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120\u0120\u0120"], + ids: [1900, 17022, 12978, 416], + decoded: "trailing space ", + }, + SURROUNDING_SPACE: { + text: BASE_TEST_STRINGS.SURROUNDING_SPACE, + tokens: ["\u0120\u0120", "\u0120surrounding", "\u0120space", "\u0120\u0120\u0120"], + ids: [250, 66599, 12978, 416], + decoded: " surrounding space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [30050, 210, 86153], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$1", "\u0120R2", "\u0120#3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2\u0124", "\u00b1", "9", "\u0120test"], + ids: [9234, 41448, 80774, 201642, 20117, 23, 40300, 24, 62153, 25, 72279, 100, 26, 120434, 27, 72279, 113, 28, 4006], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [44, 87926, 660, 101091, 613, 41448, 17, 462, 919, 368, 18706, 17], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [23438, 4346, 250], + decoded: "you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [23438, 4346, 12361], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [23438, 4346, 12361, 23438, 4346, 12361], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef\u00bd", "\u0140", "\u0120case"], + ids: [2136, 7589, 122354, 242, 29655, 122354, 242, 4462], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [26127, 213, 6168, 15299, 213, 290, 15299, 213, 68, 15299, 213, 9234, 15299, 213, 17], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141", "\u0133", "\u0131", "\u0120\u00e2\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u0134", "\u013e", "\u0120\u00f0\u0141", "\u0134", "\u013c", "\u0120\u00f0\u0141", "\u0134", "\u0139", "\u0120\u00f0\u0141", "\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141", "\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141", "\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u012b", "\u0120\u00f0\u0141", "\u0133", "\u0122", "\u0120\u00f0\u0141", "\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141", "\u0134", "\u0122", "\u0120\u00f0\u0141", "\u0133", "\u0129", "\u0120\u00f0\u0141", "\u0133", "\u012d", "\u0120\u00e2\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141", "\u0134", "\u00b0"], + ids: [127322, 214, 41234, 229, 225, 41234, 101, 100, 126342, 225, 126342, 245, 41234, 226, 221, 41234, 237, 227, 126342, 222, 41234, 232, 102, 126342, 213, 126342, 217, 41234, 101, 235, 126342, 218, 41234, 229, 227, 189367, 101, 116057, 41234, 230, 240, 41234, 230, 238, 41234, 230, 235, 41234, 230, 237, 41234, 234, 101, 126342, 226, 41234, 229, 224, 41234, 102, 115, 41234, 230, 107, 76758, 105, 41234, 229, 221, 41234, 229, 212, 41234, 230, 111, 41234, 226, 220, 41234, 237, 220, 41234, 237, 224, 41234, 230, 212, 41234, 229, 219, 41234, 229, 223, 76758, 217, 41234, 226, 213, 41234, 224, 242, 41234, 224, 120, 41234, 230, 112], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141", "\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u0133", "\u00b1", "\u00f0\u0141\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00e2\u013f", "\u00a4", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bc"], + ids: [120709, 105, 41234, 101, 235, 41234, 229, 213, 116057, 41234, 229, 113, 244635, 123, 41234, 233, 117, 1553, 15596, 214, 116057, 41234, 104, 237, 244635, 123, 1553, 15596, 214, 41234, 229, 105, 244635, 123, 1553, 22618, 224, 126, 41234, 104, 229, 1553, 22618, 101, 241, 1553, 22618, 104, 229, 41234, 229, 106, 1553, 157147, 101, 1553, 139500, 223, 1553, 22618, 229, 105, 41234, 229, 106, 1553, 22618, 229, 106, 1553, 22618, 229, 104, 1553, 22618, 229, 103, 41234, 104, 229, 244635, 123, 1553, 22618, 101, 241, 1553, 22618, 104, 229, 244635, 123, 41234, 227, 116, 177, 244, 213, 104, 177, 244, 213, 99, 177, 244, 213, 102, 177, 244, 213, 110, 177, 244, 213, 104, 177, 244, 213, 127, 41234, 229, 105, 244635, 123, 1553, 157147, 101, 116057, 1553, 139500, 223, 1553, 22618, 229, 105, 244635, 124], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + ONLY_WHITESPACE: { + text: BASE_TEST_STRINGS.ONLY_WHITESPACE, + tokens: ["\u0120\u0109", "\u010a"], + ids: [33651, 189], + decoded: " \t\n", + }, + END_OF_SENTENCE_PUNCTUATION: { + text: BLOOM_TEST_STRINGS.END_OF_SENTENCE_PUNCTUATION, + tokens: ["test", ".", "\u0120test", ",", "\u0120test", "!", "\u0120test", "?", "\u0120test", "\u00e2\u0122\u00a6", "\u0120test", "\u00e3\u0122\u0124", "\u0120test", "\u00ef\u00bc\u012e", "\u0120test", "\u00e3\u0122\u0123", "\u0120test", "\u00e0\u00a5\u00a4", "\u0120test", "\u00db\u0136", "\u0120test", "\u00d8\u012e", "\u0120test"], + ids: [9234, 17, 4006, 15, 4006, 4, 4006, 34, 4006, 4346, 4006, 420, 4006, 355, 4006, 594, 4006, 527, 4006, 1174, 4006, 687, 4006], + decoded: "test. test, test! test? test\u2026 test\u3002 test\uff0c test\u3001 test\u0964 test\u06d4 test\u060c test", + }, + SPECIAL_WITH_TRAILING_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_WITH_TRAILING_WHITESPACE, + tokens: ["", "\u010a"], + ids: [1, 189], + decoded: "\n", + }, + SPECIAL_SURROUNDED_BY_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_SURROUNDED_BY_WHITESPACE, + tokens: ["\u0120", "", "\u0120test", "\u0120", "", "\u0120"], + ids: [210, 2, 4006, 210, 2, 210], + decoded: " test ", + }, + SPECIAL_NO_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_NO_WHITESPACE, + tokens: ["", "test", ""], + ids: [2, 9234, 2], + decoded: "test", + }, + }, +}; diff --git a/packages/transformers/tests/models/chmv2/test_image_processing_chmv2.js b/packages/transformers/tests/models/chmv2/test_image_processing_chmv2.js new file mode 100644 index 0000000..28dfdb7 --- /dev/null +++ b/packages/transformers/tests/models/chmv2/test_image_processing_chmv2.js @@ -0,0 +1,48 @@ +import { AutoImageProcessor, CHMv2ImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("CHMv2ImageProcessor", () => { + const model_id = "onnx-community/dinov3-vitl16-chmv2-dpt-head-ONNX"; + + /** @type {CHMv2ImageProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "default (no resize, pad to multiple of 16)", + async () => { + // cats.jpg is 640x480, already a multiple of 16 — no padding needed + const image = await load_cached_image("cats"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 480, 640]); + expect(pixel_values.mean().item()).toBeCloseTo(0.8060207962989807, 3); + + expect(original_sizes).toEqual([[480, 640]]); + expect(reshaped_input_sizes).toEqual([[480, 640]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "pads to multiple of size_divisor", + async () => { + // tiger.jpg is 612x408 — padded to 624x416 + const image = await load_cached_image("tiger"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 416, 624]); + expect(pixel_values.mean().item()).toBeCloseTo(0.07290177792310715, 3); + + expect(original_sizes).toEqual([[408, 612]]); + expect(reshaped_input_sizes).toEqual([[408, 612]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/clap/test_feature_extraction_clap.js b/packages/transformers/tests/models/clap/test_feature_extraction_clap.js new file mode 100644 index 0000000..f0ffeec --- /dev/null +++ b/packages/transformers/tests/models/clap/test_feature_extraction_clap.js @@ -0,0 +1,71 @@ +import { AutoFeatureExtractor, ClapFeatureExtractor } from "../../../src/transformers.js"; +import { random } from "../../../src/utils/random.js"; + +import { load_cached_audio } from "../../asset_cache.js"; +import { MAX_FEATURE_EXTRACTOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // ClapFeatureExtractor + describe("ClapFeatureExtractor", () => { + const model_id = "Xenova/clap-htsat-unfused"; + + /** @type {ClapFeatureExtractor} */ + let feature_extractor; + beforeAll(async () => { + feature_extractor = await AutoFeatureExtractor.from_pretrained(model_id); + }, MAX_FEATURE_EXTRACTOR_LOAD_TIME); + + it( + "truncation", + async () => { + const audio = await load_cached_audio("mlk"); + + // Since truncation uses a random strategy, we seed + // the PRNG to ensure that the test is deterministic + random.seed(0); + + let long_audio = new Float32Array(500000); + long_audio.set(audio); + long_audio.set(audio, long_audio.length - audio.length); + + const { input_features } = await feature_extractor(long_audio); + const { dims, data } = input_features; + expect(dims).toEqual([1, 1, 1001, 64]); + + expect(input_features.mean().item()).toBeCloseTo(-37.9171257019043); + expect(data[0]).toBeCloseTo(-23.681066513061523); + expect(data[1]).toBeCloseTo(-20.759065628051758); + expect(data[65]).toBeCloseTo(-25.722291946411133); + expect(data[1002]).toBeCloseTo(-2.2271111011505127); + expect(data[10000]).toBeCloseTo(-17.393341064453125); + expect(data[60000]).toBeCloseTo(-48.419677734375); + expect(data[64062]).toBeCloseTo(-35.120487213134766); + expect(data[64063]).toBeCloseTo(-36.36371612548828); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "padding", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_features } = await feature_extractor(audio); + const { data, dims } = input_features; + expect(dims).toEqual([1, 1, 1001, 64]); + + expect(input_features.mean().item()).toBeCloseTo(-34.99049377441406); + expect(data[0]).toBeCloseTo(-21.32573890686035); + expect(data[1]).toBeCloseTo(-26.168411254882812); + expect(data[65]).toBeCloseTo(-29.716018676757812); + expect(data[1002]).toBeCloseTo(-32.16273498535156); + expect(data[10000]).toBeCloseTo(-19.9283390045166); + + // padded values + expect(data[60000]).toBeCloseTo(-100.0); + expect(data[64062]).toBeCloseTo(-100.0); + expect(data[64063]).toBeCloseTo(-100.0); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/clip/test_image_processing_clip.js b/packages/transformers/tests/models/clip/test_image_processing_clip.js new file mode 100644 index 0000000..cafa716 --- /dev/null +++ b/packages/transformers/tests/models/clip/test_image_processing_clip.js @@ -0,0 +1,33 @@ +import { AutoImageProcessor, CLIPFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // CLIPFeatureExtractor + // - tests center crop (do_center_crop=true, crop_size=224) + describe("CLIPFeatureExtractor", () => { + const model_id = "Xenova/clip-vit-base-patch16"; + + /** @type {CLIPFeatureExtractor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "default", + async () => { + const image = await load_cached_image("tiger"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 224, 224]); + expect(pixel_values.mean().item()).toBeCloseTo(-0.06678297738282096, 6); + + expect(original_sizes).toEqual([[408, 612]]); + expect(reshaped_input_sizes).toEqual([[224, 224]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/clip/test_modeling_clip.js b/packages/transformers/tests/models/clip/test_modeling_clip.js new file mode 100644 index 0000000..433517f --- /dev/null +++ b/packages/transformers/tests/models/clip/test_modeling_clip.js @@ -0,0 +1,58 @@ +import { AutoTokenizer, AutoProcessor, load_image, CLIPVisionModelWithProjection, CLIPTextModelWithProjection } from "../../../src/transformers.js"; + +import { MAX_TEST_EXECUTION_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + const models_to_test = ["hf-internal-testing/tiny-random-CLIPModel"]; + it( + `CLIPTextModelWithProjection`, + async () => { + const model_id = models_to_test[0]; + + // Load tokenizer and text model + const tokenizer = await AutoTokenizer.from_pretrained(model_id); + const text_model = await CLIPTextModelWithProjection.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + + // Run tokenization + const texts = ["a photo of a car", "a photo of a football match"]; + const text_inputs = tokenizer(texts, { padding: true, truncation: true }); + + // Compute embeddings + const { text_embeds } = await text_model(text_inputs); + + // Ensure correct shapes + const expected_shape = [texts.length, text_model.config.projection_dim]; + const actual_shape = text_embeds.dims; + expect(expected_shape).toEqual(actual_shape); + + await text_model.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + `CLIPVisionModelWithProjection`, + async () => { + const model_id = models_to_test[0]; + + // Load processor and vision model + const processor = await AutoProcessor.from_pretrained(model_id); + const vision_model = await CLIPVisionModelWithProjection.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + + // Read image and run processor + const image = await load_image("https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg"); + const image_inputs = await processor(image); + + // Compute embeddings + const { image_embeds } = await vision_model(image_inputs); + + // Ensure correct shapes + const expected_shape = [1, vision_model.config.projection_dim]; + const actual_shape = image_embeds.dims; + expect(expected_shape).toEqual(actual_shape); + + await vision_model.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); +}; diff --git a/packages/transformers/tests/models/clip/test_tokenization_clip.js b/packages/transformers/tests/models/clip/test_tokenization_clip.js new file mode 100644 index 0000000..b53f3c9 --- /dev/null +++ b/packages/transformers/tests/models/clip/test_tokenization_clip.js @@ -0,0 +1,166 @@ +import { CLIPTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = CLIPTokenizer; +export const TEST_CONFIG = { + "Xenova/clip-vit-base-patch16": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["how", "are", "you", "doing", "?"], + ids: [49406, 829, 631, 592, 1960, 286, 49407], + decoded: "<|startoftext|>how are you doing? <|endoftext|>", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["you", "should", "'ve", "done", "this"], + ids: [49406, 592, 1535, 1200, 1700, 589, 49407], + decoded: "<|startoftext|>you should've done this <|endoftext|>", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "1", "0", "1", "0", "0", "1", "0", "0", "0"], + ids: [49406, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 272, 271, 272, 271, 271, 272, 271, 271, 271, 49407], + decoded: "<|startoftext|>0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 1 0 1 0 0 1 0 0 0 <|endoftext|>", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["the", "company", "was", "founded", "in", "2", "0", "1", "6", "."], + ids: [49406, 518, 2634, 739, 12240, 530, 273, 271, 272, 277, 269, 49407], + decoded: "<|startoftext|>the company was founded in 2 0 1 6. <|endoftext|>", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["a", "'ll", "!!", "to", "?'", "d", "''", "d", "of", ",", "can", "'t", "."], + ids: [49406, 320, 1342, 748, 531, 13610, 323, 8445, 323, 539, 267, 753, 713, 269, 49407], + decoded: "<|startoftext|>a 'll!! to?' d '' d of, can 't. <|endoftext|>", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "main", "(", "):", "pass"], + ids: [49406, 11649, 2623, 7, 4143, 3511, 49407], + decoded: "<|startoftext|>def main (): pass <|endoftext|>", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "j", ".", "to", "string", "(", ");", "to", "string", "(", ");"], + ids: [49406, 1094, 320, 284, 1411, 329, 269, 580, 9696, 7, 19686, 580, 9696, 7, 19686, 49407], + decoded: "<|startoftext|>let a = obj. tostring (); tostring (); <|endoftext|>", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["this", "is", "a", "test", "."], + ids: [49406, 589, 533, 320, 1628, 269, 49407], + decoded: "<|startoftext|>this is a test. <|endoftext|>", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["un", "want", "\u00c3\u00a9", "d", ",", "running"], + ids: [49406, 569, 18356, 3459, 323, 267, 2761, 49407], + decoded: "<|startoftext|>unwant\u00e9d, running <|endoftext|>", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [49406, 272, 444, 273, 39802, 274, 49407], + decoded: "<|startoftext|>1 \u0000 2 \ufffd 3 <|endoftext|>", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["hello", "world"], + ids: [49406, 3306, 1002, 49407], + decoded: "<|startoftext|>hello world <|endoftext|>", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "world"], + ids: [49406, 3306, 1002, 49407], + decoded: "<|startoftext|>hello world <|endoftext|>", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7", "\u013c", "\u0126", "\u00e7\u013e\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a", "\u00af"], + ids: [49406, 33375, 162, 112, 119, 163, 248, 226, 41570, 164, 108, 249, 42891, 363, 49407], + decoded: "<|startoftext|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f <|endoftext|>", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["leading", "space"], + ids: [49406, 3833, 2138, 49407], + decoded: "<|startoftext|>leading space <|endoftext|>", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "space"], + ids: [49406, 37427, 2138, 49407], + decoded: "<|startoftext|>trailing space <|endoftext|>", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["hi", "hello"], + ids: [49406, 1883, 3306, 49407], + decoded: "<|startoftext|>hi hello <|endoftext|>", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r", "2", "#", "3", "\u00e2\u0124\u00ac", "4", "\u00c2\u00a3", "5", "\u00c2\u00a5", "6", "\u00e2\u0124", "\u00a3", "7", "\u00e2\u0124\u00b9", "8", "\u00e2\u0124", "\u00b1", "9", "test"], + ids: [49406, 1628, 259, 272, 337, 273, 258, 274, 6309, 275, 1950, 276, 20199, 277, 5227, 352, 278, 21777, 279, 5227, 365, 280, 1628, 49407], + decoded: "<|startoftext|>test $ 1 r 2 # 3 \u20ac 4 \u00a3 5 \u00a5 6 \u20a3 7 \u20b9 8 \u20b1 9 test <|endoftext|>", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["i", "bought", "an", "apple", "for", "$", "1", ".", "0", "0", "at", "the", "store", "."], + ids: [49406, 328, 4142, 550, 3055, 556, 259, 272, 269, 271, 271, 536, 518, 2183, 269, 49407], + decoded: "<|startoftext|>i bought an apple for $ 1. 0 0 at the store. <|endoftext|>", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6"], + ids: [49406, 592, 959, 49407], + decoded: "<|startoftext|>you \u2026 <|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6"], + ids: [49406, 592, 959, 49407], + decoded: "<|startoftext|>you \u2026 <|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "you", "\u00e2\u0122\u00a6"], + ids: [49406, 592, 959, 592, 959, 49407], + decoded: "<|startoftext|>you \u2026 you \u2026 <|endoftext|>", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "\u00ef", "\u00bd", "\u0140", "edge", "\u00ef", "\u00bd", "\u0140", "case"], + ids: [49406, 5613, 171, 121, 508, 5461, 171, 121, 508, 2068, 49407], + decoded: "<|startoftext|>weird \uff5e edge \uff5e case <|endoftext|>", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "this", "\u00e2\u0138", "\u0123", "is", "\u00e2\u0138", "\u0123", "a", "\u00e2\u0138", "\u0123", "test", "\u00e2\u0138", "\u0123", "."], + ids: [49406, 4168, 479, 589, 4168, 479, 533, 4168, 479, 320, 4168, 479, 1628, 4168, 223, 269, 49407], + decoded: "<|startoftext|>\u2581 this \u2581 is \u2581 a \u2581 test \u2581. <|endoftext|>", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a\u0124", "\u00f0\u0141\u0133\u012f", "\u00f0\u0141\u00a4\u00a3", "\u00f0\u0141\u013a\u012f", "\u00f0\u0141\u013a\u0143", "\u00f0\u0141\u0130\u012b", "\u00f0\u0141\u013b\u0131", "\u00f0\u0141\u013a\u012c", "\u00f0\u0141\u0136\u00a5", "\u00f0\u0141\u013a\u0123", "\u00f0\u0141\u013a\u0127", "\u00f0\u0141\u00a4\u0139", "\u00f0\u0141\u013a\u0128", "\u00f0\u0141\u0133\u0131", "\u00e2\u013f\u00a4\u00ef\u00b8\u0131", "\u00f0\u0141\u0134\u013e", "\u00f0\u0141\u0134\u013c", "\u00f0\u0141\u0134\u0139", "\u00f0\u0141\u0134\u013b", "\u00f0\u0141\u0138\u00a4", "\u00f0\u0141\u013a\u0130", "\u00f0\u0141\u0133\u012e", "\u00f0\u0141\u00a5\u00b3", "\u00f0\u0141\u0134\u00aa", "\u00e2\u013e\u00a8", "\u00f0\u0141\u0133\u012b", "\u00f0\u0141\u0133\u0122", "\u00f0\u0141\u0134\u00af", "\u00f0\u0141\u0130\u012a", "\u00f0\u0141\u013b\u012a", "\u00f0\u0141\u013b\u012e", "\u00f0\u0141\u0134\u0122", "\u00f0\u0141\u0133\u0129", "\u00f0\u0141\u0133\u012d", "\u00e2\u013e\u0127", "\u00f0\u0141\u0130\u0123", "\u00f0\u0141\u012e\u0140", "\u00f0\u0141\u012e\u00b8", "\u00f0\u0141\u0134\u00b0"], + ids: [49406, 1558, 4201, 9909, 1754, 3915, 3986, 5503, 3020, 3016, 4821, 9188, 10465, 10943, 4829, 1752, 4882, 6521, 6690, 4074, 10860, 4345, 4494, 28055, 6440, 3531, 3988, 5908, 7018, 14448, 9516, 4855, 12158, 7475, 17686, 5564, 13462, 12980, 10980, 14078, 49407], + decoded: "<|startoftext|>\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0 <|endoftext|>", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e\u00a8", "\u00f0\u0141\u00a4\u0139", "\u00f0\u0141\u0133\u0123", "\u00ef\u00b8\u0131", "\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141\u0131\u00bb", "\u00f0\u0141\u0137", "\u00b5", "\u00e2\u0122\u012f\u00e2\u013b\u0124\u00ef\u00b8\u0131", "\u00f0\u0141\u00a7", "\u013b", "\u00f0\u0141\u0131\u00bb", "\u00e2\u0122\u012f\u00e2\u013b", "\u0124", "\u00f0\u0141\u0133\u00a8", "\u00f0\u0141\u0131\u00bb\u00e2\u0122\u012f", "\u00f0\u0141\u012e\u00be", "\u00f0\u0141\u00a7", "\u0133", "\u00e2\u0122\u012f", "\u00f0\u0141\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141\u00a7", "\u0133", "\u00f0\u0141\u0133\u00a9\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0133\u00a9\u00e2\u0122\u012f", "\u00f0\u0141\u0133\u00a9\u00e2\u0122\u012f", "\u00f0\u0141\u0133\u00a7", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a6", "\u00f0\u0141\u00a7", "\u0133", "\u00f0\u0141\u0131\u00bb\u00e2\u0122\u012f", "\u00f0\u0141\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141\u00a7", "\u0133", "\u00f0\u0141\u0131\u00bb", "\u00f0\u0141\u0131\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u00f0\u0141\u0133\u00a8", "\u00f0\u0141\u0131\u00bb\u00e2\u0122\u012f", "\u00e2\u013f\u00a4\u00ef\u00b8\u0131", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133\u00a8", "\u00f0\u0141\u0131\u00bc"], + ids: [49406, 3531, 10465, 47796, 1001, 964, 109, 3702, 7692, 113, 10613, 8792, 247, 5042, 5177, 480, 18966, 46250, 39796, 8792, 239, 4244, 1793, 251, 4244, 8792, 495, 26304, 1266, 4244, 12217, 4244, 964, 357, 26304, 26304, 48938, 4244, 964, 355, 8792, 239, 46250, 1793, 251, 4244, 8792, 239, 3702, 39690, 175, 254, 223, 100, 175, 254, 223, 95, 175, 254, 223, 98, 175, 254, 223, 106, 175, 254, 223, 100, 175, 254, 223, 379, 18966, 46250, 2626, 4244, 12217, 4244, 18966, 4027, 49407], + decoded: "<|startoftext|>\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc <|endoftext|>", + }, + }, + "Xenova/owlvit-base-patch32": { + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["a", "'ll", "!", "!", "to", "?'", "d", "''", "d", "of", ",", "can", "'t", "."], + ids: [49406, 320, 1342, 0, 0, 531, 13610, 323, 8445, 323, 539, 267, 753, 713, 269, 49407], + decoded: "<|startoftext|>a 'll!!to?' d '' d of, can 't. <|endoftext|>", + }, + }, +}; diff --git a/packages/transformers/tests/models/codegen/test_modeling_codegen.js b/packages/transformers/tests/models/codegen/test_modeling_codegen.js new file mode 100644 index 0000000..d8044e5 --- /dev/null +++ b/packages/transformers/tests/models/codegen/test_modeling_codegen.js @@ -0,0 +1,51 @@ +import { CodeGenTokenizer, CodeGenForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("CodeGenForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-CodeGenForCausalLM"; + /** @type {CodeGenForCausalLM} */ + let model; + /** @type {CodeGenTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await CodeGenForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await CodeGenTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[258n, 863n, 79n, 437n, 334n, 450n, 294n, 621n, 375n, 385n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 258n, 863n, 79n, 437n, 334n, 450n, 294n, 621n], + [258n, 863n, 79n, 269n, 813n, 759n, 113n, 295n, 574n, 987n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/cohere/test_modeling_cohere.js b/packages/transformers/tests/models/cohere/test_modeling_cohere.js new file mode 100644 index 0000000..b3ff863 --- /dev/null +++ b/packages/transformers/tests/models/cohere/test_modeling_cohere.js @@ -0,0 +1,90 @@ +import { CohereTokenizer, CohereModel, CohereForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("CohereModel", () => { + const model_id = "hf-internal-testing/tiny-random-CohereModel"; + /** @type {CohereModel} */ + let model; + /** @type {CohereTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await CohereModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await CohereTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const { last_hidden_state } = await model(inputs); + expect(last_hidden_state.dims).toEqual([1, 4, 32]); + expect(last_hidden_state.mean().item()).toBeCloseTo(0.0, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const { last_hidden_state } = await model(inputs); + expect(last_hidden_state.dims).toEqual([2, 6, 32]); + expect(last_hidden_state.mean().item()).toBeCloseTo(9.934107758624577e-9, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("CohereForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-CohereForCausalLM"; + /** @type {CohereForCausalLM} */ + let model; + /** @type {CohereTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await CohereForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await CohereTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[5n, 203n, 790n, 87n, 87n, 87n, 87n, 87n, 87n, 87n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 5n, 203n, 790n, 87n, 87n, 87n, 87n, 87n], + [5n, 203n, 790n, 87n, 214n, 741n, 741n, 741n, 741n, 741n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/cohere_asr/test_feature_extraction_cohere_asr.js b/packages/transformers/tests/models/cohere_asr/test_feature_extraction_cohere_asr.js new file mode 100644 index 0000000..1ee0c92 --- /dev/null +++ b/packages/transformers/tests/models/cohere_asr/test_feature_extraction_cohere_asr.js @@ -0,0 +1,93 @@ +import { AutoFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_audio } from "../../asset_cache.js"; +import { MAX_FEATURE_EXTRACTOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("CohereAsrFeatureExtractor", () => { + const model_id = "onnx-community/cohere-transcribe-03-2026-ONNX"; + + let feature_extractor; + beforeAll(async () => { + feature_extractor = await AutoFeatureExtractor.from_pretrained(model_id); + }, MAX_FEATURE_EXTRACTOR_LOAD_TIME); + + it( + "default", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_features, attention_mask } = await feature_extractor(audio); + + // Shape: [1, num_frames, 128] (transposed mel spectrogram) + expect(input_features.dims).toEqual([1, 1301, 128]); + + // Attention mask: [1, num_frames], 1300 valid frames + expect(attention_mask.dims).toEqual([1, 1301]); + const mask_sum = attention_mask.data.reduce((a, b) => a + b, 0n); + expect(Number(mask_sum)).toEqual(1300); + + // Check feature values against Python reference + // NOTE: Small differences (~1e-3) are expected due to mel filter precision + // (librosa float32 vs JS float64) and different dithering PRNGs. + expect(input_features.mean().item()).toBeCloseTo(0.0, 3); + expect(input_features.data[0]).toBeCloseTo(1.9019224644, 2); // [0,0,0] + expect(input_features.data[1]).toBeCloseTo(1.4606336355, 2); // [0,0,1] + expect(input_features.data[128]).toBeCloseTo(1.6364065409, 2); // [0,1,0] + expect(input_features.data[127]).toBeCloseTo(-0.8954101205, 2); // [0,0,127] + expect(input_features.data[12800]).toBeCloseTo(0.9838520288, 2); // [0,100,0] + expect(input_features.data[64050]).toBeCloseTo(-0.6117327809, 2); // [0,500,50] + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "short audio", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_features, attention_mask } = await feature_extractor(audio.slice(0, 16000)); + + expect(input_features.dims).toEqual([1, 101, 128]); + expect(attention_mask.dims).toEqual([1, 101]); + + const mask_sum = attention_mask.data.reduce((a, b) => a + b, 0n); + expect(Number(mask_sum)).toEqual(100); + + expect(input_features.mean().item()).toBeCloseTo(0.0, 3); + expect(input_features.data[0]).toBeCloseTo(1.518887043, 2); // [0,0,0] + expect(input_features.data[1]).toBeCloseTo(1.1131993532, 2); // [0,0,1] + expect(input_features.data[128]).toBeCloseTo(1.230540514, 2); // [0,1,0] + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "split_audio for long audio", + async () => { + const audio = await load_cached_audio("mlk"); + // mlk is ~13 seconds, below 35s threshold, should not split + const chunks = feature_extractor.split_audio(audio); + expect(chunks.length).toEqual(1); + expect(chunks[0].length).toEqual(audio.length); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "split_audio triggers for very long audio", + async () => { + // Create a fake long audio (>35s at 16kHz = 560000 samples) + const long_audio = new Float32Array(600000); + for (let i = 0; i < long_audio.length; ++i) { + long_audio[i] = Math.sin(i / 100) * 0.1; + } + const chunks = feature_extractor.split_audio(long_audio); + expect(chunks.length).toBeGreaterThan(1); + + // All chunks together should cover the full audio + const total = chunks.reduce((acc, c) => acc + c.length, 0); + expect(total).toEqual(long_audio.length); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/convnext/test_image_processing_convnext.js b/packages/transformers/tests/models/convnext/test_image_processing_convnext.js new file mode 100644 index 0000000..2ff92aa --- /dev/null +++ b/packages/transformers/tests/models/convnext/test_image_processing_convnext.js @@ -0,0 +1,32 @@ +import { AutoImageProcessor, ConvNextFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // ConvNextFeatureExtractor + describe("ConvNextFeatureExtractor", () => { + const model_id = "Xenova/resnet-50"; + + /** @type {ConvNextFeatureExtractor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "default", + async () => { + const image = await load_cached_image("tiger"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 224, 224]); + expect(pixel_values.mean().item()).toBeCloseTo(0.06262318789958954, 3); + + expect(original_sizes).toEqual([[408, 612]]); + expect(reshaped_input_sizes).toEqual([[224, 224]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/dac/test_modeling_dac.js b/packages/transformers/tests/models/dac/test_modeling_dac.js new file mode 100644 index 0000000..2ec2e50 --- /dev/null +++ b/packages/transformers/tests/models/dac/test_modeling_dac.js @@ -0,0 +1,88 @@ +import { DacFeatureExtractor, DacModel, DacEncoderModel, DacDecoderModel } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("DacModel", () => { + const model_id = "hf-internal-testing/tiny-random-DacModel"; + + /** @type {DacModel} */ + let model; + /** @type {DacFeatureExtractor} */ + let feature_extractor; + let inputs; + beforeAll(async () => { + model = await DacModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + feature_extractor = await DacFeatureExtractor.from_pretrained(model_id); + inputs = await feature_extractor(new Float32Array(12000)); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + const { audio_values } = await model(inputs); + expect(audio_values.dims).toEqual([1, 1, 11832]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "encode & decode", + async () => { + const encoder_outputs = await model.encode(inputs); + expect(encoder_outputs.audio_codes.dims).toEqual([1, model.config.n_codebooks, 37]); + + const { audio_values } = await model.decode(encoder_outputs); + expect(audio_values.dims).toEqual([1, 1, 11832]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("DacEncoderModel and DacDecoderModel", () => { + const model_id = "hf-internal-testing/tiny-random-DacModel"; + + /** @type {DacEncoderModel} */ + let encoder_model; + /** @type {DacDecoderModel} */ + let decoder_model; + /** @type {DacFeatureExtractor} */ + let feature_extractor; + let inputs; + let encoder_outputs; + + beforeAll(async () => { + encoder_model = await DacEncoderModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + decoder_model = await DacDecoderModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + feature_extractor = await DacFeatureExtractor.from_pretrained(model_id); + inputs = await feature_extractor(new Float32Array(12000)); + }, MAX_MODEL_LOAD_TIME); + + it( + "encode", + async () => { + encoder_outputs = await encoder_model(inputs); + expect(encoder_outputs.audio_codes.dims).toEqual([1, encoder_model.config.n_codebooks, 37]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "decode", + async () => { + const { audio_values } = await decoder_model(encoder_outputs); + expect(audio_values.dims).toEqual([1, 1, 11832]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await encoder_model?.dispose(); + await decoder_model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/deberta_v2/test_tokenization_deberta_v2.js b/packages/transformers/tests/models/deberta_v2/test_tokenization_deberta_v2.js new file mode 100644 index 0000000..c0c72be --- /dev/null +++ b/packages/transformers/tests/models/deberta_v2/test_tokenization_deberta_v2.js @@ -0,0 +1,304 @@ +import { DebertaV2Tokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, BERT_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = DebertaV2Tokenizer; +export const TEST_CONFIG = { + "Xenova/nli-deberta-v3-small": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [1, 577, 281, 274, 653, 302, 2], + decoded: "[CLS] How are you doing?[SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [1, 367, 403, 280, 415, 619, 291, 2], + decoded: "[CLS] You should've done this[SEP]", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u25810", "123456", "789", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [1, 767, 120304, 51535, 767, 376, 392, 404, 453, 456, 525, 574, 578, 712, 466, 803, 4985, 2], + decoded: "[CLS] 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000[SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u25812016", "."], + ids: [1, 279, 483, 284, 3679, 267, 892, 260, 2], + decoded: "[CLS] The company was founded in 2016.[SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581'", "ll", "\u2581!", "!", "to", "?", "'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [1, 336, 382, 436, 1084, 300, 725, 302, 280, 407, 280, 280, 407, 265, 261, 295, 280, 297, 260, 2], + decoded: "[CLS] A 'll!!to?'d''d of, can't.[SEP]", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581def", "\u2581main", "(", ")", ":", "\u2581pass"], + ids: [1, 23097, 872, 555, 285, 294, 1633, 2], + decoded: "[CLS] def main(): pass[SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581obj", ".", "to", "String", "(", ")", ";", "\u2581to", "String", "(", ")", ";"], + ids: [1, 678, 266, 1842, 68215, 260, 725, 29867, 555, 285, 346, 264, 29867, 555, 285, 346, 2], + decoded: "[CLS] let a = obj.toString(); toString();[SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "."], + ids: [1, 329, 269, 266, 1010, 260, 2], + decoded: "[CLS] This is a test.[SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "want", "\u00e9", "d", ",", "running"], + ids: [1, 4647, 27364, 5858, 407, 261, 15243, 2], + decoded: "[CLS] UNwant\u00e9d,running[SEP]", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u25811", "\u0000", "2", "\u25813"], + ids: [1, 376, 3, 445, 404, 2], + decoded: "[CLS] 1[UNK]2 3[SEP]", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581Hello", "\u2581World"], + ids: [1, 5365, 964, 2], + decoded: "[CLS] Hello World[SEP]", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [1, 12018, 447, 2], + decoded: "[CLS] hello world[SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [1, 507, 41065, 101952, 9301, 98186, 3, 30060, 2], + decoded: "[CLS] \u751f\u6d3b\u7684\u771f[UNK]\u662f[SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581leading", "\u2581space"], + ids: [1, 1249, 754, 2], + decoded: "[CLS] leading space[SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trailing", "\u2581space"], + ids: [1, 18347, 754, 2], + decoded: "[CLS] trailing space[SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581Hi", "\u2581Hello"], + ids: [1, 2684, 5365, 2], + decoded: "[CLS] Hi Hello[SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$", "1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac4", "\u2581\u00a35", "\u2581\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581\u20b1", "9", "\u2581test"], + ids: [1, 1010, 419, 435, 909, 445, 953, 508, 56238, 14636, 56478, 765, 507, 3, 819, 34880, 804, 121499, 1088, 1010, 2], + decoded: "[CLS] test $1 R2 #3 \u20ac4 \u00a35 \u00a56 [UNK]7 \u20b98 \u20b19 test[SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1", ".", "00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [1, 273, 2031, 299, 6038, 270, 419, 435, 260, 962, 288, 262, 1106, 260, 2], + decoded: "[CLS] I bought an apple for $1.00 at the store.[SEP]", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", ".", ".", "."], + ids: [1, 274, 260, 260, 260, 2], + decoded: "[CLS] you...[SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", ".", ".", "."], + ids: [1, 274, 260, 260, 260, 2], + decoded: "[CLS] you...[SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", ".", ".", ".", "\u2581you", ".", ".", "."], + ids: [1, 274, 260, 260, 260, 274, 260, 260, 260, 2], + decoded: "[CLS] you... you...[SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], + ids: [1, 4926, 507, 96622, 2363, 507, 96622, 571, 2], + decoded: "[CLS] weird \uff5e edge \uff5e case[SEP]", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581."], + ids: [1, 329, 269, 266, 1010, 323, 2], + decoded: "[CLS] This is a test.[SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581\ud83d\ude0a", "\u2581\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [1, 97504, 507, 117545, 507, 123057, 507, 96353, 507, 123058, 507, 123169, 507, 121772, 109976, 115475, 507, 122874, 507, 124017, 507, 123983, 507, 123571, 507, 122632, 49509, 25377, 507, 123614, 507, 124105, 507, 124077, 507, 123384, 507, 124382, 507, 123340, 507, 123492, 507, 3, 507, 123306, 507, 110119, 507, 122633, 507, 123659, 507, 123765, 507, 125799, 507, 124322, 507, 122878, 507, 125843, 507, 124011, 507, 125021, 88523, 507, 124698, 507, 125612, 507, 123887, 507, 123979, 2], + decoded: "[CLS] \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c [UNK] \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0[SEP]", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4", "\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], + ids: [1, 507, 110119, 507, 123983, 507, 127294, 25377, 507, 3, 108391, 507, 3, 507, 117868, 25377, 507, 3, 108391, 507, 117868, 507, 125199, 108391, 507, 3, 507, 3, 507, 3, 507, 3, 507, 124709, 49509, 507, 124327, 507, 125199, 507, 124709, 507, 124709, 507, 126640, 507, 126853, 507, 3, 108391, 507, 3, 507, 3, 108391, 507, 126132, 3, 507, 125199, 108391, 49509, 25377, 507, 124327, 507, 125199, 118155, 2], + decoded: "[CLS] \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f [UNK]\ud83c\udffb [UNK] \u2642\ufe0f [UNK]\ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb [UNK] [UNK] [UNK] [UNK] \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83d\udc67 \ud83d\udc66 [UNK]\ud83c\udffb [UNK] [UNK]\ud83c\udffb \ud83c\udff4[UNK] \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc[SEP]", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["\u2581a", "h", "\u535a", "\u63a8", "zz"], + ids: [1, 266, 1537, 122598, 111743, 23260, 2], + decoded: "[CLS] ah\u535a\u63a8zz[SEP]", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["\u2581H\u00e9", "llo"], + ids: [1, 93519, 25341, 2], + decoded: "[CLS] H\u00e9llo[SEP]", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u2581He", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?"], + ids: [1, 383, 17145, 795, 300, 5608, 1396, 14469, 2628, 302, 2], + decoded: "[CLS] HeLLo!how Are yoU?[SEP]", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u2581H\u00e4", "LL", "o", "!", "how", "\u2581Are", "\u2581yo", "U", "?"], + ids: [1, 62693, 17145, 795, 300, 5608, 1396, 14469, 2628, 302, 2], + decoded: "[CLS] H\u00e4LLo!how Are yoU?[SEP]", + }, + }, + "Xenova/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581do", "ing", "?"], + ids: [1, 5101, 419, 522, 343, 348, 292, 2], + decoded: "[CLS] How are you doing?[SEP]", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u2581", "0123456789", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [1, 260, 170160, 498, 334, 357, 382, 420, 431, 571, 618, 631, 775, 476, 967, 3884, 2], + decoded: "[CLS] 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000[SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u25812016."], + ids: [1, 487, 5836, 640, 5898, 346, 282, 13792, 2], + decoded: "[CLS] The company was founded in 2016.[SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581", "'", "ll", "\u2581", "!!", "to", "?", "'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [1, 299, 260, 278, 1579, 260, 1524, 477, 292, 278, 286, 4461, 286, 305, 262, 739, 278, 271, 261, 2], + decoded: "[CLS] A 'll!!to?'d''d of, can't.[SEP]", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581de", "f", "\u2581main", "():", "\u2581pass"], + ids: [1, 270, 368, 4398, 78612, 4748, 2], + decoded: "[CLS] def main(): pass[SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581", "a", "\u2581", "=", "\u2581obj", ".", "toString", "();", "\u2581", "toString", "();"], + ids: [1, 3257, 260, 263, 260, 350, 50670, 261, 64577, 1994, 260, 64577, 1994, 2], + decoded: "[CLS] let a = obj.toString(); toString();[SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581This", "\u2581is", "\u2581", "a", "\u2581test", "."], + ids: [1, 1495, 340, 260, 263, 2979, 261, 2], + decoded: "[CLS] This is a test.[SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "wan", "t\u00e9", "d", ",", "running"], + ids: [1, 10970, 3016, 3986, 286, 262, 170565, 2], + decoded: "[CLS] UNwant\u00e9d,running[SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f\u6d3b\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [1, 260, 197263, 7275, 241962, 1544, 2], + decoded: "[CLS] \u751f\u6d3b\u7684\u771f\u8c1b\u662f[SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581", "leading", "\u2581space"], + ids: [1, 260, 22120, 11496, 2], + decoded: "[CLS] leading space[SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trail", "ing", "\u2581space"], + ids: [1, 66699, 348, 11496, 2], + decoded: "[CLS] trailing space[SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac4", "\u2581\u00a35", "\u2581\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [1, 2979, 21793, 532, 339, 19403, 157186, 156260, 33481, 452, 260, 242687, 488, 39568, 450, 260, 211232, 496, 2979, 2], + decoded: "[CLS] test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test[SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581b", "ought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1.00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [1, 337, 331, 22280, 462, 44791, 333, 1161, 42645, 345, 288, 5318, 261, 2], + decoded: "[CLS] I bought an apple for $1.00 at the store.[SEP]", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "..."], + ids: [1, 522, 303, 2], + decoded: "[CLS] you...[SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "..."], + ids: [1, 522, 303, 2], + decoded: "[CLS] you...[SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "..."], + ids: [1, 522, 303, 522, 303, 2], + decoded: "[CLS] you... you...[SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581w", "eird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], + ids: [1, 415, 116640, 260, 2790, 53876, 260, 2790, 4073, 2], + decoded: "[CLS] weird \uff5e edge \uff5e case[SEP]", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581", "a", "\u2581test", "\u2581", "."], + ids: [1, 1495, 340, 260, 263, 2979, 260, 261, 2], + decoded: "[CLS] This is a test.[SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [1, 260, 116844, 260, 72330, 260, 160951, 260, 78796, 260, 180546, 260, 212774, 260, 102930, 260, 71509, 260, 96089, 260, 137652, 260, 194608, 260, 182033, 260, 164467, 260, 149267, 56787, 4668, 260, 210251, 260, 195202, 260, 178523, 260, 167604, 260, 236081, 260, 157800, 260, 162843, 260, 242580, 260, 174590, 260, 65271, 113700, 260, 239652, 260, 237474, 260, 240937, 260, 239131, 260, 216701, 260, 242618, 260, 133395, 260, 240645, 82147, 260, 49599, 260, 239888, 260, 152102, 260, 239168, 2], + decoded: "[CLS] \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0[SEP]", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4", "\udb40\udc67", "\udb40\udc62", "\udb40\udc65", "\udb40\udc6e", "\udb40\udc67", "\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], + ids: [1, 260, 65271, 260, 182033, 260, 16307, 4668, 260, 244774, 75846, 260, 247133, 260, 50622, 4668, 260, 3, 75846, 260, 50622, 260, 239432, 75846, 260, 243052, 260, 244250, 260, 243394, 260, 244250, 260, 239098, 56787, 260, 223802, 260, 239432, 260, 239098, 260, 239098, 260, 241727, 260, 242446, 260, 244250, 75846, 260, 243394, 260, 244250, 75846, 260, 244177, 245994, 247023, 248837, 248531, 245994, 245953, 260, 239432, 75846, 56787, 4668, 260, 223802, 260, 239432, 159667, 2], + decoded: "[CLS] \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75 \u2642\ufe0f [UNK]\ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb \ud83c\udf3e \ud83e\uddd1 \ud83e\udd1d \ud83e\uddd1 \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83d\udc67 \ud83d\udc66 \ud83e\uddd1\ud83c\udffb \ud83e\udd1d \ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc[SEP]", + }, + }, +}; diff --git a/packages/transformers/tests/models/deepseek_v3/test_modeling_deepseek_v3.js b/packages/transformers/tests/models/deepseek_v3/test_modeling_deepseek_v3.js new file mode 100644 index 0000000..287d5b5 --- /dev/null +++ b/packages/transformers/tests/models/deepseek_v3/test_modeling_deepseek_v3.js @@ -0,0 +1,51 @@ +import { PreTrainedTokenizer, DeepseekV3ForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("DeepseekV3ForCausalLM", () => { + const model_id = "onnx-internal-testing/tiny-random-DeepseekV3ForCausalLM"; + /** @type {DeepseekV3ForCausalLM} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await DeepseekV3ForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[33310n, 26249n, 26249n, 26249n, 26249n, 26249n, 26249n, 26249n, 26249n, 26249n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [1n, 33310n, 26249n, 26249n, 26249n, 26249n, 26249n, 26249n, 26249n, 26249n], + [33310n, 2058n, 26249n, 26249n, 26249n, 26249n, 26249n, 26249n, 26249n, 26249n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/deepseek_v4/test_modeling_deepseek_v4.js b/packages/transformers/tests/models/deepseek_v4/test_modeling_deepseek_v4.js new file mode 100644 index 0000000..6aa550c --- /dev/null +++ b/packages/transformers/tests/models/deepseek_v4/test_modeling_deepseek_v4.js @@ -0,0 +1,75 @@ +import { PreTrainedTokenizer, DeepseekV4ForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("DeepseekV4ForCausalLM", () => { + const model_id = "onnx-internal-testing/tiny-random-DeepseekV4ForCausalLM"; + /** @type {DeepseekV4ForCausalLM} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await DeepseekV4ForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + do_sample: false, + }); + expect(outputs.tolist()).toEqual([[33310n, 83919n, 100040n, 101230n, 55545n, 23803n, 76294n, 119693n, 74059n, 1036n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + do_sample: false, + }); + expect(outputs.tolist()).toEqual([ + [1n, 33310n, 83919n, 100040n, 101230n, 55545n, 23803n, 76294n, 119693n, 74059n], + [33310n, 2058n, 75023n, 89073n, 120077n, 49327n, 57703n, 107564n, 3563n, 126139n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "generate w/ past_compressor and past_indexer cache", + async () => { + const inputs = tokenizer("hello"); + const { past_key_values } = await model.generate({ + ...inputs, + max_new_tokens: 1, + do_sample: false, + return_dict_in_generate: true, + }); + + expect(past_key_values.get_seq_length()).toBeGreaterThan(0); + const cache_names = ["past_key_values.0.key", "past_key_values.0.value", "past_key_values.3.key", "past_key_values.3.value", "past_compressor.2.kv", "past_compressor.2.gate", "past_indexer.2.kv", "past_indexer.2.gate", "past_compressor.3.kv", "past_compressor.3.gate"]; + for (const name of cache_names) { + expect(past_key_values[name]).toBeDefined(); + } + + await past_key_values.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/deit/test_image_processing_deit.js b/packages/transformers/tests/models/deit/test_image_processing_deit.js new file mode 100644 index 0000000..3f1c591 --- /dev/null +++ b/packages/transformers/tests/models/deit/test_image_processing_deit.js @@ -0,0 +1,31 @@ +import { AutoImageProcessor, DeiTFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("DeiTFeatureExtractor", () => { + const model_id = "Xenova/deit-tiny-distilled-patch16-224"; + + /** @type {DeiTFeatureExtractor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "default", + async () => { + const image = await load_cached_image("tiger"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 224, 224]); + expect(pixel_values.mean().item()).toBeCloseTo(-0.2760336682859463, 6); + + expect(original_sizes).toEqual([[408, 612]]); + expect(reshaped_input_sizes).toEqual([[224, 224]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/detr/test_image_processing_detr.js b/packages/transformers/tests/models/detr/test_image_processing_detr.js new file mode 100644 index 0000000..23c58d5 --- /dev/null +++ b/packages/transformers/tests/models/detr/test_image_processing_detr.js @@ -0,0 +1,34 @@ +import { AutoImageProcessor, DetrFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("DetrFeatureExtractor", () => { + const model_id = "Xenova/detr-resnet-50"; + + /** @type {DetrFeatureExtractor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "default", + async () => { + const image = await load_cached_image("tiger"); + const { pixel_values, original_sizes, reshaped_input_sizes, pixel_mask } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 888, 1333]); + expect(pixel_values.mean().item()).toBeCloseTo(-0.27840224131001773, 6); + + expect(original_sizes).toEqual([[408, 612]]); + expect(reshaped_input_sizes).toEqual([[888, 1333]]); + + expect(pixel_mask.dims).toEqual([1, 64, 64]); + expect(pixel_mask.to("float32").mean().item()).toBeCloseTo(1.0, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/distilbert/test_tokenization_distilbert.js b/packages/transformers/tests/models/distilbert/test_tokenization_distilbert.js new file mode 100644 index 0000000..ba22bc4 --- /dev/null +++ b/packages/transformers/tests/models/distilbert/test_tokenization_distilbert.js @@ -0,0 +1,315 @@ +import { DistilBertTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, BERT_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = DistilBertTokenizer; +export const TEST_CONFIG = { + "Xenova/distilbert-base-cased-distilled-squad": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "are", "you", "doing", "?"], + ids: [101, 1731, 1132, 1128, 1833, 136, 102], + decoded: "[CLS] How are you doing? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "should", "'", "ve", "done", "this"], + ids: [101, 1192, 1431, 112, 1396, 1694, 1142, 102], + decoded: "[CLS] You should've done this [SEP]", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["01", "##23", "##45", "##6", "##7", "##8", "##9", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], + ids: [101, 5187, 22737, 21336, 1545, 1559, 1604, 1580, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 1275, 1620, 6087, 102], + decoded: "[CLS] 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000 [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "company", "was", "founded", "in", "2016", "."], + ids: [101, 1109, 1419, 1108, 1771, 1107, 1446, 119, 102], + decoded: "[CLS] The company was founded in 2016. [SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [101, 138, 112, 1325, 106, 106, 1106, 136, 112, 173, 112, 112, 173, 1104, 117, 1169, 112, 189, 119, 102], + decoded: "[CLS] A'll!! to?'d'' d of, can't. [SEP]", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "main", "(", ")", ":", "pass"], + ids: [101, 19353, 1514, 113, 114, 131, 2789, 102], + decoded: "[CLS] def main ( ) : pass [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "o", "##b", "##j", ".", "to", "##S", "##tring", "(", ")", ";", "to", "##S", "##tring", "(", ")", ";"], + ids: [101, 1519, 170, 134, 184, 1830, 3361, 119, 1106, 1708, 28108, 113, 114, 132, 1106, 1708, 28108, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "is", "a", "test", "."], + ids: [101, 1188, 1110, 170, 2774, 119, 102], + decoded: "[CLS] This is a test. [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "##wan", "##t\u00e9", "##d", ",", "running"], + ids: [101, 7414, 5491, 14608, 1181, 117, 1919, 102], + decoded: "[CLS] UNwant\u00e9d, running [SEP]", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["123"], + ids: [101, 13414, 102], + decoded: "[CLS] 123 [SEP]", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "World"], + ids: [101, 8667, 1291, 102], + decoded: "[CLS] Hello World [SEP]", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "world"], + ids: [101, 19082, 1362, 102], + decoded: "[CLS] hello world [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "[UNK]", "[UNK]", "\u771f", "[UNK]", "[UNK]"], + ids: [101, 1056, 100, 100, 1061, 100, 100, 102], + decoded: "[CLS] \u751f [UNK] [UNK] \u771f [UNK] [UNK] [SEP]", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["leading", "space"], + ids: [101, 2020, 2000, 102], + decoded: "[CLS] leading space [SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "space"], + ids: [101, 13161, 2000, 102], + decoded: "[CLS] trailing space [SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "Hello"], + ids: [101, 8790, 8667, 102], + decoded: "[CLS] Hi Hello [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "R", "##2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [101, 2774, 109, 122, 155, 1477, 108, 124, 836, 1527, 202, 1571, 203, 1545, 100, 838, 1604, 837, 1580, 2774, 102], + decoded: "[CLS] test $ 1 R2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 146, 3306, 1126, 12075, 1111, 109, 122, 119, 3135, 1120, 1103, 2984, 119, 102], + decoded: "[CLS] I bought an apple for $ 1. 00 at the store. [SEP]", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u2026"], + ids: [101, 1128, 795, 102], + decoded: "[CLS] you \u2026 [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u2026"], + ids: [101, 1128, 795, 102], + decoded: "[CLS] you \u2026 [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u2026", "you", "\u2026"], + ids: [101, 1128, 795, 1128, 795, 102], + decoded: "[CLS] you \u2026 you \u2026 [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "[UNK]", "edge", "[UNK]", "case"], + ids: [101, 6994, 100, 2652, 100, 1692, 102], + decoded: "[CLS] weird [UNK] edge [UNK] case [SEP]", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "."], + ids: [101, 100, 100, 100, 100, 100, 119, 102], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK]. [SEP]", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [101, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [101, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 102], + decoded: "[CLS] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [SEP]", + }, + }, + "Xenova/distilbert-base-uncased-finetuned-sst-2-english": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["how", "are", "you", "doing", "?"], + ids: [101, 2129, 2024, 2017, 2725, 1029, 102], + decoded: "[CLS] how are you doing? [SEP]", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["you", "should", "'", "ve", "done", "this"], + ids: [101, 2017, 2323, 1005, 2310, 2589, 2023, 102], + decoded: "[CLS] you should've done this [SEP]", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["the", "company", "was", "founded", "in", "2016", "."], + ids: [101, 1996, 2194, 2001, 2631, 1999, 2355, 1012, 102], + decoded: "[CLS] the company was founded in 2016. [SEP]", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [101, 1037, 1005, 2222, 999, 999, 2000, 1029, 1005, 1040, 1005, 1005, 1040, 1997, 1010, 2064, 1005, 1056, 1012, 102], + decoded: "[CLS] a'll!! to?'d'' d of, can't. [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##st", "##ring", "(", ")", ";", "to", "##st", "##ring", "(", ")", ";"], + ids: [101, 2292, 1037, 1027, 27885, 3501, 1012, 2000, 3367, 4892, 1006, 1007, 1025, 2000, 3367, 4892, 1006, 1007, 1025, 102], + decoded: "[CLS] let a = obj. tostring ( ) ; tostring ( ) ; [SEP]", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["this", "is", "a", "test", "."], + ids: [101, 2023, 2003, 1037, 3231, 1012, 102], + decoded: "[CLS] this is a test. [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["unwanted", ",", "running"], + ids: [101, 18162, 1010, 2770, 102], + decoded: "[CLS] unwanted, running [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "[UNK]", "\u7684", "\u771f", "[UNK]", "[UNK]"], + ids: [101, 1910, 100, 1916, 1921, 100, 100, 102], + decoded: "[CLS] \u751f [UNK] \u7684 \u771f [UNK] [UNK] [SEP]", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["hi", "hello"], + ids: [101, 7632, 7592, 102], + decoded: "[CLS] hi hello [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r", "##2", "#", "3", "\u20ac", "##4", "\u00a35", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [101, 3231, 1002, 1015, 1054, 2475, 1001, 1017, 1574, 2549, 27813, 1071, 2575, 100, 1576, 2620, 1575, 2683, 3231, 102], + decoded: "[CLS] test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["i", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 1045, 4149, 2019, 6207, 2005, 1002, 1015, 1012, 4002, 2012, 1996, 3573, 1012, 102], + decoded: "[CLS] i bought an apple for $ 1. 00 at the store. [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "\uff5e", "edge", "\uff5e", "case"], + ids: [101, 6881, 1995, 3341, 1995, 2553, 102], + decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", + }, + }, + "Xenova/distiluse-base-multilingual-cased-v2": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["012", "##34", "##5", "##6", "##7", "##8", "##9", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], + ids: [101, 69878, 78301, 11166, 11211, 11305, 11396, 11373, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 10150, 10407, 12186, 102], + decoded: "[CLS] 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000 [SEP]", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##S", "##trin", "##g", "(", ")", ";", "to", "##S", "##trin", "##g", "(", ")", ";"], + ids: [101, 13595, 169, 134, 17339, 10418, 119, 10114, 10731, 109163, 10240, 113, 114, 132, 10114, 10731, 109163, 10240, 113, 114, 132, 102], + decoded: "[CLS] let a = obj. toString ( ) ; toString ( ) ; [SEP]", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "##want", "##\u00e9d", ",", "running"], + ids: [101, 26578, 104216, 84193, 117, 18020, 102], + decoded: "[CLS] UNwant\u00e9d, running [SEP]", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hell", "##o", "world"], + ids: [101, 61694, 10133, 11356, 102], + decoded: "[CLS] hello world [SEP]", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [101, 5600, 4978, 5718, 5769, 7378, 4380, 102], + decoded: "[CLS] \u751f \u6d3b \u7684 \u771f \u8c1b \u662f [SEP]", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trail", "##ing", "space"], + ids: [101, 56559, 10230, 16199, 102], + decoded: "[CLS] trailing space [SEP]", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "R2", "#", "3", "\u20ac", "##4", "\u00a3", "##5", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "[UNK]", "test"], + ids: [101, 15839, 109, 122, 94000, 108, 124, 1775, 11011, 201, 11166, 202, 11211, 100, 1776, 11396, 100, 15839, 102], + decoded: "[CLS] test $ 1 R2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 [UNK] test [SEP]", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "bought", "an", "app", "##le", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [101, 146, 28870, 10151, 72894, 10284, 10142, 109, 122, 119, 11025, 10160, 10105, 13708, 119, 102], + decoded: "[CLS] I bought an apple for $ 1. 00 at the store. [SEP]", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "[UNK]"], + ids: [101, 13028, 100, 102], + decoded: "[CLS] you [UNK] [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "[UNK]"], + ids: [101, 13028, 100, 102], + decoded: "[CLS] you [UNK] [SEP]", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "[UNK]", "you", "[UNK]"], + ids: [101, 13028, 100, 13028, 100, 102], + decoded: "[CLS] you [UNK] you [UNK] [SEP]", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["wei", "##rd", "\uff5e", "edge", "\uff5e", "case"], + ids: [101, 86981, 12023, 10096, 30599, 10096, 13474, 102], + decoded: "[CLS] weird \uff5e edge \uff5e case [SEP]", + }, + }, + // `model.type` field missing in tokenizer.json + "distilbert/distilbert-base-multilingual-cased": { + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "\u535a", "\u63a8", "z", "##z"], + ids: [101, 69863, 2684, 4163, 194, 10305, 102], + decoded: "[CLS] ah \u535a \u63a8 zz [SEP]", + }, + }, +}; diff --git a/packages/transformers/tests/models/donut/test_image_processing_donut.js b/packages/transformers/tests/models/donut/test_image_processing_donut.js new file mode 100644 index 0000000..682b358 --- /dev/null +++ b/packages/transformers/tests/models/donut/test_image_processing_donut.js @@ -0,0 +1,34 @@ +import { AutoImageProcessor, DonutFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // DonutFeatureExtractor + // - tests thumbnail resizing (do_thumbnail=true, size=[960, 1280]) + // - tests padding after normalization (image_mean=image_std=0.5) + describe("DonutFeatureExtractor", () => { + const model_id = "Xenova/donut-base-finetuned-cord-v2"; + + /** @type {DonutFeatureExtractor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "default", + async () => { + const image = await load_cached_image("receipt"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 1280, 960]); + expect(pixel_values.mean().item()).toBeCloseTo(0.1229388610053704, 6); + + expect(original_sizes).toEqual([[864, 576]]); + expect(reshaped_input_sizes).toEqual([[1280, 853]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/dpt/test_image_processing_dpt.js b/packages/transformers/tests/models/dpt/test_image_processing_dpt.js new file mode 100644 index 0000000..cbe54de --- /dev/null +++ b/packages/transformers/tests/models/dpt/test_image_processing_dpt.js @@ -0,0 +1,77 @@ +import { AutoImageProcessor, DPTFeatureExtractor, DPTImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // DPTFeatureExtractor + describe("DPTFeatureExtractor", () => { + const model_id = "Xenova/dpt-hybrid-midas"; + + /** @type {DPTFeatureExtractor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "grayscale images", + async () => { + const image = await load_cached_image("cats"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 384, 384]); + expect(pixel_values.mean().item()).toBeCloseTo(0.0372855559389454, 6); + + expect(original_sizes).toEqual([[480, 640]]); + expect(reshaped_input_sizes).toEqual([[384, 384]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + // DPTImageProcessor + // - tests ensure_multiple_of + // - tests keep_aspect_ratio + // - tests bankers rounding + describe("DPTImageProcessor", () => { + const model_id = "Xenova/depth-anything-small-hf"; + + /** @type {DPTImageProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "ensure_multiple_of w/ normal rounding", + async () => { + const image = await load_cached_image("cats"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 518, 686]); + expect(pixel_values.mean().item()).toBeCloseTo(0.30337387323379517, 3); + + expect(original_sizes).toEqual([[480, 640]]); + expect(reshaped_input_sizes).toEqual([[518, 686]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "ensure_multiple_of w/ bankers rounding", + async () => { + const image = await load_cached_image("checkerboard_64x32"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + // NOTE: without bankers rounding, this would be [1, 3, 266, 518] + expect(pixel_values.dims).toEqual([1, 3, 252, 518]); + expect(pixel_values.mean().item()).toBeCloseTo(0.2267402559518814, 1); + + expect(original_sizes).toEqual([[32, 64]]); + expect(reshaped_input_sizes).toEqual([[252, 518]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/efficientnet/test_image_processing_efficientnet.js b/packages/transformers/tests/models/efficientnet/test_image_processing_efficientnet.js new file mode 100644 index 0000000..fefa7d9 --- /dev/null +++ b/packages/transformers/tests/models/efficientnet/test_image_processing_efficientnet.js @@ -0,0 +1,46 @@ +import { EfficientNetImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // EfficientNetImageProcessor + // - tests include_top + describe("EfficientNetImageProcessor", () => { + /** @type {EfficientNetImageProcessor} */ + const processor = new EfficientNetImageProcessor({ + crop_size: { + height: 289, + width: 289, + }, + do_center_crop: false, + do_normalize: true, + do_rescale: true, + do_resize: true, + image_mean: [0.485, 0.456, 0.406], + image_processor_type: "EfficientNetImageProcessor", + image_std: [0.47853944, 0.4732864, 0.47434163], + include_top: true, + resample: 0, + rescale_factor: 0.00392156862745098, + rescale_offset: false, + size: { + height: 224, + width: 224, + }, + }); + + it( + "default", + async () => { + const image = await load_cached_image("cats"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + expect(pixel_values.dims).toEqual([1, 3, 224, 224]); + expect(pixel_values.mean().item()).toBeCloseTo(0.3015307230282871, 6); + expect(original_sizes).toEqual([[480, 640]]); + expect(reshaped_input_sizes).toEqual([[224, 224]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/esm/test_tokenization_esm.js b/packages/transformers/tests/models/esm/test_tokenization_esm.js new file mode 100644 index 0000000..e44fee3 --- /dev/null +++ b/packages/transformers/tests/models/esm/test_tokenization_esm.js @@ -0,0 +1,322 @@ +import { EsmTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, ESM_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = EsmTokenizer; +export const TEST_CONFIG = { + "Xenova/nucleotide-transformer-500m-human-ref": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + // "tokens": ["How", "are", "you", "doing?"], + ids: [3, 0, 0, 0, 0], + decoded: " ", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + // "tokens": ["You", "should've", "done", "this"], + ids: [3, 0, 0, 0, 0], + decoded: " ", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + // "tokens": ["0123456789", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], + ids: [3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + decoded: " ", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + // "tokens": ["T", "he", "company", "was", "founded", "in", "2016."], + ids: [3, 4101, 0, 0, 0, 0, 0, 0], + decoded: " T ", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + // "tokens": ["A", "'ll", "!!to?'d''d", "of,", "can't."], + ids: [3, 4100, 0, 0, 0, 0], + decoded: " A ", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + // "tokens": ["def", "main():", "pass"], + ids: [3, 0, 0, 0], + decoded: " ", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + // "tokens": ["let", "a", "=", "obj.toString();", "toString();"], + ids: [3, 0, 0, 0, 0, 0], + decoded: " ", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + // "tokens": ["T", "his", "is", "a", "test."], + ids: [3, 4101, 0, 0, 0, 0], + decoded: " T ", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + // "tokens": ["U", "N", "want\u00e9d,running"], + ids: [3, 0, 4104, 0], + decoded: " N ", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + // "tokens": ["1\u00002\ufffd3"], + ids: [3, 0], + decoded: " ", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + // "tokens": ["Hello", "World"], + ids: [3, 0, 0], + decoded: " ", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + // "tokens": ["hello", "world"], + ids: [3, 0, 0], + decoded: " ", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + // "tokens": ["\u751f\u6d3b\u7684\u771f\u8c1b\u662f"], + ids: [3, 0], + decoded: " ", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + // "tokens": ["leading", "space"], + ids: [3, 0, 0], + decoded: " ", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + // "tokens": ["trailing", "space"], + ids: [3, 0, 0], + decoded: " ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + // "tokens": ["Hi", "Hello"], + ids: [3, 0, 0], + decoded: " ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + // "tokens": ["test", "$1", "R2", "#3", "\u20ac4", "\u00a35", "\u00a56", "\u20a37", "\u20b98", "\u20b19", "test"], + ids: [3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + decoded: " ", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + // "tokens": ["I", "bought", "an", "apple", "for", "$1.00", "at", "the", "store."], + ids: [3, 0, 0, 0, 0, 0, 0, 0, 0, 0], + decoded: " ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + // "tokens": ["you\u2026"], + ids: [3, 0], + decoded: " ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + // "tokens": ["you\u2026"], + ids: [3, 0], + decoded: " ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + // "tokens": ["you\u2026", "you\u2026"], + ids: [3, 0, 0], + decoded: " ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + // "tokens": ["weird", "\uff5e", "edge", "\uff5e", "case"], + ids: [3, 0, 0, 0, 0, 0], + decoded: " ", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + // "tokens": ["\u2581", "T", "his", "\u2581is", "\u2581a", "\u2581test", "\u2581."], + ids: [3, 0, 4101, 0, 0, 0, 0, 0], + decoded: " T ", + }, + SPECIAL_TOKENS: { + text: ESM_TEST_STRINGS.SPECIAL_TOKENS, + tokens: ["", "", "", "", "", ""], + ids: [3, 0, 1, 2, 3, 4105, 4106], + decoded: " ", + }, + PROTEIN_SEQUENCES_1: { + text: ESM_TEST_STRINGS.PROTEIN_SEQUENCES_1, + tokens: ["ATTCCG", "ATTCCG", "ATTCCG"], + ids: [3, 367, 367, 367], + decoded: " ATTCCG ATTCCG ATTCCG", + }, + PROTEIN_SEQUENCES_2: { + text: ESM_TEST_STRINGS.PROTEIN_SEQUENCES_2, + tokens: ["ATTTCT", "CTCTCT", "CTCTGA", "GATCGA", "TCGATC", "G", "A", "T"], + ids: [3, 349, 2461, 2464, 3184, 1738, 4103, 4100, 4101], + decoded: " ATTTCT CTCTCT CTCTGA GATCGA TCGATC G A T", + }, + }, + "Xenova/esm2_t12_35M_UR50D": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + // "tokens": ["H", "ow", "are", "you", "doing?"], + ids: [0, 21, 3, 3, 3, 3, 2], + decoded: " H ", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + // "tokens": ["Y", "ou", "should've", "done", "this"], + ids: [0, 19, 3, 3, 3, 3, 2], + decoded: " Y ", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + // "tokens": ["0123456789", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], + ids: [0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2], + decoded: " ", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + // "tokens": ["T", "he", "company", "was", "founded", "in", "2016", "."], + ids: [0, 11, 3, 3, 3, 3, 3, 3, 29, 2], + decoded: " T . ", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + // "tokens": ["A", "'ll", "!!to?'d''d", "of,", "can't", "."], + ids: [0, 5, 3, 3, 3, 3, 29, 2], + decoded: " A . ", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + // "tokens": ["def", "main():", "pass"], + ids: [0, 3, 3, 3, 2], + decoded: " ", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + // "tokens": ["let", "a", "=", "obj", ".", "to", "S", "tring();", "to", "S", "tring();"], + ids: [0, 3, 3, 3, 3, 29, 3, 8, 3, 3, 8, 3, 2], + decoded: " . S S ", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + // "tokens": ["T", "his", "is", "a", "test", "."], + ids: [0, 11, 3, 3, 3, 3, 29, 2], + decoded: " T . ", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + // "tokens": ["U", "N", "want\u00e9d,running"], + ids: [0, 26, 17, 3, 2], + decoded: " U N ", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + // "tokens": ["1\u00002\ufffd3"], + ids: [0, 3, 2], + decoded: " ", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + // "tokens": ["H", "ello", "W", "orld"], + ids: [0, 21, 3, 22, 3, 2], + decoded: " H W ", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + // "tokens": ["hello", "world"], + ids: [0, 3, 3, 2], + decoded: " ", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + // "tokens": ["\u751f\u6d3b\u7684\u771f\u8c1b\u662f"], + ids: [0, 3, 2], + decoded: " ", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + // "tokens": ["leading", "space"], + ids: [0, 3, 3, 2], + decoded: " ", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + // "tokens": ["trailing", "space"], + ids: [0, 3, 3, 2], + decoded: " ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + // "tokens": ["H", "i", "H", "ello"], + ids: [0, 21, 3, 21, 3, 2], + decoded: " H H ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + // "tokens": ["test", "$1", "R", "2", "#3", "\u20ac4", "\u00a35", "\u00a56", "\u20a37", "\u20b98", "\u20b19", "test"], + ids: [0, 3, 3, 10, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2], + decoded: " R ", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + // "tokens": ["I", "bought", "an", "apple", "for", "$1", ".", "00", "at", "the", "store", "."], + ids: [0, 12, 3, 3, 3, 3, 3, 29, 3, 3, 3, 3, 29, 2], + decoded: " I . . ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + // "tokens": ["you\u2026"], + ids: [0, 3, 2], + decoded: " ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + // "tokens": ["you\u2026"], + ids: [0, 3, 2], + decoded: " ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + // "tokens": ["you\u2026", "you\u2026"], + ids: [0, 3, 3, 2], + decoded: " ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + // "tokens": ["weird", "\uff5e", "edge", "\uff5e", "case"], + ids: [0, 3, 3, 3, 3, 3, 2], + decoded: " ", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + // "tokens": ["\u2581", "T", "his", "\u2581is", "\u2581a", "\u2581test", "\u2581", "."], + ids: [0, 3, 11, 3, 3, 3, 3, 3, 29, 2], + decoded: " T . ", + }, + SPECIAL_TOKENS: { + text: ESM_TEST_STRINGS.SPECIAL_TOKENS, + // "tokens": ["", "", "", "", "", ""], + ids: [0, 3, 1, 32, 0, 2, 3, 2], + decoded: " ", + }, + PROTEIN_SEQUENCES_1: { + text: ESM_TEST_STRINGS.PROTEIN_SEQUENCES_1, + tokens: ["A", "T", "T", "C", "C", "G", "A", "T", "T", "C", "C", "G", "A", "T", "T", "C", "C", "G"], + ids: [0, 5, 11, 11, 23, 23, 6, 5, 11, 11, 23, 23, 6, 5, 11, 11, 23, 23, 6, 2], + decoded: " A T T C C G A T T C C G A T T C C G ", + }, + PROTEIN_SEQUENCES_2: { + text: ESM_TEST_STRINGS.PROTEIN_SEQUENCES_2, + tokens: ["A", "T", "T", "T", "C", "T", "C", "T", "C", "T", "C", "T", "C", "T", "C", "T", "G", "A", "G", "A", "T", "C", "G", "A", "T", "C", "G", "A", "T", "C", "G", "A", "T"], + ids: [0, 5, 11, 11, 11, 23, 11, 23, 11, 23, 11, 23, 11, 23, 11, 23, 11, 6, 5, 6, 5, 11, 23, 6, 5, 11, 23, 6, 5, 11, 23, 6, 5, 11, 2], + decoded: " A T T T C T C T C T C T C T C T G A G A T C G A T C G A T C G A T ", + }, + }, +}; diff --git a/packages/transformers/tests/models/eurobert/test_modeling_eurobert.js b/packages/transformers/tests/models/eurobert/test_modeling_eurobert.js new file mode 100644 index 0000000..5c1ac1c --- /dev/null +++ b/packages/transformers/tests/models/eurobert/test_modeling_eurobert.js @@ -0,0 +1,44 @@ +import { EuroBertModel, EuroBertForMaskedLM, EuroBertForSequenceClassification, EuroBertForTokenClassification, AutoTokenizer } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("EuroBertModel", () => { + const model_id = "onnx-internal-testing/tiny-random-EuroBertModel"; + + /** @type {EuroBertModel} */ + let model; + /** @type {import("../../../src/transformers.js").PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await EuroBertModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await AutoTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const { last_hidden_state } = await model(inputs); + expect(last_hidden_state.dims).toEqual([1, 2, 32]); + expect(last_hidden_state.mean().item()).toBeCloseTo(-0.1891579031944275, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const { last_hidden_state } = await model(inputs); + expect(last_hidden_state.dims).toEqual([2, 3, 32]); + expect(last_hidden_state.mean().item()).toBeCloseTo(-0.13678674399852753, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/falcon/test_tokenization_falcon.js b/packages/transformers/tests/models/falcon/test_tokenization_falcon.js new file mode 100644 index 0000000..e06c6e2 --- /dev/null +++ b/packages/transformers/tests/models/falcon/test_tokenization_falcon.js @@ -0,0 +1,244 @@ +import { FalconTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, FALCON_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = FalconTokenizer; +export const TEST_CONFIG = { + "tiiuae/falcon-7b": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [1830, 362, 299, 1836, 42], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'", "ve", "\u0120done", "\u0120this"], + ids: [1357, 808, 18, 298, 1782, 414], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["012", "345", "678", "9", "\u0120", "0", "\u0120", "1", "\u0120", "2", "\u0120", "3", "\u0120", "4", "\u0120", "5", "\u0120", "6", "\u0120", "7", "\u0120", "8", "\u0120", "9", "\u0120", "10", "\u0120", "100", "\u0120", "100", "0"], + ids: [24445, 29094, 41583, 36, 204, 27, 204, 28, 204, 29, 204, 30, 204, 31, 204, 32, 204, 33, 204, 34, 204, 35, 204, 36, 204, 696, 204, 1425, 204, 1425, 27], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "201", "6", "."], + ids: [487, 1438, 398, 9923, 272, 204, 626, 33, 25], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'", "ll", "\u0120", "!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'", "t", "."], + ids: [44, 193, 18, 567, 204, 1409, 534, 12493, 79, 7544, 79, 275, 23, 418, 18, 95, 25], + decoded: "A\n'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"], + ids: [3071, 1316, 13160, 193, 192, 5412], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120", "=", "\u0120obj", ".", "toString", "();", "\u010a", "toString", "();"], + ids: [1025, 241, 204, 40, 13756, 25, 19409, 2032, 193, 19409, 2032], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [1182, 193, 193, 259, 193, 76, 193, 4780, 25], + decoded: "This\n\nis\na\ntest.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], + ids: [4000, 32108, 5706, 23, 27386], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf", "\u00bd", "3"], + ids: [28, 186, 29, 13112, 133, 30], + decoded: "1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [9856, 2889], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [30835, 1079], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [32725, 1105, 15498, 8061, 233, 2364], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [258, 3736, 2151], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120\u0120\u0120"], + ids: [9172, 4447, 2151, 466], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [5516, 204, 23090], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120", "$", "1", "\u0120R", "2", "\u0120", "#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2\u0124", "\u00a3", "7", "\u0120\u00e2\u0124", "\u00b9", "8", "\u0120\u00e2\u0124", "\u00b1", "9", "\u0120test"], + ids: [4780, 204, 15, 28, 382, 29, 204, 14, 30, 6471, 31, 5131, 32, 3068, 110, 33, 25631, 108, 34, 25631, 129, 35, 25631, 121, 36, 1318], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120", "$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [52, 5659, 267, 12381, 312, 204, 15, 28, 25, 527, 388, 248, 2946, 25], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [5667, 898, 258], + decoded: "you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [5667, 898, 60482], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [5667, 898, 4381, 4381, 5667, 898, 60482], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [698, 1505, 204, 181, 133, 236, 5753, 204, 181, 133, 236, 1494], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [13856, 207, 1182, 26607, 207, 259, 26607, 207, 76, 26607, 207, 4780, 26607, 207, 25], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + NUMBERS_SPLIT: { + text: FALCON_TEST_STRINGS.NUMBERS_SPLIT, + tokens: ["12", "\u0120and", "\u0120", "123", "\u0120and", "\u0120", "123", "4"], + ids: [928, 273, 204, 10963, 273, 204, 10963, 31], + decoded: "12 and 123 and 1234", + }, + }, + "tiiuae/falcon-rw-1b": { + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [1639, 815, 1053, 1760, 428], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["01", "23", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [486, 1954, 2231, 3134, 4531, 657, 352, 362, 513, 604, 642, 718, 767, 807, 860, 838, 1802, 8576], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], + ids: [464, 1664, 373, 9393, 287, 1584, 13], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [32, 198, 1183, 37867, 1462, 8348, 67, 7061, 67, 286, 11, 460, 470, 13], + decoded: "A\n'll!!to?'d''d of, can't.", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "to", "String", "();", "\u010a", "to", "String", "();"], + ids: [1616, 257, 796, 26181, 13, 1462, 10100, 9783, 198, 1462, 10100, 9783], + decoded: "let a = obj.toString();\ntoString();", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9", "d", ",", "running"], + ids: [4944, 42949, 2634, 67, 11, 20270], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [16, 188, 17, 4210, 18], + decoded: "1\u00002\ufffd3", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [37955, 162, 112, 119, 21410, 40367, 253, 164, 108, 249, 42468], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], + ids: [220, 220, 3756, 2272], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], + ids: [9535, 4386, 2272, 220, 220, 220], + decoded: "trailing space ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [9288, 720, 16, 371, 17, 1303, 18, 10432, 19, 4248, 20, 38221, 21, 2343, 224, 96, 22, 2343, 224, 117, 23, 2343, 224, 109, 24, 1332], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [40, 5839, 281, 17180, 329, 720, 16, 13, 405, 379, 262, 3650, 13], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], + ids: [5832, 1399, 220, 220], + decoded: "you\u2026 ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [732, 1447, 27332, 121, 252, 5743, 27332, 121, 252, 1339], + decoded: "weird \uff5e edge \uff5e case", + }, + NUMBERS_SPLIT: { + text: FALCON_TEST_STRINGS.NUMBERS_SPLIT, + tokens: ["12", "\u0120and", "\u0120123", "\u0120and", "\u012012", "34"], + ids: [1065, 290, 17031, 290, 1105, 2682], + decoded: "12 and 123 and 1234", + }, + }, +}; diff --git a/packages/transformers/tests/models/falcon_h1/test_modeling_falcon_h1.js b/packages/transformers/tests/models/falcon_h1/test_modeling_falcon_h1.js new file mode 100644 index 0000000..a1590e8 --- /dev/null +++ b/packages/transformers/tests/models/falcon_h1/test_modeling_falcon_h1.js @@ -0,0 +1,51 @@ +import { PreTrainedTokenizer, FalconH1ForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("FalconH1ForCausalLM", () => { + const model_id = "onnx-internal-testing/tiny-random-FalconH1ForCausalLM"; + /** @type {FalconH1ForCausalLM} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await FalconH1ForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[30939n, 175290n, 186817n, 203743n, 186817n, 203743n, 186817n, 203743n, 186817n, 203743n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 30939n, 175290n, 186817n, 203743n, 186817n, 203743n, 186817n, 203743n, 186817n], + [30939n, 3870n, 194940n, 195605n, 240105n, 240105n, 240105n, 240105n, 240105n, 240105n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/florence2/test_modeling_florence2.js b/packages/transformers/tests/models/florence2/test_modeling_florence2.js new file mode 100644 index 0000000..449140a --- /dev/null +++ b/packages/transformers/tests/models/florence2/test_modeling_florence2.js @@ -0,0 +1,83 @@ +import { Florence2Processor, Florence2ForConditionalGeneration, RawImage, full } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + const texts = ["Describe with a paragraph what is shown in the image.", "Locate the objects with category name in the image."]; + + // Empty white image + const dims = [224, 224, 3]; + const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); + + describe("Florence2ForConditionalGeneration", () => { + const model_id = "Xenova/tiny-random-Florence2ForConditionalGeneration"; + + /** @type {Florence2ForConditionalGeneration} */ + let model; + /** @type {Florence2Processor} */ + let processor; + beforeAll(async () => { + model = await Florence2ForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + processor = await Florence2Processor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + const inputs = await processor(image, texts[0]); + + const { logits } = await model({ + ...inputs, + decoder_input_ids: full([1, 1], 2n), + }); + expect(logits.dims).toEqual([1, 1, 51289]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it.skip( + "batch_size=1", + async () => { + { + const text_inputs = processor.tokenizer(texts[0]); + const generate_ids = await model.generate({ ...text_inputs, max_new_tokens: 10 }); + expect(generate_ids.tolist()).toEqual([[2n, 0n, 0n, 0n, 1n, 0n, 0n, 2n]]); + } + { + const inputs = await processor(image, texts[0]); + const generate_ids = await model.generate({ ...inputs, max_new_tokens: 10 }); + expect(generate_ids.tolist()).toEqual([[2n, 0n, 0n, 0n, 1n, 0n, 0n, 2n]]); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + it.skip( + "batch_size>1", + async () => { + { + const text_inputs = processor.tokenizer(texts, { padding: true }); + const generate_ids = await model.generate({ ...text_inputs, max_new_tokens: 10 }); + expect(generate_ids.tolist()).toEqual([ + [2n, 0n, 0n, 0n, 1n, 0n, 0n, 2n], + [2n, 0n, 0n, 0n, 1n, 0n, 0n, 2n], + ]); + } + { + const inputs = await processor([image, image], texts, { padding: true }); + + const generate_ids = await model.generate({ ...inputs, max_new_tokens: 10 }); + expect(generate_ids.tolist()).toEqual([ + [2n, 0n, 0n, 0n, 1n, 0n, 0n, 2n], + [2n, 0n, 0n, 0n, 1n, 0n, 0n, 2n], + ]); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/florence2/test_processor_florence2.js b/packages/transformers/tests/models/florence2/test_processor_florence2.js new file mode 100644 index 0000000..75ea703 --- /dev/null +++ b/packages/transformers/tests/models/florence2/test_processor_florence2.js @@ -0,0 +1,257 @@ +import { AutoProcessor, Florence2Processor } from "../../../src/transformers.js"; +import { MAX_TEST_EXECUTION_TIME, MAX_PROCESSOR_LOAD_TIME } from "../../init.js"; +import { load_cached_image } from "../../asset_cache.js"; +export default () => { + describe("Florence2Processor", () => { + const model_id = "Xenova/tiny-random-Florence2ForConditionalGeneration"; + + /** @type {Florence2Processor} */ + let processor; + let images = {}; + + beforeAll(async () => { + processor = await AutoProcessor.from_pretrained(model_id); + images = { + beetle: await load_cached_image("beetle"), + book_cover: await load_cached_image("book_cover"), + white_image: await load_cached_image("white_image"), + }; + }, MAX_PROCESSOR_LOAD_TIME); + + describe("Processing", () => { + it( + "Process image and text (no task)", + async () => { + const inputs = await processor(images.white_image, "describe"); + expect(inputs.input_ids.dims).toEqual([1, 4]); + expect(inputs.input_ids.tolist()).toEqual([[0n, 45091n, 21700n, 2n]]); + + expect(inputs.attention_mask.dims).toEqual([1, 4]); + expect(inputs.attention_mask.tolist()).toEqual([[1n, 1n, 1n, 1n]]); + + expect(inputs.pixel_values.dims).toEqual([1, 3, 768, 768]); + expect(inputs.pixel_values.mean().item()).toBeCloseTo(2.439159870147705, 1); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "Process image and text (with task)", + async () => { + const inputs = await processor(images.white_image, "cat"); + expect(inputs.input_ids.dims).toEqual([1, 9]); + expect(inputs.input_ids.tolist()).toEqual([[0n, 574n, 22486n, 4758n, 11n, 5n, 2274n, 4n, 2n]]); + + expect(inputs.attention_mask.dims).toEqual([1, 9]); + expect(inputs.attention_mask.tolist()).toEqual([[1n, 1n, 1n, 1n, 1n, 1n, 1n, 1n, 1n]]); + + expect(inputs.pixel_values.dims).toEqual([1, 3, 768, 768]); + expect(inputs.pixel_values.mean().item()).toBeCloseTo(2.439159870147705, 1); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Prompt construction", () => { + it( + "Construct prompt", + async () => { + const text = ""; + const prompts = processor.construct_prompts(text); + const target = ["Locate the objects with category name in the image."]; + expect(prompts).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "Construct prompts", + async () => { + const texts = ["", "Locate the objects with category name in the image.", "cat"]; + const prompts = processor.construct_prompts(texts); + const target = ["Describe with a paragraph what is shown in the image.", "Locate the objects with category name in the image.", "Locate cat in the image."]; + expect(prompts).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Post-process generation", () => { + const TESTS = [ + { + task: "", + generated_text: "A green car parked in front of a yellow building.", + target: { "": "A green car parked in front of a yellow building." }, + image: "beetle", + }, + { + task: "", + generated_text: "The image shows a green Volkswagen Beetle parked in front of a yellow building with two brown doors. The sky is a mix of blue and white, and there are a few green trees in the background.", + target: { "": "The image shows a green Volkswagen Beetle parked in front of a yellow building with two brown doors. The sky is a mix of blue and white, and there are a few green trees in the background." }, + image: "beetle", + }, + { + task: "", + generated_text: "The image shows a vintage Volkswagen Beetle car parked on a cobblestone street in front of a yellow building with two wooden doors. The car is painted in a bright turquoise color and has a white stripe running along the side. It has two doors on either side of the car, one on top of the other, and a small window on the front. The building appears to be old and dilapidated, with peeling paint and crumbling walls. The sky is blue and there are trees in the background.", + target: { "": "The image shows a vintage Volkswagen Beetle car parked on a cobblestone street in front of a yellow building with two wooden doors. The car is painted in a bright turquoise color and has a white stripe running along the side. It has two doors on either side of the car, one on top of the other, and a small window on the front. The building appears to be old and dilapidated, with peeling paint and crumbling walls. The sky is blue and there are trees in the background." }, + image: "beetle", + }, + { + task: "", + generated_text: "cardoorwheel", + target: { + "": { + bboxes: [ + [34.24, 160.08, 597.44, 371.76], + [456.0, 97.68, 580.16, 261.84], + [450.88, 276.72, 554.56, 370.8], + [95.68, 280.56, 198.72, 371.28], + ], + labels: ["car", "door", "wheel", "wheel"], + }, + }, + image: "beetle", + }, + { + task: "", + generated_text: "turquoise Volkswagen Beetlewheel", + target: { + "": { + bboxes: [ + [33.6, 160.08, 596.8, 371.76], + [450.88, 276.72, 553.28, 370.8], + [95.04, 280.56, 197.44, 371.28], + ], + labels: ["turquoise Volkswagen Beetle", "wheel", "wheel"], + }, + }, + image: "beetle", + }, + { + task: "", + generated_text: "", + target: { + "": { + bboxes: [ + [33.6, 160.08, 596.8, 371.76], + [455.36, 97.68, 579.52, 261.84], + [450.88, 276.72, 553.28, 370.8], + [95.04, 280.56, 198.08, 371.28], + [226.88, 88.56, 332.48, 164.4], + [65.6, 266.64, 86.72, 295.92], + [271.68, 241.68, 302.4, 246.96], + [408.0, 308.4, 413.76, 320.88], + ], + labels: ["", "", "", "", "", "", "", ""], + }, + }, + image: "beetle", + }, + { + task: "", + text_input: "A green car parked in front of a yellow building.", + generated_text: "A green cara yellow building", + target: { + "": { + bboxes: [ + [34.88, 158.64, 583.36, 374.64], + [0.32, 4.08, 639.04, 305.04], + ], + labels: ["A green car", "a yellow building"], + }, + }, + image: "beetle", + }, + // { + // task: "", + // text_input: "a green car", + // generated_text: "", + // target: { + // '': { + // polygons: [[[[178.88, 181.68, 180.8, 180.72, 182.72, 180.72, 187.84, 177.84, 189.76, 177.84, 192.96, 175.92, 194.88, 175.92, 198.08, 174, 200.64, 173.04, 203.84, 172.08, 207.04, 170.64, 209.6, 169.68, 214.08, 168.72, 217.92, 167.76, 221.76, 166.8, 226.24, 165.84, 230.72, 164.88, 237.12, 163.92, 244.16, 162.96, 253.12, 162, 265.28, 161.04, 311.36, 161.04, 329.28, 162, 338.24, 162.96, 345.28, 163.92, 350.4, 164.88, 354.24, 165.84, 358.72, 166.8, 362.56, 167.76, 366.4, 168.72, 370.24, 169.68, 373.44, 170.64, 375.36, 172.08, 377.28, 174, 379.2, 176.88, 380.48, 179.76, 382.4, 181.68, 384.32, 185.04, 386.24, 187.92, 387.52, 190.8, 389.44, 192.72, 390.08, 196.08, 392, 198.96, 394.56, 201.84, 396.48, 204.72, 398.4, 208.08, 403.52, 212.88, 406.08, 213.84, 409.28, 216.72, 412.48, 220.08, 431.68, 220.08, 432.32, 221.04, 442.56, 222, 456.64, 222, 465.6, 222.96, 472.64, 223.92, 478.4, 224.88, 484.8, 225.84, 489.92, 226.8, 493.76, 227.76, 497.6, 228.72, 501.44, 229.68, 504.64, 231.12, 507.84, 232.08, 510.4, 233.04, 513.6, 234, 516.8, 235.92, 518.72, 235.92, 523.84, 238.8, 525.76, 238.8, 527.68, 239.76, 529.6, 241.68, 532.8, 242.64, 536, 245.04, 538.56, 247.92, 541.76, 249.84, 545.6, 251.76, 548.8, 252.72, 550.72, 252.72, 553.92, 253.68, 556.48, 255.6, 558.4, 255.6, 564.8, 258.96, 566.72, 260.88, 568.64, 260.88, 570.56, 261.84, 572.48, 263.76, 573.76, 265.68, 574.4, 268.56, 574.4, 271.92, 573.76, 272.88, 572.48, 275.76, 572.48, 279.6, 573.76, 285.84, 574.4, 286.8, 575.68, 289.68, 576.32, 292.56, 577.6, 298.8, 577.6, 301.68, 576.32, 302.64, 575.68, 310.8, 575.68, 312.72, 576.32, 313.68, 577.6, 316.56, 577.6, 320.88, 574.4, 321.84, 568.64, 322.8, 559.68, 322.8, 553.92, 323.76, 552.64, 332.88, 552, 336.72, 550.72, 339.6, 550.08, 342.96, 548.8, 344.88, 546.88, 346.8, 545.6, 349.68, 543.68, 352.56, 541.76, 355.92, 534.72, 362.64, 531.52, 364.56, 525.76, 367.92, 522.56, 368.88, 518.72, 369.84, 495.68, 369.84, 489.92, 368.88, 486.72, 367.92, 483.52, 366.96, 479.68, 364.56, 476.48, 362.64, 472.64, 359.76, 465.6, 352.56, 463.68, 349.68, 461.76, 346.8, 460.48, 344.88, 460.48, 342.96, 458.56, 339.6, 457.92, 336.72, 457.92, 334.8, 456.64, 332.88, 454.72, 330.96, 452.8, 331.92, 448.32, 336.72, 446.4, 337.68, 426.56, 336.72, 424.64, 336.72, 423.36, 337.68, 420.8, 338.64, 414.4, 339.6, 412.48, 339.6, 411.2, 338.64, 380.48, 337.68, 217.28, 337.68, 216, 338.64, 210.88, 339.6, 207.04, 339.6, 203.84, 338.64, 201.92, 337.68, 200, 335.76, 198.08, 334.8, 194.88, 334.8, 192.96, 336.72, 191.68, 338.64, 191.68, 340.56, 191.04, 342.96, 189.12, 344.88, 187.84, 347.76, 185.92, 349.68, 184.64, 352.56, 182.72, 355.92, 176.96, 361.68, 173.76, 363.6, 170.56, 365.52, 166.72, 367.92, 163.52, 368.88, 160.96, 369.84, 153.92, 370.8, 131.52, 370.8, 127.68, 369.84, 124.48, 368.88, 118.72, 365.52, 115.52, 363.6, 111.68, 360.72, 106.56, 355.92, 104.64, 352.56, 103.36, 349.68, 101.44, 347.76, 100.8, 345.84, 99.52, 342.96, 99.52, 339.6, 98.88, 337.68, 95.68, 334.8, 93.76, 333.84, 86.72, 333.84, 80.32, 334.8, 79.68, 335.76, 74.56, 336.72, 66.24, 336.72, 63.68, 334.8, 53.44, 334.8, 50.24, 333.84, 48.32, 331.92, 48.32, 328.56, 50.24, 326.64, 51.52, 324.72, 51.52, 322.8, 44.48, 321.84, 40.64, 320.88, 38.72, 319.92, 37.44, 317.52, 36.16, 313.68, 36.16, 306.96, 38.72, 304.56, 42.56, 303.6, 46.4, 302.64, 55.36, 301.68, 65.6, 301.68, 67.52, 300.72, 69.44, 298.8, 70.72, 296.88, 70.72, 292.56, 69.44, 291.6, 68.8, 288.72, 67.52, 284.88, 67.52, 276.72, 68.8, 273.84, 69.44, 271.92, 72.64, 268.56, 74.56, 267.6, 77.76, 266.64, 79.68, 266.64, 81.6, 264.72, 80.32, 260.88, 81.6, 258.96, 83.52, 256.56, 88.64, 256.56, 90.56, 255.6, 92.48, 253.68, 92.48, 252.72, 97.6, 246.96, 114.88, 229.68, 117.44, 226.8, 122.56, 222.96, 125.76, 221.04, 126.4, 221.04, 129.6, 219.12, 133.44, 215.76, 138.56, 211.92, 143.68, 208.08, 149.44, 201.84, 153.92, 198.96, 154.56, 198.96, 157.76, 197.04, 162.88, 192.72, 168.64, 186.96, 171.84, 185.04, 176.96, 183.12, 178.88, 180.72]]]], + // labels: [''], + // } + // }, + // image: 'beetle', + // }, + // { + // task: "", + // text_input: "", + // generated_text: "", + // target: { + // '': { + // polygons: [[[[470.08, 288.24, 473.92, 285.36, 477.12, 283.44, 479.04, 282.48, 480.96, 282.48, 484.16, 280.56, 486.72, 279.6, 489.92, 278.64, 495.04, 277.68, 512.32, 277.68, 514.88, 278.64, 518.08, 279.6, 521.28, 281.52, 523.2, 281.52, 525.12, 283.44, 528.32, 284.4, 530.88, 286.32, 534.08, 288.24, 543.04, 297.36, 544.96, 300.24, 546.88, 303.12, 550.08, 309.36, 551.36, 312.24, 552, 315.12, 553.28, 319.44, 553.28, 332.4, 552, 337.2, 551.36, 340.08, 550.08, 343.44, 548.16, 347.28, 546.24, 350.16, 544.32, 353.04, 541.12, 357.36, 537.28, 361.2, 532.16, 365.04, 528.96, 366.96, 527.04, 367.92, 523.84, 368.88, 521.28, 369.84, 516.16, 371.28, 500.8, 371.28, 491.84, 369.84, 488, 368.88, 484.8, 367.92, 479.04, 365.04, 475.84, 363.12, 472, 360.24, 464.96, 353.04, 463.04, 350.16, 461.12, 347.28, 459.84, 345.36, 459.84, 343.44, 457.92, 340.08, 456.64, 337.2, 456, 334.32, 454.72, 330.48, 454.72, 316.08, 456, 311.28, 456.64, 307.44, 457.92, 304.08, 459.84, 301.2, 459.84, 299.28, 461.12, 297.36, 463.04, 294.48]]]], + // labels: [''], + // } + // }, + // image: 'beetle', + // }, + // { + // task: "", + // text_input: "a green car", + // generated_text: "a green car", + // target: { + // '': { + // bboxes: [[34.24, 158.64, 582.72, 374.16]], + // bboxes_labels: ['a green car'], + // polygons: [], + // polygons_labels: [], + // } + // }, + // image: 'beetle', + // }, + { + task: "", + text_input: "", + generated_text: "car", + target: { "": "car" }, + image: "beetle", + }, + { + task: "", + text_input: "", + generated_text: "turquoise Volkswagen Beetle", + target: { "": "turquoise Volkswagen Beetle" }, + image: "beetle", + }, + { + task: "", + generated_text: "CUDAFOR ENGINEERSAn Introduction to High-PerformanceParallel ComputingDUANE STORTIMETE YURTOGLU", + target: { "": "CUDAFOR ENGINEERSAn Introduction to High-PerformanceParallel ComputingDUANE STORTIMETE YURTOGLU" }, + image: "book_cover", + }, + { + task: "", + generated_text: "CUDAFOR ENGINEERSAn Introduction to High-PerformanceParallel ComputingDUANE STORTIMETE YURTOGLU", + target: { + "": { + quad_boxes: [ + [167.0435028076172, 50.25, 375.7974853515625, 50.25, 375.7974853515625, 114.75, 167.0435028076172, 114.75], + [144.8784942626953, 120.75, 375.7974853515625, 120.75, 375.7974853515625, 149.25, 144.8784942626953, 149.25], + [115.86249542236328, 165.25, 376.6034851074219, 166.25, 376.6034851074219, 184.25, 115.86249542236328, 183.25], + [239.9864959716797, 184.25, 376.6034851074219, 186.25, 376.6034851074219, 204.25, 239.9864959716797, 202.25], + [266.1814880371094, 441.25, 376.6034851074219, 441.25, 376.6034851074219, 456.25, 266.1814880371094, 456.25], + [252.0764923095703, 460.25, 376.6034851074219, 460.25, 376.6034851074219, 475.25, 252.0764923095703, 475.25], + ], + + // NOTE: Python version has a bug here, it should be "CUDA" instead of "CUDA" + labels: [/* 'CUDA' */ "CUDA", "FOR ENGINEERS", "An Introduction to High-Performance", "Parallel Computing", "DUANE STORTI", "METE YURTOGLU"], + }, + }, + image: "book_cover", + }, + ]; + + for (const { task, generated_text, target, image } of TESTS) { + it( + task, + () => { + const result = processor.post_process_generation(generated_text, task, images[image].size); + expect(result).toBeCloseToNested(target, 4); + }, + MAX_TEST_EXECUTION_TIME, + ); + } + }); + }); +}; diff --git a/packages/transformers/tests/models/gemma/test_modeling_gemma.js b/packages/transformers/tests/models/gemma/test_modeling_gemma.js new file mode 100644 index 0000000..47501f2 --- /dev/null +++ b/packages/transformers/tests/models/gemma/test_modeling_gemma.js @@ -0,0 +1,51 @@ +import { GemmaTokenizer, GemmaForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("GemmaForCausalLM", () => { + const model_id = "Xenova/tiny-random-GemmaForCausalLM"; + /** @type {GemmaForCausalLM} */ + let model; + /** @type {GemmaTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GemmaForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await GemmaTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[2n, 17534n, 254059n, 254059n, 254059n, 254059n, 254059n, 254059n, 254059n, 254059n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 2n, 17534n, 254059n, 254059n, 254059n, 254059n, 254059n, 254059n, 254059n], + [2n, 17534n, 2134n, 71055n, 71055n, 71055n, 71055n, 71055n, 71055n, 71055n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/gemma/test_tokenization_gemma.js b/packages/transformers/tests/models/gemma/test_tokenization_gemma.js new file mode 100644 index 0000000..5bfd085 --- /dev/null +++ b/packages/transformers/tests/models/gemma/test_tokenization_gemma.js @@ -0,0 +1,231 @@ +import { GemmaTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, LLAMA_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = GemmaTokenizer; +export const TEST_CONFIG = { + // Xenova/gemma-tokenizer + "Xenova/gemma2-tokenizer": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [2, 2299, 708, 692, 3900, 235336], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [2, 2045, 1412, 235303, 524, 3015, 736], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u2581", "0", "\u2581", "1", "\u2581", "2", "\u2581", "3", "\u2581", "4", "\u2581", "5", "\u2581", "6", "\u2581", "7", "\u2581", "8", "\u2581", "9", "\u2581", "1", "0", "\u2581", "1", "0", "0", "\u2581", "1", "0", "0", "0"], + ids: [2, 235276, 235274, 235284, 235304, 235310, 235308, 235318, 235324, 235321, 235315, 235248, 235276, 235248, 235274, 235248, 235284, 235248, 235304, 235248, 235310, 235248, 235308, 235248, 235318, 235248, 235324, 235248, 235321, 235248, 235315, 235248, 235274, 235276, 235248, 235274, 235276, 235276, 235248, 235274, 235276, 235276, 235276], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u2581", "2", "0", "1", "6", "."], + ids: [2, 651, 3277, 729, 18942, 575, 235248, 235284, 235276, 235274, 235318, 235265], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\n", "'", "ll", "\u2581!!", "to", "?'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [2, 235280, 108, 235303, 529, 9063, 511, 18016, 235258, 3404, 235258, 576, 235269, 798, 235303, 235251, 235265], + decoded: "A\n'll !!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u2581main", "():", "\n", "\t", "pass"], + ids: [2, 1293, 1872, 4409, 108, 226, 3095], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u2581a", "\u2581=", "\u2581obj", ".", "toString", "();", "\n", "toString", "();"], + ids: [2, 1243, 476, 589, 6555, 235265, 7114, 821, 108, 7114, 821], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: LLAMA_TEST_STRINGS.NEWLINES, + tokens: ["ax", "\n", "####", "\n", "boo"], + ids: [2, 1247, 108, 3308, 108, 31931], + decoded: "ax\n####\nboo", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00e9d", ",", "running"], + ids: [2, 2019, 29007, 45346, 235269, 23655], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "<0x00>", "2", "\ufffd", "3"], + ids: [2, 235274, 217, 235284, 236193, 235304], + decoded: "1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u2581World"], + ids: [2, 4521, 3855], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u2581world"], + ids: [2, 17534, 2134], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f\u6d3b\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [2, 122182, 235710, 245467, 235427], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581\u2581\u2581", "leading", "\u2581space"], + ids: [2, 140, 26650, 3641], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "\u2581space", "\u2581\u2581\u2581"], + ids: [2, 100504, 3641, 140], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u2581\u2581", "Hello"], + ids: [2, 2151, 139, 4521], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u2581$", "1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [2, 2195, 697, 235274, 625, 235284, 1700, 235304, 8296, 235310, 5955, 235308, 74393, 235318, 235248, 252058, 235324, 56712, 235321, 235248, 243132, 235315, 2121], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1", ".", "0", "0", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [2, 235285, 8989, 671, 15491, 604, 697, 235274, 235265, 235276, 235276, 696, 573, 4659, 235265], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u2026", "\u2581\u2581"], + ids: [2, 4747, 235417, 139], + decoded: "you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u2026", "\u00a0\u00a0"], + ids: [2, 4747, 235417, 25445], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u2026", "\u00a0\u00a0", "you", "\u2026", "\u00a0\u00a0"], + ids: [2, 4747, 235417, 25445, 4747, 235417, 25445], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "\u2581\uff5e", "\u2581edge", "\u2581\uff5e", "\u2581case"], + ids: [2, 102422, 134012, 8541, 134012, 2270], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581\u2581", "is", "\u2581\u2581", "a", "\u2581\u2581", "test", "\u2581\u2581", "."], + ids: [2, 1417, 139, 502, 139, 235250, 139, 2195, 139, 235265], + decoded: " This is a test .", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\ud83d\ude02", "\u2581\ud83d\udc4d", "\u2581\ud83e\udd23", "\u2581\ud83d\ude0d", "\u2581\ud83d\ude2d", "\u2581\ud83c\udf89", "\u2581\ud83d\ude4f", "\u2581\ud83d\ude0a", "\u2581\ud83d\udd25", "\u2581\ud83d\ude01", "\u2581\ud83d\ude05", "\u2581\ud83e\udd17", "\u2581\ud83d\ude06", "\u2581\ud83d\udc4f", "\u2581\u2764\ufe0f", "\u2581\ud83d\udc9c", "\u2581\ud83d\udc9a", "\u2581\ud83d\udc97", "\u2581\ud83d\udc99", "\u2581\ud83d\udda4", "\u2581\ud83d\ude0e", "\u2581\ud83d\udc4c", "\u2581\ud83e\udd73", "\u2581\ud83d\udcaa", "\u2581\u2728", "\u2581\ud83d\udc49", "\u2581\ud83d\udc40", "\u2581\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581\ud83d\ude4c", "\u2581\ud83d\udc80", "\u2581\ud83d\udc47", "\u2581\ud83d\udc4b", "\u2581\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [2, 236471, 38104, 55937, 46434, 55605, 160588, 68226, 44416, 72373, 70636, 75298, 156808, 120433, 104492, 35373, 131674, 191384, 204903, 146773, 166620, 87949, 83860, 211978, 142816, 64726, 166368, 108892, 174882, 235248, 242431, 235248, 241259, 134540, 106918, 154601, 169692, 92641, 235248, 241227, 235248, 241971, 233958, 235248, 241034], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2728", "\u2581\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u200d\u2642\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u200d\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u200d", "\u2764", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u200d", "\ud83d\udc69", "\u200d", "\ud83d\udc67", "\u200d", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4", "\udb40\udc67", "\udb40\udc62", "\udb40\udc65", "\udb40\udc6e", "\udb40\udc67", "\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\u2764\ufe0f", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\ud83c\udffc"], + ids: [2, 236309, 156808, 235248, 241666, 235969, 235248, 247216, 237933, 235248, 246522, 68399, 235248, 246422, 237933, 63233, 235248, 241568, 237933, 235879, 244448, 235248, 243634, 235879, 241668, 235879, 243634, 235248, 241355, 235879, 236457, 235879, 240887, 235879, 241568, 235248, 241355, 235879, 241355, 235879, 244355, 235879, 244670, 235248, 243634, 237933, 235879, 241668, 235879, 243634, 237933, 235248, 244443, 246738, 247704, 250142, 250123, 246738, 247662, 235248, 241568, 237933, 235879, 16176, 235879, 240887, 235879, 241568, 238683], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + BPE_SCORES_PRIORITY_1: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, + tokens: ["grab", "bed"], + ids: [2, 59031, 2907], + decoded: "grabbed", + }, + BPE_SCORES_PRIORITY_2: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, + tokens: ["\u2581grabbed"], + ids: [2, 41939], + decoded: " grabbed", + }, + BPE_SCORES_PRIORITY_3: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, + tokens: ["\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581", "grab", "bed"], + ids: [2, 148, 59031, 2907], + decoded: " grabbed", + }, + NEWLINE: { + text: LLAMA_TEST_STRINGS.NEWLINE, + tokens: ["\n"], + ids: [2, 108], + decoded: "\n", + }, + NEWLINE_WITH_LEADING_SPACE: { + text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, + tokens: ["\u2581", "\n"], + ids: [2, 235248, 108], + decoded: " \n", + }, + TABS: { + text: LLAMA_TEST_STRINGS.TABS, + tokens: ["\t", "tabs", "\t\t\t\t", "out", "\u2581here"], + ids: [2, 226, 31973, 255971, 745, 1517], + decoded: "\ttabs\t\t\t\tout here", + }, + NEWLINE_AND_TAB: { + text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, + tokens: ["\n", "\t", "\n"], + ids: [2, 108, 226, 108], + decoded: "\n\t\n", + }, + CHINESE_LETTER: { + text: LLAMA_TEST_STRINGS.CHINESE_LETTER, + tokens: ["\u9547"], + ids: [2, 237796], + decoded: "\u9547", + }, + EMOJIS_1: { + text: LLAMA_TEST_STRINGS.EMOJIS_1, + tokens: ["\ud83e\udd99"], + ids: [2, 250645], + decoded: "\ud83e\udd99", + }, + EMOJIS_2: { + text: LLAMA_TEST_STRINGS.EMOJIS_2, + tokens: ["\ud83e\udd99", "<0xEA>", "<0x99>", "<0x8A>"], + ids: [2, 250645, 451, 370, 355], + decoded: "\ud83e\udd99\ua64a", + }, + EMOJIS_3: { + text: LLAMA_TEST_STRINGS.EMOJIS_3, + tokens: ["<0xEA>", "<0x99>", "<0x8A>", "\ud83e\udd99"], + ids: [2, 451, 370, 355, 250645], + decoded: "\ua64a\ud83e\udd99", + }, + PARAGRAPH: { + text: LLAMA_TEST_STRINGS.PARAGRAPH, + tokens: ["The", "\u2581llama", "\u2581(/", "\u02c8", "l", "\u0251", "\u02d0", "m\u0259", "/;", "\u2581", "\ud83e\udd99", "Spanish", "\u2581pronunciation", ":", "\u2581[", "\u02c8", "\u028e", "ama", "])", "\u2581(", "Lama", "\u2581g", "lama", ")", "\u2581is", "\u2581a", "\u2581domesticated", "\u2581South", "\u2581American", "\u2581came", "lid", ",", "\u2581widely", "\u2581used", "\u2581as", "\u2581a", "\u2581meat", "\u2581and", "\u2581pack", "\u2581animal", "\u2581by", "\u2581Andean", "\u2581cultures", "\u2581since", "\u2581the", "\u2581Pre", "-", "Columb", "ian", "\u2581era", ".", "\u2581Lla", "mas", "\u2581are", "\u2581social", "\u2581animals", "\u2581and", "\u2581live", "\u2581with", "\u2581others", "\u2581as", "\u2581a", "\u2581herd", ".", "\u2581Their", "\u2581wool", "\u2581is", "\u2581soft", "\u2581and", "\u2581contains", "\u2581only", "\u2581a", "\u2581small", "\u2581amount", "\u2581of", "\u2581lan", "olin", ".[", "2", "]", "\u2581Lla", "mas", "\u2581can", "\u2581learn", "\u2581simple", "\u2581tasks", "\u2581after", "\u2581a", "\u2581few", "\u2581repetitions", ".", "\u2581When", "\u2581using", "\u2581a", "\u2581pack", ",", "\u2581they", "\u2581can", "\u2581carry", "\u2581about", "\u2581", "2", "5", "\u2581to", "\u2581", "3", "0", "%", "\u2581of", "\u2581their", "\u2581body", "\u2581weight", "\u2581for", "\u2581", "8", "\u2581to", "\u2581", "1", "3", "\u2581km", "\u2581(", "5", "\u2013", "8", "\u2581miles", ").[", "3", "]", "\u2581The", "\u2581name", "\u2581llama", "\u2581(", "in", "\u2581the", "\u2581past", "\u2581also", "\u2581spelled", '\u2581"', "lama", '"', "\u2581or", '\u2581"', "g", "lama", '")', "\u2581was", "\u2581adopted", "\u2581by", "\u2581European", "\u2581settlers", "\u2581from", "\u2581native", "\u2581Peru", "vi", "ans", ".[", "4", "]", "\u2581The", "\u2581ancestors", "\u2581of", "\u2581llamas", "\u2581are", "\u2581thought", "\u2581to", "\u2581have", "\u2581originated", "\u2581from", "\u2581the", "\u2581Great", "\u2581Plains", "\u2581of", "\u2581North", "\u2581America", "\u2581about", "\u2581", "4", "0", "\u2581million", "\u2581years", "\u2581ago", ",", "\u2581and", "\u2581subsequently", "\u2581migrated", "\u2581to", "\u2581South", "\u2581America", "\u2581about", "\u2581three", "\u2581million", "\u2581years", "\u2581ago", "\u2581during", "\u2581the", "\u2581Great", "\u2581American", "\u2581Interchange", ".", "\u2581By", "\u2581the", "\u2581end", "\u2581of", "\u2581the", "\u2581last", "\u2581ice", "\u2581age", "\u2581(", "1", "0", ",", "0", "0", "0", "\u2013", "1", "2", ",", "0", "0", "0", "\u2581years", "\u2581ago", "),", "\u2581came", "lids", "\u2581were", "\u2581extinct", "\u2581in", "\u2581North", "\u2581America", ".[", "3", "]", "\u2581As", "\u2581of", "\u2581", "2", "0", "0", "7", ",", "\u2581there", "\u2581were", "\u2581over", "\u2581seven", "\u2581million", "\u2581llamas", "\u2581and", "\u2581al", "pac", "as", "\u2581in", "\u2581South", "\u2581America", "\u2581and", "\u2581over", "\u2581", "1", "5", "8", ",", "0", "0", "0", "\u2581llamas", "\u2581and", "\u2581", "1", "0", "0", ",", "0", "0", "0", "<0xEA>", "<0x99>", "<0x8A>", "\ud83e\udd99", "\u2581al", "pac", "as", ",", "\u2581descended", "\u2581from", "\u2581progen", "itors", "\u2581imported", "\u2581late", "\u2581in", "\u2581the", "\u2581", "2", "0", "th", "\u2581century", ",", "\u2581in", "\u2581the", "\u2581United", "\u2581States", "\u2581and", "\u2581Canada", ".[", "5", "]", "\u2581In", "\u2581A", "ym", "ara", "\u2581mythology", ",", "\u2581llamas", "\u2581are", "\u2581important", "\u2581beings", ".", "\u2581The", "\u2581Heavenly", "\u2581Llama", "\u2581is", "\u2581said", "\u2581to", "\u2581drink", "\u2581water", "\u2581from", "\u2581the", "\u2581ocean", "\u2581and", "\u2581urin", "ates", "\u2581as", "\u2581it", "\u2581rains", ".[", "6", "]", "\u2581According", "\u2581to", "\u2581A", "ym", "ara", "\u2581es", "ch", "atology", ",", "\u2581llamas", "\u2581will", "\u2581return", "\u2581to", "\u2581the", "\u2581water", "\u2581springs", "\u2581and", "\u2581lagoons", "\u2581where", "\u2581they", "\u2581come", "\u2581from", "\u2581at", "\u2581the", "\u2581end", "\u2581of", "\u2581time", ".[", "6", "]"], + ids: [2, 651, 19001, 101949, 239229, 235257, 240527, 240342, 128631, 102430, 235248, 250645, 51590, 74569, 235292, 892, 239229, 246752, 2867, 3013, 591, 221520, 583, 10450, 235275, 603, 476, 183304, 4316, 3725, 3392, 3353, 235269, 16714, 1671, 685, 476, 11827, 578, 3386, 8205, 731, 207552, 24541, 2754, 573, 2769, 235290, 222963, 1282, 6063, 235265, 172809, 2616, 708, 3127, 8398, 578, 3685, 675, 3588, 685, 476, 48010, 235265, 10368, 23834, 603, 4072, 578, 7744, 1297, 476, 2301, 3619, 576, 7607, 28424, 19047, 235284, 235307, 172809, 2616, 798, 3918, 3890, 13333, 1452, 476, 2619, 126286, 235265, 3194, 2177, 476, 3386, 235269, 984, 798, 6383, 1105, 235248, 235284, 235308, 577, 235248, 235304, 235276, 235358, 576, 1024, 2971, 5171, 604, 235248, 235321, 577, 235248, 235274, 235304, 5821, 591, 235308, 235389, 235321, 7112, 232524, 235304, 235307, 714, 1503, 19001, 591, 473, 573, 3433, 1170, 73003, 664, 10450, 235281, 689, 664, 235264, 10450, 1388, 729, 13861, 731, 7737, 57710, 774, 11634, 30160, 893, 779, 19047, 235310, 235307, 714, 44106, 576, 129953, 708, 3421, 577, 791, 52102, 774, 573, 6553, 55118, 576, 4612, 5783, 1105, 235248, 235310, 235276, 4416, 1658, 3958, 235269, 578, 27956, 106398, 577, 4316, 5783, 1105, 2149, 4416, 1658, 3958, 2290, 573, 6553, 3725, 193879, 235265, 3339, 573, 1580, 576, 573, 2001, 8357, 3911, 591, 235274, 235276, 235269, 235276, 235276, 235276, 235389, 235274, 235284, 235269, 235276, 235276, 235276, 1658, 3958, 823, 3392, 41253, 1049, 78561, 575, 4612, 5783, 19047, 235304, 235307, 1877, 576, 235248, 235284, 235276, 235276, 235324, 235269, 1104, 1049, 1163, 6861, 4416, 129953, 578, 717, 23337, 508, 575, 4316, 5783, 578, 1163, 235248, 235274, 235308, 235321, 235269, 235276, 235276, 235276, 129953, 578, 235248, 235274, 235276, 235276, 235269, 235276, 235276, 235276, 451, 370, 355, 250645, 717, 23337, 508, 235269, 64700, 774, 66279, 15517, 29271, 5245, 575, 573, 235248, 235284, 235276, 489, 7861, 235269, 575, 573, 3520, 3858, 578, 6591, 19047, 235308, 235307, 878, 586, 3985, 1610, 76701, 235269, 129953, 708, 2845, 27290, 235265, 714, 89830, 170669, 603, 1180, 577, 7182, 2003, 774, 573, 13940, 578, 111204, 1204, 685, 665, 50852, 19047, 235318, 235307, 11926, 577, 586, 3985, 1610, 875, 530, 92764, 235269, 129953, 877, 2203, 577, 573, 2003, 31104, 578, 221493, 1570, 984, 2063, 774, 696, 573, 1580, 576, 1069, 19047, 235318, 235307], + decoded: 'The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + }, + }, +}; diff --git a/packages/transformers/tests/models/gemma2/test_modeling_gemma2.js b/packages/transformers/tests/models/gemma2/test_modeling_gemma2.js new file mode 100644 index 0000000..768c116 --- /dev/null +++ b/packages/transformers/tests/models/gemma2/test_modeling_gemma2.js @@ -0,0 +1,51 @@ +import { GemmaTokenizer, Gemma2ForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("Gemma2ForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-Gemma2ForCausalLM"; + /** @type {Gemma2ForCausalLM} */ + let model; + /** @type {GemmaTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await Gemma2ForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await GemmaTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[2n, 17534n, 127534n, 160055n, 160055n, 160055n, 160055n, 160055n, 160055n, 160055n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 2n, 17534n, 127534n, 127534n, 215341n, 215341n, 215341n, 215341n, 215341n], + [2n, 17534n, 2134n, 107508n, 160055n, 160055n, 160055n, 160055n, 160055n, 160055n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/gemma3/test_modeling_gemma3.js b/packages/transformers/tests/models/gemma3/test_modeling_gemma3.js new file mode 100644 index 0000000..c9e4020 --- /dev/null +++ b/packages/transformers/tests/models/gemma3/test_modeling_gemma3.js @@ -0,0 +1,128 @@ +import { Gemma3ForConditionalGeneration, Gemma3ForCausalLM, AutoProcessor, AutoTokenizer, RawImage } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + const CONVERSATION = [ + { + role: "user", + content: [{ type: "text", text: "Hello" }], + }, + ]; + + const CONVERSATION_WITH_IMAGE = [ + { + role: "user", + content: [{ type: "image" }, { type: "text", text: "Describe this image." }], + }, + ]; + + // Empty white image + const dims = [224, 224, 3]; + const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); + + describe("Gemma3ForConditionalGeneration", () => { + const model_id = "onnx-internal-testing/tiny-random-Gemma3ForConditionalGeneration"; + + /** @type {Gemma3ForConditionalGeneration} */ + let model; + /** @type {AutoProcessor} */ + let processor; + beforeAll(async () => { + model = await Gemma3ForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + processor = await AutoProcessor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "text-only forward", + async () => { + const text = processor.apply_chat_template(CONVERSATION, { add_generation_prompt: true }); + const inputs = await processor(text); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 11, 262208]); + expect(logits.mean().item()).toBeCloseTo(-0.004435515962541103, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "text + image forward", + async () => { + const text = processor.apply_chat_template(CONVERSATION_WITH_IMAGE, { add_generation_prompt: true }); + const inputs = await processor(text, image); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 21, 262208]); + expect(logits.mean().item()).toBeCloseTo(-0.0029795959126204252, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "text-only (batch_size=1)", + async () => { + const text = processor.apply_chat_template(CONVERSATION, { add_generation_prompt: true }); + const inputs = await processor(text); + const generate_ids = await model.generate({ + ...inputs, + max_new_tokens: 10, + do_sample: false, + }); + const new_tokens = generate_ids.slice(null, [inputs.input_ids.dims.at(-1), null]); + expect(new_tokens.tolist()).toEqual([[107n, 107n, 107n, 107n, 107n, 107n, 107n, 107n, 107n, 107n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "text + image (batch_size=1)", + async () => { + const text = processor.apply_chat_template(CONVERSATION_WITH_IMAGE, { add_generation_prompt: true }); + const inputs = await processor(text, image); + const generate_ids = await model.generate({ + ...inputs, + max_new_tokens: 10, + do_sample: false, + }); + const new_tokens = generate_ids.slice(null, [inputs.input_ids.dims.at(-1), null]); + expect(new_tokens.tolist()).toEqual([[107n, 107n, 107n, 107n, 107n, 107n, 107n, 107n, 107n, 107n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("Gemma3ForCausalLM", () => { + const model_id = "onnx-internal-testing/tiny-random-Gemma3ForCausalLM"; + + /** @type {Gemma3ForCausalLM} */ + let model; + /** @type {AutoTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await Gemma3ForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await AutoTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_new_tokens: 5, + do_sample: false, + }); + const new_tokens = outputs.slice(null, [inputs.input_ids.dims.at(-1), null]); + expect(new_tokens.tolist()).toEqual([[23391n, 23391n, 23391n, 23391n, 23391n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/gemma4/test_feature_extraction_gemma4.js b/packages/transformers/tests/models/gemma4/test_feature_extraction_gemma4.js new file mode 100644 index 0000000..d1aa2b3 --- /dev/null +++ b/packages/transformers/tests/models/gemma4/test_feature_extraction_gemma4.js @@ -0,0 +1,91 @@ +import { AutoProcessor, Gemma4AudioFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_audio } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe.skip("Gemma4AudioFeatureExtractor", () => { + const model_id = "onnx-community/gemma-4-E2B-it-ONNX"; + + /** @type {Gemma4AudioFeatureExtractor} */ + let feature_extractor; + beforeAll(async () => { + const processor = await AutoProcessor.from_pretrained(model_id); + feature_extractor = processor.feature_extractor; + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "full audio", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_features, input_features_mask } = await feature_extractor(audio); + + // Python ref: shape=(1299, 128), mean=-1.606543, mask_sum=1299 + expect(input_features.dims).toEqual([1, 1299, 128]); + expect(input_features_mask.dims).toEqual([1, 1299]); + + expect(input_features.mean().item()).toBeCloseTo(-1.6065434217453, 5); + expect(input_features.data[0]).toBeCloseTo(-6.907755374908447, 5); + expect(input_features.data[1]).toBeCloseTo(-1.892147541046143, 5); + expect(input_features.data[127]).toBeCloseTo(-3.07513427734375, 5); + expect(input_features.data[128]).toBeCloseTo(-6.907755374908447, 5); + expect(input_features.data[1000]).toBeCloseTo(-1.210815072059631, 5); + expect(input_features.data[input_features.data.length - 1]).toBeCloseTo(-3.489276647567749, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "short audio (1 second)", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_features, input_features_mask } = await feature_extractor(audio.slice(0, 16000)); + + // Python ref: shape=(99, 128), mean=-1.938566 + expect(input_features.dims).toEqual([1, 99, 128]); + expect(input_features_mask.dims).toEqual([1, 99]); + + expect(input_features.mean().item()).toBeCloseTo(-1.938566446304321, 5); + expect(input_features.data[0]).toBeCloseTo(-6.907755374908447, 5); + expect(input_features.data[input_features.data.length - 1]).toBeCloseTo(-2.342944622039795, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "very short audio (50ms)", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_features, input_features_mask } = await feature_extractor(audio.slice(0, 800)); + + // Python ref: shape=(5, 128), mean=-1.612146, mask=[T,T,T,T,F] + expect(input_features.dims).toEqual([1, 5, 128]); + expect(input_features_mask.dims).toEqual([1, 5]); + + expect(input_features.mean().item()).toBeCloseTo(-1.612145781517029, 5); + expect(input_features.data[0]).toBeCloseTo(-6.907755374908447, 5); + // Last frame is masked (zeroed out) + expect(input_features.data[input_features.data.length - 1]).toBeCloseTo(0.0, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "3 second audio", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_features, input_features_mask } = await feature_extractor(audio.slice(0, 48000)); + + // Python ref: shape=(299, 128), mean=-1.686108 + expect(input_features.dims).toEqual([1, 299, 128]); + expect(input_features_mask.dims).toEqual([1, 299]); + + expect(input_features.mean().item()).toBeCloseTo(-1.686108350753784, 5); + expect(input_features.data[0]).toBeCloseTo(-6.907755374908447, 5); + expect(input_features.data[1000]).toBeCloseTo(-1.210815072059631, 5); + expect(input_features.data[input_features.data.length - 1]).toBeCloseTo(-2.798041820526123, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/gemma4/test_image_processing_gemma4.js b/packages/transformers/tests/models/gemma4/test_image_processing_gemma4.js new file mode 100644 index 0000000..6a0c3bb --- /dev/null +++ b/packages/transformers/tests/models/gemma4/test_image_processing_gemma4.js @@ -0,0 +1,110 @@ +import { AutoProcessor, Gemma4ImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe.skip("Gemma4ImageProcessor", () => { + // Load image processor via processor (config is nested in processor_config.json) + const model_id = "onnx-community/gemma-4-E2B-it-ONNX"; + + /** @type {Gemma4ImageProcessor} */ + let image_processor; + beforeAll(async () => { + const processor = await AutoProcessor.from_pretrained(model_id); + image_processor = processor.image_processor; + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "cats (640x480)", + async () => { + const image = await load_cached_image("cats"); + const { pixel_values, image_position_ids, num_soft_tokens_per_image } = await image_processor(image); + + // max_patches = 280 * 9 = 2520, patch_dim = 16*16*3 = 768 + expect(pixel_values.dims).toEqual([1, 2520, 768]); + expect(image_position_ids.dims).toEqual([1, 2520, 2]); + expect(num_soft_tokens_per_image).toEqual([266]); + expect(pixel_values.mean().item()).toBeCloseTo(0.4925, 2); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "tiger (612x408)", + async () => { + const image = await load_cached_image("tiger"); + const { pixel_values, num_soft_tokens_per_image } = await image_processor(image); + + expect(pixel_values.dims).toEqual([1, 2520, 768]); + expect(num_soft_tokens_per_image).toEqual([260]); + expect(pixel_values.mean().item()).toBeCloseTo(0.3586, 2); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "white image (224x224)", + async () => { + const image = await load_cached_image("white_image"); + const { pixel_values, num_soft_tokens_per_image } = await image_processor(image); + + expect(pixel_values.dims).toEqual([1, 2520, 768]); + expect(num_soft_tokens_per_image).toEqual([256]); + // White image: mostly 1.0 values with zero-padding → mean ~0.91 + expect(pixel_values.mean().item()).toBeCloseTo(0.9114, 2); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "gradient (1280x640)", + async () => { + const image = await load_cached_image("gradient_1280x640"); + const { pixel_values, num_soft_tokens_per_image } = await image_processor(image); + + expect(pixel_values.dims).toEqual([1, 2520, 768]); + expect(num_soft_tokens_per_image).toEqual([253]); + expect(pixel_values.mean().item()).toBeCloseTo(0.4514, 2); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batched (cats + tiger)", + async () => { + const cats = await load_cached_image("cats"); + const tiger = await load_cached_image("tiger"); + const { pixel_values, image_position_ids, num_soft_tokens_per_image } = await image_processor([cats, tiger]); + + expect(pixel_values.dims).toEqual([2, 2520, 768]); + expect(image_position_ids.dims).toEqual([2, 2520, 2]); + expect(num_soft_tokens_per_image).toEqual([266, 260]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "position IDs are correct", + async () => { + const image = await load_cached_image("cats"); + const { image_position_ids, num_soft_tokens_per_image } = await image_processor(image); + + // First position should be [0, 0] (col=0, row=0) + expect(Number(image_position_ids.data[0])).toEqual(0); + expect(Number(image_position_ids.data[1])).toEqual(0); + + // Second position should be [1, 0] (col=1, row=0) + expect(Number(image_position_ids.data[2])).toEqual(1); + expect(Number(image_position_ids.data[3])).toEqual(0); + + // Padding positions should be -1 + const num_real_patches = num_soft_tokens_per_image[0] * 9; // 266 * 9 = 2394 + const first_pad_idx = num_real_patches * 2; + expect(Number(image_position_ids.data[first_pad_idx])).toEqual(-1); + expect(Number(image_position_ids.data[first_pad_idx + 1])).toEqual(-1); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/gemma4/test_processor_gemma4.js b/packages/transformers/tests/models/gemma4/test_processor_gemma4.js new file mode 100644 index 0000000..96f8bfa --- /dev/null +++ b/packages/transformers/tests/models/gemma4/test_processor_gemma4.js @@ -0,0 +1,180 @@ +import { AutoProcessor, Gemma4Processor } from "../../../src/transformers.js"; + +import { load_cached_image, load_cached_audio } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe.skip("Gemma4Processor", () => { + const model_id = "onnx-community/gemma-4-E2B-it-ONNX"; + + /** @type {Gemma4Processor} */ + let processor; + + beforeAll(async () => { + processor = await AutoProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + // ===== IMAGE TESTS ===== + + it( + "single image (cats, 640x480)", + async () => { + const image = await load_cached_image("cats"); + const messages = [{ role: "user", content: [{ type: "image" }, { type: "text", text: "Describe." }] }]; + const text = processor.apply_chat_template(messages, { add_generation_prompt: true }); + const { input_ids, pixel_values, image_position_ids, num_soft_tokens_per_image } = await processor(text, image, null, { add_special_tokens: false }); + + // Python ref: pv_shape=(1, 2520, 768), num_soft=[266] + expect(pixel_values.dims).toEqual([1, 2520, 768]); + expect(image_position_ids.dims).toEqual([1, 2520, 2]); + expect(num_soft_tokens_per_image).toEqual([266]); + + // input_ids should contain expanded image tokens + expect(input_ids.dims[0]).toEqual(1); + expect(input_ids.dims[1]).toBeGreaterThan(266); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "single image (tiger, 612x408)", + async () => { + const image = await load_cached_image("tiger"); + const messages = [{ role: "user", content: [{ type: "image" }, { type: "text", text: "Describe." }] }]; + const text = processor.apply_chat_template(messages, { add_generation_prompt: true }); + const { pixel_values, num_soft_tokens_per_image } = await processor(text, image, null, { add_special_tokens: false }); + + // Python ref: pv_shape=(1, 2520, 768), num_soft=[260] + expect(pixel_values.dims).toEqual([1, 2520, 768]); + expect(num_soft_tokens_per_image).toEqual([260]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "single image (white, 224x224)", + async () => { + const image = await load_cached_image("white_image"); + const messages = [{ role: "user", content: [{ type: "image" }, { type: "text", text: "Describe." }] }]; + const text = processor.apply_chat_template(messages, { add_generation_prompt: true }); + const { pixel_values, num_soft_tokens_per_image } = await processor(text, image, null, { add_special_tokens: false }); + + // Python ref: pv_shape=(1, 2520, 768), num_soft=[256] + expect(pixel_values.dims).toEqual([1, 2520, 768]); + expect(num_soft_tokens_per_image).toEqual([256]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "single image (gradient, 1280x640)", + async () => { + const image = await load_cached_image("gradient_1280x640"); + const messages = [{ role: "user", content: [{ type: "image" }, { type: "text", text: "Describe." }] }]; + const text = processor.apply_chat_template(messages, { add_generation_prompt: true }); + const { pixel_values, num_soft_tokens_per_image } = await processor(text, image, null, { add_special_tokens: false }); + + // Python ref: pv_shape=(1, 2520, 768), num_soft=[253] + expect(pixel_values.dims).toEqual([1, 2520, 768]); + expect(num_soft_tokens_per_image).toEqual([253]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batched images (cats + tiger)", + async () => { + const cats = await load_cached_image("cats"); + const tiger = await load_cached_image("tiger"); + const messages = [ + { + role: "user", + content: [{ type: "image" }, { type: "image" }, { type: "text", text: "Compare." }], + }, + ]; + const text = processor.apply_chat_template(messages, { add_generation_prompt: true }); + const { pixel_values, num_soft_tokens_per_image } = await processor(text, [cats, tiger], null, { add_special_tokens: false }); + + // Python ref: pv_shape=(2, 2520, 768), num_soft=[266, 260] + expect(pixel_values.dims).toEqual([2, 2520, 768]); + expect(num_soft_tokens_per_image).toEqual([266, 260]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + // ===== AUDIO TOKEN COUNT TESTS ===== + + it("_compute_audio_num_tokens", () => { + // Python reference values + expect(processor._compute_audio_num_tokens(208000, 16000)).toEqual(325); + expect(processor._compute_audio_num_tokens(16000, 16000)).toEqual(25); + expect(processor._compute_audio_num_tokens(800, 16000)).toEqual(1); + expect(processor._compute_audio_num_tokens(48000, 16000)).toEqual(75); + expect(processor._compute_audio_num_tokens(160000, 16000)).toEqual(250); + expect(processor._compute_audio_num_tokens(480000, 16000)).toEqual(750); + expect(processor._compute_audio_num_tokens(100, 16000)).toEqual(0); + }); + + // ===== AUDIO + TEXT TESTS ===== + + it( + "audio + text", + async () => { + const audio = await load_cached_audio("mlk"); + const messages = [{ role: "user", content: [{ type: "audio" }, { type: "text", text: "Transcribe." }] }]; + const text = processor.apply_chat_template(messages, { add_generation_prompt: true }); + + const { input_ids, input_features, input_features_mask } = await processor(text, null, audio, { add_special_tokens: false }); + + expect(input_features.dims).toEqual([1, 1299, 128]); + expect(input_features_mask.dims).toEqual([1, 1299]); + + // 325 audio tokens should be expanded in input_ids + expect(input_ids.dims[1]).toBeGreaterThan(325); + }, + MAX_TEST_EXECUTION_TIME, + ); + + // ===== IMAGE + AUDIO + TEXT ===== + + it( + "image + audio + text", + async () => { + const image = await load_cached_image("cats"); + const audio = await load_cached_audio("mlk"); + const messages = [ + { + role: "user", + content: [{ type: "image" }, { type: "audio" }, { type: "text", text: "Describe and transcribe." }], + }, + ]; + const text = processor.apply_chat_template(messages, { add_generation_prompt: true }); + + const { input_ids, pixel_values, image_position_ids, num_soft_tokens_per_image, input_features } = await processor(text, image, audio, { add_special_tokens: false }); + + // Image outputs + expect(pixel_values.dims).toEqual([1, 2520, 768]); + expect(image_position_ids.dims).toEqual([1, 2520, 2]); + expect(num_soft_tokens_per_image).toEqual([266]); + + // Audio outputs + expect(input_features.dims).toEqual([1, 1299, 128]); + + // Combined input_ids should have both expanded image (266) and audio (325) tokens + expect(input_ids.dims[1]).toBeGreaterThan(266 + 325); + }, + MAX_TEST_EXECUTION_TIME, + ); + + // ===== TEXT-ONLY ===== + + it("text only", async () => { + const messages = [{ role: "user", content: [{ type: "text", text: "Hello" }] }]; + const text = processor.apply_chat_template(messages, { add_generation_prompt: true }); + const { input_ids } = await processor(text, null, null, { add_special_tokens: false }); + + // Python ref: ids_shape=(1, 10) + expect(input_ids.dims).toEqual([1, 10]); + }); + }); +}; diff --git a/packages/transformers/tests/models/glm/test_modeling_glm.js b/packages/transformers/tests/models/glm/test_modeling_glm.js new file mode 100644 index 0000000..14660bb --- /dev/null +++ b/packages/transformers/tests/models/glm/test_modeling_glm.js @@ -0,0 +1,51 @@ +import { PreTrainedTokenizer, GlmForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("GlmForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-GlmForCausalLM"; + /** @type {GlmForCausalLM} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GlmForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[23582n, 5797n, 38238n, 24486n, 36539n, 34489n, 6948n, 34489n, 6948n, 16014n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [59246n, 23582n, 5797n, 38238n, 24486n, 36539n, 34489n, 6948n, 34489n, 6948n], + [23582n, 2901n, 39936n, 25036n, 55411n, 10337n, 3424n, 39183n, 30430n, 37285n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/glm46v/test_processor_glm46v.js b/packages/transformers/tests/models/glm46v/test_processor_glm46v.js new file mode 100644 index 0000000..3594f06 --- /dev/null +++ b/packages/transformers/tests/models/glm46v/test_processor_glm46v.js @@ -0,0 +1,43 @@ +import { AutoProcessor, Glm46VProcessor, RawImage } from "../../../src/transformers.js"; + +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("Glm46VProcessor", () => { + const model_id = "onnx-community/GLM-OCR-ONNX"; + + /** @type {Glm46VProcessor} */ + let processor; + let image; + + beforeAll(async () => { + processor = await AutoProcessor.from_pretrained(model_id); + // Create a test image of size 666 (height) x 540 (width) + image = new RawImage(new Uint8ClampedArray(666 * 540 * 3), 540, 666, 3); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "Image and text", + async () => { + const conversation = [ + { + role: "user", + content: [{ type: "image" }, { type: "text", text: "Text Recognition:" }], + }, + ]; + + const text = processor.apply_chat_template(conversation, { + add_generation_prompt: true, + }); + const { input_ids, attention_mask, pixel_values, image_grid_thw } = await processor(text, image); + + expect(input_ids.dims).toEqual([1, 468]); + expect(attention_mask.dims).toEqual([1, 468]); + expect(pixel_values.dims).toEqual([1824, 1176]); + expect(image_grid_thw.dims).toEqual([1, 3]); + expect(image_grid_thw.tolist()).toEqual([[1n, 48n, 38n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/glm_moe_dsa/test_modeling_glm_moe_dsa.js b/packages/transformers/tests/models/glm_moe_dsa/test_modeling_glm_moe_dsa.js new file mode 100644 index 0000000..5464b16 --- /dev/null +++ b/packages/transformers/tests/models/glm_moe_dsa/test_modeling_glm_moe_dsa.js @@ -0,0 +1,51 @@ +import { PreTrainedTokenizer, GlmMoeDsaForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("GlmMoeDsaForCausalLM", () => { + const model_id = "onnx-internal-testing/tiny-random-GlmMoeDsaForCausalLM"; + /** @type {GlmMoeDsaForCausalLM} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GlmMoeDsaForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[14978n, 147266n, 147266n, 147266n, 147266n, 147266n, 147266n, 147266n, 147266n, 147266n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [154820n, 14978n, 147266n, 147266n, 147266n, 147266n, 147266n, 147266n, 147266n, 147266n], + [14978n, 1879n, 114263n, 57689n, 61053n, 61053n, 61053n, 61053n, 61053n, 61053n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/glm_ocr/test_modeling_glm_ocr.js b/packages/transformers/tests/models/glm_ocr/test_modeling_glm_ocr.js new file mode 100644 index 0000000..7090f26 --- /dev/null +++ b/packages/transformers/tests/models/glm_ocr/test_modeling_glm_ocr.js @@ -0,0 +1,64 @@ +import { GlmOcrForConditionalGeneration, Glm46VProcessor, RawImage } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + const CONVERSATION_WITH_IMAGE = [ + { + role: "user", + content: [{ type: "image" }, { type: "text", text: "Describe this image." }], + }, + ]; + + // Empty white image + const dims = [224, 224, 3]; + const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); + + describe("GlmOcrForConditionalGeneration", () => { + const model_id = "onnx-internal-testing/tiny-random-GlmOcrForConditionalGeneration"; + + /** @type {GlmOcrForConditionalGeneration} */ + let model; + /** @type {Glm46VProcessor} */ + let processor; + beforeAll(async () => { + model = await GlmOcrForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + processor = await Glm46VProcessor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + const text = processor.apply_chat_template(CONVERSATION_WITH_IMAGE, { + add_generation_prompt: true, + }); + const inputs = await processor(text, image); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 76, 59392]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "text + image (batch_size=1)", + async () => { + const text = processor.apply_chat_template(CONVERSATION_WITH_IMAGE, { + add_generation_prompt: true, + }); + const inputs = await processor(text, image); + const generate_ids = await model.generate({ + ...inputs, + max_new_tokens: 10, + }); + + const new_tokens = generate_ids.slice(null, [inputs.input_ids.dims.at(-1), null]); + expect(new_tokens.tolist()).toEqual([[3875n, 22214n, 21946n, 27197n, 15231n, 15231n, 15231n, 15231n, 15231n, 15231n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/glpn/test_image_processing_glpn.js b/packages/transformers/tests/models/glpn/test_image_processing_glpn.js new file mode 100644 index 0000000..9348716 --- /dev/null +++ b/packages/transformers/tests/models/glpn/test_image_processing_glpn.js @@ -0,0 +1,48 @@ +import { AutoImageProcessor, GLPNFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // GLPNFeatureExtractor + // - tests `size_divisor` and no size (size_divisor=32) + describe("GLPNFeatureExtractor", () => { + const model_id = "Xenova/glpn-kitti"; + + /** @type {GLPNFeatureExtractor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "multiple of size_divisor", + async () => { + const image = await load_cached_image("cats"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + expect(pixel_values.dims).toEqual([1, 3, 480, 640]); + expect(pixel_values.mean().item()).toBeCloseTo(0.5186172404123327, 6); + + expect(original_sizes).toEqual([[480, 640]]); + expect(reshaped_input_sizes).toEqual([[480, 640]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "non-multiple of size_divisor", + async () => { + // Tests input which is not a multiple of 32 ([408, 612] -> [384, 608]) + const image = await load_cached_image("tiger"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 384, 608]); + expect(pixel_values.mean().item()).toBeCloseTo(0.38628831535989555, 6); + + expect(original_sizes).toEqual([[408, 612]]); + expect(reshaped_input_sizes).toEqual([[384, 608]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/gpt2/test_modeling_gpt2.js b/packages/transformers/tests/models/gpt2/test_modeling_gpt2.js new file mode 100644 index 0000000..a2f9374 --- /dev/null +++ b/packages/transformers/tests/models/gpt2/test_modeling_gpt2.js @@ -0,0 +1,51 @@ +import { GPT2Tokenizer, GPT2LMHeadModel } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("GPT2LMHeadModel", () => { + const model_id = "hf-internal-testing/tiny-random-GPT2LMHeadModel"; + /** @type {GPT2LMHeadModel} */ + let model; + /** @type {GPT2Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GPT2LMHeadModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await GPT2Tokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[258n, 863n, 79n, 79n, 79n, 79n, 79n, 79n, 79n, 243n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 258n, 863n, 79n, 79n, 79n, 79n, 79n, 79n], + [258n, 863n, 79n, 269n, 813n, 813n, 813n, 813n, 813n, 813n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/gpt2/test_tokenization_gpt2.js b/packages/transformers/tests/models/gpt2/test_tokenization_gpt2.js new file mode 100644 index 0000000..6e72676 --- /dev/null +++ b/packages/transformers/tests/models/gpt2/test_tokenization_gpt2.js @@ -0,0 +1,465 @@ +import { GPT2Tokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, SENTENCEPIECE_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = GPT2Tokenizer; +export const TEST_CONFIG = { + // - clean_up_tokenization_spaces=true + // - default pretokenization regex + "Xenova/gpt2": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [2437, 389, 345, 1804, 30], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [1639, 815, 1053, 1760, 428], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["01", "23", "45", "67", "89", "Ġ0", "Ġ1", "Ġ2", "Ġ3", "Ġ4", "Ġ5", "Ġ6", "Ġ7", "Ġ8", "Ġ9", "Ġ10", "Ġ100", "Ġ1000"], + ids: [486, 1954, 2231, 3134, 4531, 657, 352, 362, 513, 604, 642, 718, 767, 807, 860, 838, 1802, 8576], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], + ids: [464, 1664, 373, 9393, 287, 1584, 13], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [32, 198, 1183, 37867, 1462, 8348, 67, 7061, 67, 286, 11, 460, 470, 13], + decoded: "A\n'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"], + ids: [4299, 1388, 33529, 198, 197, 6603], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "to", "String", "();", "\u010a", "to", "String", "();"], + ids: [1616, 257, 796, 26181, 13, 1462, 10100, 9783, 198, 1462, 10100, 9783], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [1212, 198, 198, 271, 198, 64, 198, 9288, 13], + decoded: "This\n\nis\na\ntest.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9", "d", ",", "running"], + ids: [4944, 42949, 2634, 67, 11, 20270], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [16, 188, 17, 4210, 18], + decoded: "1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [15496, 2159], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [31373, 995], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [37955, 162, 112, 119, 21410, 40367, 253, 164, 108, 249, 42468], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], + ids: [220, 220, 3756, 2272], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], + ids: [9535, 4386, 2272, 220, 220, 220], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [17250, 220, 18435], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [9288, 720, 16, 371, 17, 1303, 18, 10432, 19, 4248, 20, 38221, 21, 2343, 224, 96, 22, 2343, 224, 117, 23, 2343, 224, 109, 24, 1332], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [40, 5839, 281, 17180, 329, 720, 16, 13, 405, 379, 262, 3650, 13], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], + ids: [5832, 1399, 220, 220], + decoded: "you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [5832, 1399, 4603], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [5832, 1399, 1849, 1849, 5832, 1399, 4603], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [732, 1447, 27332, 121, 252, 5743, 27332, 121, 252, 1339], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [5008, 223, 1212, 11019, 223, 271, 11019, 223, 64, 11019, 223, 9288, 11019, 223, 13], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + SPECIAL_WITH_TRAILING_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_WITH_TRAILING_WHITESPACE, + tokens: ["<", "s", ">", "\u010a"], + ids: [27, 82, 29, 198], + decoded: "\n", + }, + SPECIAL_SURROUNDED_BY_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_SURROUNDED_BY_WHITESPACE, + tokens: ["\u0120", "\u0120test", "\u0120", "\u0120"], + ids: [7359, 82, 29, 1332, 7359, 82, 29, 220], + decoded: " test ", + }, + SPECIAL_NO_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_NO_WHITESPACE, + tokens: ["", "test", ""], + ids: [3556, 82, 29, 9288, 3556, 82, 29], + decoded: "test", + }, + }, + // - clean_up_tokenization_spaces=false + // - custom pretokenization regex + "Xenova/gpt-4": { + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [32, 198, 3358, 11261, 998, 20837, 67, 4708, 67, 315, 11, 649, 956, 13], + decoded: "A\n'll !!to?'d''d of, can't.", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".toString", "();\u010a", "toString", "();"], + ids: [1169, 264, 284, 2909, 5180, 545, 6712, 2178], + decoded: "let a = obj.toString();\ntoString();", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [1985, 400, 16, 432, 17, 674, 18, 13281, 19, 7083, 20, 72588, 21, 2928, 224, 96, 22, 90891, 23, 2928, 224, 109, 24, 1296], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120", "\u00ef\u00bd\u0140", "\u0120edge", "\u0120", "\u00ef\u00bd\u0140", "\u0120case"], + ids: [906, 2668, 220, 21909, 6964, 220, 21909, 1162], + decoded: "weird \uff5e edge \uff5e case", + }, + }, + "Xenova/gpt-4o": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["012", "345", "678", "9", "Ġ", "0", "Ġ", "1", "Ġ", "2", "Ġ", "3", "Ġ", "4", "Ġ", "5", "Ġ", "6", "Ġ", "7", "Ġ", "8", "Ġ", "9", "Ġ", "10", "Ġ", "100", "Ġ", "100", "0"], + ids: [19267, 22901, 30833, 24, 220, 15, 220, 16, 220, 17, 220, 18, 220, 19, 220, 20, 220, 21, 220, 22, 220, 23, 220, 24, 220, 702, 220, 1353, 220, 1353, 15], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "201", "6", "."], + ids: [976, 3175, 673, 24303, 306, 220, 667, 21, 13], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can't", "."], + ids: [32, 198, 6090, 17131, 935, 48511, 67, 5830, 67, 328, 11, 8535, 13], + decoded: "A\n'll !!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():\u010a", "\u0109pass"], + ids: [1314, 2758, 8595, 100653], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".to", "String", "();\u010a", "to", "String", "();"], + ids: [1347, 261, 314, 4099, 3552, 916, 740, 935, 916, 4177], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [2500, 279, 276, 198, 64, 198, 3190, 13], + decoded: "This\n\nis\na\ntest.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",r", "unning"], + ids: [2926, 72517, 6383, 33654, 11244], + decoded: "UNwant\u00e9d,running", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [32479, 1616, 7910, 7856, 249, 3221], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [256, 8117, 4918], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tr", "ailing", "\u0120space", "\u0120\u0120\u0120"], + ids: [371, 24408, 4918, 271], + decoded: "trailing space ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2\u0124", "\u00b1", "9", "\u0120test"], + ids: [3190, 548, 16, 460, 17, 1069, 18, 7950, 19, 8989, 20, 123814, 21, 59790, 96, 22, 73406, 23, 59790, 109, 24, 1746], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [13320, 1131, 256], + decoded: "you\u2026 ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef\u00bd\u0140", "\u0120edge", "\u0120\u00ef\u00bd\u0140", "\u0120case"], + ids: [854, 2716, 105665, 11165, 105665, 1890], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138\u0123", "is", "\u0120\u00e2\u0138\u0123", "a", "\u0120\u00e2\u0138\u0123", "test", "\u0120\u00e2\u0138\u0123", "."], + ids: [6762, 223, 2500, 39960, 276, 39960, 64, 39960, 3190, 39960, 13], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + SPECIAL_WITH_TRAILING_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_WITH_TRAILING_WHITESPACE, + tokens: ["\u010a"], + ids: [101950, 523], + decoded: "\n", + }, + }, + "Xenova/claude-tokenizer": { + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "toString", "();", "\u010a", "toString", "();"], + ids: [1785, 269, 284, 2652, 18, 26492, 4370, 203, 26492, 4370], + decoded: "let a = obj.toString();\ntoString();", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], + ids: [2359, 17571, 37911, 16, 7889], + decoded: "UNwant\u00e9d,running", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [14706, 37675, 2471, 56904, 15959, 254, 5977], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "\u0120space", "\u0120\u0120\u0120"], + ids: [40110, 3384, 264], + decoded: "trailing space ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [765, 734, 21, 487, 22, 379, 23, 36714, 24, 13206, 25, 2455, 103, 26, 4937, 229, 101, 27, 4937, 229, 122, 28, 4937, 229, 114, 29, 722], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "...", "\u0120\u0120"], + ids: [6773, 1174, 261], + decoded: "you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "...", "\u0120\u0120"], + ids: [6773, 1174, 261], + decoded: "you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "...", "\u0120", "\u0120you", "...", "\u0120\u0120"], + ids: [6773, 1174, 225, 583, 1174, 261], + decoded: "you... you... ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120~", "\u0120edge", "\u0120~", "\u0120case"], + ids: [798, 2650, 6217, 4915, 6217, 1544], + decoded: "weird ~ edge ~ case", + }, + }, + "bigcode/santacoder": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "Ġ", "0", "Ġ", "1", "Ġ", "2", "Ġ", "3", "Ġ", "4", "Ġ", "5", "Ġ", "6", "Ġ", "7", "Ġ", "8", "Ġ", "9", "Ġ", "1", "0", "Ġ", "1", "0", "0", "Ġ", "1", "0", "0", "0"], + ids: [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 207, 15, 207, 16, 207, 17, 207, 18, 207, 19, 207, 20, 207, 21, 207, 22, 207, 23, 207, 24, 207, 16, 15, 207, 16, 15, 15, 207, 16, 15, 15, 15], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120fo", "unded", "\u0120in", "\u0120", "2", "0", "1", "6", "."], + ids: [2111, 10107, 2501, 17436, 7584, 319, 207, 17, 15, 16, 21, 13], + decoded: "The company was founded in 2016.", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [8715, 24543, 1825, 34717, 37452, 236, 4343], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bo", "ught", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "0", "0", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [40, 12307, 10310, 743, 29806, 408, 763, 16, 13, 15, 15, 869, 331, 2823, 13], + decoded: "I bought an apple for $1.00 at the store.", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120edge", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120case"], + ids: [1850, 4427, 207, 29217, 239, 4959, 207, 29217, 239, 1210], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120", "\u00e2\u0138", "\u0123", "is", "\u0120", "\u00e2\u0138", "\u0123", "a", "\u0120", "\u00e2\u0138", "\u0123", "test", "\u0120", "\u00e2\u0138", "\u0123", "."], + ids: [3718, 210, 3456, 207, 3718, 210, 280, 207, 3718, 210, 64, 207, 3718, 210, 706, 207, 3718, 210, 13], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + }, + "Xenova/CodeGPT-tokenizer": { + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [25506, 165, 115, 122, 5137, 43415, 256, 20679, 252, 13283], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "\u0120space", "\u0120", "\u0120", "\u0120"], + ids: [15584, 3497, 223, 223, 223], + decoded: "trailing space ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [13953, 29502, 129, 257, 129, 257], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [13953, 29502, 129, 257, 129, 257, 13953, 29502, 129, 257, 129, 257], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + }, + "huggingface-course/codeparrot-ds": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0123456789", "Ġ0", "Ġ1", "Ġ2", "Ġ3", "Ġ4", "Ġ5", "Ġ6", "Ġ7", "Ġ8", "Ġ9", "Ġ10", "Ġ100", "Ġ1000"], + ids: [25218, 443, 396, 554, 869, 1163, 1462, 1911, 2624, 2070, 2837, 2009, 3038, 4764], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120fo", "unded", "\u0120in", "\u01202016", "."], + ids: [2096, 16502, 1442, 11689, 7865, 253, 8780, 14], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!", "!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [33, 173, 6402, 905, 1, 403, 15227, 68, 589, 68, 311, 12, 796, 1059, 14], + decoded: "A\n'll!!to?'d''d of, can't.", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "toString", "();", "\u010a", "toString", "();"], + ids: [2047, 231, 233, 1300, 14, 30494, 16248, 173, 30494, 16248], + decoded: "let a = obj.toString();\ntoString();", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [20185, 43799, 120, 3994, 37782, 211, 15933, 207, 11130], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "\u0120space", "\u0120\u0120\u0120"], + ids: [17031, 3000, 216], + decoded: "trailing space ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2", "\u0124\u00ac", "4", "\u0120\u00c2", "\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [1824, 3748, 17, 683, 18, 294, 19, 5161, 28898, 20, 23446, 97, 21, 23446, 99, 22, 5161, 182, 97, 23, 5161, 182, 118, 24, 5161, 182, 110, 25, 1737], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bo", "ught", "\u0120an", "\u0120app", "le", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [41, 772, 8272, 309, 870, 239, 296, 3748, 17, 14, 543, 815, 256, 2689, 14], + decoded: "I bought an apple for $1.00 at the store.", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [955, 6075, 179, 166, 122, 210, 2703, 179, 166, 122, 210, 1539], + decoded: "weird \uff5e edge \uff5e case", + }, + }, +}; + +// Test that tokenizer type can be inferred (`type: "BPE"` is missing) +TEST_CONFIG["openai-community/gpt2"] = TEST_CONFIG["Xenova/gpt2"]; diff --git a/packages/transformers/tests/models/gpt_bigcode/test_modeling_gpt_bigcode.js b/packages/transformers/tests/models/gpt_bigcode/test_modeling_gpt_bigcode.js new file mode 100644 index 0000000..3cc5c4f --- /dev/null +++ b/packages/transformers/tests/models/gpt_bigcode/test_modeling_gpt_bigcode.js @@ -0,0 +1,51 @@ +import { GPT2Tokenizer, GPTBigCodeForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("GPTBigCodeForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-GPTBigCodeForCausalLM"; + /** @type {GPTBigCodeForCausalLM} */ + let model; + /** @type {GPT2Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GPTBigCodeForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await GPT2Tokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[258n, 863n, 79n, 79n, 79n, 79n, 79n, 79n, 79n, 79n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 258n, 863n, 79n, 79n, 79n, 79n, 79n, 79n], + [258n, 863n, 79n, 269n, 813n, 832n, 93n, 93n, 93n, 93n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/gpt_neo/test_modeling_gpt_neo.js b/packages/transformers/tests/models/gpt_neo/test_modeling_gpt_neo.js new file mode 100644 index 0000000..3557b6a --- /dev/null +++ b/packages/transformers/tests/models/gpt_neo/test_modeling_gpt_neo.js @@ -0,0 +1,51 @@ +import { GPT2Tokenizer, GPTNeoForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("GPTNeoForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-GPTNeoForCausalLM"; + /** @type {GPTNeoForCausalLM} */ + let model; + /** @type {GPT2Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GPTNeoForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await GPT2Tokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[258n, 863n, 79n, 79n, 79n, 949n, 949n, 949n, 949n, 949n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 258n, 863n, 79n, 79n, 79n, 949n, 949n, 949n], + [258n, 863n, 79n, 269n, 813n, 849n, 849n, 849n, 849n, 849n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/gpt_neox/test_modeling_gpt_neox.js b/packages/transformers/tests/models/gpt_neox/test_modeling_gpt_neox.js new file mode 100644 index 0000000..9d340f9 --- /dev/null +++ b/packages/transformers/tests/models/gpt_neox/test_modeling_gpt_neox.js @@ -0,0 +1,51 @@ +import { GPTNeoXTokenizer, GPTNeoXForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("GPTNeoXForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-GPTNeoXForCausalLM"; + /** @type {GPTNeoXForCausalLM} */ + let model; + /** @type {GPTNeoXTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GPTNeoXForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await GPTNeoXTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[259n, 864n, 80n, 881n, 502n, 895n, 938n, 668n, 502n, 895n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 259n, 864n, 80n, 881n, 502n, 895n, 938n, 668n], + [259n, 864n, 80n, 270n, 814n, 522n, 112n, 268n, 503n, 468n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/gptj/test_modeling_gptj.js b/packages/transformers/tests/models/gptj/test_modeling_gptj.js new file mode 100644 index 0000000..ee5a742 --- /dev/null +++ b/packages/transformers/tests/models/gptj/test_modeling_gptj.js @@ -0,0 +1,51 @@ +import { GPTNeoXTokenizer, GPTJForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("GPTJForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-GPTJForCausalLM"; + /** @type {GPTJForCausalLM} */ + let model; + /** @type {GPTNeoXTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GPTJForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await GPTNeoXTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[258n, 863n, 79n, 102n, 401n, 773n, 889n, 159n, 957n, 869n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 258n, 863n, 79n, 102n, 401n, 773n, 889n, 159n], + [258n, 863n, 79n, 269n, 813n, 879n, 175n, 39n, 141n, 1000n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/granite/test_modeling_granite.js b/packages/transformers/tests/models/granite/test_modeling_granite.js new file mode 100644 index 0000000..82e6b8c --- /dev/null +++ b/packages/transformers/tests/models/granite/test_modeling_granite.js @@ -0,0 +1,50 @@ +import { GPT2Tokenizer, GraniteForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("GraniteForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-GraniteForCausalLM"; + /** @type {GraniteForCausalLM} */ + let model; + /** @type {GPT2Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GraniteForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await GPT2Tokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[7656n, 39727n, 33077n, 9643n, 30539n, 47869n, 48739n, 15085n, 9203n, 14020n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 7656n, 39727n, 33077n, 9643n, 30539n, 47869n, 48739n, 15085n, 9203n], + [7656n, 5788n, 17835n, 13234n, 7592n, 21471n, 30537n, 23023n, 43450n, 4824n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/granite_speech/test_feature_extraction_granite_speech.js b/packages/transformers/tests/models/granite_speech/test_feature_extraction_granite_speech.js new file mode 100644 index 0000000..2e473ff --- /dev/null +++ b/packages/transformers/tests/models/granite_speech/test_feature_extraction_granite_speech.js @@ -0,0 +1,53 @@ +import { AutoFeatureExtractor, GraniteSpeechFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_audio } from "../../asset_cache.js"; +import { MAX_FEATURE_EXTRACTOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("GraniteSpeechFeatureExtractor", () => { + const model_id = "onnx-community/granite-4.0-1b-speech-ONNX"; + + /** @type {GraniteSpeechFeatureExtractor} */ + let feature_extractor; + beforeAll(async () => { + feature_extractor = await AutoFeatureExtractor.from_pretrained(model_id); + }, MAX_FEATURE_EXTRACTOR_LOAD_TIME); + + it( + "full audio", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_features } = await feature_extractor(audio); + + // Shape: [1, 650, 160] + expect(input_features.dims).toEqual([1, 650, 160]); + + expect(input_features.mean().item()).toBeCloseTo(0.709274411201477, 3); + expect(input_features.data[0]).toBeCloseTo(0.726300835609436, 3); + expect(input_features.data[1]).toBeCloseTo(0.683963894844055, 3); + expect(input_features.data[159]).toBeCloseTo(0.264412879943848, 3); + expect(input_features.data[160]).toBeCloseTo(0.544037103652954, 3); + expect(input_features.data[1000]).toBeCloseTo(0.773821651935577, 3); + expect(input_features.data[10000]).toBeCloseTo(-0.2131507396698, 3); + expect(input_features.data[input_features.data.length - 1]).toBeCloseTo(0.233379304409027, 3); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "short audio", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_features } = await feature_extractor(audio.slice(0, 1000)); + + // Shape: [1, 3, 160] + expect(input_features.dims).toEqual([1, 3, 160]); + + expect(input_features.mean().item()).toBeCloseTo(0.634890496730804, 3); + expect(input_features.data[0]).toBeCloseTo(0.726300835609436, 3); + expect(input_features.data[input_features.data.length - 1]).toBeCloseTo(0.301110625267029, 3); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/granite_speech/test_modeling_granite_speech.js b/packages/transformers/tests/models/granite_speech/test_modeling_granite_speech.js new file mode 100644 index 0000000..e35dcad --- /dev/null +++ b/packages/transformers/tests/models/granite_speech/test_modeling_granite_speech.js @@ -0,0 +1,65 @@ +import { GraniteSpeechProcessor, GraniteSpeechForConditionalGeneration } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("GraniteSpeechForConditionalGeneration", () => { + const model_id = "onnx-internal-testing/tiny-random-GraniteSpeechForConditionalGeneration"; + + /** @type {GraniteSpeechForConditionalGeneration} */ + let model; + /** @type {GraniteSpeechProcessor} */ + let processor; + beforeAll(async () => { + model = await GraniteSpeechForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + processor = await GraniteSpeechProcessor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "text-only generation", + async () => { + const messages = [{ role: "user", content: "What is the capital of France?" }]; + const text = processor.apply_chat_template(messages, { + add_generation_prompt: true, + tokenize: false, + }); + + const inputs = processor.tokenizer(text, { add_special_tokens: false }); + const outputs = await model.generate({ + ...inputs, + max_new_tokens: 10, + }); + + expect(outputs.tolist()).toEqual([[6584n, 25n, 3639n, 374n, 279n, 6864n, 315n, 9822n, 30n, 198n, 36660n, 3931n, 2891n, 25n, 84130n, 50307n, 50306n, 77183n, 9714n, 7869n, 2959n, 36251n, 63351n, 16719n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "text + audio generation", + async () => { + // 9600 samples → 60 mel frames → 30 encoder frames (= context_size) + const audio = new Float32Array(9600); + + const messages = [{ role: "user", content: "<|audio|>hello" }]; + const text = processor.apply_chat_template(messages, { + add_generation_prompt: true, + tokenize: false, + }); + + const inputs = await processor(text, audio); + const outputs = await model.generate({ + ...inputs, + max_new_tokens: 10, + }); + + expect(outputs.tolist()).toEqual([[6584n, 25n, 220n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 100352n, 15339n, 198n, 36660n, 3931n, 2891n, 25n, 91004n, 27904n, 16003n, 2165n, 96148n, 27934n, 44295n, 29186n, 7485n, 87373n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/granite_speech/test_processor_granite_speech.js b/packages/transformers/tests/models/granite_speech/test_processor_granite_speech.js new file mode 100644 index 0000000..29fcfff --- /dev/null +++ b/packages/transformers/tests/models/granite_speech/test_processor_granite_speech.js @@ -0,0 +1,59 @@ +import { AutoProcessor, GraniteSpeechProcessor } from "../../../src/transformers.js"; + +import { load_cached_audio } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("GraniteSpeechProcessor", () => { + const model_id = "onnx-community/granite-4.0-1b-speech-ONNX"; + + /** @type {GraniteSpeechProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "text + audio", + async () => { + const audio = await load_cached_audio("mlk"); + + const messages = [{ role: "user", content: "<|audio|>can you transcribe the speech into a written format?" }]; + const text = processor.tokenizer.apply_chat_template(messages, { + add_generation_prompt: true, + tokenize: false, + }); + + const { input_ids, input_features } = await processor(text, audio); + + // input_ids: 151 tokens (text + expanded audio tokens) + expect(input_ids.dims).toEqual([1, 151]); + + // input_features: [1, 650, 160] + expect(input_features.dims).toEqual([1, 650, 160]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "tokens and features stay aligned for boundary audio lengths", + async () => { + const boundary_lengths = [4960, 9760, 139360]; // For L = 160 * (30k + 1) + const { projector_window_size, projector_downsample_rate } = processor.feature_extractor.config; + const effective_window_size = Math.floor(projector_window_size / projector_downsample_rate); + + for (const L of boundary_lengths) { + const audio = new Float32Array(L); + const { input_features } = await processor.feature_extractor(audio); + + const predicted_tokens = processor._get_num_audio_features(L); + const time_steps = input_features.dims[1]; + const projector_tokens = Math.ceil(time_steps / projector_window_size) * effective_window_size; + + expect(projector_tokens).toEqual(predicted_tokens); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/granitemoehybrid/test_modeling_granitemoehybrid.js b/packages/transformers/tests/models/granitemoehybrid/test_modeling_granitemoehybrid.js new file mode 100644 index 0000000..e2a834d --- /dev/null +++ b/packages/transformers/tests/models/granitemoehybrid/test_modeling_granitemoehybrid.js @@ -0,0 +1,51 @@ +import { PreTrainedTokenizer, GraniteMoeHybridForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("GraniteMoeHybridForCausalLM", () => { + const model_id = "onnx-internal-testing/tiny-random-GraniteMoeHybridForCausalLM"; + /** @type {GraniteMoeHybridForCausalLM} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await GraniteMoeHybridForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[15339n, 70197n, 8290n, 70197n, 8290n, 70197n, 8290n, 70197n, 8290n, 70197n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [100256n, 15339n, 70197n, 8290n, 70197n, 8290n, 70197n, 8290n, 70197n, 8290n], + [15339n, 1917n, 88135n, 6324n, 88135n, 6324n, 88135n, 6324n, 88135n, 6324n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/grounding_dino/test_modeling_grounding_dino.js b/packages/transformers/tests/models/grounding_dino/test_modeling_grounding_dino.js new file mode 100644 index 0000000..177f62a --- /dev/null +++ b/packages/transformers/tests/models/grounding_dino/test_modeling_grounding_dino.js @@ -0,0 +1,44 @@ +import { GroundingDinoProcessor, GroundingDinoForObjectDetection, RawImage } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + const text = "a cat."; // NB: text query needs to be lowercased + end with a dot + + // Empty white image + const dims = [224, 224, 3]; + const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); + + describe("GroundingDinoForObjectDetection", () => { + const model_id = "hf-internal-testing/tiny-random-GroundingDinoForObjectDetection"; + + /** @type {GroundingDinoForObjectDetection} */ + let model; + /** @type {GroundingDinoProcessor} */ + let processor; + beforeAll(async () => { + model = await GroundingDinoForObjectDetection.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + processor = await GroundingDinoProcessor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + const inputs = await processor(image, text); + const { d_model, num_queries } = model.config; + + const { logits, pred_boxes } = await model(inputs); + expect(logits.dims).toEqual([1, num_queries, d_model]); + expect(pred_boxes.dims).toEqual([1, num_queries, 4]); + expect(logits.max().item()).toBeCloseTo(56.237613677978516, 2); + expect(logits.min().item()).toEqual(-Infinity); + expect(pred_boxes.mean().item()).toBeCloseTo(0.2500016987323761, 4); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/grounding_dino/test_processor_grounding_dino.js b/packages/transformers/tests/models/grounding_dino/test_processor_grounding_dino.js new file mode 100644 index 0000000..e3de567 --- /dev/null +++ b/packages/transformers/tests/models/grounding_dino/test_processor_grounding_dino.js @@ -0,0 +1,57 @@ +import { AutoProcessor, full, GroundingDinoProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + const model_id = "hf-internal-testing/tiny-random-GroundingDinoForObjectDetection"; + + describe("GroundingDinoProcessor", () => { + /** @type {GroundingDinoProcessor} */ + let processor; + let images = {}; + + beforeAll(async () => { + processor = await AutoProcessor.from_pretrained(model_id); + images = { + white_image: await load_cached_image("white_image"), + }; + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "Single image & text", + async () => { + const { input_ids, pixel_values } = await processor(images.white_image, "a cat."); + expect(input_ids.dims).toEqual([1, 5]); + expect(pixel_values.dims).toEqual([1, 3, 800, 800]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "post_process_grounded_object_detection", + async () => { + const outputs = { + logits: full([1, 900, 256], 0.5), + pred_boxes: full([1, 900, 4], 0.5), + }; + const inputs = { + input_ids: full([1, 5], 1n), + }; + + const results = processor.post_process_grounded_object_detection(outputs, inputs.input_ids, { + box_threshold: 0.3, + text_threshold: 0.3, + target_sizes: [[360, 240]], + }); + const { scores, boxes, labels } = results[0]; + expect(scores).toHaveLength(900); + expect(boxes).toHaveLength(900); + expect(labels).toHaveLength(900); + expect(boxes[0]).toEqual([60, 90, 180, 270]); + expect(scores[0]).toBeCloseTo(0.622459352016449, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/helium/test_modeling_helium.js b/packages/transformers/tests/models/helium/test_modeling_helium.js new file mode 100644 index 0000000..1bdf886 --- /dev/null +++ b/packages/transformers/tests/models/helium/test_modeling_helium.js @@ -0,0 +1,51 @@ +import { PreTrainedTokenizer, HeliumForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("HeliumForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-HeliumForCausalLM"; + /** @type {HeliumForCausalLM} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await HeliumForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[1n, 456n, 5660n, 1700n, 1486n, 37744n, 35669n, 39396n, 12024n, 32253n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [3n, 1n, 456n, 5660n, 1700n, 1486n, 37744n, 35669n, 39396n, 12024n], + [1n, 456n, 5660n, 998n, 6136n, 2080n, 172n, 8843n, 40579n, 23953n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/idefics3/test_image_processing_idefics3.js b/packages/transformers/tests/models/idefics3/test_image_processing_idefics3.js new file mode 100644 index 0000000..a9a1509 --- /dev/null +++ b/packages/transformers/tests/models/idefics3/test_image_processing_idefics3.js @@ -0,0 +1,107 @@ +import { AutoImageProcessor, Idefics3ImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // Idefics3ImageProcessor + // - custom image processing (patching) + describe("Idefics3ImageProcessor", () => { + const model_id = "hf-internal-testing/tiny-random-Idefics3ForConditionalGeneration"; + + /** @type {Record} */ + const images = {}; + /** @type {Idefics3ImageProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + + // Load images + const image = await load_cached_image("gradient_1280x640"); + const white_image = await load_cached_image("white_image"); + + images.image = image; + images.image_1 = await image.resize(1600, 1067); + images.image_2 = await image.resize(224, 224); + images.white_image = white_image; + images.white_image_1 = await white_image.resize(1600, 1067); + images.white_image_2 = await white_image.resize(224, 224); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "no image splitting", + async () => { + const { pixel_values, rows, cols } = await processor(images.image, { do_image_splitting: false, return_row_col_info: true }); + expect(pixel_values.dims).toEqual([1, 1, 3, 364, 364]); + expect(pixel_values.mean().item()).toBeCloseTo(-0.001035306602716446, 2); + expect(rows).toEqual([[0]]); + expect(cols).toEqual([[0]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batched no image splitting", + async () => { + const { pixel_values, pixel_attention_mask, rows, cols } = await processor([[images.white_image_1], [images.white_image_2], [images.white_image_1, images.white_image_2]], { do_image_splitting: false, return_row_col_info: true }); + expect(pixel_values.dims).toEqual([3, 2, 3, 364, 364]); + expect(pixel_values.mean().item()).toBeCloseTo(2 / 3, 2); + expect(pixel_attention_mask.dims).toEqual([3, 2, 364, 364]); + expect(pixel_attention_mask.to("float32").mean().item()).toBeCloseTo(2 / 3, 3); + expect(rows).toEqual([[0], [0], [0, 0]]); + expect(cols).toEqual([[0], [0], [0, 0]]); + + // Test that the order of the pixel attention mask matches the python implementation + expect(pixel_attention_mask.data.reduce((a, b, i) => a + i * b, 0)).toEqual(228217205216); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "correct patching", + async () => { + const { pixel_values, rows, cols } = await processor(images.image, { return_row_col_info: true }); + expect(pixel_values.dims).toEqual([1, 9, 3, 364, 364]); + expect(pixel_values.flatten(2).mean(2).tolist()).toBeCloseToNested([[-0.7012196183204651, -0.30104631185531616, 0.09912905097007751, 0.49929487705230713, -0.5011996626853943, -0.10103467106819153, 0.2991456389427185, 0.6993265151977539, -0.0010353063698858023]], 1); + expect(rows).toEqual([[2]]); + expect(cols).toEqual([[4]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "unbatched, single image", + async () => { + const { pixel_values, rows, cols } = await processor(images.image_1, { return_row_col_info: true }); + expect(pixel_values.dims).toEqual([1, 13, 3, 364, 364]); + + expect(rows).toEqual([[3]]); + expect(cols).toEqual([[4]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "unbatched, multiple images", + async () => { + const { pixel_values, rows, cols } = await processor([images.image_1, images.image_2], { return_row_col_info: true }); + expect(pixel_values.dims).toEqual([1, 30, 3, 364, 364]); + + expect(rows).toEqual([[3, 4]]); + expect(cols).toEqual([[4, 4]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batched, multiple images", + async () => { + const { pixel_values, rows, cols } = await processor([[images.image_1], [images.image_1, images.image_2]], { return_row_col_info: true }); + expect(pixel_values.dims).toEqual([2, 30, 3, 364, 364]); + expect(rows).toEqual([[3], [3, 4]]); + expect(cols).toEqual([[4], [4, 4]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/idefics3/test_modeling_idefics3.js b/packages/transformers/tests/models/idefics3/test_modeling_idefics3.js new file mode 100644 index 0000000..92b163e --- /dev/null +++ b/packages/transformers/tests/models/idefics3/test_modeling_idefics3.js @@ -0,0 +1,142 @@ +import { Idefics3Processor, Idefics3ForConditionalGeneration, RawImage } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + const conversation = [ + { + role: "user", + content: [{ type: "image" }, { type: "text", text: "Can you describe this image?" }], + }, + ]; + + // Empty white and black images + const white_image_dims = [224, 224, 3]; + const white_image = new RawImage(new Uint8ClampedArray(white_image_dims[0] * white_image_dims[1] * white_image_dims[2]).fill(255), ...white_image_dims); + const black_image_dims = [720, 360, 3]; + const black_image = new RawImage(new Uint8ClampedArray(black_image_dims[0] * black_image_dims[1] * black_image_dims[2]).fill(0), ...black_image_dims); + + describe("Idefics3ForConditionalGeneration", () => { + const model_id = "hf-internal-testing/tiny-random-Idefics3ForConditionalGeneration"; + + /** @type {Idefics3ForConditionalGeneration} */ + let model; + /** @type {Idefics3Processor} */ + let processor; + /** @type {string} */ + let text; + beforeAll(async () => { + model = await Idefics3ForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + processor = await Idefics3Processor.from_pretrained(model_id); + + text = processor.apply_chat_template(conversation, { + add_generation_prompt: true, + }); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward w/ image splitting (default)", + async () => { + const inputs = await processor(text, white_image, { + do_image_splitting: true, + }); + + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 3041, 128259]); + expect(logits.mean().item()).toBeCloseTo(-0.0002692154666874558, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "forward w/o image splitting", + async () => { + const inputs = await processor(text, white_image, { + do_image_splitting: false, + }); + + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 189, 128259]); + expect(logits.mean().item()).toBeCloseTo(-0.00019743280427064747, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size=1 w/ image splitting", + async () => { + const inputs = await processor(text, white_image, { + do_image_splitting: true, + }); + const generate_ids = await model.generate({ + ...inputs, + max_new_tokens: 10, + + // To obtain unique output tokens, deterministically + repetition_penalty: 2.0, + }); + expect(generate_ids.dims).toEqual([1, 3051]); + + const new_tokens = generate_ids.slice(null, [inputs.input_ids.dims.at(-1), null]); + expect(new_tokens.tolist()).toEqual([[64531n, 121777n, 70370n, 105334n, 12720n, 113356n, 47739n, 59240n, 102001n, 60344n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size=1 w/o image splitting", + async () => { + const inputs = await processor(text, white_image, { + do_image_splitting: false, + }); + const generate_ids = await model.generate({ + ...inputs, + max_new_tokens: 10, + + // To obtain unique output tokens, deterministically + repetition_penalty: 2.0, + }); + expect(generate_ids.dims).toEqual([1, 199]); + + const new_tokens = generate_ids.slice(null, [inputs.input_ids.dims.at(-1), null]); + expect(new_tokens.tolist()).toEqual([[64531n, 121777n, 70370n, 105334n, 12720n, 113356n, 47739n, 59240n, 59697n, 65246n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size=1 multi-image w/o image splitting", + async () => { + const multi_image_conversation = [ + { + role: "user", + content: [{ type: "image" }, { type: "image" }, { type: "text", text: "Can you describe these images?" }], + }, + ]; + + const multi_image_text = processor.apply_chat_template(multi_image_conversation, { + add_generation_prompt: true, + }); + const inputs = await processor(multi_image_text, [white_image, black_image], { + do_image_splitting: false, + }); + const generate_ids = await model.generate({ + ...inputs, + max_new_tokens: 10, + + // To obtain unique output tokens, deterministically + repetition_penalty: 2.0, + }); + expect(generate_ids.dims).toEqual([1, 374]); + + const new_tokens = generate_ids.slice(null, [inputs.input_ids.dims.at(-1), null]); + expect(new_tokens.tolist()).toEqual([[73189n, 99346n, 113252n, 51743n, 33499n, 66430n, 78739n, 89539n, 121023n, 14474n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/jais/test_modeling_jais.js b/packages/transformers/tests/models/jais/test_modeling_jais.js new file mode 100644 index 0000000..77a3a44 --- /dev/null +++ b/packages/transformers/tests/models/jais/test_modeling_jais.js @@ -0,0 +1,51 @@ +import { PreTrainedTokenizer, JAISLMHeadModel } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("JAISLMHeadModel", () => { + const model_id = "onnx-community/tiny-random-jais"; + /** @type {JAISLMHeadModel} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await JAISLMHeadModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n, 55422n], + [55422n, 2838n, 2838n, 2838n, 2838n, 2838n, 2838n, 2838n, 2838n, 2838n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/janus/test_processor_janus.js b/packages/transformers/tests/models/janus/test_processor_janus.js new file mode 100644 index 0000000..3092fb9 --- /dev/null +++ b/packages/transformers/tests/models/janus/test_processor_janus.js @@ -0,0 +1,47 @@ +import { AutoProcessor, VLChatProcessor } from "../../../src/transformers.js"; + +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("VLChatProcessor", () => { + const model_id = "onnx-community/Janus-1.3B-ONNX"; + + /** @type {VLChatProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "Image and text", + async () => { + // Prepare inputs + const conversation = [ + { + role: "User", + content: "\nConvert the formula into latex code.", + images: ["https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/quadratic_formula.png"], + }, + ]; + + const { input_ids, attention_mask, images_seq_mask, images_emb_mask, pixel_values, original_sizes, reshaped_input_sizes } = await processor(conversation); + const num_tokens = 631; + const { num_image_tokens } = processor.config; // 576 + const { image_size } = processor.image_processor.config; // 384 + + expect(input_ids.dims).toEqual([1, num_tokens]); + expect(attention_mask.dims).toEqual([1, num_tokens]); + expect(images_seq_mask.dims).toEqual([1, num_tokens]); + expect(images_seq_mask.to("float32").mean().item()).toBeCloseTo(num_image_tokens / num_tokens, 6); + expect(images_emb_mask.dims).toEqual([1, 1, num_image_tokens]); + expect(images_emb_mask.to("float32").mean().item()).toBeCloseTo(1); + expect(pixel_values.dims).toEqual([1, 1, 3, image_size, image_size]); + expect(pixel_values.mean().item()).toBeCloseTo(0.5999642610549927, 6); + + expect(original_sizes).toEqual([[206, 767]]); + expect(reshaped_input_sizes).toEqual([[103, image_size]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/jina_clip/test_image_processing_jina_clip.js b/packages/transformers/tests/models/jina_clip/test_image_processing_jina_clip.js new file mode 100644 index 0000000..6d59dd6 --- /dev/null +++ b/packages/transformers/tests/models/jina_clip/test_image_processing_jina_clip.js @@ -0,0 +1,33 @@ +import { AutoImageProcessor, JinaCLIPImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // JinaCLIPImageProcessor + // - custom config overrides + describe("JinaCLIPImageProcessor", () => { + const model_id = "jinaai/jina-clip-v2"; + + /** @type {JinaCLIPImageProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "default", + async () => { + const image = await load_cached_image("tiger"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 512, 512]); + expect(pixel_values.mean().item()).toBeCloseTo(-0.06637834757566452, 3); + + expect(original_sizes).toEqual([[408, 612]]); + expect(reshaped_input_sizes).toEqual([[512, 512]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/jina_clip/test_processor_jina_clip.js b/packages/transformers/tests/models/jina_clip/test_processor_jina_clip.js new file mode 100644 index 0000000..07b11c9 --- /dev/null +++ b/packages/transformers/tests/models/jina_clip/test_processor_jina_clip.js @@ -0,0 +1,44 @@ +import { AutoProcessor, JinaCLIPProcessor } from "../../../src/transformers.js"; +import { load_cached_image } from "../../asset_cache.js"; + +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("JinaCLIPProcessor", () => { + const model_id = "jinaai/jina-clip-v2"; + + /** @type {JinaCLIPProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "Image and text", + async () => { + // Prepare inputs + const images = [await load_cached_image("white_image"), await load_cached_image("blue_image")]; + const sentences = [ + "غروب جميل على الشاطئ", // Arabic + "海滩上美丽的日落", // Chinese + "Un beau coucher de soleil sur la plage", // French + "Ein wunderschöner Sonnenuntergang am Strand", // German + "Ένα όμορφο ηλιοβασίλεμα πάνω από την παραλία", // Greek + "समुद्र तट पर एक खूबसूरत सूर्यास्त", // Hindi + "Un bellissimo tramonto sulla spiaggia", // Italian + "浜辺に沈む美しい夕日", // Japanese + "해변 위로 아름다운 일몰", // Korean + ]; + + // Encode text and images + const { input_ids, attention_mask, pixel_values } = await processor(sentences, images, { padding: true, truncation: true }); + + expect(input_ids.dims).toEqual([sentences.length, 14]); + expect(attention_mask.dims).toEqual([sentences.length, 14]); + expect(pixel_values.dims).toEqual([images.length, 3, 512, 512]); + expect(pixel_values.mean().item()).toBeCloseTo(0.7857685685157776, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/lfm2/test_modeling_lfm2.js b/packages/transformers/tests/models/lfm2/test_modeling_lfm2.js new file mode 100644 index 0000000..c41dabf --- /dev/null +++ b/packages/transformers/tests/models/lfm2/test_modeling_lfm2.js @@ -0,0 +1,51 @@ +import { PreTrainedTokenizer, Lfm2ForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("Lfm2ForCausalLM", () => { + const model_id = "onnx-internal-testing/tiny-random-Lfm2ForCausalLM"; + /** @type {Lfm2ForCausalLM} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await Lfm2ForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[1n, 52572n, 38892n, 6902n, 53329n, 33092n, 13656n, 49822n, 6902n, 52520n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 1n, 52572n, 60239n, 57205n, 6790n, 58292n, 30935n, 5959n, 6902n], + [1n, 52572n, 2031n, 59572n, 43345n, 42427n, 31142n, 41100n, 5321n, 5816n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/lfm2_vl/test_image_processing_lfm2_vl.js b/packages/transformers/tests/models/lfm2_vl/test_image_processing_lfm2_vl.js new file mode 100644 index 0000000..ae4d5d0 --- /dev/null +++ b/packages/transformers/tests/models/lfm2_vl/test_image_processing_lfm2_vl.js @@ -0,0 +1,120 @@ +import { AutoImageProcessor, Lfm2VlImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("Lfm2VlImageProcessor", () => { + const model_id = "onnx-community/LFM2-VL-450M-ONNX"; + + /** @type {Record} */ + const images = {}; + /** @type {Lfm2VlImageProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + + images.white_image = await load_cached_image("white_image"); + images.gradient_image = await load_cached_image("gradient_1280x640"); + images.cats = await load_cached_image("cats"); + }, MAX_PROCESSOR_LOAD_TIME); + + it("should be an instance of Lfm2VlImageProcessor", () => { + expect(processor).toBeInstanceOf(Lfm2VlImageProcessor); + }); + + it( + "small square image (single tile)", + async () => { + // White image: 224x224 -> smart_resize -> 256x256, single tile + const { pixel_values, pixel_attention_mask, spatial_shapes } = await processor(images.white_image); + + // Shape: [1 tile, max_num_patches=1024, patch_dim=768] + expect(pixel_values.dims).toEqual([1, 1024, 768]); + expect(pixel_values.mean().item()).toBeCloseTo(0.25, 2); + + // Attention mask: 256 real patches out of 1024 + expect(pixel_attention_mask.dims).toEqual([1, 1024]); + expect(pixel_attention_mask.sum().item()).toBe(256n); + + // Spatial shapes: 16x16 patches + expect(spatial_shapes.dims).toEqual([1, 2]); + expect(spatial_shapes.tolist()).toEqual([[16n, 16n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "large non-square image (multi-tile with thumbnail)", + async () => { + // Gradient 1280x640 -> tiled 2x1 grid + thumbnail = 3 tiles + const result = await processor(images.gradient_image, { return_row_col_info: true }); + const { pixel_values, pixel_attention_mask, spatial_shapes } = result; + + // Shape: [3 tiles, max_num_patches=1024, patch_dim=768] + expect(pixel_values.dims).toEqual([3, 1024, 768]); + expect(pixel_values.mean().item()).toBeCloseTo(-0.0009292, 3); + + // Attention mask per tile: tiles are fully packed (1024), thumbnail has 968 + expect(pixel_attention_mask.dims).toEqual([3, 1024]); + expect(pixel_attention_mask.sum(1).tolist()).toEqual([1024n, 1024n, 968n]); + + // Spatial shapes: two 32x32 tiles + 22x44 thumbnail + expect(spatial_shapes.tolist()).toEqual([ + [32n, 32n], + [32n, 32n], + [22n, 44n], + ]); + + // Row/col info + expect(result.image_rows).toEqual([1]); + expect(result.image_cols).toEqual([2]); + expect(result.image_sizes).toEqual([[352, 704]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "non-square image (single tile)", + async () => { + // Cats 640x480 -> smart_resize -> 416x576, single tile + const { pixel_values, pixel_attention_mask, spatial_shapes } = await processor(images.cats); + + expect(pixel_values.dims).toEqual([1, 1024, 768]); + expect(pixel_values.mean().item()).toBeCloseTo(0.0340382, 2); + + // 26*36 = 936 real patches + expect(pixel_attention_mask.dims).toEqual([1, 1024]); + expect(pixel_attention_mask.sum().item()).toBe(936n); + + expect(spatial_shapes.tolist()).toEqual([[26n, 36n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "multiple images", + async () => { + const result = await processor([images.white_image, images.cats], { return_row_col_info: true }); + const { pixel_values, pixel_attention_mask, spatial_shapes } = result; + + // 2 images, each single tile + expect(pixel_values.dims).toEqual([2, 1024, 768]); + expect(pixel_attention_mask.dims).toEqual([2, 1024]); + + expect(spatial_shapes.tolist()).toEqual([ + [16n, 16n], + [26n, 36n], + ]); + + expect(result.image_rows).toEqual([1, 1]); + expect(result.image_cols).toEqual([1, 1]); + expect(result.image_sizes).toEqual([ + [256, 256], + [416, 576], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/llama/test_modeling_llama.js b/packages/transformers/tests/models/llama/test_modeling_llama.js new file mode 100644 index 0000000..f4d9e85 --- /dev/null +++ b/packages/transformers/tests/models/llama/test_modeling_llama.js @@ -0,0 +1,180 @@ +import { LlamaTokenizer, LlamaForCausalLM, pipeline, Tensor } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, MAX_TEST_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("LlamaForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; + /** @type {LlamaForCausalLM} */ + let model; + /** @type {LlamaTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await LlamaForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await LlamaTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[1n, 22172n, 18547n, 8143n, 22202n, 9456n, 17213n, 15330n, 26591n, 15721n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 1n, 22172n, 18547n, 8143n, 22202n, 9456n, 17213n, 15330n, 26591n], + [1n, 22172n, 3186n, 24786n, 19169n, 20222n, 29993n, 27146n, 27426n, 24562n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("LlamaForCausalLM (onnxruntime-genai) w/ num_logits_to_keep", () => { + const model_id = "onnx-internal-testing/tiny-random-LlamaForCausalLM_num_logits_to_keep"; + + for (const dtype of ["fp32", "fp16"]) { + describe(`dtype=${dtype}`, () => { + /** @type {LlamaForCausalLM} */ + let model; + /** @type {LlamaTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await LlamaForCausalLM.from_pretrained(model_id, { + ...DEFAULT_MODEL_OPTIONS, + dtype, + }); + tokenizer = await LlamaTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward (default num_logits_to_keep)", + async () => { + const inputs = tokenizer("hello"); + const { logits } = await model.forward(inputs); + // Default: returns logits for all tokens + expect(logits.dims).toEqual([1, 2, 128256]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "forward (num_logits_to_keep=0)", + async () => { + const inputs = tokenizer("hello"); + const num_logits_to_keep = new Tensor("int64", BigInt64Array.from([0n]), []); + const { logits } = await model.forward({ ...inputs, num_logits_to_keep }); + // num_logits_to_keep=0 returns all logits (same as default) + expect(logits.dims).toEqual([1, 2, 128256]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "forward (num_logits_to_keep=1)", + async () => { + const inputs = tokenizer("hello"); + const num_logits_to_keep = new Tensor("int64", BigInt64Array.from([1n]), []); + const { logits } = await model.forward({ ...inputs, num_logits_to_keep }); + // Only last token's logits + expect(logits.dims).toEqual([1, 1, 128256]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "forward (num_logits_to_keep=3)", + async () => { + const inputs = tokenizer("hello world"); + const num_logits_to_keep = new Tensor("int64", BigInt64Array.from([3n]), []); + const { logits } = await model.forward({ ...inputs, num_logits_to_keep }); + // Last 3 tokens' logits + expect(logits.dims).toEqual([1, 3, 128256]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "generate", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 5, + }); + expect(outputs.tolist()).toEqual([[128000n, 15339n, 73474n, 38648n, 56375n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + } + + it( + "pipeline", + async () => { + const pipe = await pipeline("text-generation", model_id, DEFAULT_MODEL_OPTIONS); + const result = await pipe("hello", { max_new_tokens: 3 }); + expect(result).toEqual([{ generated_text: "hello-close heroic actively" }]); + await pipe.dispose(); + }, + MAX_TEST_TIME, + ); + }); + + describe("LlamaForCausalLM (onnxruntime-genai)", () => { + const model_id = "onnx-community/tiny-random-LlamaForCausalLM-ONNX"; + /** @type {LlamaTokenizer} */ + let tokenizer; + let inputs; + beforeAll(async () => { + tokenizer = await LlamaTokenizer.from_pretrained(model_id); + inputs = tokenizer("hello"); + }, MAX_MODEL_LOAD_TIME); + + const dtypes = ["fp32", "fp16", "q4", "q4f16"]; + + for (const dtype of dtypes) { + it( + `dtype=${dtype}`, + async () => { + /** @type {LlamaForCausalLM} */ + const model = await LlamaForCausalLM.from_pretrained(model_id, { + ...DEFAULT_MODEL_OPTIONS, + dtype, + }); + + const outputs = await model.generate({ + ...inputs, + max_length: 5, + }); + expect(outputs.tolist()).toEqual([[128000n, 15339n, 15339n, 15339n, 15339n]]); + + await model?.dispose(); + }, + MAX_TEST_TIME, + ); + } + }); +}; diff --git a/packages/transformers/tests/models/llama/test_tokenization_llama.js b/packages/transformers/tests/models/llama/test_tokenization_llama.js new file mode 100644 index 0000000..ab5b63e --- /dev/null +++ b/packages/transformers/tests/models/llama/test_tokenization_llama.js @@ -0,0 +1,1185 @@ +import { LlamaTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, LLAMA_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = LlamaTokenizer; +export const TEST_CONFIG = { + "Xenova/llama-tokenizer": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [1, 1128, 526, 366, 2599, 29973], + decoded: " How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [1, 887, 881, 29915, 345, 2309, 445], + decoded: " You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u2581", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u2581", "0", "\u2581", "1", "\u2581", "2", "\u2581", "3", "\u2581", "4", "\u2581", "5", "\u2581", "6", "\u2581", "7", "\u2581", "8", "\u2581", "9", "\u2581", "1", "0", "\u2581", "1", "0", "0", "\u2581", "1", "0", "0", "0"], + ids: [1, 29871, 29900, 29896, 29906, 29941, 29946, 29945, 29953, 29955, 29947, 29929, 29871, 29900, 29871, 29896, 29871, 29906, 29871, 29941, 29871, 29946, 29871, 29945, 29871, 29953, 29871, 29955, 29871, 29947, 29871, 29929, 29871, 29896, 29900, 29871, 29896, 29900, 29900, 29871, 29896, 29900, 29900, 29900], + decoded: " 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u2581", "2", "0", "1", "6", "."], + ids: [1, 450, 5001, 471, 11091, 297, 29871, 29906, 29900, 29896, 29953, 29889], + decoded: " The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "<0x0A>", "'", "ll", "\u2581!!", "to", "?'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [1, 319, 13, 29915, 645, 21443, 517, 17901, 29881, 4907, 29881, 310, 29892, 508, 29915, 29873, 29889], + decoded: " A\n'll !!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581def", "\u2581main", "():", "<0x0A>", "<0x09>", "pass"], + ids: [1, 822, 1667, 7295, 13, 12, 3364], + decoded: " def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581obj", ".", "toString", "();", "<0x0A>", "toString", "();"], + ids: [1, 1235, 263, 353, 5446, 29889, 7711, 890, 13, 7711, 890], + decoded: " let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: LLAMA_TEST_STRINGS.NEWLINES, + tokens: ["\u2581ax", "<0x0A>", "####", "<0x0A>", "bo", "o"], + ids: [1, 4853, 13, 4136, 13, 833, 29877], + decoded: " ax\n####\nboo", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "w", "ant", "\u00e9d", ",", "running"], + ids: [1, 8291, 29893, 424, 2487, 29892, 21094], + decoded: " UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u2581", "1", "<0x00>", "2", "\ufffd", "3"], + ids: [1, 29871, 29896, 3, 29906, 30140, 29941], + decoded: " 1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581Hello", "\u2581World"], + ids: [1, 15043, 2787], + decoded: " Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [1, 22172, 3186], + decoded: " hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f", "\u6d3b", "\u7684", "\u771f", "<0xE8>", "<0xB0>", "<0x9B>", "\u662f"], + ids: [1, 29871, 30486, 31704, 30210, 30848, 235, 179, 158, 30392], + decoded: " \u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581\u2581\u2581", "\u2581leading", "\u2581space"], + ids: [1, 1678, 8236, 2913], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trailing", "\u2581space", "\u2581\u2581\u2581"], + ids: [1, 25053, 2913, 1678], + decoded: " trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581Hi", "\u2581", "\u2581Hello"], + ids: [1, 6324, 29871, 15043], + decoded: " Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$", "1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "<0xE2>", "<0x82>", "<0xA3>", "7", "\u2581", "\u20b9", "8", "\u2581", "<0xE2>", "<0x82>", "<0xB1>", "9", "\u2581test"], + ids: [1, 1243, 395, 29896, 390, 29906, 396, 29941, 25540, 29946, 15151, 29945, 29871, 30563, 29953, 29871, 229, 133, 166, 29955, 29871, 30620, 29947, 29871, 229, 133, 180, 29929, 1243], + decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1", ".", "0", "0", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [1, 306, 18093, 385, 26163, 363, 395, 29896, 29889, 29900, 29900, 472, 278, 3787, 29889], + decoded: " I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "\u2026", "\u2581\u2581"], + ids: [1, 366, 30098, 259], + decoded: " you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "\u2026", "\u00a0\u00a0"], + ids: [1, 366, 30098, 8655], + decoded: " you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "\u2026", "\u00a0\u00a0", "you", "\u2026", "\u00a0\u00a0"], + ids: [1, 366, 30098, 8655, 6293, 30098, 8655], + decoded: " you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], + ids: [1, 13543, 29871, 30739, 7636, 29871, 30739, 1206], + decoded: " weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581", "\u2581This", "\u2581", "\u2581is", "\u2581", "\u2581a", "\u2581", "\u2581test", "\u2581", "\u2581."], + ids: [1, 29871, 910, 29871, 338, 29871, 263, 29871, 1243, 29871, 869], + decoded: " This is a test .", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x82>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8D>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0xA3>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8D>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0xAD>", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x89>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x8F>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8A>", "\u2581", "<0xF0>", "<0x9F>", "<0x94>", "<0xA5>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x81>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x85>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x86>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8F>", "\u2581", "<0xE2>", "<0x9D>", "<0xA4>", "\ufe0f", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x9C>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x9A>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x99>", "\u2581", "<0xF0>", "<0x9F>", "<0x96>", "<0xA4>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8E>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8C>", "\u2581", "<0xF0>", "<0x9F>", "<0xA5>", "<0xB3>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xAA>", "\u2581", "<0xE2>", "<0x9C>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x89>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x80>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xAF>", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x88>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x88>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x8C>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x80>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x87>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8B>", "\u2581", "\u2705", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x81>", "\u2581", "<0xF0>", "<0x9F>", "<0x8C>", "<0x9E>", "\u2581", "<0xF0>", "<0x9F>", "<0x8C>", "<0xB8>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xB0>"], + ids: [1, 29871, 243, 162, 155, 133, 29871, 243, 162, 148, 144, 29871, 243, 162, 167, 166, 29871, 243, 162, 155, 144, 29871, 243, 162, 155, 176, 29871, 243, 162, 145, 140, 29871, 243, 162, 156, 146, 29871, 243, 162, 155, 141, 29871, 243, 162, 151, 168, 29871, 243, 162, 155, 132, 29871, 243, 162, 155, 136, 29871, 243, 162, 167, 154, 29871, 243, 162, 155, 137, 29871, 243, 162, 148, 146, 29871, 229, 160, 167, 30598, 29871, 243, 162, 149, 159, 29871, 243, 162, 149, 157, 29871, 243, 162, 149, 154, 29871, 243, 162, 149, 156, 29871, 243, 162, 153, 167, 29871, 243, 162, 155, 145, 29871, 243, 162, 148, 143, 29871, 243, 162, 168, 182, 29871, 243, 162, 149, 173, 29871, 229, 159, 171, 29871, 243, 162, 148, 140, 29871, 243, 162, 148, 131, 29871, 243, 162, 149, 178, 29871, 243, 162, 145, 139, 29871, 243, 162, 156, 139, 29871, 243, 162, 156, 143, 29871, 243, 162, 149, 131, 29871, 243, 162, 148, 138, 29871, 243, 162, 148, 142, 29871, 31681, 29871, 243, 162, 145, 132, 29871, 243, 162, 143, 161, 29871, 243, 162, 143, 187, 29871, 243, 162, 149, 179], + decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "<0xE2>", "<0x9C>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x81>", "\ufe0f", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xB1>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u2581", "<0xF0>", "<0x9F>", "<0x95>", "<0xB5>", "\u200d", "\u2642", "\ufe0f", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x99>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u200d", "\u2642", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u200d", "<0xF0>", "<0x9F>", "<0x8C>", "<0xBE>", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "\u200d", "<0xF0>", "<0x9F>", "<0xA4>", "<0x9D>", "\u200d", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "\u200d", "<0xE2>", "<0x9D>", "<0xA4>", "\u200d", "<0xF0>", "<0x9F>", "<0x92>", "<0x8B>", "\u200d", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "\u200d", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "\u200d", "<0xF0>", "<0x9F>", "<0x91>", "<0xA7>", "\u200d", "<0xF0>", "<0x9F>", "<0x91>", "<0xA6>", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u200d", "<0xF0>", "<0x9F>", "<0xA4>", "<0x9D>", "\u200d", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u2581", "<0xF0>", "<0x9F>", "<0x8F>", "<0xB4>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA7>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA2>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA5>", "<0xF3>", "<0xA0>", "<0x81>", "<0xAE>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA7>", "<0xF3>", "<0xA0>", "<0x81>", "<0xBF>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u200d", "<0xE2>", "<0x9D>", "<0xA4>", "\ufe0f", "\u200d", "<0xF0>", "<0x9F>", "<0x92>", "<0x8B>", "\u200d", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBC>"], + ids: [1, 29871, 229, 159, 171, 29871, 243, 162, 167, 154, 29871, 243, 162, 148, 132, 30598, 29871, 243, 162, 148, 180, 243, 162, 146, 190, 29871, 243, 162, 152, 184, 30722, 31135, 30598, 29871, 243, 162, 170, 156, 243, 162, 146, 190, 30722, 31135, 29871, 243, 162, 148, 171, 243, 162, 146, 190, 30722, 243, 162, 143, 193, 29871, 243, 162, 170, 148, 30722, 243, 162, 167, 160, 30722, 243, 162, 170, 148, 29871, 243, 162, 148, 172, 30722, 229, 160, 167, 30722, 243, 162, 149, 142, 30722, 243, 162, 148, 171, 29871, 243, 162, 148, 172, 30722, 243, 162, 148, 172, 30722, 243, 162, 148, 170, 30722, 243, 162, 148, 169, 29871, 243, 162, 170, 148, 243, 162, 146, 190, 30722, 243, 162, 167, 160, 30722, 243, 162, 170, 148, 243, 162, 146, 190, 29871, 243, 162, 146, 183, 246, 163, 132, 170, 246, 163, 132, 165, 246, 163, 132, 168, 246, 163, 132, 177, 246, 163, 132, 170, 246, 163, 132, 194, 29871, 243, 162, 148, 171, 243, 162, 146, 190, 30722, 229, 160, 167, 30598, 30722, 243, 162, 149, 142, 30722, 243, 162, 148, 171, 243, 162, 146, 191], + decoded: " \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + BPE_SCORES_PRIORITY_1: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, + tokens: ["\u2581gra", "bb", "ed"], + ids: [1, 2646, 1327, 287], + decoded: " grabbed", + }, + BPE_SCORES_PRIORITY_2: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, + tokens: ["\u2581", "\u2581gra", "bb", "ed"], + ids: [1, 29871, 2646, 1327, 287], + decoded: " grabbed", + }, + BPE_SCORES_PRIORITY_3: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, + tokens: ["\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581", "\u2581gra", "bb", "ed"], + ids: [1, 9651, 2646, 1327, 287], + decoded: " grabbed", + }, + NEWLINE: { + text: LLAMA_TEST_STRINGS.NEWLINE, + tokens: ["\u2581", "<0x0A>"], + ids: [1, 29871, 13], + decoded: " \n", + }, + NEWLINE_WITH_LEADING_SPACE: { + text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, + tokens: ["\u2581\u2581", "<0x0A>"], + ids: [1, 259, 13], + decoded: " \n", + }, + TABS: { + text: LLAMA_TEST_STRINGS.TABS, + tokens: ["\u2581", "<0x09>", "tabs", "<0x09>", "<0x09>", "<0x09>", "<0x09>", "out", "\u2581here"], + ids: [1, 29871, 12, 21175, 12, 12, 12, 12, 449, 1244], + decoded: " \ttabs\t\t\t\tout here", + }, + NEWLINE_AND_TAB: { + text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, + tokens: ["\u2581", "<0x0A>", "<0x09>", "<0x0A>"], + ids: [1, 29871, 13, 12, 13], + decoded: " \n\t\n", + }, + CHINESE_LETTER: { + text: LLAMA_TEST_STRINGS.CHINESE_LETTER, + tokens: ["\u2581", "\u9547"], + ids: [1, 29871, 30411], + decoded: " \u9547", + }, + EMOJIS_1: { + text: LLAMA_TEST_STRINGS.EMOJIS_1, + tokens: ["\u2581", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>"], + ids: [1, 29871, 243, 162, 169, 156], + decoded: " \ud83e\udd99", + }, + EMOJIS_2: { + text: LLAMA_TEST_STRINGS.EMOJIS_2, + tokens: ["\u2581", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>", "<0xEA>", "<0x99>", "<0x8A>"], + ids: [1, 29871, 243, 162, 169, 156, 237, 156, 141], + decoded: " \ud83e\udd99\ua64a", + }, + EMOJIS_3: { + text: LLAMA_TEST_STRINGS.EMOJIS_3, + tokens: ["\u2581", "<0xEA>", "<0x99>", "<0x8A>", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>"], + ids: [1, 29871, 237, 156, 141, 243, 162, 169, 156], + decoded: " \ua64a\ud83e\udd99", + }, + PARAGRAPH: { + text: LLAMA_TEST_STRINGS.PARAGRAPH, + tokens: ["\u2581The", "\u2581ll", "ama", "\u2581(/", "\u02c8", "l", "\u0251", "\u02d0", "m", "\u0259", "/", ";", "\u2581", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>", "Span", "ish", "\u2581pron", "unci", "ation", ":", "\u2581[", "\u02c8", "\u028e", "ama", "])", "\u2581(", "L", "ama", "\u2581gl", "ama", ")", "\u2581is", "\u2581a", "\u2581domestic", "ated", "\u2581South", "\u2581American", "\u2581cam", "el", "id", ",", "\u2581widely", "\u2581used", "\u2581as", "\u2581a", "\u2581meat", "\u2581and", "\u2581pack", "\u2581animal", "\u2581by", "\u2581And", "e", "an", "\u2581cult", "ures", "\u2581since", "\u2581the", "\u2581Pre", "-", "Col", "umb", "ian", "\u2581era", ".", "\u2581L", "lam", "as", "\u2581are", "\u2581social", "\u2581animals", "\u2581and", "\u2581live", "\u2581with", "\u2581others", "\u2581as", "\u2581a", "\u2581her", "d", ".", "\u2581Their", "\u2581w", "ool", "\u2581is", "\u2581soft", "\u2581and", "\u2581contains", "\u2581only", "\u2581a", "\u2581small", "\u2581amount", "\u2581of", "\u2581lan", "olin", ".[", "2", "]", "\u2581L", "lam", "as", "\u2581can", "\u2581learn", "\u2581simple", "\u2581tasks", "\u2581after", "\u2581a", "\u2581few", "\u2581repet", "itions", ".", "\u2581When", "\u2581using", "\u2581a", "\u2581pack", ",", "\u2581they", "\u2581can", "\u2581carry", "\u2581about", "\u2581", "2", "5", "\u2581to", "\u2581", "3", "0", "%", "\u2581of", "\u2581their", "\u2581body", "\u2581weight", "\u2581for", "\u2581", "8", "\u2581to", "\u2581", "1", "3", "\u2581km", "\u2581(", "5", "\u2013", "8", "\u2581miles", ").", "[", "3", "]", "\u2581The", "\u2581name", "\u2581ll", "ama", "\u2581(", "in", "\u2581the", "\u2581past", "\u2581also", "\u2581sp", "elled", '\u2581"', "l", "ama", '"', "\u2581or", '\u2581"', "gl", "ama", '")', "\u2581was", "\u2581adopted", "\u2581by", "\u2581European", "\u2581sett", "lers", "\u2581from", "\u2581native", "\u2581Peru", "vi", "ans", ".[", "4", "]", "\u2581The", "\u2581ancest", "ors", "\u2581of", "\u2581llam", "as", "\u2581are", "\u2581thought", "\u2581to", "\u2581have", "\u2581origin", "ated", "\u2581from", "\u2581the", "\u2581Great", "\u2581Pla", "ins", "\u2581of", "\u2581North", "\u2581America", "\u2581about", "\u2581", "4", "0", "\u2581million", "\u2581years", "\u2581ago", ",", "\u2581and", "\u2581subsequently", "\u2581migr", "ated", "\u2581to", "\u2581South", "\u2581America", "\u2581about", "\u2581three", "\u2581million", "\u2581years", "\u2581ago", "\u2581during", "\u2581the", "\u2581Great", "\u2581American", "\u2581Inter", "change", ".", "\u2581By", "\u2581the", "\u2581end", "\u2581of", "\u2581the", "\u2581last", "\u2581ice", "\u2581age", "\u2581(", "1", "0", ",", "0", "0", "0", "\u2013", "1", "2", ",", "0", "0", "0", "\u2581years", "\u2581ago", "),", "\u2581cam", "el", "ids", "\u2581were", "\u2581ext", "inct", "\u2581in", "\u2581North", "\u2581America", ".[", "3", "]", "\u2581As", "\u2581of", "\u2581", "2", "0", "0", "7", ",", "\u2581there", "\u2581were", "\u2581over", "\u2581seven", "\u2581million", "\u2581llam", "as", "\u2581and", "\u2581al", "p", "ac", "as", "\u2581in", "\u2581South", "\u2581America", "\u2581and", "\u2581over", "\u2581", "1", "5", "8", ",", "0", "0", "0", "\u2581llam", "as", "\u2581and", "\u2581", "1", "0", "0", ",", "0", "0", "0", "<0xEA>", "<0x99>", "<0x8A>", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>", "\u2581al", "p", "ac", "as", ",", "\u2581desc", "ended", "\u2581from", "\u2581pro", "gen", "itors", "\u2581imported", "\u2581late", "\u2581in", "\u2581the", "\u2581", "2", "0", "th", "\u2581century", ",", "\u2581in", "\u2581the", "\u2581United", "\u2581States", "\u2581and", "\u2581Canada", ".[", "5", "]", "\u2581In", "\u2581A", "ym", "ara", "\u2581myth", "ology", ",", "\u2581llam", "as", "\u2581are", "\u2581important", "\u2581be", "ings", ".", "\u2581The", "\u2581Heaven", "ly", "\u2581L", "l", "ama", "\u2581is", "\u2581said", "\u2581to", "\u2581drink", "\u2581water", "\u2581from", "\u2581the", "\u2581ocean", "\u2581and", "\u2581ur", "in", "ates", "\u2581as", "\u2581it", "\u2581ra", "ins", ".[", "6", "]", "\u2581According", "\u2581to", "\u2581A", "ym", "ara", "\u2581es", "chat", "ology", ",", "\u2581llam", "as", "\u2581will", "\u2581return", "\u2581to", "\u2581the", "\u2581water", "\u2581spr", "ings", "\u2581and", "\u2581l", "ago", "ons", "\u2581where", "\u2581they", "\u2581come", "\u2581from", "\u2581at", "\u2581the", "\u2581end", "\u2581of", "\u2581time", ".[", "6", "]"], + ids: [1, 450, 11148, 3304, 20374, 30176, 29880, 30426, 30215, 29885, 30184, 29914, 29936, 29871, 243, 162, 169, 156, 15495, 728, 11504, 11173, 362, 29901, 518, 30176, 31743, 3304, 2314, 313, 29931, 3304, 3144, 3304, 29897, 338, 263, 21849, 630, 4275, 3082, 3949, 295, 333, 29892, 17644, 1304, 408, 263, 27654, 322, 4870, 13019, 491, 1126, 29872, 273, 4185, 1973, 1951, 278, 4721, 29899, 1625, 3774, 713, 3152, 29889, 365, 5288, 294, 526, 5264, 15006, 322, 5735, 411, 4045, 408, 263, 902, 29881, 29889, 11275, 281, 1507, 338, 4964, 322, 3743, 871, 263, 2319, 5253, 310, 10906, 22878, 7226, 29906, 29962, 365, 5288, 294, 508, 5110, 2560, 9595, 1156, 263, 2846, 21159, 2187, 29889, 1932, 773, 263, 4870, 29892, 896, 508, 8677, 1048, 29871, 29906, 29945, 304, 29871, 29941, 29900, 29995, 310, 1009, 3573, 7688, 363, 29871, 29947, 304, 29871, 29896, 29941, 2383, 313, 29945, 29994, 29947, 7800, 467, 29961, 29941, 29962, 450, 1024, 11148, 3304, 313, 262, 278, 4940, 884, 805, 14356, 376, 29880, 3304, 29908, 470, 376, 3820, 3304, 1159, 471, 16356, 491, 7824, 3604, 9306, 515, 7531, 25493, 1403, 550, 7226, 29946, 29962, 450, 19525, 943, 310, 11829, 294, 526, 2714, 304, 505, 3978, 630, 515, 278, 7027, 13494, 1144, 310, 4644, 6813, 1048, 29871, 29946, 29900, 7284, 2440, 8020, 29892, 322, 17602, 9725, 630, 304, 4275, 6813, 1048, 2211, 7284, 2440, 8020, 2645, 278, 7027, 3082, 4124, 3167, 29889, 2648, 278, 1095, 310, 278, 1833, 14890, 5046, 313, 29896, 29900, 29892, 29900, 29900, 29900, 29994, 29896, 29906, 29892, 29900, 29900, 29900, 2440, 8020, 511, 3949, 295, 4841, 892, 1294, 5562, 297, 4644, 6813, 7226, 29941, 29962, 1094, 310, 29871, 29906, 29900, 29900, 29955, 29892, 727, 892, 975, 9881, 7284, 11829, 294, 322, 394, 29886, 562, 294, 297, 4275, 6813, 322, 975, 29871, 29896, 29945, 29947, 29892, 29900, 29900, 29900, 11829, 294, 322, 29871, 29896, 29900, 29900, 29892, 29900, 29900, 29900, 237, 156, 141, 243, 162, 169, 156, 394, 29886, 562, 294, 29892, 5153, 2760, 515, 410, 1885, 17259, 19673, 5683, 297, 278, 29871, 29906, 29900, 386, 6462, 29892, 297, 278, 3303, 3900, 322, 7400, 7226, 29945, 29962, 512, 319, 962, 2518, 22082, 3002, 29892, 11829, 294, 526, 4100, 367, 886, 29889, 450, 22977, 368, 365, 29880, 3304, 338, 1497, 304, 13748, 4094, 515, 278, 23474, 322, 5065, 262, 1078, 408, 372, 1153, 1144, 7226, 29953, 29962, 7579, 304, 319, 962, 2518, 831, 13496, 3002, 29892, 11829, 294, 674, 736, 304, 278, 4094, 7689, 886, 322, 301, 4425, 787, 988, 896, 2041, 515, 472, 278, 1095, 310, 931, 7226, 29953, 29962], + decoded: ' The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + }, + }, + "Xenova/llama3-tokenizer": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [4438, 527, 499, 3815, 30], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [2675, 1288, 3077, 2884, 420], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["012", "345", "678", "9", "\u0120", "0", "\u0120", "1", "\u0120", "2", "\u0120", "3", "\u0120", "4", "\u0120", "5", "\u0120", "6", "\u0120", "7", "\u0120", "8", "\u0120", "9", "\u0120", "10", "\u0120", "100", "\u0120", "100", "0"], + ids: [11531, 12901, 17458, 24, 220, 15, 220, 16, 220, 17, 220, 18, 220, 19, 220, 20, 220, 21, 220, 22, 220, 23, 220, 24, 220, 605, 220, 1041, 220, 1041, 15], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "201", "6", "."], + ids: [791, 2883, 574, 18538, 304, 220, 679, 21, 13], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [32, 198, 3358, 11261, 998, 20837, 67, 4708, 67, 315, 11, 649, 956, 13], + decoded: "A\n'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():\u010a", "\u0109pass"], + ids: [755, 1925, 4019, 42531], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".toString", "();\u010a", "toString", "();"], + ids: [1169, 264, 284, 2909, 5180, 545, 6712, 2178], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: LLAMA_TEST_STRINGS.NEWLINES, + tokens: ["ax", "\u010a", "####\u010a", "boo"], + ids: [710, 198, 71050, 34093], + decoded: "ax\n####\nboo", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], + ids: [1899, 53757, 15433, 11, 28272], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [16, 188, 17, 5809, 18], + decoded: "1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [9906, 4435], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [15339, 1917], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [104654, 9554, 89151, 39013, 249, 21043], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [256, 6522, 3634], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tr", "ailing", "\u0120space", "\u0120\u0120\u0120"], + ids: [376, 14612, 3634, 262], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [13347, 220, 22691], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2\u0124", "\u00b1", "9", "\u0120test"], + ids: [1985, 400, 16, 432, 17, 674, 18, 13281, 19, 7083, 20, 72588, 21, 113384, 96, 22, 90891, 23, 113384, 109, 24, 1296], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [40, 11021, 459, 24149, 369, 400, 16, 13, 410, 520, 279, 3637, 13], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [9514, 1981, 256], + decoded: "you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [9514, 1981, 9421], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [9514, 1981, 4194, 4194, 9514, 1981, 9421], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef\u00bd\u0140", "\u0120edge", "\u0120\u00ef\u00bd\u0140", "\u0120case"], + ids: [906, 2668, 111942, 6964, 111942, 1162], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [10634, 223, 2028, 14860, 223, 285, 14860, 223, 64, 14860, 223, 1985, 14860, 223, 13], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0134", "\u013e", "\u0120\u00f0\u0141\u0134", "\u013c", "\u0120\u00f0\u0141\u0134", "\u0139", "\u0120\u00f0\u0141\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141\u0134", "\u00b0"], + ids: [76460, 224, 62904, 235, 11410, 97, 96, 27623, 235, 27623, 255, 11410, 236, 231, 11410, 247, 237, 27623, 232, 96169, 98, 27623, 223, 27623, 227, 11410, 97, 245, 27623, 228, 62904, 237, 71570, 31643, 64139, 250, 64139, 248, 64139, 245, 64139, 247, 11410, 244, 97, 27623, 236, 62904, 234, 11410, 98, 111, 64139, 103, 26602, 101, 62904, 231, 62904, 222, 64139, 107, 11410, 236, 230, 11410, 247, 230, 11410, 247, 234, 64139, 222, 62904, 229, 62904, 233, 26602, 227, 11410, 236, 223, 11410, 234, 252, 11410, 234, 116, 64139, 108], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00a2", "\u00f3", "\u0142\u0123", "\u00a5", "\u00f3", "\u0142\u0123", "\u00ae", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], + ids: [38798, 101, 11410, 97, 245, 62904, 223, 31643, 62904, 109, 9468, 237, 119, 11410, 243, 113, 102470, 17245, 224, 31643, 11410, 100, 247, 9468, 237, 119, 102470, 17245, 224, 62904, 101, 9468, 237, 119, 102470, 9468, 234, 122, 11410, 100, 239, 102470, 9468, 97, 251, 102470, 9468, 100, 239, 62904, 102, 102470, 121643, 102470, 93273, 233, 102470, 9468, 239, 101, 62904, 102, 102470, 9468, 239, 102, 102470, 9468, 239, 100, 102470, 9468, 239, 99, 11410, 100, 239, 9468, 237, 119, 102470, 9468, 97, 251, 102470, 9468, 100, 239, 9468, 237, 119, 11410, 237, 112, 175, 16050, 100, 175, 16050, 95, 175, 16050, 98, 175, 16050, 106, 175, 16050, 100, 175, 16050, 123, 62904, 101, 9468, 237, 119, 102470, 121643, 31643, 102470, 93273, 233, 102470, 9468, 239, 101, 9468, 237, 120], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + BPE_SCORES_PRIORITY_1: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, + tokens: ["grab", "bed"], + ids: [59312, 2788], + decoded: "grabbed", + }, + BPE_SCORES_PRIORITY_2: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, + tokens: ["\u0120grabbed"], + ids: [30418], + decoded: " grabbed", + }, + BPE_SCORES_PRIORITY_3: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, + tokens: ["\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120", "\u0120grabbed"], + ids: [1881, 30418], + decoded: " grabbed", + }, + NEWLINE: { + text: LLAMA_TEST_STRINGS.NEWLINE, + tokens: ["\u010a"], + ids: [198], + decoded: "\n", + }, + NEWLINE_WITH_LEADING_SPACE: { + text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, + tokens: ["\u0120\u010a"], + ids: [720], + decoded: " \n", + }, + TABS: { + text: LLAMA_TEST_STRINGS.TABS, + tokens: ["\u0109t", "abs", "\u0109\u0109\u0109", "\u0109out", "\u0120here"], + ids: [3324, 3518, 573, 14294, 1618], + decoded: "\ttabs\t\t\t\tout here", + }, + NEWLINE_AND_TAB: { + text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, + tokens: ["\u010a\u0109\u010a"], + ids: [18108], + decoded: "\n\t\n", + }, + CHINESE_LETTER: { + text: LLAMA_TEST_STRINGS.CHINESE_LETTER, + tokens: ["\u00e9\u0137\u0129"], + ids: [104643], + decoded: "\u9547", + }, + EMOJIS_1: { + text: LLAMA_TEST_STRINGS.EMOJIS_1, + tokens: ["\u00f0\u0141", "\u00a6", "\u013b"], + ids: [9468, 99, 247], + decoded: "\ud83e\udd99", + }, + EMOJIS_2: { + text: LLAMA_TEST_STRINGS.EMOJIS_2, + tokens: ["\u00f0\u0141", "\u00a6", "\u013b", "\u00ea", "\u013b", "\u012c"], + ids: [9468, 99, 247, 166, 247, 232], + decoded: "\ud83e\udd99\ua64a", + }, + EMOJIS_3: { + text: LLAMA_TEST_STRINGS.EMOJIS_3, + tokens: ["\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b"], + ids: [166, 247, 232, 9468, 99, 247], + decoded: "\ua64a\ud83e\udd99", + }, + PARAGRAPH: { + text: LLAMA_TEST_STRINGS.PARAGRAPH, + tokens: ["The", "\u0120llama", "\u0120(/", "\u00cb", "\u012a", "l", "\u00c9", "\u0133", "\u00cb", "\u0132", "m", "\u00c9\u013b", "/", ";", "\u0120\u00f0\u0141", "\u00a6", "\u013b", "Spanish", "\u0120pronunciation", ":", "\u0120[", "\u00cb", "\u012a", "\u00ca", "\u0130", "ama", "])", "\u0120(", "L", "ama", "\u0120gl", "ama", ")", "\u0120is", "\u0120a", "\u0120domestic", "ated", "\u0120South", "\u0120American", "\u0120camel", "id", ",", "\u0120widely", "\u0120used", "\u0120as", "\u0120a", "\u0120meat", "\u0120and", "\u0120pack", "\u0120animal", "\u0120by", "\u0120And", "ean", "\u0120cultures", "\u0120since", "\u0120the", "\u0120Pre", "-C", "olum", "bian", "\u0120era", ".", "\u0120L", "lam", "as", "\u0120are", "\u0120social", "\u0120animals", "\u0120and", "\u0120live", "\u0120with", "\u0120others", "\u0120as", "\u0120a", "\u0120herd", ".", "\u0120Their", "\u0120wool", "\u0120is", "\u0120soft", "\u0120and", "\u0120contains", "\u0120only", "\u0120a", "\u0120small", "\u0120amount", "\u0120of", "\u0120lan", "olin", ".[", "2", "]", "\u0120L", "lam", "as", "\u0120can", "\u0120learn", "\u0120simple", "\u0120tasks", "\u0120after", "\u0120a", "\u0120few", "\u0120repetitions", ".", "\u0120When", "\u0120using", "\u0120a", "\u0120pack", ",", "\u0120they", "\u0120can", "\u0120carry", "\u0120about", "\u0120", "25", "\u0120to", "\u0120", "30", "%", "\u0120of", "\u0120their", "\u0120body", "\u0120weight", "\u0120for", "\u0120", "8", "\u0120to", "\u0120", "13", "\u0120km", "\u0120(", "5", "\u00e2\u0122\u0135", "8", "\u0120miles", ").[", "3", "]", "\u0120The", "\u0120name", "\u0120llama", "\u0120(", "in", "\u0120the", "\u0120past", "\u0120also", "\u0120spelled", '\u0120"', "lama", '"', "\u0120or", '\u0120"', "gl", "ama", '")', "\u0120was", "\u0120adopted", "\u0120by", "\u0120European", "\u0120settlers", "\u0120from", "\u0120native", "\u0120Per", "uv", "ians", ".[", "4", "]", "\u0120The", "\u0120ancestors", "\u0120of", "\u0120ll", "amas", "\u0120are", "\u0120thought", "\u0120to", "\u0120have", "\u0120originated", "\u0120from", "\u0120the", "\u0120Great", "\u0120Plains", "\u0120of", "\u0120North", "\u0120America", "\u0120about", "\u0120", "40", "\u0120million", "\u0120years", "\u0120ago", ",", "\u0120and", "\u0120subsequently", "\u0120migrated", "\u0120to", "\u0120South", "\u0120America", "\u0120about", "\u0120three", "\u0120million", "\u0120years", "\u0120ago", "\u0120during", "\u0120the", "\u0120Great", "\u0120American", "\u0120Inter", "change", ".", "\u0120By", "\u0120the", "\u0120end", "\u0120of", "\u0120the", "\u0120last", "\u0120ice", "\u0120age", "\u0120(", "10", ",", "000", "\u00e2\u0122\u0135", "12", ",", "000", "\u0120years", "\u0120ago", "),", "\u0120camel", "ids", "\u0120were", "\u0120extinct", "\u0120in", "\u0120North", "\u0120America", ".[", "3", "]", "\u0120As", "\u0120of", "\u0120", "200", "7", ",", "\u0120there", "\u0120were", "\u0120over", "\u0120seven", "\u0120million", "\u0120ll", "amas", "\u0120and", "\u0120al", "pac", "as", "\u0120in", "\u0120South", "\u0120America", "\u0120and", "\u0120over", "\u0120", "158", ",", "000", "\u0120ll", "amas", "\u0120and", "\u0120", "100", ",", "000", "\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b", "\u0120al", "pac", "as", ",", "\u0120descended", "\u0120from", "\u0120progen", "itors", "\u0120imported", "\u0120late", "\u0120in", "\u0120the", "\u0120", "20", "th", "\u0120century", ",", "\u0120in", "\u0120the", "\u0120United", "\u0120States", "\u0120and", "\u0120Canada", ".[", "5", "]", "\u0120In", "\u0120A", "ym", "ara", "\u0120mythology", ",", "\u0120ll", "amas", "\u0120are", "\u0120important", "\u0120beings", ".", "\u0120The", "\u0120Heavenly", "\u0120L", "lama", "\u0120is", "\u0120said", "\u0120to", "\u0120drink", "\u0120water", "\u0120from", "\u0120the", "\u0120ocean", "\u0120and", "\u0120ur", "in", "ates", "\u0120as", "\u0120it", "\u0120rains", ".[", "6", "]", "\u0120According", "\u0120to", "\u0120A", "ym", "ara", "\u0120es", "chat", "ology", ",", "\u0120ll", "amas", "\u0120will", "\u0120return", "\u0120to", "\u0120the", "\u0120water", "\u0120springs", "\u0120and", "\u0120l", "ago", "ons", "\u0120where", "\u0120they", "\u0120come", "\u0120from", "\u0120at", "\u0120the", "\u0120end", "\u0120of", "\u0120time", ".[", "6", "]"], + ids: [791, 94776, 47325, 135, 230, 75, 133, 239, 135, 238, 76, 99638, 14, 26, 11410, 99, 247, 62897, 71722, 25, 510, 135, 230, 134, 236, 3105, 2526, 320, 43, 3105, 2840, 3105, 8, 374, 264, 13018, 660, 4987, 3778, 50252, 307, 11, 13882, 1511, 439, 264, 13339, 323, 3854, 10065, 555, 1628, 5420, 27833, 2533, 279, 5075, 7813, 1152, 13464, 11639, 13, 445, 24705, 300, 527, 3674, 10099, 323, 3974, 449, 3885, 439, 264, 59213, 13, 11205, 39640, 374, 8579, 323, 5727, 1193, 264, 2678, 3392, 315, 31791, 37737, 8032, 17, 60, 445, 24705, 300, 649, 4048, 4382, 9256, 1306, 264, 2478, 86066, 13, 3277, 1701, 264, 3854, 11, 814, 649, 6920, 922, 220, 914, 311, 220, 966, 4, 315, 872, 2547, 4785, 369, 220, 23, 311, 220, 1032, 13437, 320, 20, 4235, 23, 8931, 94638, 18, 60, 578, 836, 94776, 320, 258, 279, 3347, 1101, 68918, 330, 81101, 1, 477, 330, 6200, 3105, 909, 574, 18306, 555, 7665, 61107, 505, 10068, 3700, 12328, 5493, 8032, 19, 60, 578, 38618, 315, 9507, 29189, 527, 3463, 311, 617, 44853, 505, 279, 8681, 63911, 315, 4892, 5270, 922, 220, 1272, 3610, 1667, 4227, 11, 323, 28520, 73691, 311, 4987, 5270, 922, 2380, 3610, 1667, 4227, 2391, 279, 8681, 3778, 5783, 3455, 13, 3296, 279, 842, 315, 279, 1566, 10054, 4325, 320, 605, 11, 931, 4235, 717, 11, 931, 1667, 4227, 705, 50252, 3447, 1051, 69918, 304, 4892, 5270, 8032, 18, 60, 1666, 315, 220, 1049, 22, 11, 1070, 1051, 927, 8254, 3610, 9507, 29189, 323, 453, 46051, 300, 304, 4987, 5270, 323, 927, 220, 11286, 11, 931, 9507, 29189, 323, 220, 1041, 11, 931, 166, 247, 232, 9468, 99, 247, 453, 46051, 300, 11, 58842, 505, 84360, 12170, 25973, 3389, 304, 279, 220, 508, 339, 9478, 11, 304, 279, 3723, 4273, 323, 7008, 8032, 20, 60, 763, 362, 1631, 5169, 59492, 11, 9507, 29189, 527, 3062, 23837, 13, 578, 88150, 445, 81101, 374, 1071, 311, 7172, 3090, 505, 279, 18435, 323, 4433, 258, 988, 439, 433, 62555, 8032, 21, 60, 10771, 311, 362, 1631, 5169, 1560, 9884, 2508, 11, 9507, 29189, 690, 471, 311, 279, 3090, 42242, 323, 326, 6438, 2439, 1405, 814, 2586, 505, 520, 279, 842, 315, 892, 8032, 21, 60], + decoded: 'The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + }, + }, + + // - Sequence PostProcessor + // - "ignore_merges": true + "Xenova/llama3-tokenizer-new": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [128000, 4438, 527, 499, 3815, 30], + decoded: "<|begin_of_text|>How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [128000, 2675, 1288, 3077, 2884, 420], + decoded: "<|begin_of_text|>You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["012", "345", "678", "9", "\u0120", "0", "\u0120", "1", "\u0120", "2", "\u0120", "3", "\u0120", "4", "\u0120", "5", "\u0120", "6", "\u0120", "7", "\u0120", "8", "\u0120", "9", "\u0120", "10", "\u0120", "100", "\u0120", "100", "0"], + ids: [128000, 11531, 12901, 17458, 24, 220, 15, 220, 16, 220, 17, 220, 18, 220, 19, 220, 20, 220, 21, 220, 22, 220, 23, 220, 24, 220, 605, 220, 1041, 220, 1041, 15], + decoded: "<|begin_of_text|>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "201", "6", "."], + ids: [128000, 791, 2883, 574, 18538, 304, 220, 679, 21, 13], + decoded: "<|begin_of_text|>The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [128000, 32, 198, 3358, 11261, 998, 20837, 67, 4708, 67, 315, 11, 649, 956, 13], + decoded: "<|begin_of_text|>A\n'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():\u010a", "\u0109pass"], + ids: [128000, 755, 1925, 4019, 42531], + decoded: "<|begin_of_text|>def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".toString", "();\u010a", "toString", "();"], + ids: [128000, 1169, 264, 284, 2909, 5180, 545, 6712, 2178], + decoded: "<|begin_of_text|>let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: LLAMA_TEST_STRINGS.NEWLINES, + tokens: ["ax", "\u010a", "####\u010a", "boo"], + ids: [128000, 710, 198, 71050, 34093], + decoded: "<|begin_of_text|>ax\n####\nboo", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], + ids: [128000, 1899, 53757, 15433, 11, 28272], + decoded: "<|begin_of_text|>UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [128000, 16, 188, 17, 5809, 18], + decoded: "<|begin_of_text|>1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [128000, 9906, 4435], + decoded: "<|begin_of_text|>Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [128000, 15339, 1917], + decoded: "<|begin_of_text|>hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [128000, 104654, 9554, 89151, 39013, 249, 21043], + decoded: "<|begin_of_text|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [128000, 256, 6522, 3634], + decoded: "<|begin_of_text|> leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tr", "ailing", "\u0120space", "\u0120\u0120\u0120"], + ids: [128000, 376, 14612, 3634, 262], + decoded: "<|begin_of_text|>trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [128000, 13347, 220, 22691], + decoded: "<|begin_of_text|>Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2\u0124", "\u00b1", "9", "\u0120test"], + ids: [128000, 1985, 400, 16, 432, 17, 674, 18, 13281, 19, 7083, 20, 72588, 21, 113384, 96, 22, 90891, 23, 113384, 109, 24, 1296], + decoded: "<|begin_of_text|>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [128000, 40, 11021, 459, 24149, 369, 400, 16, 13, 410, 520, 279, 3637, 13], + decoded: "<|begin_of_text|>I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [128000, 9514, 1981, 256], + decoded: "<|begin_of_text|>you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [128000, 9514, 1981, 9421], + decoded: "<|begin_of_text|>you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [128000, 9514, 1981, 4194, 4194, 9514, 1981, 9421], + decoded: "<|begin_of_text|>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef\u00bd\u0140", "\u0120edge", "\u0120\u00ef\u00bd\u0140", "\u0120case"], + ids: [128000, 906, 2668, 111942, 6964, 111942, 1162], + decoded: "<|begin_of_text|>weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [128000, 10634, 223, 2028, 14860, 223, 285, 14860, 223, 64, 14860, 223, 1985, 14860, 223, 13], + decoded: "<|begin_of_text|>\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0134", "\u013e", "\u0120\u00f0\u0141\u0134", "\u013c", "\u0120\u00f0\u0141\u0134", "\u0139", "\u0120\u00f0\u0141\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141\u0134", "\u00b0"], + ids: [128000, 76460, 224, 62904, 235, 11410, 97, 96, 27623, 235, 27623, 255, 11410, 236, 231, 11410, 247, 237, 27623, 232, 96169, 98, 27623, 223, 27623, 227, 11410, 97, 245, 27623, 228, 62904, 237, 71570, 31643, 64139, 250, 64139, 248, 64139, 245, 64139, 247, 11410, 244, 97, 27623, 236, 62904, 234, 11410, 98, 111, 64139, 103, 26602, 101, 62904, 231, 62904, 222, 64139, 107, 11410, 236, 230, 11410, 247, 230, 11410, 247, 234, 64139, 222, 62904, 229, 62904, 233, 26602, 227, 11410, 236, 223, 11410, 234, 252, 11410, 234, 116, 64139, 108], + decoded: "<|begin_of_text|>\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00a2", "\u00f3", "\u0142\u0123", "\u00a5", "\u00f3", "\u0142\u0123", "\u00ae", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], + ids: [128000, 38798, 101, 11410, 97, 245, 62904, 223, 31643, 62904, 109, 9468, 237, 119, 11410, 243, 113, 102470, 17245, 224, 31643, 11410, 100, 247, 9468, 237, 119, 102470, 17245, 224, 62904, 101, 9468, 237, 119, 102470, 9468, 234, 122, 11410, 100, 239, 102470, 9468, 97, 251, 102470, 9468, 100, 239, 62904, 102, 102470, 121643, 102470, 93273, 233, 102470, 9468, 239, 101, 62904, 102, 102470, 9468, 239, 102, 102470, 9468, 239, 100, 102470, 9468, 239, 99, 11410, 100, 239, 9468, 237, 119, 102470, 9468, 97, 251, 102470, 9468, 100, 239, 9468, 237, 119, 11410, 237, 112, 175, 16050, 100, 175, 16050, 95, 175, 16050, 98, 175, 16050, 106, 175, 16050, 100, 175, 16050, 123, 62904, 101, 9468, 237, 119, 102470, 121643, 31643, 102470, 93273, 233, 102470, 9468, 239, 101, 9468, 237, 120], + decoded: "<|begin_of_text|>\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + BPE_SCORES_PRIORITY_1: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, + tokens: ["grab", "bed"], + ids: [128000, 59312, 2788], + decoded: "<|begin_of_text|>grabbed", + }, + BPE_SCORES_PRIORITY_2: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, + tokens: ["\u0120grabbed"], + ids: [128000, 30418], + decoded: "<|begin_of_text|> grabbed", + }, + BPE_SCORES_PRIORITY_3: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, + tokens: ["\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120", "\u0120grabbed"], + ids: [128000, 1881, 30418], + decoded: "<|begin_of_text|> grabbed", + }, + NEWLINE: { + text: LLAMA_TEST_STRINGS.NEWLINE, + tokens: ["\u010a"], + ids: [128000, 198], + decoded: "<|begin_of_text|>\n", + }, + NEWLINE_WITH_LEADING_SPACE: { + text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, + tokens: ["\u0120\u010a"], + ids: [128000, 720], + decoded: "<|begin_of_text|> \n", + }, + TABS: { + text: LLAMA_TEST_STRINGS.TABS, + tokens: ["\u0109t", "abs", "\u0109\u0109\u0109", "\u0109out", "\u0120here"], + ids: [128000, 3324, 3518, 573, 14294, 1618], + decoded: "<|begin_of_text|>\ttabs\t\t\t\tout here", + }, + NEWLINE_AND_TAB: { + text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, + tokens: ["\u010a\u0109\u010a"], + ids: [128000, 18108], + decoded: "<|begin_of_text|>\n\t\n", + }, + CHINESE_LETTER: { + text: LLAMA_TEST_STRINGS.CHINESE_LETTER, + tokens: ["\u00e9\u0137\u0129"], + ids: [128000, 104643], + decoded: "<|begin_of_text|>\u9547", + }, + EMOJIS_1: { + text: LLAMA_TEST_STRINGS.EMOJIS_1, + tokens: ["\u00f0\u0141", "\u00a6", "\u013b"], + ids: [128000, 9468, 99, 247], + decoded: "<|begin_of_text|>\ud83e\udd99", + }, + EMOJIS_2: { + text: LLAMA_TEST_STRINGS.EMOJIS_2, + tokens: ["\u00f0\u0141", "\u00a6", "\u013b", "\u00ea", "\u013b", "\u012c"], + ids: [128000, 9468, 99, 247, 166, 247, 232], + decoded: "<|begin_of_text|>\ud83e\udd99\ua64a", + }, + EMOJIS_3: { + text: LLAMA_TEST_STRINGS.EMOJIS_3, + tokens: ["\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b"], + ids: [128000, 166, 247, 232, 9468, 99, 247], + decoded: "<|begin_of_text|>\ua64a\ud83e\udd99", + }, + PARAGRAPH: { + text: LLAMA_TEST_STRINGS.PARAGRAPH, + tokens: ["The", "\u0120llama", "\u0120(/", "\u00cb", "\u012a", "l", "\u00c9", "\u0133", "\u00cb", "\u0132", "m", "\u00c9\u013b", "/", ";", "\u0120\u00f0\u0141", "\u00a6", "\u013b", "Spanish", "\u0120pronunciation", ":", "\u0120[", "\u00cb", "\u012a", "\u00ca", "\u0130", "ama", "])", "\u0120(", "L", "ama", "\u0120gl", "ama", ")", "\u0120is", "\u0120a", "\u0120domestic", "ated", "\u0120South", "\u0120American", "\u0120camel", "id", ",", "\u0120widely", "\u0120used", "\u0120as", "\u0120a", "\u0120meat", "\u0120and", "\u0120pack", "\u0120animal", "\u0120by", "\u0120And", "ean", "\u0120cultures", "\u0120since", "\u0120the", "\u0120Pre", "-C", "olum", "bian", "\u0120era", ".", "\u0120L", "lam", "as", "\u0120are", "\u0120social", "\u0120animals", "\u0120and", "\u0120live", "\u0120with", "\u0120others", "\u0120as", "\u0120a", "\u0120herd", ".", "\u0120Their", "\u0120wool", "\u0120is", "\u0120soft", "\u0120and", "\u0120contains", "\u0120only", "\u0120a", "\u0120small", "\u0120amount", "\u0120of", "\u0120lan", "olin", ".[", "2", "]", "\u0120L", "lam", "as", "\u0120can", "\u0120learn", "\u0120simple", "\u0120tasks", "\u0120after", "\u0120a", "\u0120few", "\u0120repetitions", ".", "\u0120When", "\u0120using", "\u0120a", "\u0120pack", ",", "\u0120they", "\u0120can", "\u0120carry", "\u0120about", "\u0120", "25", "\u0120to", "\u0120", "30", "%", "\u0120of", "\u0120their", "\u0120body", "\u0120weight", "\u0120for", "\u0120", "8", "\u0120to", "\u0120", "13", "\u0120km", "\u0120(", "5", "\u00e2\u0122\u0135", "8", "\u0120miles", ").[", "3", "]", "\u0120The", "\u0120name", "\u0120llama", "\u0120(", "in", "\u0120the", "\u0120past", "\u0120also", "\u0120spelled", '\u0120"', "lama", '"', "\u0120or", '\u0120"', "gl", "ama", '")', "\u0120was", "\u0120adopted", "\u0120by", "\u0120European", "\u0120settlers", "\u0120from", "\u0120native", "\u0120Per", "uv", "ians", ".[", "4", "]", "\u0120The", "\u0120ancestors", "\u0120of", "\u0120ll", "amas", "\u0120are", "\u0120thought", "\u0120to", "\u0120have", "\u0120originated", "\u0120from", "\u0120the", "\u0120Great", "\u0120Plains", "\u0120of", "\u0120North", "\u0120America", "\u0120about", "\u0120", "40", "\u0120million", "\u0120years", "\u0120ago", ",", "\u0120and", "\u0120subsequently", "\u0120migrated", "\u0120to", "\u0120South", "\u0120America", "\u0120about", "\u0120three", "\u0120million", "\u0120years", "\u0120ago", "\u0120during", "\u0120the", "\u0120Great", "\u0120American", "\u0120Inter", "change", ".", "\u0120By", "\u0120the", "\u0120end", "\u0120of", "\u0120the", "\u0120last", "\u0120ice", "\u0120age", "\u0120(", "10", ",", "000", "\u00e2\u0122\u0135", "12", ",", "000", "\u0120years", "\u0120ago", "),", "\u0120camel", "ids", "\u0120were", "\u0120extinct", "\u0120in", "\u0120North", "\u0120America", ".[", "3", "]", "\u0120As", "\u0120of", "\u0120", "200", "7", ",", "\u0120there", "\u0120were", "\u0120over", "\u0120seven", "\u0120million", "\u0120ll", "amas", "\u0120and", "\u0120al", "pac", "as", "\u0120in", "\u0120South", "\u0120America", "\u0120and", "\u0120over", "\u0120", "158", ",", "000", "\u0120ll", "amas", "\u0120and", "\u0120", "100", ",", "000", "\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b", "\u0120al", "pac", "as", ",", "\u0120descended", "\u0120from", "\u0120progen", "itors", "\u0120imported", "\u0120late", "\u0120in", "\u0120the", "\u0120", "20", "th", "\u0120century", ",", "\u0120in", "\u0120the", "\u0120United", "\u0120States", "\u0120and", "\u0120Canada", ".[", "5", "]", "\u0120In", "\u0120A", "ym", "ara", "\u0120mythology", ",", "\u0120ll", "amas", "\u0120are", "\u0120important", "\u0120beings", ".", "\u0120The", "\u0120Heavenly", "\u0120L", "lama", "\u0120is", "\u0120said", "\u0120to", "\u0120drink", "\u0120water", "\u0120from", "\u0120the", "\u0120ocean", "\u0120and", "\u0120ur", "in", "ates", "\u0120as", "\u0120it", "\u0120rains", ".[", "6", "]", "\u0120According", "\u0120to", "\u0120A", "ym", "ara", "\u0120es", "chat", "ology", ",", "\u0120ll", "amas", "\u0120will", "\u0120return", "\u0120to", "\u0120the", "\u0120water", "\u0120springs", "\u0120and", "\u0120l", "ago", "ons", "\u0120where", "\u0120they", "\u0120come", "\u0120from", "\u0120at", "\u0120the", "\u0120end", "\u0120of", "\u0120time", ".[", "6", "]"], + ids: [128000, 791, 94776, 47325, 135, 230, 75, 133, 239, 135, 238, 76, 99638, 14, 26, 11410, 99, 247, 62897, 71722, 25, 510, 135, 230, 134, 236, 3105, 2526, 320, 43, 3105, 2840, 3105, 8, 374, 264, 13018, 660, 4987, 3778, 50252, 307, 11, 13882, 1511, 439, 264, 13339, 323, 3854, 10065, 555, 1628, 5420, 27833, 2533, 279, 5075, 7813, 1152, 13464, 11639, 13, 445, 24705, 300, 527, 3674, 10099, 323, 3974, 449, 3885, 439, 264, 59213, 13, 11205, 39640, 374, 8579, 323, 5727, 1193, 264, 2678, 3392, 315, 31791, 37737, 8032, 17, 60, 445, 24705, 300, 649, 4048, 4382, 9256, 1306, 264, 2478, 86066, 13, 3277, 1701, 264, 3854, 11, 814, 649, 6920, 922, 220, 914, 311, 220, 966, 4, 315, 872, 2547, 4785, 369, 220, 23, 311, 220, 1032, 13437, 320, 20, 4235, 23, 8931, 94638, 18, 60, 578, 836, 94776, 320, 258, 279, 3347, 1101, 68918, 330, 81101, 1, 477, 330, 6200, 3105, 909, 574, 18306, 555, 7665, 61107, 505, 10068, 3700, 12328, 5493, 8032, 19, 60, 578, 38618, 315, 9507, 29189, 527, 3463, 311, 617, 44853, 505, 279, 8681, 63911, 315, 4892, 5270, 922, 220, 1272, 3610, 1667, 4227, 11, 323, 28520, 73691, 311, 4987, 5270, 922, 2380, 3610, 1667, 4227, 2391, 279, 8681, 3778, 5783, 3455, 13, 3296, 279, 842, 315, 279, 1566, 10054, 4325, 320, 605, 11, 931, 4235, 717, 11, 931, 1667, 4227, 705, 50252, 3447, 1051, 69918, 304, 4892, 5270, 8032, 18, 60, 1666, 315, 220, 1049, 22, 11, 1070, 1051, 927, 8254, 3610, 9507, 29189, 323, 453, 46051, 300, 304, 4987, 5270, 323, 927, 220, 11286, 11, 931, 9507, 29189, 323, 220, 1041, 11, 931, 166, 247, 232, 9468, 99, 247, 453, 46051, 300, 11, 58842, 505, 84360, 12170, 25973, 3389, 304, 279, 220, 508, 339, 9478, 11, 304, 279, 3723, 4273, 323, 7008, 8032, 20, 60, 763, 362, 1631, 5169, 59492, 11, 9507, 29189, 527, 3062, 23837, 13, 578, 88150, 445, 81101, 374, 1071, 311, 7172, 3090, 505, 279, 18435, 323, 4433, 258, 988, 439, 433, 62555, 8032, 21, 60, 10771, 311, 362, 1631, 5169, 1560, 9884, 2508, 11, 9507, 29189, 690, 471, 311, 279, 3090, 42242, 323, 326, 6438, 2439, 1405, 814, 2586, 505, 520, 279, 842, 315, 892, 8032, 21, 60], + decoded: '<|begin_of_text|>The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + }, + }, + "Xenova/TinyLLama-v0": { + NEWLINES: { + text: LLAMA_TEST_STRINGS.NEWLINES, + tokens: ["\u2581ax", "<0x0A>", "####", "<0x0A>", "b", "oo"], + ids: [1, 9013, 13, 20411, 13, 31842, 2742], + decoded: " ax\n####\nboo", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "<0xE7>", "<0x94>", "<0x9F>", "<0xE6>", "<0xB4>", "<0xBB>", "<0xE7>", "<0x9A>", "<0x84>", "<0xE7>", "<0x9C>", "<0x9F>", "<0xE8>", "<0xB0>", "<0x9B>", "<0xE6>", "<0x98>", "<0xAF>"], + ids: [1, 31822, 234, 151, 162, 233, 183, 190, 234, 157, 135, 234, 159, 162, 235, 179, 158, 233, 155, 178], + decoded: " \u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trailing", "\u2581space", "\u2581", "\u2581", "\u2581"], + ids: [1, 30174, 2138, 31822, 31822, 31822], + decoded: " trailing space ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$", "1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "<0xC2>", "<0xA5>", "6", "\u2581", "<0xE2>", "<0x82>", "<0xA3>", "7", "\u2581", "<0xE2>", "<0x82>", "<0xB9>", "8", "\u2581", "<0xE2>", "<0x82>", "<0xB1>", "9", "\u2581test"], + ids: [1, 1397, 569, 31853, 360, 31855, 1257, 31878, 9390, 31882, 3922, 31880, 31822, 197, 168, 31887, 31822, 229, 133, 166, 31888, 31822, 229, 133, 188, 31886, 31822, 229, 133, 180, 31877, 1397], + decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "\u2026", "\u2581", "\u2581"], + ids: [1, 365, 31925, 31822, 31822], + decoded: " you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "\u2026", "\u00a0", "\u00a0"], + ids: [1, 365, 31925, 31963, 31963], + decoded: " you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "\u2026", "\u00a0", "\u00a0", "you", "\u2026", "\u00a0", "\u00a0"], + ids: [1, 365, 31925, 31963, 31963, 7936, 31925, 31963, 31963], + decoded: " you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "<0xEF>", "<0xBD>", "<0x9E>", "\u2581edge", "\u2581", "<0xEF>", "<0xBD>", "<0x9E>", "\u2581case"], + ids: [1, 9907, 31822, 242, 192, 161, 5991, 31822, 242, 192, 161, 1372], + decoded: " weird \uff5e edge \uff5e case", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x82>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8D>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0xA3>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8D>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0xAD>", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x89>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x8F>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8A>", "\u2581", "<0xF0>", "<0x9F>", "<0x94>", "<0xA5>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x81>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x85>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x86>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8F>", "\u2581", "<0xE2>", "<0x9D>", "<0xA4>", "<0xEF>", "<0xB8>", "<0x8F>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x9C>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x9A>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x99>", "\u2581", "<0xF0>", "<0x9F>", "<0x96>", "<0xA4>", "\u2581", "<0xF0>", "<0x9F>", "<0x98>", "<0x8E>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8C>", "\u2581", "<0xF0>", "<0x9F>", "<0xA5>", "<0xB3>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xAA>", "\u2581", "<0xE2>", "<0x9C>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x89>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x80>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xAF>", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x88>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x88>", "\u2581", "<0xF0>", "<0x9F>", "<0x99>", "<0x8C>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0x80>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x87>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x8B>", "\u2581", "<0xE2>", "<0x9C>", "<0x85>", "\u2581", "<0xF0>", "<0x9F>", "<0x8E>", "<0x81>", "\u2581", "<0xF0>", "<0x9F>", "<0x8C>", "<0x9E>", "\u2581", "<0xF0>", "<0x9F>", "<0x8C>", "<0xB8>", "\u2581", "<0xF0>", "<0x9F>", "<0x92>", "<0xB0>"], + ids: [1, 31822, 243, 162, 155, 133, 31822, 243, 162, 148, 144, 31822, 243, 162, 167, 166, 31822, 243, 162, 155, 144, 31822, 243, 162, 155, 176, 31822, 243, 162, 145, 140, 31822, 243, 162, 156, 146, 31822, 243, 162, 155, 141, 31822, 243, 162, 151, 168, 31822, 243, 162, 155, 132, 31822, 243, 162, 155, 136, 31822, 243, 162, 167, 154, 31822, 243, 162, 155, 137, 31822, 243, 162, 148, 146, 31822, 229, 160, 167, 242, 187, 146, 31822, 243, 162, 149, 159, 31822, 243, 162, 149, 157, 31822, 243, 162, 149, 154, 31822, 243, 162, 149, 156, 31822, 243, 162, 153, 167, 31822, 243, 162, 155, 145, 31822, 243, 162, 148, 143, 31822, 243, 162, 168, 182, 31822, 243, 162, 149, 173, 31822, 229, 159, 171, 31822, 243, 162, 148, 140, 31822, 243, 162, 148, 131, 31822, 243, 162, 149, 178, 31822, 243, 162, 145, 139, 31822, 243, 162, 156, 139, 31822, 243, 162, 156, 143, 31822, 243, 162, 149, 131, 31822, 243, 162, 148, 138, 31822, 243, 162, 148, 142, 31822, 229, 159, 136, 31822, 243, 162, 145, 132, 31822, 243, 162, 143, 161, 31822, 243, 162, 143, 187, 31822, 243, 162, 149, 179], + decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "<0xE2>", "<0x9C>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0xA4>", "<0x97>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0x81>", "<0xEF>", "<0xB8>", "<0x8F>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xB1>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u2581", "<0xF0>", "<0x9F>", "<0x95>", "<0xB5>", "<0xE2>", "<0x80>", "<0x8D>", "<0xE2>", "<0x99>", "<0x82>", "<0xEF>", "<0xB8>", "<0x8F>", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x99>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "<0xE2>", "<0x80>", "<0x8D>", "<0xE2>", "<0x99>", "<0x82>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x8C>", "<0xBE>", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0xA4>", "<0x9D>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "<0xE2>", "<0x80>", "<0x8D>", "<0xE2>", "<0x9D>", "<0xA4>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x92>", "<0x8B>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x91>", "<0xA9>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x91>", "<0xA7>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x91>", "<0xA6>", "\u2581", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0xA4>", "<0x9D>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0xA7>", "<0x91>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "\u2581", "<0xF0>", "<0x9F>", "<0x8F>", "<0xB4>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA7>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA2>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA5>", "<0xF3>", "<0xA0>", "<0x81>", "<0xAE>", "<0xF3>", "<0xA0>", "<0x81>", "<0xA7>", "<0xF3>", "<0xA0>", "<0x81>", "<0xBF>", "\u2581", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBB>", "<0xE2>", "<0x80>", "<0x8D>", "<0xE2>", "<0x9D>", "<0xA4>", "<0xEF>", "<0xB8>", "<0x8F>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x92>", "<0x8B>", "<0xE2>", "<0x80>", "<0x8D>", "<0xF0>", "<0x9F>", "<0x91>", "<0xA8>", "<0xF0>", "<0x9F>", "<0x8F>", "<0xBC>"], + ids: [1, 31822, 229, 159, 171, 31822, 243, 162, 167, 154, 31822, 243, 162, 148, 132, 242, 187, 146, 31822, 243, 162, 148, 180, 243, 162, 146, 190, 31822, 243, 162, 152, 184, 229, 131, 144, 229, 156, 133, 242, 187, 146, 31822, 243, 162, 170, 156, 243, 162, 146, 190, 229, 131, 144, 229, 156, 133, 31822, 243, 162, 148, 171, 243, 162, 146, 190, 229, 131, 144, 243, 162, 143, 193, 31822, 243, 162, 170, 148, 229, 131, 144, 243, 162, 167, 160, 229, 131, 144, 243, 162, 170, 148, 31822, 243, 162, 148, 172, 229, 131, 144, 229, 160, 167, 229, 131, 144, 243, 162, 149, 142, 229, 131, 144, 243, 162, 148, 171, 31822, 243, 162, 148, 172, 229, 131, 144, 243, 162, 148, 172, 229, 131, 144, 243, 162, 148, 170, 229, 131, 144, 243, 162, 148, 169, 31822, 243, 162, 170, 148, 243, 162, 146, 190, 229, 131, 144, 243, 162, 167, 160, 229, 131, 144, 243, 162, 170, 148, 243, 162, 146, 190, 31822, 243, 162, 146, 183, 246, 163, 132, 170, 246, 163, 132, 165, 246, 163, 132, 168, 246, 163, 132, 177, 246, 163, 132, 170, 246, 163, 132, 194, 31822, 243, 162, 148, 171, 243, 162, 146, 190, 229, 131, 144, 229, 160, 167, 242, 187, 146, 229, 131, 144, 243, 162, 149, 142, 229, 131, 144, 243, 162, 148, 171, 243, 162, 146, 191], + decoded: " \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + NEWLINE_WITH_LEADING_SPACE: { + text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, + tokens: ["\u2581", "\u2581", "<0x0A>"], + ids: [1, 31822, 31822, 13], + decoded: " \n", + }, + CHINESE_LETTER: { + text: LLAMA_TEST_STRINGS.CHINESE_LETTER, + tokens: ["\u2581", "<0xE9>", "<0x95>", "<0x87>"], + ids: [1, 31822, 236, 152, 138], + decoded: " \u9547", + }, + PARAGRAPH: { + text: LLAMA_TEST_STRINGS.PARAGRAPH, + tokens: ["\u2581The", "\u2581ll", "ama", "\u2581(", "/", "<0xCB>", "<0x88>", "l", "<0xC9>", "<0x91>", "<0xCB>", "<0x90>", "m", "<0xC9>", "<0x99>", "/", ";", "\u2581", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>", "Sp", "anish", "\u2581pron", "unciation", ":", "\u2581[", "<0xCB>", "<0x88>", "<0xCA>", "<0x8E>", "ama", "])", "\u2581(", "L", "ama", "\u2581gl", "ama", ")", "\u2581is", "\u2581a", "\u2581domest", "icated", "\u2581South", "\u2581American", "\u2581cam", "el", "id", ",", "\u2581widely", "\u2581used", "\u2581as", "\u2581a", "\u2581meat", "\u2581and", "\u2581pack", "\u2581animal", "\u2581by", "\u2581And", "ean", "\u2581cultures", "\u2581since", "\u2581the", "\u2581Pre", "-", "Col", "umb", "ian", "\u2581era", ".", "\u2581L", "lam", "as", "\u2581are", "\u2581social", "\u2581animals", "\u2581and", "\u2581live", "\u2581with", "\u2581others", "\u2581as", "\u2581a", "\u2581herd", ".", "\u2581Their", "\u2581wool", "\u2581is", "\u2581soft", "\u2581and", "\u2581contains", "\u2581only", "\u2581a", "\u2581small", "\u2581amount", "\u2581of", "\u2581l", "anol", "in", ".[", "2", "]", "\u2581L", "lam", "as", "\u2581can", "\u2581learn", "\u2581simple", "\u2581", "t", "asks", "\u2581after", "\u2581a", "\u2581few", "\u2581repet", "itions", ".", "\u2581When", "\u2581using", "\u2581a", "\u2581pack", ",", "\u2581they", "\u2581can", "\u2581carry", "\u2581about", "\u2581", "2", "5", "\u2581to", "\u2581", "3", "0", "%", "\u2581of", "\u2581their", "\u2581body", "\u2581weight", "\u2581for", "\u2581", "8", "\u2581to", "\u2581", "1", "3", "\u2581km", "\u2581(", "5", "\u2013", "8", "\u2581miles", ").", "[", "3", "]", "\u2581The", "\u2581name", "\u2581ll", "ama", "\u2581(", "in", "\u2581the", "\u2581past", "\u2581also", "\u2581sp", "elled", '\u2581"', "l", "ama", '"', "\u2581or", '\u2581"', "gl", "ama", '")', "\u2581was", "\u2581adopted", "\u2581by", "\u2581European", "\u2581settlers", "\u2581from", "\u2581native", "\u2581Per", "uv", "ians", ".[", "4", "]", "\u2581The", "\u2581ancestors", "\u2581of", "\u2581l", "lam", "as", "\u2581are", "\u2581thought", "\u2581to", "\u2581have", "\u2581originated", "\u2581from", "\u2581the", "\u2581Great", "\u2581Plains", "\u2581of", "\u2581North", "\u2581America", "\u2581about", "\u2581", "4", "0", "\u2581million", "\u2581years", "\u2581ago", ",", "\u2581and", "\u2581subsequently", "\u2581mig", "rated", "\u2581to", "\u2581South", "\u2581America", "\u2581about", "\u2581three", "\u2581million", "\u2581years", "\u2581ago", "\u2581during", "\u2581the", "\u2581Great", "\u2581American", "\u2581Inter", "change", ".", "\u2581By", "\u2581the", "\u2581end", "\u2581of", "\u2581the", "\u2581last", "\u2581ice", "\u2581age", "\u2581(", "1", "0", ",", "0", "0", "0", "\u2013", "1", "2", ",", "0", "0", "0", "\u2581years", "\u2581ago", "),", "\u2581cam", "el", "ids", "\u2581were", "\u2581extinct", "\u2581in", "\u2581North", "\u2581America", ".[", "3", "]", "\u2581As", "\u2581of", "\u2581", "2", "0", "0", "7", ",", "\u2581there", "\u2581were", "\u2581over", "\u2581seven", "\u2581million", "\u2581l", "lam", "as", "\u2581and", "\u2581al", "p", "ac", "as", "\u2581in", "\u2581South", "\u2581America", "\u2581and", "\u2581over", "\u2581", "1", "5", "8", ",", "0", "0", "0", "\u2581l", "lam", "as", "\u2581and", "\u2581", "1", "0", "0", ",", "0", "0", "0", "<0xEA>", "<0x99>", "<0x8A>", "<0xF0>", "<0x9F>", "<0xA6>", "<0x99>", "\u2581al", "p", "ac", "as", ",", "\u2581descended", "\u2581from", "\u2581pro", "gen", "itors", "\u2581imported", "\u2581late", "\u2581in", "\u2581the", "\u2581", "2", "0", "th", "\u2581century", ",", "\u2581in", "\u2581the", "\u2581United", "\u2581States", "\u2581and", "\u2581Canada", ".[", "5", "]", "\u2581In", "\u2581A", "ym", "ara", "\u2581mythology", ",", "\u2581l", "lam", "as", "\u2581are", "\u2581important", "\u2581beings", ".", "\u2581The", "\u2581Heaven", "ly", "\u2581Ll", "ama", "\u2581is", "\u2581said", "\u2581to", "\u2581drink", "\u2581water", "\u2581from", "\u2581the", "\u2581ocean", "\u2581and", "\u2581ur", "inates", "\u2581as", "\u2581it", "\u2581rains", ".[", "6", "]", "\u2581According", "\u2581to", "\u2581A", "ym", "ara", "\u2581es", "chat", "ology", ",", "\u2581l", "lam", "as", "\u2581will", "\u2581return", "\u2581to", "\u2581the", "\u2581water", "\u2581springs", "\u2581and", "\u2581l", "ago", "ons", "\u2581where", "\u2581they", "\u2581come", "\u2581from", "\u2581at", "\u2581the", "\u2581end", "\u2581of", "\u2581time", ".[", "6", "]"], + ids: [1, 347, 31763, 2269, 352, 31873, 206, 139, 31832, 204, 148, 206, 147, 31836, 204, 156, 31873, 31891, 31822, 243, 162, 169, 156, 8889, 5817, 11155, 26128, 31871, 836, 206, 139, 205, 145, 2269, 9772, 352, 31867, 2269, 1192, 2269, 31861, 322, 260, 27940, 2672, 1897, 1454, 3764, 307, 317, 31844, 7055, 1065, 362, 260, 8659, 291, 2667, 6075, 417, 787, 14083, 10775, 1314, 266, 2345, 31854, 4848, 2234, 620, 5998, 31843, 372, 3082, 295, 397, 1619, 5220, 291, 1983, 351, 1892, 362, 260, 27172, 31843, 4585, 22729, 322, 2647, 291, 5140, 744, 260, 1435, 2399, 287, 309, 18426, 261, 3564, 31855, 31908, 372, 3082, 295, 473, 1977, 3102, 31822, 31824, 5577, 768, 260, 1346, 17042, 1479, 31843, 1408, 1340, 260, 2667, 31844, 526, 473, 3875, 562, 31822, 31855, 31880, 289, 31822, 31878, 31852, 31914, 287, 518, 2108, 4182, 329, 31822, 31886, 289, 31822, 31853, 31878, 6512, 352, 31880, 31906, 31886, 4465, 656, 31907, 31878, 31908, 347, 1382, 31763, 2269, 352, 261, 266, 1646, 615, 612, 5902, 495, 31832, 2269, 31875, 405, 495, 4261, 2269, 4290, 393, 7574, 417, 2821, 23343, 427, 6412, 2083, 10099, 1580, 3564, 31882, 31908, 347, 18294, 287, 309, 3082, 295, 397, 1991, 289, 435, 20355, 427, 266, 3172, 26744, 287, 1975, 2139, 562, 31822, 31882, 31852, 1577, 778, 2236, 31844, 291, 11786, 21052, 3397, 289, 1897, 2139, 562, 1166, 1577, 778, 2236, 1177, 266, 3172, 1454, 3029, 3604, 31843, 1433, 266, 928, 287, 266, 1060, 5707, 2253, 352, 31853, 31852, 31844, 31852, 31852, 31852, 31906, 31853, 31855, 31844, 31852, 31852, 31852, 778, 2236, 698, 3764, 307, 1982, 577, 30610, 288, 1975, 2139, 3564, 31878, 31908, 717, 287, 31822, 31855, 31852, 31852, 31888, 31844, 635, 577, 648, 3931, 1577, 309, 3082, 295, 291, 366, 31837, 380, 295, 288, 1897, 2139, 291, 648, 31822, 31853, 31880, 31886, 31844, 31852, 31852, 31852, 309, 3082, 295, 291, 31822, 31853, 31852, 31852, 31844, 31852, 31852, 31852, 237, 156, 141, 243, 162, 169, 156, 366, 31837, 380, 295, 31844, 27627, 427, 375, 3353, 4705, 17798, 2732, 288, 266, 31822, 31855, 31852, 388, 3373, 31844, 288, 266, 1494, 1769, 291, 3008, 3564, 31880, 31908, 455, 308, 1276, 2776, 24143, 31844, 309, 3082, 295, 397, 1480, 11844, 31843, 347, 15836, 326, 11321, 2269, 322, 664, 289, 5065, 1579, 427, 266, 8622, 291, 4328, 11466, 362, 357, 28738, 3564, 31887, 31908, 3252, 289, 308, 1276, 2776, 1582, 20068, 1058, 31844, 309, 3082, 295, 482, 1199, 289, 266, 1579, 24250, 291, 309, 3405, 680, 804, 526, 1412, 427, 389, 266, 928, 287, 647, 3564, 31887, 31908], + decoded: ' The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + }, + }, + "Xenova/deepseek-coder-1.3b-instruct": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [32013, 2808, 417, 340, 3207, 30], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'", "ve", "\u0120done", "\u0120this"], + ids: [32013, 2042, 1020, 6, 312, 2359, 437], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u0120", "0", "\u0120", "1", "\u0120", "2", "\u0120", "3", "\u0120", "4", "\u0120", "5", "\u0120", "6", "\u0120", "7", "\u0120", "8", "\u0120", "9", "\u0120", "1", "0", "\u0120", "1", "0", "0", "\u0120", "1", "0", "0", "0"], + ids: [32013, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 207, 15, 207, 16, 207, 17, 207, 18, 207, 19, 207, 20, 207, 21, 207, 22, 207, 23, 207, 24, 207, 16, 15, 207, 16, 15, 15, 207, 16, 15, 15, 15], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "2", "0", "1", "6", "."], + ids: [32013, 546, 2595, 438, 16316, 279, 207, 17, 15, 16, 21, 13], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'", "ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'", "t", "."], + ids: [32013, 32, 185, 6, 642, 24466, 577, 11665, 67, 4191, 67, 280, 11, 482, 6, 83, 13], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>A\n'll !!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"], + ids: [32013, 1551, 1959, 10942, 185, 184, 4805], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "toString", "();", "\u010a", "toString", "();"], + ids: [32013, 1160, 245, 405, 6528, 13, 12617, 1293, 185, 12617, 1293], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: LLAMA_TEST_STRINGS.NEWLINES, + tokens: ["ax", "\u010a", "####", "\u010a", "bo", "o"], + ids: [32013, 1099, 185, 3576, 185, 952, 78], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>ax\n####\nboo", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], + ids: [32013, 4348, 28626, 31898, 11, 22785], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [32013, 16, 175, 17, 10006, 18], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [32013, 17535, 5414], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [32013, 31702, 1835], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [32013, 23393, 2651, 1534, 236, 502], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [32013, 243, 5877, 2507], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c> leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120\u0120\u0120"], + ids: [32013, 7246, 5964, 2507, 315], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120H", "ello"], + ids: [32013, 11041, 207, 414, 9489], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120", "\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120", "\u00e2\u0124", "\u00a3", "7", "\u0120", "\u00e2\u0124", "\u00b9", "8", "\u0120", "\u00e2\u0124", "\u00b1", "9", "\u0120test"], + ids: [32013, 2806, 371, 16, 432, 17, 1494, 18, 207, 11010, 19, 8761, 20, 2688, 98, 21, 207, 7935, 96, 22, 207, 7935, 117, 23, 207, 7935, 109, 24, 1719], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "0", "0", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [32013, 40, 8942, 274, 15902, 327, 371, 16, 13, 15, 15, 429, 254, 4730, 13], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [32013, 4209, 2484, 243], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [32013, 4209, 2484, 10447], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [32013, 4209, 2484, 1200, 1200, 4209, 2484, 10447], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120", "\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [32013, 828, 2369, 207, 169, 121, 239, 5935, 207, 169, 121, 239, 1452], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>weird \uff5e edge \uff5e case", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141", "\u013a", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141", "\u013a", "\u012f", "\u0120\u00f0\u0141", "\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141\u013b", "\u0131", "\u0120\u00f0\u0141", "\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141", "\u013a", "\u0123", "\u0120\u00f0\u0141", "\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141", "\u013a", "\u0128", "\u0120\u00f0\u0141", "\u0133", "\u0131", "\u0120", "\u00e2", "\u013f", "\u00a4", "\u00ef", "\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u0134", "\u013e", "\u0120\u00f0\u0141", "\u0134", "\u013c", "\u0120\u00f0\u0141", "\u0134", "\u0139", "\u0120\u00f0\u0141", "\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141", "\u013a", "\u0130", "\u0120\u00f0\u0141", "\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141", "\u0134", "\u00aa", "\u0120", "\u00e2", "\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u012b", "\u0120\u00f0\u0141", "\u0133", "\u0122", "\u0120\u00f0\u0141", "\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141\u013b", "\u012a", "\u0120\u00f0\u0141\u013b", "\u012e", "\u0120\u00f0\u0141", "\u0134", "\u0122", "\u0120\u00f0\u0141", "\u0133", "\u0129", "\u0120\u00f0\u0141", "\u0133", "\u012d", "\u0120", "\u00e2", "\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141", "\u0134", "\u00b0"], + ids: [32013, 10047, 233, 211, 12394, 226, 222, 12394, 97, 96, 12394, 233, 222, 12394, 233, 242, 12394, 223, 218, 22709, 224, 12394, 233, 219, 12394, 229, 98, 12394, 233, 210, 12394, 233, 214, 12394, 97, 232, 12394, 233, 215, 12394, 226, 224, 207, 156, 238, 97, 169, 116, 224, 12394, 227, 237, 12394, 227, 235, 12394, 227, 232, 12394, 227, 234, 12394, 231, 97, 12394, 233, 223, 12394, 226, 221, 12394, 98, 111, 12394, 227, 103, 207, 156, 237, 101, 12394, 226, 218, 12394, 226, 209, 12394, 227, 107, 12394, 223, 217, 22709, 217, 22709, 221, 12394, 227, 209, 12394, 226, 216, 12394, 226, 220, 207, 156, 237, 214, 12394, 223, 210, 12394, 221, 239, 12394, 221, 116, 12394, 227, 108], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2", "\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141", "\u0133", "\u0123", "\u00ef", "\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122", "\u012f", "\u00e2", "\u013b", "\u0124", "\u00ef", "\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2", "\u013b", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00ef", "\u00b8", "\u0131", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], + ids: [32013, 156, 237, 101, 12394, 97, 232, 12394, 226, 210, 169, 116, 224, 12394, 226, 109, 10047, 224, 119, 12394, 230, 113, 350, 222, 156, 234, 211, 169, 116, 224, 12394, 100, 234, 10047, 224, 119, 350, 222, 156, 234, 211, 12394, 226, 101, 10047, 224, 119, 350, 222, 10047, 221, 122, 12394, 100, 226, 350, 222, 10047, 97, 238, 350, 222, 10047, 100, 226, 12394, 226, 102, 350, 222, 156, 238, 97, 350, 222, 10047, 227, 220, 350, 222, 10047, 226, 101, 12394, 226, 102, 350, 222, 10047, 226, 102, 350, 222, 10047, 226, 100, 350, 222, 10047, 226, 99, 12394, 100, 226, 10047, 224, 119, 350, 222, 10047, 97, 238, 350, 222, 10047, 100, 226, 10047, 224, 119, 12394, 224, 112, 173, 241, 210, 100, 173, 241, 210, 95, 173, 241, 210, 98, 173, 241, 210, 106, 173, 241, 210, 100, 173, 241, 210, 123, 12394, 226, 101, 10047, 224, 119, 350, 222, 156, 238, 97, 169, 116, 224, 350, 222, 10047, 227, 220, 350, 222, 10047, 226, 101, 10047, 224, 120], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120", "\u00e2\u0138", "\u0123", "is", "\u0120", "\u00e2\u0138", "\u0123", "a", "\u0120", "\u00e2\u0138", "\u0123", "test", "\u0120", "\u00e2\u0138", "\u0123", "."], + ids: [32013, 11028, 210, 1559, 207, 11028, 210, 262, 207, 11028, 210, 64, 207, 11028, 210, 2806, 207, 11028, 210, 13], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + BPE_SCORES_PRIORITY_1: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, + tokens: ["gr", "ab", "bed"], + ids: [32013, 877, 356, 3861], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>grabbed", + }, + BPE_SCORES_PRIORITY_2: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, + tokens: ["\u0120grab", "bed"], + ids: [32013, 14596, 3861], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c> grabbed", + }, + BPE_SCORES_PRIORITY_3: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, + tokens: ["\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120", "\u0120grab", "bed"], + ids: [32013, 3137, 14596, 3861], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c> grabbed", + }, + NEWLINE: { + text: LLAMA_TEST_STRINGS.NEWLINE, + tokens: ["\u010a"], + ids: [32013, 185], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\n", + }, + NEWLINE_WITH_LEADING_SPACE: { + text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, + tokens: ["\u0120", "\u010a"], + ids: [32013, 207, 185], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c> \n", + }, + TABS: { + text: LLAMA_TEST_STRINGS.TABS, + tokens: ["\u0109", "tabs", "\u0109\u0109\u0109", "\u0109", "out", "\u0120here"], + ids: [32013, 184, 20611, 1749, 184, 406, 1283], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\ttabs\t\t\t\tout here", + }, + NEWLINE_AND_TAB: { + text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, + tokens: ["\u010a", "\u0109", "\u010a"], + ids: [32013, 185, 184, 185], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\n\t\n", + }, + CHINESE_LETTER: { + text: LLAMA_TEST_STRINGS.CHINESE_LETTER, + tokens: ["\u00e9\u0137\u0129"], + ids: [32013, 6759], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\u9547", + }, + EMOJIS_1: { + text: LLAMA_TEST_STRINGS.EMOJIS_1, + tokens: ["\u00f0\u0141", "\u00a6", "\u013b"], + ids: [32013, 10047, 99, 234], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\ud83e\udd99", + }, + EMOJIS_2: { + text: LLAMA_TEST_STRINGS.EMOJIS_2, + tokens: ["\u00f0\u0141", "\u00a6", "\u013b", "\u00ea", "\u013b", "\u012c"], + ids: [32013, 10047, 99, 234, 164, 234, 219], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\ud83e\udd99\ua64a", + }, + EMOJIS_3: { + text: LLAMA_TEST_STRINGS.EMOJIS_3, + tokens: ["\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b"], + ids: [32013, 164, 234, 219, 10047, 99, 234], + decoded: "<\uff5cbegin\u2581of\u2581sentence\uff5c>\ua64a\ud83e\udd99", + }, + PARAGRAPH: { + text: LLAMA_TEST_STRINGS.PARAGRAPH, + tokens: ["The", "\u0120ll", "ama", "\u0120(/", "\u00cb\u012a", "l", "\u00c9", "\u0133", "\u00cb", "\u0132", "m", "\u00c9\u013b", "/", ";", "\u0120\u00f0\u0141", "\u00a6", "\u013b", "Span", "ish", "\u0120pron", "unciation", ":", "\u0120[", "\u00cb\u012a", "\u00ca", "\u0130", "ama", "])", "\u0120(", "L", "ama", "\u0120gl", "ama", ")", "\u0120is", "\u0120a", "\u0120domestic", "ated", "\u0120South", "\u0120American", "\u0120cam", "el", "id", ",", "\u0120widely", "\u0120used", "\u0120as", "\u0120a", "\u0120meat", "\u0120and", "\u0120pack", "\u0120animal", "\u0120by", "\u0120And", "ean", "\u0120cultures", "\u0120since", "\u0120the", "\u0120Pre", "-", "Col", "umb", "ian", "\u0120era", ".", "\u0120L", "lam", "as", "\u0120are", "\u0120social", "\u0120animals", "\u0120and", "\u0120live", "\u0120with", "\u0120others", "\u0120as", "\u0120a", "\u0120her", "d", ".", "\u0120Their", "\u0120wool", "\u0120is", "\u0120soft", "\u0120and", "\u0120contains", "\u0120only", "\u0120a", "\u0120small", "\u0120amount", "\u0120of", "\u0120lan", "ol", "in", ".[", "2", "]", "\u0120L", "lam", "as", "\u0120can", "\u0120learn", "\u0120simple", "\u0120tasks", "\u0120after", "\u0120a", "\u0120few", "\u0120repet", "itions", ".", "\u0120When", "\u0120using", "\u0120a", "\u0120pack", ",", "\u0120they", "\u0120can", "\u0120carry", "\u0120about", "\u0120", "2", "5", "\u0120to", "\u0120", "3", "0", "%", "\u0120of", "\u0120their", "\u0120body", "\u0120weight", "\u0120for", "\u0120", "8", "\u0120to", "\u0120", "1", "3", "\u0120km", "\u0120(", "5", "\u00e2\u0122\u0135", "8", "\u0120miles", ").", "[", "3", "]", "\u0120The", "\u0120name", "\u0120ll", "ama", "\u0120(", "in", "\u0120the", "\u0120past", "\u0120also", "\u0120sp", "elled", '\u0120"', "l", "ama", '"', "\u0120or", '\u0120"', "gl", "ama", '")', "\u0120was", "\u0120adopted", "\u0120by", "\u0120European", "\u0120sett", "lers", "\u0120from", "\u0120native", "\u0120Per", "uv", "ians", ".[", "4", "]", "\u0120The", "\u0120ancest", "ors", "\u0120of", "\u0120llam", "as", "\u0120are", "\u0120thought", "\u0120to", "\u0120have", "\u0120origin", "ated", "\u0120from", "\u0120the", "\u0120Great", "\u0120Pl", "ains", "\u0120of", "\u0120North", "\u0120America", "\u0120about", "\u0120", "4", "0", "\u0120million", "\u0120years", "\u0120ago", ",", "\u0120and", "\u0120subsequently", "\u0120mig", "rated", "\u0120to", "\u0120South", "\u0120America", "\u0120about", "\u0120three", "\u0120million", "\u0120years", "\u0120ago", "\u0120during", "\u0120the", "\u0120Great", "\u0120American", "\u0120Inter", "change", ".", "\u0120By", "\u0120the", "\u0120end", "\u0120of", "\u0120the", "\u0120last", "\u0120ice", "\u0120age", "\u0120(", "1", "0", ",", "0", "0", "0", "\u00e2\u0122\u0135", "1", "2", ",", "0", "0", "0", "\u0120years", "\u0120ago", "),", "\u0120cam", "el", "ids", "\u0120were", "\u0120ext", "inct", "\u0120in", "\u0120North", "\u0120America", ".[", "3", "]", "\u0120As", "\u0120of", "\u0120", "2", "0", "0", "7", ",", "\u0120there", "\u0120were", "\u0120over", "\u0120seven", "\u0120million", "\u0120llam", "as", "\u0120and", "\u0120al", "p", "ac", "as", "\u0120in", "\u0120South", "\u0120America", "\u0120and", "\u0120over", "\u0120", "1", "5", "8", ",", "0", "0", "0", "\u0120llam", "as", "\u0120and", "\u0120", "1", "0", "0", ",", "0", "0", "0", "\u00ea", "\u013b", "\u012c", "\u00f0\u0141", "\u00a6", "\u013b", "\u0120al", "p", "ac", "as", ",", "\u0120desc", "ended", "\u0120from", "\u0120pro", "gen", "itors", "\u0120imported", "\u0120late", "\u0120in", "\u0120the", "\u0120", "2", "0", "th", "\u0120century", ",", "\u0120in", "\u0120the", "\u0120United", "\u0120States", "\u0120and", "\u0120Canada", ".[", "5", "]", "\u0120In", "\u0120A", "ym", "ara", "\u0120myth", "ology", ",", "\u0120llam", "as", "\u0120are", "\u0120important", "\u0120beings", ".", "\u0120The", "\u0120Heaven", "ly", "\u0120Ll", "ama", "\u0120is", "\u0120said", "\u0120to", "\u0120drink", "\u0120water", "\u0120from", "\u0120the", "\u0120ocean", "\u0120and", "\u0120ur", "in", "ates", "\u0120as", "\u0120it", "\u0120ra", "ins", ".[", "6", "]", "\u0120According", "\u0120to", "\u0120A", "ym", "ara", "\u0120es", "chat", "ology", ",", "\u0120llam", "as", "\u0120will", "\u0120return", "\u0120to", "\u0120the", "\u0120water", "\u0120springs", "\u0120and", "\u0120l", "ago", "ons", "\u0120where", "\u0120they", "\u0120come", "\u0120from", "\u0120at", "\u0120the", "\u0120end", "\u0120of", "\u0120time", ".[", "6", "]"], + ids: [32013, 546, 1703, 4204, 31905, 31459, 75, 131, 226, 133, 225, 76, 28747, 14, 26, 12394, 99, 234, 20786, 840, 9119, 25307, 25, 821, 31459, 132, 223, 4204, 5589, 334, 43, 4204, 1649, 4204, 8, 317, 245, 13569, 612, 5168, 4115, 4370, 282, 304, 11, 13620, 1219, 372, 245, 12342, 285, 2379, 9542, 457, 1306, 24391, 24783, 1952, 254, 7606, 12, 2608, 4313, 987, 2895, 13, 412, 8265, 281, 417, 3601, 8469, 285, 3516, 365, 3060, 372, 245, 706, 67, 13, 9195, 24547, 317, 2829, 285, 5396, 885, 245, 1752, 3733, 280, 27264, 313, 246, 9469, 17, 60, 412, 8265, 281, 482, 3059, 2966, 9227, 1164, 245, 1853, 15747, 2160, 13, 2463, 1242, 245, 2379, 11, 653, 482, 5642, 782, 207, 17, 20, 276, 207, 18, 15, 4, 280, 699, 3110, 4285, 327, 207, 23, 276, 207, 16, 18, 9004, 334, 20, 887, 23, 6595, 628, 58, 18, 60, 428, 1208, 1703, 4204, 334, 246, 254, 2872, 835, 731, 6679, 440, 75, 4204, 1, 409, 440, 2521, 4204, 2456, 438, 13509, 457, 8717, 6762, 12104, 473, 8118, 3043, 12466, 3091, 9469, 19, 60, 428, 18901, 710, 280, 15410, 281, 417, 2207, 276, 463, 6948, 612, 473, 254, 6984, 2284, 2200, 280, 5216, 6092, 782, 207, 19, 15, 4866, 1547, 4074, 11, 285, 23909, 8290, 9831, 276, 5168, 6092, 782, 1846, 4866, 1547, 4074, 2310, 254, 6984, 4115, 6660, 4865, 13, 3550, 254, 1223, 280, 254, 1554, 9405, 4489, 334, 16, 15, 11, 15, 15, 15, 887, 16, 17, 11, 15, 15, 15, 1547, 4074, 650, 4370, 282, 2929, 773, 1309, 5729, 279, 5216, 6092, 9469, 18, 60, 1725, 280, 207, 17, 15, 15, 22, 11, 741, 773, 851, 7970, 4866, 15410, 281, 285, 360, 79, 305, 281, 279, 5168, 6092, 285, 851, 207, 16, 20, 23, 11, 15, 15, 15, 15410, 281, 285, 207, 16, 15, 15, 11, 15, 15, 15, 164, 234, 219, 10047, 99, 234, 360, 79, 305, 281, 11, 1774, 2611, 473, 381, 4920, 6041, 26357, 5179, 279, 254, 207, 17, 15, 392, 8299, 11, 279, 254, 4783, 5098, 285, 8905, 9469, 20, 60, 680, 338, 1254, 3367, 25157, 2333, 11, 15410, 281, 417, 2364, 22792, 13, 428, 18933, 326, 9140, 4204, 317, 989, 276, 7371, 2345, 473, 254, 15439, 285, 8580, 246, 980, 372, 359, 1809, 1231, 9469, 21, 60, 10068, 276, 338, 1254, 3367, 707, 24570, 2333, 11, 15410, 281, 540, 967, 276, 254, 2345, 30851, 285, 284, 5980, 875, 1064, 653, 1857, 473, 429, 254, 1223, 280, 761, 9469, 21, 60], + decoded: '<\uff5cbegin\u2581of\u2581sentence\uff5c>The llama (/\u02c8l\u0251\u02d0m\u0259/; \ud83e\udd99Spanish pronunciation: [\u02c8\u028eama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000\ua64a\ud83e\udd99 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + }, + }, + "Xenova/tamillama_tiny_30m": { + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u2581", "2", "0", "1", "6", "."], + ids: [1, 147, 10984, 139, 949, 78, 198, 31654, 13, 21, 12, 17, 34], + decoded: " The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\n", "'", "ll", "\u2581", "!", "!", "to", "?", "'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [1, 231, 5, 31, 370, 31654, 31715, 31715, 5140, 31725, 31, 31679, 31, 31, 31679, 251, 35, 645, 31, 31665, 34], + decoded: " A\n'll !!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581def", "\u2581main", "(", ")", ":", "\n", "", "p", "ass"], + ids: [1, 12849, 17375, 32, 33, 29, 5, 0, 31694, 1917], + decoded: " def main():\npass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581", "=", "\u2581ob", "j", ".", "to", "St", "ring", "(", ")", ";", "\n", "to", "St", "ring", "(", ")", ";"], + ids: [1, 1996, 48, 31654, 25, 4083, 31733, 34, 5140, 23417, 6631, 32, 33, 30, 5, 5140, 23417, 6631, 32, 33, 30], + decoded: " let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: LLAMA_TEST_STRINGS.NEWLINES, + tokens: ["\u2581ax", "\n", "#", "#", "#", "#", "\n", "boo"], + ids: [1, 11441, 5, 22, 22, 22, 22, 5, 21260], + decoded: " ax\n####\nboo", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581U", "N", "w", "ant", "\u00e9", "d", ",", "r", "un", "ning"], + ids: [1, 5841, 31748, 31689, 1027, 31771, 31679, 35, 31678, 367, 1855], + decoded: " UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u2581", "1", "", "2", "", "3"], + ids: [1, 31654, 12, 0, 13, 0, 14], + decoded: " 123", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581H", "ello", "\u2581World"], + ids: [1, 207, 3589, 25544], + decoded: " Hello World", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", ""], + ids: [1, 31654, 0], + decoded: " ", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581", "\u2581", "\u2581", "\u2581leading", "\u2581space"], + ids: [1, 31654, 31654, 31654, 7951, 7259], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581tra", "iling", "\u2581space", "\u2581", "\u2581", "\u2581"], + ids: [1, 2036, 9850, 7259, 31654, 31654, 31654], + decoded: " trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581H", "i", "\u2581", "\u2581H", "ello"], + ids: [1, 207, 31673, 31654, 207, 3589], + decoded: " Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581", "$", "1", "\u2581R", "2", "\u2581", "#", "3", "\u2581", "\u20ac", "4", "\u2581", "\u00a3", "5", "\u2581", "", "6", "\u2581", "", "7", "\u2581", "\u20b9", "8", "\u2581", "", "9", "\u2581test"], + ids: [1, 6370, 31654, 9, 12, 947, 13, 31654, 22, 14, 31654, 31746, 15, 31654, 31792, 16, 31654, 0, 17, 31654, 0, 18, 31654, 31999, 19, 31654, 0, 20, 6370], + decoded: " test $1 R2 #3 \u20ac4 \u00a35 6 7 \u20b98 9 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581", "$", "1", ".", "0", "0", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [1, 320, 4685, 446, 4223, 347, 31654, 9, 12, 34, 21, 21, 586, 70, 2023, 34], + decoded: " I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "", "\u2581", "\u2581"], + ids: [1, 356, 0, 31654, 31654], + decoded: " you ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", ""], + ids: [1, 356, 0], + decoded: " you", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "", "you", ""], + ids: [1, 356, 0, 21984, 0], + decoded: " youyou", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "", "\u2581edge", "\u2581", "", "\u2581case"], + ids: [1, 7865, 31654, 0, 11148, 31654, 0, 10143], + decoded: " weird edge case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581", "\u2581This", "\u2581", "\u2581is", "\u2581", "\u2581a", "\u2581", "\u2581test", "\u2581", "\u2581", "."], + ids: [1, 31654, 3827, 31654, 344, 31654, 48, 31654, 6370, 31654, 31654, 34], + decoded: " This is a test .", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", ""], + ids: [1, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0], + decoded: " ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u200d", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u200d", "", "\u200d", "", "\u2581", "", "\u200d", "", "\u200d", "", "\u2581", "", "\u2581", "", "\u200d", "", "\u200d", "", "\u200d", ""], + ids: [1, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31654, 0, 31928, 0, 31654, 0, 31928, 0, 31654, 0, 31928, 0, 31654, 0, 31928, 0, 31928, 0, 31654, 0, 31928, 0, 31928, 0, 31928, 0, 31654, 0, 31928, 0, 31928, 0, 31928, 0, 31654, 0, 31928, 0, 31928, 0, 31654, 0, 31654, 0, 31928, 0, 31928, 0, 31928, 0], + decoded: " \u200d \u200d \u200d \u200d\u200d \u200d\u200d\u200d \u200d\u200d\u200d \u200d\u200d \u200d\u200d\u200d", + }, + BPE_SCORES_PRIORITY_1: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_1, + tokens: ["\u2581grabbed"], + ids: [1, 3618], + decoded: " grabbed", + }, + BPE_SCORES_PRIORITY_2: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_2, + tokens: ["\u2581", "\u2581grabbed"], + ids: [1, 31654, 3618], + decoded: " grabbed", + }, + BPE_SCORES_PRIORITY_3: { + text: LLAMA_TEST_STRINGS.BPE_SCORES_PRIORITY_3, + tokens: ["\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581", "\u2581grabbed"], + ids: [1, 31654, 31654, 31654, 31654, 31654, 31654, 31654, 31654, 31654, 31654, 31654, 3618], + decoded: " grabbed", + }, + NEWLINE: { + text: LLAMA_TEST_STRINGS.NEWLINE, + tokens: ["\u2581", "\n"], + ids: [1, 31654, 5], + decoded: " \n", + }, + NEWLINE_WITH_LEADING_SPACE: { + text: LLAMA_TEST_STRINGS.NEWLINE_WITH_LEADING_SPACE, + tokens: ["\u2581", "\u2581", "\n"], + ids: [1, 31654, 31654, 5], + decoded: " \n", + }, + TABS: { + text: LLAMA_TEST_STRINGS.TABS, + tokens: ["\u2581", "", "t", "ab", "s", "", "out", "\u2581here"], + ids: [1, 31654, 0, 31665, 878, 31675, 0, 415, 3278], + decoded: " tabsout here", + }, + NEWLINE_AND_TAB: { + text: LLAMA_TEST_STRINGS.NEWLINE_AND_TAB, + tokens: ["\u2581", "\n", "", "\n"], + ids: [1, 31654, 5, 0, 5], + decoded: " \n\n", + }, + CHINESE_LETTER: { + text: LLAMA_TEST_STRINGS.CHINESE_LETTER, + tokens: ["\u2581", ""], + ids: [1, 31654, 0], + decoded: " ", + }, + EMOJIS_1: { + text: LLAMA_TEST_STRINGS.EMOJIS_1, + tokens: ["\u2581", ""], + ids: [1, 31654, 0], + decoded: " ", + }, + EMOJIS_2: { + text: LLAMA_TEST_STRINGS.EMOJIS_2, + tokens: ["\u2581", ""], + ids: [1, 31654, 0], + decoded: " ", + }, + EMOJIS_3: { + text: LLAMA_TEST_STRINGS.EMOJIS_3, + tokens: ["\u2581", ""], + ids: [1, 31654, 0], + decoded: " ", + }, + PARAGRAPH: { + text: LLAMA_TEST_STRINGS.PARAGRAPH, + tokens: ["\u2581The", "\u2581l", "l", "ama", "\u2581", "(", "/", "\u02c8", "l", "\u0251", "\u02d0", "m", "\u0259", "/", ";", "\u2581", "", "Sp", "an", "ish", "\u2581pr", "on", "un", "ci", "ation", ":", "\u2581", "[", "\u02c8", "", "ama", "]", ")", "\u2581", "(", "L", "ama", "\u2581gl", "ama", ")", "\u2581is", "\u2581a", "\u2581d", "om", "est", "ic", "ated", "\u2581South", "\u2581American", "\u2581cam", "el", "id", ",", "\u2581wid", "ely", "\u2581used", "\u2581as", "\u2581a", "\u2581meat", "\u2581and", "\u2581pack", "\u2581animal", "\u2581by", "\u2581And", "e", "an", "\u2581c", "ult", "ures", "\u2581since", "\u2581the", "\u2581P", "re", "-", "C", "ol", "umb", "ian", "\u2581", "era", ".", "\u2581L", "l", "am", "as", "\u2581are", "\u2581social", "\u2581animals", "\u2581and", "\u2581live", "\u2581with", "\u2581others", "\u2581as", "\u2581a", "\u2581her", "d", ".", "\u2581Their", "\u2581wool", "\u2581is", "\u2581soft", "\u2581and", "\u2581contains", "\u2581only", "\u2581a", "\u2581small", "\u2581amount", "\u2581of", "\u2581l", "an", "ol", "in", ".", "[", "2", "]", "\u2581L", "l", "am", "as", "\u2581can", "\u2581learn", "\u2581simple", "\u2581tasks", "\u2581after", "\u2581a", "\u2581few", "\u2581rep", "et", "itions", ".", "\u2581When", "\u2581using", "\u2581a", "\u2581pack", ",", "\u2581they", "\u2581can", "\u2581carry", "\u2581about", "\u2581", "2", "5", "\u2581to", "\u2581", "3", "0", "%", "\u2581of", "\u2581their", "\u2581body", "\u2581weight", "\u2581for", "\u2581", "8", "\u2581to", "\u2581", "1", "3", "\u2581km", "\u2581", "(", "5", "\u2013", "8", "\u2581miles", ")", ".", "[", "3", "]", "\u2581The", "\u2581name", "\u2581l", "l", "ama", "\u2581", "(", "in", "\u2581the", "\u2581past", "\u2581also", "\u2581spell", "ed", '\u2581"', "l", "ama", '"', "\u2581or", '\u2581"', "gl", "ama", '"', ")", "\u2581was", "\u2581adop", "ted", "\u2581by", "\u2581E", "urope", "an", "\u2581sett", "l", "ers", "\u2581from", "\u2581n", "ative", "\u2581Per", "u", "v", "ians", ".", "[", "4", "]", "\u2581The", "\u2581an", "c", "est", "ors", "\u2581of", "\u2581l", "l", "am", "as", "\u2581are", "\u2581thought", "\u2581to", "\u2581have", "\u2581origin", "ated", "\u2581from", "\u2581the", "\u2581Great", "\u2581Pl", "ain", "s", "\u2581of", "\u2581North", "\u2581America", "\u2581about", "\u2581", "4", "0", "\u2581million", "\u2581years", "\u2581ago", ",", "\u2581and", "\u2581sub", "sequ", "ently", "\u2581m", "ig", "r", "ated", "\u2581to", "\u2581South", "\u2581America", "\u2581about", "\u2581three", "\u2581million", "\u2581years", "\u2581ago", "\u2581during", "\u2581the", "\u2581Great", "\u2581American", "\u2581Int", "er", "ch", "ange", ".", "\u2581By", "\u2581the", "\u2581end", "\u2581of", "\u2581the", "\u2581last", "\u2581ice", "\u2581age", "\u2581", "(", "1", "0", ",", "0", "0", "0", "\u2013", "1", "2", ",", "0", "0", "0", "\u2581years", "\u2581ago", ")", ",", "\u2581cam", "el", "ids", "\u2581were", "\u2581ext", "inct", "\u2581in", "\u2581North", "\u2581America", ".", "[", "3", "]", "\u2581As", "\u2581of", "\u2581", "2", "0", "0", "7", ",", "\u2581there", "\u2581were", "\u2581over", "\u2581seven", "\u2581million", "\u2581l", "l", "am", "as", "\u2581and", "\u2581al", "p", "ac", "as", "\u2581in", "\u2581South", "\u2581America", "\u2581and", "\u2581over", "\u2581", "1", "5", "8", ",", "0", "0", "0", "\u2581l", "l", "am", "as", "\u2581and", "\u2581", "1", "0", "0", ",", "0", "0", "0", "", "\u2581al", "p", "ac", "as", ",", "\u2581des", "ce", "nd", "ed", "\u2581from", "\u2581pro", "gen", "it", "ors", "\u2581import", "ed", "\u2581late", "\u2581in", "\u2581the", "\u2581", "2", "0", "th", "\u2581cent", "ury", ",", "\u2581in", "\u2581the", "\u2581United", "\u2581States", "\u2581and", "\u2581Can", "ada", ".", "[", "5", "]", "\u2581In", "\u2581A", "ym", "ara", "\u2581my", "th", "ology", ",", "\u2581l", "l", "am", "as", "\u2581are", "\u2581important", "\u2581be", "ings", ".", "\u2581The", "\u2581He", "aven", "ly", "\u2581L", "l", "ama", "\u2581is", "\u2581said", "\u2581to", "\u2581drink", "\u2581water", "\u2581from", "\u2581the", "\u2581ocean", "\u2581and", "\u2581ur", "in", "ates", "\u2581as", "\u2581it", "\u2581rains", ".", "[", "6", "]", "\u2581Acc", "ord", "ing", "\u2581to", "\u2581A", "ym", "ara", "\u2581es", "ch", "at", "ology", ",", "\u2581l", "l", "am", "as", "\u2581will", "\u2581return", "\u2581to", "\u2581the", "\u2581water", "\u2581spr", "ings", "\u2581and", "\u2581l", "ag", "oons", "\u2581where", "\u2581they", "\u2581come", "\u2581from", "\u2581at", "\u2581the", "\u2581end", "\u2581of", "\u2581time", ".", "[", "6", "]"], + ids: [1, 147, 105, 31683, 4464, 31654, 32, 31753, 31774, 31683, 31813, 31779, 31687, 31781, 31753, 30, 31654, 0, 30106, 142, 531, 1823, 111, 367, 8762, 633, 29, 31654, 31778, 31774, 0, 4464, 31780, 33, 31654, 32, 31717, 4464, 1861, 4464, 33, 344, 48, 108, 120, 504, 515, 3062, 29052, 18424, 8829, 256, 153, 35, 20517, 2001, 2680, 488, 48, 9910, 83, 4314, 1448, 1015, 1736, 31660, 142, 103, 3441, 605, 13397, 70, 1629, 86, 7, 31739, 819, 4618, 1685, 31654, 7129, 34, 218, 31683, 235, 691, 617, 23632, 1707, 83, 5860, 249, 2905, 488, 48, 192, 31679, 34, 5290, 11964, 344, 3077, 83, 12959, 2859, 48, 1388, 7238, 251, 105, 142, 819, 81, 34, 31778, 13, 31780, 218, 31683, 235, 691, 645, 907, 16188, 22936, 1609, 48, 4505, 4706, 183, 29049, 34, 1354, 5247, 48, 4314, 35, 338, 645, 4923, 1096, 31654, 13, 16, 84, 31654, 14, 21, 10, 251, 626, 6011, 9152, 347, 31654, 19, 84, 31654, 12, 14, 29496, 31654, 32, 16, 31760, 19, 7843, 33, 34, 31778, 14, 31780, 147, 3516, 105, 31683, 4464, 31654, 32, 81, 70, 4829, 2320, 9948, 78, 245, 31683, 4464, 31690, 1187, 245, 686, 4464, 31690, 33, 139, 25228, 2490, 1015, 465, 25799, 142, 16405, 31683, 983, 825, 152, 12724, 24466, 31688, 31711, 26361, 34, 31778, 15, 31780, 147, 446, 31692, 504, 4166, 251, 105, 31683, 235, 691, 617, 1302, 84, 649, 7206, 3062, 825, 70, 27718, 12966, 588, 31675, 251, 26698, 27393, 1096, 31654, 15, 21, 23109, 3514, 17246, 35, 83, 5097, 17541, 19560, 114, 258, 31678, 3062, 84, 29052, 27393, 1096, 2765, 23109, 3514, 17246, 5823, 70, 27718, 18424, 25473, 98, 345, 3292, 34, 15498, 70, 1645, 251, 70, 6103, 2802, 13463, 31654, 32, 12, 21, 35, 21, 21, 21, 31760, 12, 13, 35, 21, 21, 21, 3514, 17246, 33, 35, 8829, 256, 16185, 579, 7522, 21465, 198, 26698, 27393, 34, 31778, 14, 31780, 1822, 251, 31654, 13, 21, 21, 18, 35, 478, 579, 1407, 20358, 23109, 105, 31683, 235, 691, 83, 789, 31694, 1324, 691, 198, 29052, 27393, 83, 1407, 31654, 12, 16, 19, 35, 21, 21, 21, 105, 31683, 235, 691, 83, 31654, 12, 21, 21, 35, 21, 21, 21, 0, 789, 31694, 1324, 691, 35, 3601, 215, 65, 78, 825, 2482, 8170, 93, 4166, 1777, 78, 5359, 198, 70, 31654, 13, 21, 1671, 11823, 11325, 35, 198, 70, 17562, 18843, 83, 3226, 19507, 34, 31778, 16, 31780, 2266, 231, 10586, 1362, 1286, 1671, 25316, 35, 105, 31683, 235, 691, 617, 2288, 233, 826, 34, 147, 264, 21794, 321, 218, 31683, 4464, 344, 309, 84, 4057, 1357, 825, 70, 5187, 83, 9947, 81, 4897, 488, 182, 24761, 34, 31778, 17, 31780, 28616, 4173, 127, 84, 231, 10586, 1362, 4469, 345, 122, 25316, 35, 105, 31683, 235, 691, 1214, 3520, 84, 70, 1357, 12312, 826, 83, 105, 762, 31431, 1930, 338, 1909, 825, 586, 70, 1645, 251, 470, 34, 31778, 17, 31780], + decoded: ' The llama (/\u02c8l\u0251\u02d0m\u0259/; Spanish pronunciation: [\u02c8ama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5\u20138 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000\u201312,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + }, + }, +}; diff --git a/packages/transformers/tests/models/llava/test_modeling_llava.js b/packages/transformers/tests/models/llava/test_modeling_llava.js new file mode 100644 index 0000000..5f632ba --- /dev/null +++ b/packages/transformers/tests/models/llava/test_modeling_llava.js @@ -0,0 +1,114 @@ +import { LlavaForConditionalGeneration, RawImage, LlavaProcessor } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + const prompts = [ + // Example adapted from https://huggingface.co/docs/transformers/model_doc/llava#transformers.LlavaForConditionalGeneration.forward.example + "USER: \nWhat's the content of the image? ASSISTANT:", + "Hi", + ]; + + // Empty white image + const dims = [224, 224, 3]; + const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); + + describe("LlavaForConditionalGeneration", () => { + const model_id = "Xenova/tiny-random-LlavaForConditionalGeneration"; + + /** @type {LlavaForConditionalGeneration} */ + let model; + /** @type {LlavaProcessor} */ + let processor; + beforeAll(async () => { + model = await LlavaForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + processor = await LlavaProcessor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + const inputs = await processor(image, prompts[0]); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 246, 32002]); + expect(logits.mean().item()).toBeCloseTo(-0.0005688573000952601, 8); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size=1", + async () => { + const inputs = await processor(image, prompts[0]); + const generate_ids = await model.generate({ ...inputs, max_new_tokens: 10 }); + expect(generate_ids.dims).toEqual([1, 256]); + const new_ids = generate_ids.slice(null, [inputs.input_ids.dims[1], null]); + expect(new_ids.tolist()).toEqual([[21557n, 16781n, 27238n, 8279n, 20454n, 11927n, 12462n, 12306n, 2414n, 7561n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = await processor([image, image], prompts, { + padding: true, + }); + const generate_ids = await model.generate({ ...inputs, max_new_tokens: 10 }); + const new_ids = generate_ids.slice(null, [inputs.input_ids.dims[1], null]); + expect(new_ids.tolist()).toEqual([ + [21557n, 16781n, 27238n, 8279n, 20454n, 11927n, 12462n, 12306n, 2414n, 7561n], + [1217n, 22958n, 22913n, 10381n, 148n, 31410n, 31736n, 7358n, 9150n, 28635n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "generate w/ past_key_values", + async () => { + // Empty white image + const dims = [224, 224, 3]; + const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); + const inputs = await processor(image, prompts[0]); + + // Generate first sequence w/o PKV + // NOTE: `return_dict_in_generate=true` is required to get PKV + const { past_key_values, sequences } = await model.generate({ + ...inputs, + max_new_tokens: 5, + do_sample: false, + return_dict_in_generate: true, + }); + + // Run w/o PKV + const generated_ids = await model.generate({ + ...inputs, + max_new_tokens: 8, + do_sample: false, + }); + + // Run w/ PKV + const generated_ids_pkv = await model.generate({ + input_ids: sequences, + past_key_values, + max_new_tokens: 3, + do_sample: false, + }); + + const result = generated_ids.slice(null, [inputs.input_ids.dims[1], null]).tolist(); + const result_pkv = generated_ids_pkv.slice(null, [inputs.input_ids.dims[1], null]).tolist(); + + // Ensure output is the same and correct + const target = [[21557n, 16781n, 27238n, 8279n, 20454n, 11927n, 12462n, 12306n]]; + expect(result).toEqual(target); + expect(result_pkv).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/m2m_100/test_tokenization_m2m_100.js b/packages/transformers/tests/models/m2m_100/test_tokenization_m2m_100.js new file mode 100644 index 0000000..8532568 --- /dev/null +++ b/packages/transformers/tests/models/m2m_100/test_tokenization_m2m_100.js @@ -0,0 +1,179 @@ +import { M2M100Tokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, M2M_100_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = M2M100Tokenizer; + +// NOTE: The slow tokenizer (used by transformers) has minor inconsistencies against the fast tokenizer. +// For this reason, we may override the expected results for certain tests. +export const TEST_CONFIG = { + "Xenova/m2m100_418M": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [128022, 34226, 4234, 8251, 123047, 24, 2], + decoded: "__en__ How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [128022, 14921, 119092, 12, 470, 111108, 15911, 2], + decoded: "__en__ You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u25810", "123", "45", "6", "78", "9", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [128022, 847, 78596, 3834, 435, 7049, 718, 847, 161, 168, 205, 273, 265, 376, 442, 455, 572, 301, 1245, 7336, 2], + decoded: "__en__ 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u25812016."], + ids: [128022, 1658, 66486, 1513, 118728, 241, 28, 8860, 2], + decoded: "__en__ The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581'", "ll", "\u2581!!", "to", "?'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [128022, 58, 244, 2279, 9403, 428, 72956, 173, 8471, 173, 432, 4, 3154, 12, 88, 5, 2], + decoded: "__en__ A 'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581def", "\u2581main", "(", "):", "\u2581pass"], + ids: [128022, 8268, 9359, 249, 2825, 4799, 2], + decoded: "__en__ def main(): pass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581ob", "j", ".", "to", "Str", "ing", "(", ");", "\u2581to", "Str", "ing", "(", ");"], + ids: [128022, 2507, 8, 3255, 607, 189, 5, 428, 41549, 150, 249, 5294, 128, 41549, 150, 249, 5294, 2], + decoded: "__en__ let a = obj.toString(); toString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "."], + ids: [128022, 36606, 117, 8, 4183, 5, 2], + decoded: "__en__ This is a test.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "want", "\u00e9d", ",", "run", "ning"], + ids: [128022, 6984, 108054, 7151, 4, 18634, 656, 2], + decoded: "__en__ UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u25811", "\u0000", "2", "\u25813"], + ids: [128022, 161, 4163, 339, 205, 2], + decoded: "__en__ 1\u00002 3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581Hello", "\u2581World"], + ids: [128022, 65761, 10581, 2], + decoded: "__en__ Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [128022, 110013, 55185, 2], + decoded: "__en__ hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f\u6d3b", "\u7684", "\u771f", /* "\u8c1b" */ "", "\u662f"], + ids: [128022, 22, 8523, 80, 10418, 3, 775, 2], + decoded: "__en__ \u751f\u6d3b\u7684\u771f\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581leading", "\u2581space"], + ids: [128022, 124476, 118561, 2], + decoded: "__en__ leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581tra", "iling", "\u2581space"], + ids: [128022, 1368, 19217, 118561, 2], + decoded: "__en__ trailing space", + }, + SURROUNDING_SPACE: { + text: BASE_TEST_STRINGS.SURROUNDING_SPACE, + tokens: ["\u2581surround", "ing", "\u2581space"], + ids: [128022, 124728, 150, 118561, 2], + decoded: "__en__ surrounding space", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581Hi", "\u2581Hello"], + ids: [128022, 7676, 65761, 2], + decoded: "__en__ Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$", "1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", /* "\u20a3" */ "", "7", "\u2581", "\u20b9", "8", "\u2581", /* "\u20b1" */ "", "9", "\u2581test"], + ids: [128022, 4183, 4352, 451, 180, 339, 584, 425, 4257, 465, 13506, 679, 22, 43832, 435, 22, 3, 622, 22, 115056, 677, 22, 3, 718, 4183, 2], + decoded: "__en__ test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 9 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581ap", "ple", "\u2581for", "\u2581$", "1.", "00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [128022, 203, 127797, 48, 722, 6857, 193, 4352, 2023, 1365, 120, 1197, 9160, 5, 2], + decoded: "__en__ I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "..."], + ids: [128022, 8251, 26, 2], + decoded: "__en__ you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "..."], + ids: [128022, 8251, 26, 2], + decoded: "__en__ you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "..."], + ids: [128022, 8251, 26, 8251, 26, 2], + decoded: "__en__ you... you...", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581we", "ird", "\u2581", "\uff5e", "\u2581ed", "ge", "\u2581", "\uff5e", "\u2581case"], + ids: [128022, 1710, 13067, 22, 14691, 1500, 568, 22, 14691, 24306, 2], + decoded: "__en__ weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581."], + ids: [128022, 36606, 117, 8, 4183, 237, 2], + decoded: "__en__ This is a test.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", /* "\ud83e\udd73" */ "", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", /* "\ud83d\udc80" */ "", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", /* "\ud83c\udf1e" */ "", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [128022, 22, 74222, 22, 118514, 22, 124385, 22, 99683, 22, 123842, 22, 124821, 22, 117689, 22, 103111, 22, 121924, 22, 121088, 22, 124207, 22, 123955, 22, 120137, 22, 123534, 66038, 18905, 22, 125385, 22, 125317, 22, 126071, 22, 124787, 22, 127396, 22, 120119, 22, 122813, 22, 3, 22, 123482, 22, 120563, 22, 117995, 22, 127978, 22, 126507, 22, 127269, 22, 126179, 22, 125300, 22, 3, 22, 120807, 22, 127143, 22, 118682, 22, 125350, 22, 3, 22, 123790, 22, 126948, 2], + decoded: /* "__en__ \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf38 \ud83d\udcb0" */ "__en__ \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", /* "\ud83d\udc71" */ "", "\ud83c\udffb", "\u2581", /* "\ud83d\udd75" */ "", "\u2581", "\u2642", "\ufe0f", "\u2581", /* "\ud83e\uddd9" */ "", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", /* "\ud83c\udf3e" */ "", "\u2581", /* "\ud83e\uddd1" */ "", "\u2581", /* "\ud83e\udd1d" */ "", "\u2581", /* "\ud83e\uddd1" */ "", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", /* "\ud83d\udc67" */ "", "\u2581", /* "\ud83d\udc66" */ "", "\u2581", /* "\ud83e\uddd1" */ "", "\ud83c\udffb", "\u2581", /* "\ud83e\udd1d" */ "", "\u2581", /* "\ud83e\uddd1" */ "", "\ud83c\udffb", "\u2581", /* "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f"*/ "", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], + ids: [128022, 22, 120563, 22, 123955, 22, 121442, 18905, 22, 3, 116617, 22, 3, 22, 122517, 18905, 22, 3, 116617, 22, 122517, 22, 127603, 116617, 22, 3, 22, 3, 22, 3, 22, 3, 22, 126739, 66038, 22, 126237, 22, 127603, 22, 126739, 22, 126739, 22, 3, 22, 3, 22, 3, 116617, 22, 3, 22, 3, 116617, 22, 3, 22, 127603, 116617, 66038, 18905, 22, 126237, 22, 127603, 123285, 2], + decoded: /* "__en__ \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83c\udffb \u2642\ufe0f \ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83c\udffb \ud83c\udffb \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc" */ "__en__ \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83c\udffb \u2642\ufe0f \ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83c\udffb \ud83c\udffb \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc", + }, + ONLY_WHITESPACE: { + text: BASE_TEST_STRINGS.ONLY_WHITESPACE, + tokens: [], + ids: [128022, 2], + decoded: /* "__en__ " */ "__en__", + }, + TRANSLATION_INPUTS: { + text: M2M_100_TEST_STRINGS.TRANSLATION_INPUTS, + tokens: ["__en__", "\u2581hello", "\u2581world", ""], + ids: [128022, 128022, 110013, 55185, 2, 2], + decoded: /* "__en__ __en__ hello world" */ "__en____en__ hello world", + }, + }, +}; diff --git a/packages/transformers/tests/models/marian/test_modeling_marian.js b/packages/transformers/tests/models/marian/test_modeling_marian.js new file mode 100644 index 0000000..186951e --- /dev/null +++ b/packages/transformers/tests/models/marian/test_modeling_marian.js @@ -0,0 +1,51 @@ +import { MarianTokenizer, MarianMTModel } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("MarianMTModel", () => { + const model_id = "onnx-community/tiny-random-MarianMTModel"; + + /** @type {MarianMTModel} */ + let model; + /** @type {MarianTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await MarianMTModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await MarianTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[3n, 40672n, 8358n, 32810n, 32810n, 32810n, 32810n, 35687n, 33073n, 6870n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [3n, 40672n, 8358n, 32810n, 32810n, 32810n, 32810n, 35687n, 33073n, 6870n], + [3n, 40672n, 8358n, 32810n, 32810n, 32810n, 32810n, 35687n, 33073n, 6870n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/mgp_str/test_modeling_mgp_str.js b/packages/transformers/tests/models/mgp_str/test_modeling_mgp_str.js new file mode 100644 index 0000000..b0122f0 --- /dev/null +++ b/packages/transformers/tests/models/mgp_str/test_modeling_mgp_str.js @@ -0,0 +1,84 @@ +import { MgpstrProcessor, MgpstrForSceneTextRecognition } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("MgpstrForSceneTextRecognition", () => { + const model_id = "onnx-community/tiny-random-MgpstrForSceneTextRecognition"; + /** @type {MgpstrForSceneTextRecognition} */ + let model; + /** @type {MgpstrProcessor} */ + let processor; + beforeAll(async () => { + model = await MgpstrForSceneTextRecognition.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + processor = await MgpstrProcessor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + const TARGETS = { + white_image: { + generated_text: ["mmmmmmmmmmmmmmmmmmmmmmmmmm"], + scores: [3.5553885547065065e-27], + char_preds: ["mmmmmmmmmmmmmmmmmmmmmmmmmm"], + bpe_preds: ["wwwwwwwwwwwwwwwwwwwwwwwwww"], + wp_preds: ["[unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65][unused65]"], + }, + blue_image: { + generated_text: ["11111111111111111111111111"], + scores: [9.739909092663214e-32], + char_preds: ["11111111111111111111111111"], + bpe_preds: ["22222222222222222222222222"], + wp_preds: ["[unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59][unused59]"], + }, + }; + + it( + "batch_size=1", + async () => { + const image_id = "white_image"; + const image = await load_cached_image(image_id); + + const inputs = await processor(image); + const outputs = await model(inputs); + + const { max_token_length, num_character_labels, num_bpe_labels, num_wordpiece_labels } = model.config; + expect(outputs.char_logits.dims).toEqual([1, /* 27 */ max_token_length, /* 38 */ num_character_labels]); + expect(outputs.bpe_logits.dims).toEqual([1, /* 27 */ max_token_length, /* 99 */ num_bpe_labels]); + expect(outputs.wp_logits.dims).toEqual([1, /* 27 */ max_token_length, /* 99 */ num_wordpiece_labels]); + + const decoded = processor.batch_decode(outputs.logits); + expect(decoded).toBeCloseToNested(TARGETS[image_id]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const image_ids = ["white_image", "blue_image"]; + const images = await Promise.all(image_ids.map((image_id) => load_cached_image(image_id))); + + const inputs = await processor(images); + const outputs = await model(inputs); + + const { max_token_length, num_character_labels, num_bpe_labels, num_wordpiece_labels } = model.config; + expect(outputs.char_logits.dims).toEqual([images.length, /* 27 */ max_token_length, /* 38 */ num_character_labels]); + expect(outputs.bpe_logits.dims).toEqual([images.length, /* 27 */ max_token_length, /* 99 */ num_bpe_labels]); + expect(outputs.wp_logits.dims).toEqual([images.length, /* 27 */ max_token_length, /* 99 */ num_wordpiece_labels]); + + const decoded = processor.batch_decode(outputs.logits); + const target = image_ids.reduce((acc, image_id) => { + for (const key in TARGETS[image_id]) (acc[key] ??= []).push(...TARGETS[image_id][key]); + return acc; + }, {}); + + expect(decoded).toBeCloseToNested(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/mimi/test_modeling_mimi.js b/packages/transformers/tests/models/mimi/test_modeling_mimi.js new file mode 100644 index 0000000..2731caa --- /dev/null +++ b/packages/transformers/tests/models/mimi/test_modeling_mimi.js @@ -0,0 +1,88 @@ +import { EncodecFeatureExtractor, MimiModel, MimiEncoderModel, MimiDecoderModel } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("MimiModel", () => { + const model_id = "hf-internal-testing/tiny-random-MimiModel"; + + /** @type {MimiModel} */ + let model; + /** @type {EncodecFeatureExtractor} */ + let feature_extractor; + let inputs; + beforeAll(async () => { + model = await MimiModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + feature_extractor = await EncodecFeatureExtractor.from_pretrained(model_id); + inputs = await feature_extractor(new Float32Array(12000)); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + const { audio_values } = await model(inputs); + expect(audio_values.dims).toEqual([1, 1, 13440]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "encode & decode", + async () => { + const encoder_outputs = await model.encode(inputs); + expect(encoder_outputs.audio_codes.dims).toEqual([1, model.config.num_quantizers, 7]); + + const { audio_values } = await model.decode(encoder_outputs); + expect(audio_values.dims).toEqual([1, 1, 13440]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("MimiEncoderModel and MimiDecoderModel", () => { + const model_id = "hf-internal-testing/tiny-random-MimiModel"; + + /** @type {MimiEncoderModel} */ + let encoder_model; + /** @type {MimiDecoderModel} */ + let decoder_model; + /** @type {EncodecFeatureExtractor} */ + let feature_extractor; + let inputs; + let encoder_outputs; + + beforeAll(async () => { + encoder_model = await MimiEncoderModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + decoder_model = await MimiDecoderModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + feature_extractor = await EncodecFeatureExtractor.from_pretrained(model_id); + inputs = await feature_extractor(new Float32Array(12000)); + }, MAX_MODEL_LOAD_TIME); + + it( + "encode", + async () => { + encoder_outputs = await encoder_model(inputs); + expect(encoder_outputs.audio_codes.dims).toEqual([1, encoder_model.config.num_quantizers, 7]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "decode", + async () => { + const { audio_values } = await decoder_model(encoder_outputs); + expect(audio_values.dims).toEqual([1, 1, 13440]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await encoder_model?.dispose(); + await decoder_model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/mistral/test_modeling_mistral.js b/packages/transformers/tests/models/mistral/test_modeling_mistral.js new file mode 100644 index 0000000..294e904 --- /dev/null +++ b/packages/transformers/tests/models/mistral/test_modeling_mistral.js @@ -0,0 +1,50 @@ +import { LlamaTokenizer, MistralForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("MistralForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-MistralForCausalLM"; + /** @type {MistralForCausalLM} */ + let model; + /** @type {LlamaTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await MistralForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await LlamaTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[1n, 6312n, 28709n, 24704n, 8732n, 1310n, 9808n, 13771n, 27309n, 4779n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [2n, 1n, 6312n, 28709n, 24704n, 8732n, 1310n, 9808n, 13771n, 27309n], + [1n, 6312n, 28709n, 1526n, 8687n, 5690n, 1770n, 30811n, 12501n, 3325n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/mistral4/test_modeling_mistral4.js b/packages/transformers/tests/models/mistral4/test_modeling_mistral4.js new file mode 100644 index 0000000..4484823 --- /dev/null +++ b/packages/transformers/tests/models/mistral4/test_modeling_mistral4.js @@ -0,0 +1,51 @@ +import { PreTrainedTokenizer, Mistral4ForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("Mistral4ForCausalLM", () => { + const model_id = "onnx-internal-testing/tiny-random-Mistral4ForCausalLM"; + /** @type {Mistral4ForCausalLM} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await Mistral4ForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[1n, 6312n, 28709n, 29976n, 15596n, 6412n, 30122n, 7218n, 14257n, 31552n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [2n, 1n, 6312n, 28709n, 29976n, 15596n, 6412n, 30122n, 7218n, 14257n], + [1n, 6312n, 28709n, 1526n, 5101n, 6648n, 20332n, 30854n, 27266n, 21938n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/mobilevit/test_image_processing_mobilevit.js b/packages/transformers/tests/models/mobilevit/test_image_processing_mobilevit.js new file mode 100644 index 0000000..a737cc8 --- /dev/null +++ b/packages/transformers/tests/models/mobilevit/test_image_processing_mobilevit.js @@ -0,0 +1,90 @@ +import { AutoImageProcessor, MobileViTFeatureExtractor, MobileViTImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // MobileViTFeatureExtractor + describe("MobileViTFeatureExtractor (default)", () => { + const model_id = "Xenova/mobilevit-small"; + + /** @type {MobileViTFeatureExtractor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "default", + async () => { + const image = await load_cached_image("tiger"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 256, 256]); + expect(pixel_values.mean().item()).toBeCloseTo(0.4599160496887033, 6); + + expect(original_sizes).toEqual([[408, 612]]); + expect(reshaped_input_sizes).toEqual([[256, 256]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + // MobileViTFeatureExtractor + // - tests not converting to rgb (do_convert_rgb=false) + describe("MobileViTFeatureExtractor (do_convert_rgb=false)", () => { + const model_id = "Xenova/quickdraw-mobilevit-small"; + + /** @type {MobileViTFeatureExtractor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "grayscale image", + async () => { + const image = await load_cached_image("skateboard"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 1, 28, 28]); + expect(pixel_values.mean().item()).toBeCloseTo(0.08558923671585128, 6); + + expect(original_sizes).toEqual([[28, 28]]); + expect(reshaped_input_sizes).toEqual([[28, 28]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + // MobileViTImageProcessor + // - tests converting RGB to BGR (do_flip_channel_order=true) + describe("MobileViTImageProcessor (do_flip_channel_order=true)", () => { + const model_id = "Xenova/mobilevitv2-1.0-imagenet1k-256"; + + /** @type {MobileViTImageProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "RGB to BGR", + async () => { + const image = await load_cached_image("cats"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + const { data, dims } = pixel_values; + + expect(dims).toEqual([1, 3, 256, 256]); + expect(pixel_values.mean().item()).toBeCloseTo(0.5215385556221008, 2); + + expect(original_sizes).toEqual([[480, 640]]); + expect(reshaped_input_sizes).toEqual([[256, 256]]); + + // Ensure RGB to BGR conversion + expect(data.slice(0, 3)).toBeCloseToNested([0.24313725531101227, 0.250980406999588, 0.364705890417099], 4); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/modernbert/test_modeling_modernbert.js b/packages/transformers/tests/models/modernbert/test_modeling_modernbert.js new file mode 100644 index 0000000..4773c60 --- /dev/null +++ b/packages/transformers/tests/models/modernbert/test_modeling_modernbert.js @@ -0,0 +1,180 @@ +import { PreTrainedTokenizer, ModernBertModel, ModernBertForMaskedLM, ModernBertForSequenceClassification, ModernBertForTokenClassification } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("ModernBertModel", () => { + const model_id = "hf-internal-testing/tiny-random-ModernBertModel"; + + /** @type {ModernBertModel} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await ModernBertModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const { last_hidden_state } = await model(inputs); + expect(last_hidden_state.dims).toEqual([1, 3, 32]); + expect(last_hidden_state.mean().item()).toBeCloseTo(-0.08922556787729263, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const { last_hidden_state } = await model(inputs); + expect(last_hidden_state.dims).toEqual([2, 4, 32]); + expect(last_hidden_state.mean().item()).toBeCloseTo(0.048988230526447296, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "sequence_length > local_attention_window", + async () => { + const text = "The sun cast long shadows across the weathered cobblestones as Thomas made his way through the ancient city. The evening air carried whispers of autumn, rustling through the golden leaves that danced and swirled around his feet. His thoughts wandered to the events that had brought him here, to this moment, in this forgotten corner of the world. The old buildings loomed above him, their facades telling stories of centuries past. Windows reflected the dying light of day, creating a kaleidoscope of amber and rose that painted the narrow streets. The distant sound of church bells echoed through the maze of alleyways, marking time's steady march forward. In his pocket, he fingered the small brass key that had belonged to his grandfather. Its weight seemed to grow heavier with each step, a tangible reminder of the promise he had made. The mystery of its purpose had consumed his thoughts for weeks, leading him through archives and dusty libraries, through conversations with local historians and elderly residents who remembered the old days. As the evening deepened into dusk, streetlamps flickered to life one by one, creating pools of warm light that guided his way. The smell of wood smoke and distant cooking fires drifted through the air, reminding him of childhood evenings spent by the hearth, listening to his grandfather's tales of hidden treasures and secret passages. His footsteps echoed against the stone walls, a rhythmic accompaniment to his journey. Each step brought him closer to his destination, though uncertainty still clouded his mind about what he might find. The old map in his other pocket, creased and worn from constant consultation, had led him this far. The street ahead narrowed, and the buildings seemed to lean in closer, their upper stories nearly touching above his head. The air grew cooler in this shadowed passage, and his breath formed small clouds in front of him. Something about this place felt different, charged with possibility and ancient secrets. He walked down the [MASK]"; + const inputs = tokenizer(text); + const { last_hidden_state } = await model(inputs); + expect(last_hidden_state.dims).toEqual([1, 397, 32]); + expect(last_hidden_state.mean().item()).toBeCloseTo(-0.06889555603265762, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("ModernBertForMaskedLM", () => { + const model_id = "hf-internal-testing/tiny-random-ModernBertForMaskedLM"; + + const texts = ["The goal of life is [MASK].", "Paris is the [MASK] of France."]; + + /** @type {ModernBertForMaskedLM} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await ModernBertForMaskedLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer(texts[0]); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 9, 50368]); + expect(logits.mean().item()).toBeCloseTo(0.0053214821964502335, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(texts, { padding: true }); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([2, 9, 50368]); + expect(logits.mean().item()).toBeCloseTo(0.009154772385954857, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("ModernBertForSequenceClassification", () => { + const model_id = "hf-internal-testing/tiny-random-ModernBertForSequenceClassification"; + + /** @type {ModernBertForSequenceClassification} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await ModernBertForSequenceClassification.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const { logits } = await model(inputs); + const target = [[-0.7050137519836426, 2.343430519104004]]; + expect(logits.dims).toEqual([1, 2]); + expect(logits.tolist()).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const { logits } = await model(inputs); + const target = [ + [-0.7050137519836426, 2.343430519104004], + [-2.6860175132751465, 3.993380546569824], + ]; + expect(logits.dims).toEqual([2, 2]); + expect(logits.tolist()).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("ModernBertForTokenClassification", () => { + const model_id = "hf-internal-testing/tiny-random-ModernBertForTokenClassification"; + + /** @type {ModernBertForTokenClassification} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await ModernBertForTokenClassification.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 3, 2]); + expect(logits.mean().item()).toBeCloseTo(1.0337047576904297, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([2, 4, 2]); + expect(logits.mean().item()).toBeCloseTo(-1.3397092819213867, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/moonshine/test_feature_extraction_moonshine.js b/packages/transformers/tests/models/moonshine/test_feature_extraction_moonshine.js new file mode 100644 index 0000000..df2c533 --- /dev/null +++ b/packages/transformers/tests/models/moonshine/test_feature_extraction_moonshine.js @@ -0,0 +1,30 @@ +import { AutoFeatureExtractor, MoonshineFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_audio } from "../../asset_cache.js"; +import { MAX_FEATURE_EXTRACTOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // MoonshineFeatureExtractor + describe("MoonshineFeatureExtractor", () => { + const model_id = "onnx-community/moonshine-tiny-ONNX"; + + /** @type {MoonshineFeatureExtractor} */ + let feature_extractor; + beforeAll(async () => { + feature_extractor = await AutoFeatureExtractor.from_pretrained(model_id); + }, MAX_FEATURE_EXTRACTOR_LOAD_TIME); + + it( + "default", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_values } = await feature_extractor(audio); + expect(input_values.dims).toEqual([1, 208000]); + expect(input_values.mean().item()).toBeCloseTo(-1.5654930507480458e-7, 6); + expect(input_values.data[0]).toBeCloseTo(0.0067138671875, 6); + expect(input_values.data.at(-1)).toBeCloseTo(-0.013427734375, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/moonshine/test_modeling_moonshine.js b/packages/transformers/tests/models/moonshine/test_modeling_moonshine.js new file mode 100644 index 0000000..24b920e --- /dev/null +++ b/packages/transformers/tests/models/moonshine/test_modeling_moonshine.js @@ -0,0 +1,50 @@ +import { Wav2Vec2Processor, MoonshineForConditionalGeneration, full, ones } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("MoonshineForConditionalGeneration", () => { + const model_id = "hf-internal-testing/tiny-random-MoonshineForConditionalGeneration"; + + /** @type {MoonshineForConditionalGeneration} */ + let model; + /** @type {Wav2Vec2Processor} */ + let processor; + beforeAll(async () => { + model = await MoonshineForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + processor = await Wav2Vec2Processor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + const input_values = new Float32Array(16000); + + it( + "forward", + async () => { + const inputs = await processor(input_values); + const { logits } = await model({ + ...inputs, + decoder_input_ids: ones([1, 1]), + }); + expect(logits.dims).toEqual([1, 1, 32768]); + expect(logits.mean().item()).toBeCloseTo(0.016709428280591965, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size=1", + async () => { + const inputs = await processor(input_values); + const generate_ids = await model.generate({ ...inputs, max_new_tokens: 3 }); + + const new_tokens = generate_ids; + expect(new_tokens.tolist()).toEqual([[/* Decoder start token */ 1n, /* Generated */ 6891n, 21892n, 14850n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/mpnet/test_tokenization_mpnet.js b/packages/transformers/tests/models/mpnet/test_tokenization_mpnet.js new file mode 100644 index 0000000..33fd7b6 --- /dev/null +++ b/packages/transformers/tests/models/mpnet/test_tokenization_mpnet.js @@ -0,0 +1,158 @@ +import { MPNetTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = MPNetTokenizer; +export const TEST_CONFIG = { + "Xenova/all-mpnet-base-v2": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["how", "are", "you", "doing", "?"], + ids: [0, 2133, 2028, 2021, 2729, 1033, 2], + decoded: " how are you doing? ", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["you", "should", "'", "ve", "done", "this"], + ids: [0, 2021, 2327, 1009, 2314, 2593, 2027, 2], + decoded: " you should've done this ", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["01", "##23", "##45", "##6", "##7", "##8", "##9", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], + ids: [0, 5894, 21930, 19965, 2579, 2585, 2624, 2687, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027, 2188, 2535, 6698, 2], + decoded: " 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000 ", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["the", "company", "was", "founded", "in", "2016", "."], + ids: [0, 2000, 2198, 2005, 2635, 2003, 2359, 1016, 2], + decoded: " the company was founded in 2016. ", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "'", "'", "d", "of", ",", "can", "'", "t", "."], + ids: [0, 1041, 1009, 2226, 1003, 1003, 2004, 1033, 1009, 1044, 1009, 1009, 1044, 2001, 1014, 2068, 1009, 1060, 1016, 2], + decoded: " a'll!! to?'d'' d of, can't. ", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "main", "(", ")", ":", "pass"], + ids: [0, 13370, 2368, 1010, 1011, 1028, 3417, 2], + decoded: " def main ( ) : pass ", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "##j", ".", "to", "##st", "##ring", "(", ")", ";", "to", "##st", "##ring", "(", ")", ";"], + ids: [0, 2296, 1041, 1031, 27889, 3505, 1016, 2004, 3371, 4896, 1010, 1011, 1029, 2004, 3371, 4896, 1010, 1011, 1029, 2], + decoded: " let a = obj. tostring ( ) ; tostring ( ) ; ", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["this", "is", "a", "test", "."], + ids: [0, 2027, 2007, 1041, 3235, 1016, 2], + decoded: " this is a test. ", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["unwanted", ",", "running"], + ids: [0, 18166, 1014, 2774, 2], + decoded: " unwanted, running ", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["123"], + ids: [0, 13142, 2], + decoded: " 123 ", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["hello", "world"], + ids: [0, 7596, 2092, 2], + decoded: " hello world ", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "world"], + ids: [0, 7596, 2092, 2], + decoded: " hello world ", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "[UNK]", "\u7684", "\u771f", "[UNK]", "[UNK]"], + ids: [0, 1914, 104, 1920, 1925, 104, 104, 2], + decoded: " \u751f [UNK] \u7684 \u771f [UNK] [UNK] ", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["leading", "space"], + ids: [0, 2881, 2690, 2], + decoded: " leading space ", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "space"], + ids: [0, 12546, 2690, 2], + decoded: " trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["hi", "hello"], + ids: [0, 7636, 7596, 2], + decoded: " hi hello ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r", "##2", "#", "3", "\u20ac", "##4", "\u00a35", "\u00a5", "##6", "[UNK]", "\u20b9", "##8", "\u20b1", "##9", "test"], + ids: [0, 3235, 1006, 1019, 1058, 2479, 1005, 1021, 1578, 2553, 27817, 1075, 2579, 104, 1580, 2624, 1579, 2687, 3235, 2], + decoded: " test $ 1 r2 # 3 \u20ac4 \u00a35 \u00a56 [UNK] \u20b98 \u20b19 test ", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["i", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [0, 1049, 4153, 2023, 6211, 2009, 1006, 1019, 1016, 4006, 2016, 2000, 3577, 1016, 2], + decoded: " i bought an apple for $ 1. 00 at the store. ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u2026"], + ids: [0, 2021, 1533, 2], + decoded: " you \u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u2026"], + ids: [0, 2021, 1533, 2], + decoded: " you \u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u2026", "you", "\u2026"], + ids: [0, 2021, 1533, 2021, 1533, 2], + decoded: " you \u2026 you \u2026 ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "\uff5e", "edge", "\uff5e", "case"], + ids: [0, 6885, 1999, 3345, 1999, 2557, 2], + decoded: " weird \uff5e edge \uff5e case ", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "."], + ids: [0, 104, 104, 104, 104, 104, 1016, 2], + decoded: " [UNK] [UNK] [UNK] [UNK] [UNK]. ", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [0, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 2], + decoded: " [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]", "[UNK]"], + ids: [0, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 2], + decoded: " [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] ", + }, + }, +}; diff --git a/packages/transformers/tests/models/mpt/test_modeling_mpt.js b/packages/transformers/tests/models/mpt/test_modeling_mpt.js new file mode 100644 index 0000000..7f6e914 --- /dev/null +++ b/packages/transformers/tests/models/mpt/test_modeling_mpt.js @@ -0,0 +1,51 @@ +import { GPTNeoXTokenizer, MptForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("MptForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-MptForCausalLM"; + /** @type {MptForCausalLM} */ + let model; + /** @type {GPTNeoXTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await MptForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await GPTNeoXTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[259n, 864n, 80n, 80n, 80n, 80n, 80n, 80n, 80n, 80n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 259n, 864n, 80n, 80n, 80n, 80n, 80n, 80n], + [259n, 864n, 80n, 270n, 814n, 293n, 293n, 293n, 293n, 293n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/musicgen/test_modeling_musicgen.js b/packages/transformers/tests/models/musicgen/test_modeling_musicgen.js new file mode 100644 index 0000000..e68f83b --- /dev/null +++ b/packages/transformers/tests/models/musicgen/test_modeling_musicgen.js @@ -0,0 +1,61 @@ +import { T5Tokenizer, MusicgenForConditionalGeneration, full } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("MusicgenForConditionalGeneration", () => { + const model_id = "hf-internal-testing/tiny-random-MusicgenForConditionalGeneration"; + + // Example adapted from https://huggingface.co/docs/transformers/model_doc/musicgen#text-conditional-generation + const texts = ["80s pop track with bassy drums and synth", "90s rock song with loud guitars and heavy drums"]; + + /** @type {MusicgenForConditionalGeneration} */ + let model; + /** @type {T5Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await MusicgenForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await T5Tokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + // Example from https://huggingface.co/docs/transformers/model_doc/musicgen#transformers.MusicgenForConditionalGeneration.forward.example + const inputs = tokenizer(texts, { padding: true }); + const pad_token_id = BigInt(model.generation_config.pad_token_id); + const decoder_input_ids = full([inputs.input_ids.dims[0] * model.config.decoder.num_codebooks, 1], pad_token_id); + const { logits } = await model({ ...inputs, decoder_input_ids }); + expect(logits.dims).toEqual([8, 1, 99]); + expect(logits.mean().item()).toBeCloseTo(-0.0018370470497757196, 4); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer(texts[0]); + const audio_values = await model.generate({ ...inputs, max_length: 10 }); + expect(audio_values.dims).toEqual([1, 1, 1920]); + expect(audio_values.mean().item()).toBeCloseTo(0.16644218564033508, 4); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(texts, { padding: true }); + const audio_values = await model.generate({ ...inputs, max_length: 10 }); + expect(audio_values.dims).toEqual([2, 1, 1920]); + expect(audio_values.mean().item()).toBeCloseTo(0.16644218564033508, 4); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/nemotron_h/test_modeling_nemotron_h.js b/packages/transformers/tests/models/nemotron_h/test_modeling_nemotron_h.js new file mode 100644 index 0000000..b015a6a --- /dev/null +++ b/packages/transformers/tests/models/nemotron_h/test_modeling_nemotron_h.js @@ -0,0 +1,51 @@ +import { PreTrainedTokenizer, NemotronHForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("NemotronHForCausalLM", () => { + const model_id = "onnx-internal-testing/tiny-random-NemotronHForCausalLM"; + /** @type {NemotronHForCausalLM} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await NemotronHForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[29706n, 97336n, 121288n, 82722n, 25950n, 99280n, 131027n, 20440n, 94437n, 28124n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [11n, 29706n, 97336n, 121288n, 82722n, 25950n, 99280n, 131027n, 20440n, 94437n], + [29706n, 4304n, 128810n, 63172n, 70343n, 47180n, 52275n, 46732n, 66887n, 62566n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/nllb/test_tokenization_nllb.js b/packages/transformers/tests/models/nllb/test_tokenization_nllb.js new file mode 100644 index 0000000..768b755 --- /dev/null +++ b/packages/transformers/tests/models/nllb/test_tokenization_nllb.js @@ -0,0 +1,158 @@ +import { NllbTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = NllbTokenizer; +export const TEST_CONFIG = { + "Xenova/nllb-200-distilled-600M": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [256047, 13374, 2442, 1259, 34512, 248130, 2], + decoded: "eng_Latn How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [256047, 3555, 12516, 248116, 279, 27236, 3423, 2], + decoded: "eng_Latn You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u25810", "123", "45", "67", "89", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [256047, 4097, 232903, 25497, 37462, 42763, 4097, 94, 140, 315, 436, 481, 617, 757, 799, 855, 772, 3037, 18041, 2], + decoded: "eng_Latn 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u25812016."], + ids: [256047, 1617, 32796, 1398, 26710, 76, 108, 31889, 2], + decoded: "eng_Latn The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581'", "ll", "\u2581!!", "to", "?'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [256047, 70, 238, 1015, 12434, 208, 7358, 248072, 248116, 248116, 248072, 452, 248079, 2125, 248116, 248065, 248075, 2], + decoded: "eng_Latn A 'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581def", "\u2581main", "(", "):", "\u2581pass"], + ids: [256047, 9274, 8385, 248168, 9481, 5800, 2], + decoded: "eng_Latn def main(): pass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581ob", "j", ".", "to", "Str", "ing", "(", ");", "\u2581to", "Str", "ing", "(", ");"], + ids: [256047, 3190, 9, 5636, 859, 248086, 248075, 208, 134293, 87, 248168, 12387, 202, 134293, 87, 248168, 12387, 2], + decoded: "eng_Latn let a = obj.toString(); toString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "."], + ids: [256047, 9680, 248, 9, 7356, 248075, 2], + decoded: "eng_Latn This is a test.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "want", "\u00e9d", ",", "run", "ning"], + ids: [256047, 16297, 41691, 11317, 248079, 8464, 888, 2], + decoded: "eng_Latn UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u25811", "", "2", "\u25813"], + ids: [256047, 94, 3, 248147, 315, 2], + decoded: "eng_Latn 12 3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581Hello", "\u2581World"], + ids: [256047, 94124, 13855, 2], + decoded: "eng_Latn Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [256047, 133863, 15697, 2], + decoded: "eng_Latn hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581\u751f\u6d3b", "\u7684", "\u771f", "", "\u662f"], + ids: [256047, 182892, 248506, 249573, 3, 249221, 2], + decoded: "eng_Latn \u751f\u6d3b\u7684\u771f\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581leading", "\u2581space"], + ids: [256047, 151175, 72147, 2], + decoded: "eng_Latn leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581tra", "iling", "\u2581space", "\u2581"], + ids: [256047, 1372, 21263, 72147, 248059, 2], + decoded: "eng_Latn trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581Hi", "\u2581Hello"], + ids: [256047, 2867, 94124, 2], + decoded: "eng_Latn Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "", "7", "\u2581", "\u20b9", "8", "\u2581", "", "9", "\u2581test"], + ids: [256047, 7356, 68462, 250, 248147, 186447, 22935, 248215, 25400, 248210, 248059, 252351, 248262, 248059, 3, 248283, 248059, 254867, 248268, 248059, 3, 248212, 7356, 2], + decoded: "eng_Latn test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 9 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1.", "00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [256047, 117, 177233, 111, 203152, 351, 4589, 3044, 460, 230, 349, 21087, 248075, 2], + decoded: "eng_Latn I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [256047, 1259, 284, 248059, 2], + decoded: "eng_Latn you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [256047, 1259, 284, 248059, 2], + decoded: "eng_Latn you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "...", "\u2581"], + ids: [256047, 1259, 284, 1259, 284, 248059, 2], + decoded: "eng_Latn you... you... ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "", "\u2581ed", "ge", "\u2581", "", "\u2581case"], + ids: [256047, 197348, 248059, 3, 1074, 479, 248059, 3, 23555, 2], + decoded: "eng_Latn weird edge case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581."], + ids: [256047, 9680, 248, 9, 7356, 81, 2], + decoded: "eng_Latn This is a test.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "", "\u2581", "\ud83d\udc99", "\u2581", "", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "\ud83d\ude48", "\u2581", "", "\u2581", "", "\u2581", "\ud83d\udc47", "\u2581", "", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "", "\u2581", "\ud83c\udf38", "\u2581", ""], + ids: [256047, 104709, 248059, 253416, 248059, 253516, 241830, 248059, 253476, 248059, 3, 248059, 253443, 248059, 253515, 248059, 254402, 248059, 253288, 248059, 253776, 248059, 255232, 147677, 248059, 255420, 82495, 251759, 248059, 255742, 248059, 255949, 248059, 3, 248059, 255649, 248059, 3, 248059, 254297, 248059, 254723, 248059, 3, 248059, 255515, 248059, 254957, 248059, 253985, 248059, 3, 248059, 3, 248059, 3, 248059, 255855, 248059, 3, 248059, 3, 248059, 255354, 248059, 3, 248059, 254268, 248059, 255879, 248059, 3, 248059, 255952, 248059, 3, 2], + decoded: "eng_Latn \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc99 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\ude48 \ud83d\udc47 \u2705 \ud83c\udf81 \ud83c\udf38 ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "", "\ud83c\udffb", "\u2581", "", "\u2581", "\u2642", "\ufe0f", "\u2581", "", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "", "\ud83c\udffb", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581\u2764", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\u2581", "", "\ud83c\udffb", "\u2581", "", "\u2581", "", "\ud83c\udffb", "\u2581", "", "\u2581", "", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "", "\u2581", "", "\ud83c\udffc"], + ids: [256047, 248059, 254957, 248059, 255232, 248059, 255123, 251759, 248059, 3, 254422, 248059, 3, 248059, 255331, 251759, 248059, 3, 254422, 248059, 255331, 248059, 3, 254422, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 82495, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 248059, 3, 254422, 248059, 3, 248059, 3, 254422, 248059, 3, 248059, 3, 254422, 82495, 251759, 248059, 3, 248059, 3, 255832, 2], + decoded: "eng_Latn \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83c\udffb \u2642\ufe0f \ud83c\udffb \u2642 \ud83c\udffb \u2764 \ud83c\udffb \ud83c\udffb \ud83c\udffb \u2764\ufe0f \ud83c\udffc", + }, + }, +}; diff --git a/packages/transformers/tests/models/nougat/test_image_processing_nougat.js b/packages/transformers/tests/models/nougat/test_image_processing_nougat.js new file mode 100644 index 0000000..1cbdff6 --- /dev/null +++ b/packages/transformers/tests/models/nougat/test_image_processing_nougat.js @@ -0,0 +1,33 @@ +import { AutoImageProcessor, NougatImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // NougatImageProcessor + // - tests padding after normalization (image_mean != 0.5, image_std != 0.5) + describe("NougatImageProcessor", () => { + const model_id = "Xenova/nougat-small"; + + /** @type {NougatImageProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "padding after normalization", + async () => { + const image = await load_cached_image("paper"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 896, 672]); + expect(pixel_values.mean().item()).toBeCloseTo(1.8447155005897355, 6); + + expect(original_sizes).toEqual([[850, 685]]); + expect(reshaped_input_sizes).toEqual([[833, 672]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/olmo/test_modeling_olmo.js b/packages/transformers/tests/models/olmo/test_modeling_olmo.js new file mode 100644 index 0000000..14f70f9 --- /dev/null +++ b/packages/transformers/tests/models/olmo/test_modeling_olmo.js @@ -0,0 +1,51 @@ +import { GPTNeoXTokenizer, OlmoForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("OlmoForCausalLM", () => { + const model_id = "onnx-community/tiny-random-olmo-hf"; + /** @type {OlmoForCausalLM} */ + let model; + /** @type {GPTNeoXTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await OlmoForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await GPTNeoXTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[25521n, 10886n, 44936n, 38777n, 33038n, 18557n, 1810n, 33853n, 9517n, 28892n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [1n, 25521n, 10886n, 44936n, 38777n, 33038n, 18557n, 1810n, 33853n, 9517n], + [25521n, 1533n, 37199n, 27362n, 30594n, 39261n, 8824n, 19175n, 8545n, 29335n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/olmo2/test_modeling_olmo2.js b/packages/transformers/tests/models/olmo2/test_modeling_olmo2.js new file mode 100644 index 0000000..1c6f2ff --- /dev/null +++ b/packages/transformers/tests/models/olmo2/test_modeling_olmo2.js @@ -0,0 +1,51 @@ +import { GPT2Tokenizer, Olmo2ForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("Olmo2ForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-Olmo2ForCausalLM"; + /** @type {Olmo2ForCausalLM} */ + let model; + /** @type {GPT2Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await Olmo2ForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await GPT2Tokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[15339n, 50957n, 43410n, 77030n, 91444n, 99516n, 80720n, 4608n, 90428n, 22806n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [100277n, 15339n, 50957n, 43410n, 77030n, 91444n, 99516n, 80720n, 4608n, 90428n], + [15339n, 1917n, 12095n, 21350n, 61586n, 19306n, 39486n, 91527n, 59768n, 31934n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/opt/test_modeling_opt.js b/packages/transformers/tests/models/opt/test_modeling_opt.js new file mode 100644 index 0000000..9a505bc --- /dev/null +++ b/packages/transformers/tests/models/opt/test_modeling_opt.js @@ -0,0 +1,51 @@ +import { GPT2Tokenizer, OPTForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("OPTForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-OPTForCausalLM"; + /** @type {OPTForCausalLM} */ + let model; + /** @type {GPT2Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await OPTForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await GPT2Tokenizer.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[2n, 42891n, 39144n, 39144n, 39144n, 39144n, 39144n, 39144n, 39144n, 39144n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [1n, 2n, 42891n, 39144n, 39144n, 39144n, 39144n, 39144n, 39144n, 39144n], + [2n, 42891n, 232n, 24680n, 24680n, 24680n, 24680n, 24680n, 24680n, 24680n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/owlvit/test_image_processing_owlvit.js b/packages/transformers/tests/models/owlvit/test_image_processing_owlvit.js new file mode 100644 index 0000000..77320c0 --- /dev/null +++ b/packages/transformers/tests/models/owlvit/test_image_processing_owlvit.js @@ -0,0 +1,31 @@ +import { AutoImageProcessor, OwlViTFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("OwlViTFeatureExtractor", () => { + const model_id = "Xenova/owlvit-base-patch32"; + + /** @type {OwlViTFeatureExtractor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "default", + async () => { + const image = await load_cached_image("cats"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 768, 768]); + expect(pixel_values.mean().item()).toBeCloseTo(0.250620447910435, 6); + + expect(original_sizes).toEqual([[480, 640]]); + expect(reshaped_input_sizes).toEqual([[768, 768]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/paligemma/test_modeling_paligemma.js b/packages/transformers/tests/models/paligemma/test_modeling_paligemma.js new file mode 100644 index 0000000..e2cfa3f --- /dev/null +++ b/packages/transformers/tests/models/paligemma/test_modeling_paligemma.js @@ -0,0 +1,52 @@ +import { PaliGemmaProcessor, PaliGemmaForConditionalGeneration, RawImage } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + const text = "What is on the flower?"; + + // Empty white image + const dims = [224, 224, 3]; + const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); + + describe("PaliGemmaForConditionalGeneration", () => { + const model_id = "hf-internal-testing/tiny-random-PaliGemmaForConditionalGeneration"; + + /** @type {PaliGemmaForConditionalGeneration} */ + let model; + /** @type {PaliGemmaProcessor} */ + let processor; + beforeAll(async () => { + model = await PaliGemmaForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + processor = await PaliGemmaProcessor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + const inputs = await processor(image, text); + + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 264, 257216]); + expect(logits.mean().item()).toBeCloseTo(-0.0023024685215204954, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size=1", + async () => { + const inputs = await processor(image, text); + const generate_ids = await model.generate({ ...inputs, max_new_tokens: 10 }); + + const new_tokens = generate_ids.slice(null, [inputs.input_ids.dims.at(-1), null]); + expect(new_tokens.tolist()).toEqual([[91711n, 24904n, 144054n, 124983n, 83862n, 124983n, 124983n, 124983n, 141236n, 124983n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/paligemma/test_processor_paligemma.js b/packages/transformers/tests/models/paligemma/test_processor_paligemma.js new file mode 100644 index 0000000..4096c5f --- /dev/null +++ b/packages/transformers/tests/models/paligemma/test_processor_paligemma.js @@ -0,0 +1,51 @@ +import { AutoProcessor, PaliGemmaProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + const model_id = "hf-internal-testing/tiny-random-PaliGemmaForConditionalGeneration"; + + describe("PaliGemmaProcessor", () => { + /** @type {PaliGemmaProcessor} */ + let processor; + let images = {}; + + beforeAll(async () => { + processor = await AutoProcessor.from_pretrained(model_id); + images = { + white_image: await load_cached_image("white_image"), + }; + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "Image-only (default text)", + async () => { + const { input_ids, pixel_values } = await processor(images.white_image); + expect(input_ids.dims).toEqual([1, 258]); + expect(pixel_values.dims).toEqual([1, 3, 224, 224]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "Single image & text", + async () => { + const { input_ids, pixel_values } = await processor(images.white_image, "What is on the flower?"); + expect(input_ids.dims).toEqual([1, 264]); + expect(pixel_values.dims).toEqual([1, 3, 224, 224]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "Multiple images & text", + async () => { + const { input_ids, pixel_values } = await processor([images.white_image, images.white_image], "Describe the images."); + expect(input_ids.dims).toEqual([1, 518]); + expect(pixel_values.dims).toEqual([2, 3, 224, 224]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/patchtsmixer/test_modeling_patchtsmixer.js b/packages/transformers/tests/models/patchtsmixer/test_modeling_patchtsmixer.js new file mode 100644 index 0000000..6eee6fb --- /dev/null +++ b/packages/transformers/tests/models/patchtsmixer/test_modeling_patchtsmixer.js @@ -0,0 +1,65 @@ +import { PatchTSMixerModel, PatchTSMixerForPrediction, Tensor } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + const dims = [64, 512, 7]; + const prod = dims.reduce((a, b) => a * b, 1); + const past_values = new Tensor( + "float32", + Float32Array.from({ length: prod }, (_, i) => i / prod), + dims, + ); + + describe("PatchTSMixerModel", () => { + const model_id = "hf-internal-testing/tiny-random-PatchTSMixerModel"; + + /** @type {PatchTSMixerModel} */ + let model; + beforeAll(async () => { + model = await PatchTSMixerModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it( + "default", + async () => { + const { last_hidden_state } = await model({ past_values }); + + const { num_input_channels, num_patches, d_model } = model.config; + expect(last_hidden_state.dims).toEqual([dims[0], num_input_channels, num_patches, d_model]); + expect(last_hidden_state.mean().item()).toBeCloseTo(0.03344963490962982, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("PatchTSMixerForPrediction", () => { + const model_id = "onnx-community/granite-timeseries-patchtsmixer"; + + /** @type {PatchTSMixerForPrediction} */ + let model; + beforeAll(async () => { + model = await PatchTSMixerForPrediction.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it( + "default", + async () => { + const { prediction_outputs } = await model({ past_values }); + + const { prediction_length, num_input_channels } = model.config; + expect(prediction_outputs.dims).toEqual([dims[0], prediction_length, num_input_channels]); + expect(prediction_outputs.mean().item()).toBeCloseTo(0.5064773559570312, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/patchtst/test_modeling_patchtst.js b/packages/transformers/tests/models/patchtst/test_modeling_patchtst.js new file mode 100644 index 0000000..398538e --- /dev/null +++ b/packages/transformers/tests/models/patchtst/test_modeling_patchtst.js @@ -0,0 +1,65 @@ +import { PatchTSTModel, PatchTSTForPrediction, Tensor } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + const dims = [64, 512, 7]; + const prod = dims.reduce((a, b) => a * b, 1); + const past_values = new Tensor( + "float32", + Float32Array.from({ length: prod }, (_, i) => i / prod), + dims, + ); + + describe("PatchTSTModel", () => { + const model_id = "hf-internal-testing/tiny-random-PatchTSTModel"; + + /** @type {PatchTSTModel} */ + let model; + beforeAll(async () => { + model = await PatchTSTModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it( + "default", + async () => { + const { last_hidden_state } = await model({ past_values }); + + const { num_input_channels, d_model } = model.config; + expect(last_hidden_state.dims).toEqual([dims[0], num_input_channels, 43, d_model]); + expect(last_hidden_state.mean().item()).toBeCloseTo(0.016672514379024506, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("PatchTSTForPrediction", () => { + const model_id = "onnx-community/granite-timeseries-patchtst"; + + /** @type {PatchTSTForPrediction} */ + let model; + beforeAll(async () => { + model = await PatchTSTForPrediction.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it( + "default", + async () => { + const { prediction_outputs } = await model({ past_values }); + + const { prediction_length, num_input_channels } = model.config; + expect(prediction_outputs.dims).toEqual([dims[0], prediction_length, num_input_channels]); + expect(prediction_outputs.mean().item()).toBeCloseTo(0.506528377532959, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/phi3_v/test_image_processing_phi3_v.js b/packages/transformers/tests/models/phi3_v/test_image_processing_phi3_v.js new file mode 100644 index 0000000..cbf21e8 --- /dev/null +++ b/packages/transformers/tests/models/phi3_v/test_image_processing_phi3_v.js @@ -0,0 +1,93 @@ +import { AutoImageProcessor, Phi3VImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +const TARGET_IMAGE_SIZE = [3, 336, 336]; + +export default () => { + // Phi3VImageProcessor + // - custom image processing (patching) + describe("Phi3VImageProcessor", () => { + const model_id = "onnx-community/Phi-3.5-vision-instruct"; + + /** @type {Record} */ + const images = {}; + /** @type {Phi3VImageProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + + // Load images + const gradient_image = await load_cached_image("gradient_1280x640"); + const white_image = await load_cached_image("white_image"); + + images.gradient_image = gradient_image; + images.white_image = white_image; + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "square image (num_crops=4)", + async () => { + const num_crops = 4; + const { pixel_values, image_sizes, num_img_tokens } = await processor(images.white_image, { num_crops }); + expect(pixel_values.dims).toEqual([1, 1 + num_crops, ...TARGET_IMAGE_SIZE]); + expect(pixel_values.flatten(2).mean(2).tolist()).toBeCloseToNested([[2.050372362136841, 2.050372362136841, 2.050372362136841, 2.050372362136841, 2.050372362136841]], 1); + expect(pixel_values.mean().item()).toBeCloseTo(2.050372362136841, 1); + + expect(image_sizes.tolist()).toEqual([[672n, 672n]]); + expect(num_img_tokens).toEqual([757]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "non-square image (num_crops=4)", + async () => { + const num_crops = 4; + const { pixel_values, image_sizes, num_img_tokens } = await processor(images.gradient_image, { num_crops }); + expect(pixel_values.dims).toEqual([1, 1 + num_crops, ...TARGET_IMAGE_SIZE]); + + // NOTE: We use a slighly different cropping strategy to the python implementation, + // meaning the following tests would fail. + // expect(pixel_values.flatten(2).mean(2).tolist()).toBeCloseToNested([[ + // 0.18679802119731903, -0.5585645437240601, 0.9321606755256653, 0.0, 0.0, + // ]], 1); + // expect(pixel_values.mean().item()).toBeCloseTo(0.11207880824804306, 6); + + expect(image_sizes.tolist()).toEqual([[336n, 672n]]); + expect(num_img_tokens).toEqual([457]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "single image (num_crops=16)", + async () => { + const num_crops = 16; + const { pixel_values, image_sizes, num_img_tokens } = await processor(images.gradient_image, { num_crops }); + expect(pixel_values.dims).toEqual([1, 1 + num_crops, 3, 336, 336]); + expect(pixel_values.mean().item()).toBeCloseTo(0.4677375257015228, 1); + + expect(image_sizes.tolist()).toEqual([[1008n, 1680n]]); + expect(num_img_tokens).toEqual([2353]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "multiple images (num_crops=4)", + async () => { + const num_crops = 4; + const { pixel_values, image_sizes, num_img_tokens } = await processor([images.gradient_image, images.white_image], { num_crops }); + expect(pixel_values.dims).toEqual([2, 1 + num_crops, ...TARGET_IMAGE_SIZE]); + expect(image_sizes.tolist()).toEqual([ + [336n, 672n], + [672n, 672n], + ]); + expect(num_img_tokens).toEqual([457, 757]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/phi3_v/test_processor_phi3_v.js b/packages/transformers/tests/models/phi3_v/test_processor_phi3_v.js new file mode 100644 index 0000000..b513339 --- /dev/null +++ b/packages/transformers/tests/models/phi3_v/test_processor_phi3_v.js @@ -0,0 +1,84 @@ +import { AutoProcessor, Phi3VProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + const model_id = "onnx-community/Phi-3.5-vision-instruct"; + + describe("Phi3VProcessor", () => { + /** @type {Phi3VProcessor} */ + let processor; + let images = {}; + + beforeAll(async () => { + processor = await AutoProcessor.from_pretrained(model_id); + images = { + white_image: await load_cached_image("white_image"), + }; + }, MAX_PROCESSOR_LOAD_TIME); + + const create_prompt = (text, images = []) => { + const placeholder = images.map((_, i) => `<|image_${i + 1}|>\n`).join(""); + const messages = [{ role: "user", content: placeholder + text }]; + const prompt = processor.tokenizer.apply_chat_template(messages, { tokenize: false, add_generation_prompt: true }); + return prompt; + }; + + it( + "Text-only", + async () => { + const prompt = create_prompt("Hi there."); + const { input_ids, pixel_values } = await processor(prompt); + expect(input_ids.dims).toEqual([1, 11]); + expect(pixel_values).toBeUndefined(); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "Single image & text", + async () => { + const imgs = [images.white_image]; + const prompt = create_prompt("Describe this image.", imgs); + const { input_ids, attention_mask, pixel_values, image_sizes } = await processor(prompt, imgs); + expect(input_ids.dims).toEqual([1, /* 773 */ 770]); + expect(attention_mask.dims).toEqual(input_ids.dims); + expect(pixel_values.dims).toEqual([1, 5, 3, 336, 336]); + expect(image_sizes.tolist()).toEqual([[672n, 672n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "Single image (num_crops=16) & text", + async () => { + const imgs = [images.white_image]; + const prompt = create_prompt("Describe this image.", imgs); + const { input_ids, attention_mask, pixel_values, image_sizes } = await processor(prompt, imgs, { num_crops: 16 }); + expect(input_ids.dims).toEqual([1, /* 2525 */ 2522]); + expect(attention_mask.dims).toEqual(input_ids.dims); + expect(pixel_values.dims).toEqual([1, 17, 3, 336, 336]); + expect(image_sizes.tolist()).toEqual([[1344n, 1344n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "Multiple images & text", + async () => { + const imgs = [images.white_image, images.white_image]; + const prompt = create_prompt("Describe these images.", imgs); + const { input_ids, attention_mask, pixel_values, image_sizes } = await processor(prompt, imgs); + expect(input_ids.dims).toEqual([1, /* 1533 */ 1527]); + expect(attention_mask.dims).toEqual(input_ids.dims); + expect(pixel_values.dims).toEqual([2, 5, 3, 336, 336]); + expect(image_sizes.tolist()).toEqual([ + [672n, 672n], + [672n, 672n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/pyannote/test_modeling_pyannote.js b/packages/transformers/tests/models/pyannote/test_modeling_pyannote.js new file mode 100644 index 0000000..38dfd4b --- /dev/null +++ b/packages/transformers/tests/models/pyannote/test_modeling_pyannote.js @@ -0,0 +1,50 @@ +import { AutoProcessor, AutoModelForAudioFrameClassification } from "../../../src/transformers.js"; + +import { MAX_TEST_EXECUTION_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; +import { load_cached_audio } from "../../asset_cache.js"; + +export default () => { + const models_to_test = ["onnx-community/pyannote-segmentation-3.0"]; + + it( + `PyAnnoteForAudioFrameClassification`, + async () => { + const model_id = models_to_test[0]; + + // Load model and processor + const model = await AutoModelForAudioFrameClassification.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + const processor = await AutoProcessor.from_pretrained(model_id); + + // Check processor config + expect(processor.sampling_rate).toEqual(16000); + + // Preprocess audio + const audio = await load_cached_audio("mlk"); + const inputs = await processor(audio); + + // Run model with inputs + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 767, 7]); + expect(logits.mean().item()).toBeCloseTo(-4.822614669799805, 6); + + const result = processor.post_process_speaker_diarization(logits, audio.length); + const target = [ + [ + { id: 0, start: 0, end: 1.0512535626298245, confidence: 0.7898106738171984 }, + { id: 2, start: 1.0512535626298245, end: 2.373798367228636, confidence: 0.8923380609065887 }, + { id: 0, start: 2.373798367228636, end: 3.5776532534660155, confidence: 0.6920057005438546 }, + { id: 2, start: 3.5776532534660155, end: 4.578039708226655, confidence: 0.8169249580865657 }, + { id: 3, start: 4.578039708226655, end: 6.2396985652867, confidence: 0.6921662061495533 }, + { id: 2, start: 6.2396985652867, end: 8.664364040384521, confidence: 0.705263573835628 }, + { id: 0, start: 8.664364040384521, end: 10.071687358098641, confidence: 0.6650650397924295 }, + { id: 2, start: 10.071687358098641, end: 12.598087048934833, confidence: 0.8999033333468749 }, + { id: 0, start: 12.598087048934833, end: 13.005023911888312, confidence: 0.37838892004965197 }, + ], + ]; + expect(result).toBeCloseToNested(target); + + await model.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); +}; diff --git a/packages/transformers/tests/models/qwen2/test_tokenization_qwen2.js b/packages/transformers/tests/models/qwen2/test_tokenization_qwen2.js new file mode 100644 index 0000000..855d0d5 --- /dev/null +++ b/packages/transformers/tests/models/qwen2/test_tokenization_qwen2.js @@ -0,0 +1,158 @@ +import { Qwen2Tokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = Qwen2Tokenizer; +export const TEST_CONFIG = { + "Xenova/Qwen1.5-0.5B-Chat": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [4340, 525, 498, 3730, 30], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [2610, 1265, 3003, 2814, 419], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u0120", "0", "\u0120", "1", "\u0120", "2", "\u0120", "3", "\u0120", "4", "\u0120", "5", "\u0120", "6", "\u0120", "7", "\u0120", "8", "\u0120", "9", "\u0120", "1", "0", "\u0120", "1", "0", "0", "\u0120", "1", "0", "0", "0"], + ids: [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 220, 15, 220, 16, 220, 17, 220, 18, 220, 19, 220, 20, 220, 21, 220, 22, 220, 23, 220, 24, 220, 16, 15, 220, 16, 15, 15, 220, 16, 15, 15, 15], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u0120", "2", "0", "1", "6", "."], + ids: [785, 2813, 572, 18047, 304, 220, 17, 15, 16, 21, 13], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [32, 198, 3278, 11015, 983, 20224, 67, 4605, 67, 315, 11, 646, 944, 13], + decoded: "A\n'll !!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():\u010a", "\u0109pass"], + ids: [750, 1887, 3932, 41431], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".toString", "();\u010a", "toString", "();"], + ids: [1149, 264, 284, 2839, 5070, 543, 6575, 2129], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [1986, 271, 285, 198, 64, 198, 1944, 13], + decoded: "This\n\nis\na\ntest.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9d", ",", "running"], + ids: [1861, 52657, 15083, 11, 27173], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [16, 188, 17, 5691, 18], + decoded: "1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [9707, 4337], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [14990, 1879], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0\u013d", "\u00e6\u013a\u00af"], + ids: [105301, 88051, 116109, 20412], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [256, 6388, 3550], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tr", "ailing", "\u0120space", "\u0120\u0120\u0120"], + ids: [376, 14277, 3550, 262], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [13048, 220, 21927], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [1944, 400, 16, 431, 17, 671, 18, 12984, 19, 6938, 20, 71488, 21, 2858, 224, 96, 22, 89791, 23, 2858, 224, 109, 24, 1273], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "0", "0", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [40, 10788, 458, 23268, 369, 400, 16, 13, 15, 15, 518, 279, 3553, 13], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [9330, 1940, 256], + decoded: "you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [9330, 1940, 9238], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [9330, 1940, 4102, 4102, 9330, 1940, 9238], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120", "\u00ef\u00bd\u0140", "\u0120edge", "\u0120", "\u00ef\u00bd\u0140", "\u0120case"], + ids: [896, 2603, 220, 21216, 6821, 220, 21216, 1142], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [10417, 223, 1986, 14520, 223, 285, 14520, 223, 64, 14520, 223, 1944, 14520, 223, 13], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0134", "\u013e", "\u0120\u00f0\u0141\u0134", "\u013c", "\u0120\u00f0\u0141\u0134", "\u0139", "\u0120\u00f0\u0141\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141\u0134", "\u00b0"], + ids: [144185, 61804, 235, 11162, 97, 96, 26525, 235, 26525, 255, 11162, 236, 231, 11162, 247, 237, 26525, 232, 95069, 98, 26525, 223, 26525, 227, 11162, 97, 245, 26525, 228, 61804, 237, 70470, 30543, 63039, 250, 63039, 248, 63039, 245, 63039, 247, 11162, 244, 97, 26525, 236, 61804, 234, 11162, 98, 111, 63039, 103, 25521, 101, 61804, 231, 61804, 222, 63039, 107, 11162, 236, 230, 11162, 247, 230, 11162, 247, 234, 63039, 222, 61804, 229, 61804, 233, 25521, 227, 11162, 236, 223, 11162, 234, 252, 11162, 234, 116, 63039, 108], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141\u0131\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122", "\u012f", "\u00e2\u013b\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141\u0131\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013b\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0131\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u012e\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u00a4\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u00a7\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00e2\u013f\u00a4", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0134\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133\u00a7", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141\u0131\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u00a4\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u00a7\u0133", "\u00f0\u0141\u0131\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00a2", "\u00f3", "\u0142\u0123", "\u00a5", "\u00f3", "\u0142\u0123", "\u00ae", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0131\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0134\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133\u00a8", "\u00f0\u0141\u0131\u00bc"], + ids: [144232, 11162, 97, 245, 61804, 223, 30543, 61804, 109, 144321, 11162, 243, 113, 378, 235, 144693, 30543, 11162, 100, 247, 144321, 378, 235, 144693, 61804, 101, 144321, 378, 235, 146467, 11162, 100, 239, 378, 235, 146392, 378, 235, 148738, 61804, 102, 378, 235, 141390, 378, 235, 145002, 378, 235, 145367, 61804, 102, 378, 235, 145233, 378, 235, 145665, 378, 235, 145988, 11162, 100, 239, 144321, 378, 235, 146392, 378, 235, 148738, 144321, 11162, 237, 112, 175, 15675, 100, 175, 15675, 95, 175, 15675, 98, 175, 15675, 106, 175, 15675, 100, 175, 15675, 123, 61804, 101, 144321, 378, 235, 141390, 30543, 378, 235, 145002, 378, 235, 145367, 144784], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + }, +}; diff --git a/packages/transformers/tests/models/qwen2_vl/test_image_processing_qwen2_vl.js b/packages/transformers/tests/models/qwen2_vl/test_image_processing_qwen2_vl.js new file mode 100644 index 0000000..a348ec4 --- /dev/null +++ b/packages/transformers/tests/models/qwen2_vl/test_image_processing_qwen2_vl.js @@ -0,0 +1,34 @@ +import { AutoImageProcessor, Qwen2VLImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // Qwen2VLImageProcessor + // - custom image processing (min_pixels, max_pixels) + describe("Qwen2VLImageProcessor", () => { + const model_id = "hf-internal-testing/tiny-random-Qwen2VLForConditionalGeneration"; + + /** @type {Qwen2VLImageProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "custom image processing", + async () => { + const image = await load_cached_image("white_image"); + const { pixel_values, image_grid_thw, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([256, 1176]); + expect(pixel_values.mean().item()).toBeCloseTo(2.050372362136841, 6); + expect(image_grid_thw.tolist()).toEqual([[1n, 16n, 16n]]); + + expect(original_sizes).toEqual([[224, 224]]); + expect(reshaped_input_sizes).toEqual([[224, 224]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/qwen2_vl/test_modeling_qwen2_vl.js b/packages/transformers/tests/models/qwen2_vl/test_modeling_qwen2_vl.js new file mode 100644 index 0000000..5c5d067 --- /dev/null +++ b/packages/transformers/tests/models/qwen2_vl/test_modeling_qwen2_vl.js @@ -0,0 +1,121 @@ +import { Qwen2VLProcessor, Qwen2VLForConditionalGeneration, Qwen2VLForCausalLM, AutoTokenizer, RawImage } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + const CONVERSATION = [ + { + role: "user", + content: [{ type: "text", text: "Hello" }], + }, + ]; + + // Example adapted from https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct + const CONVERSATION_WITH_IMAGE = [ + { + role: "user", + content: [{ type: "image" }, { type: "text", text: "Describe this image." }], + }, + ]; + // Empty white image + const dims = [224, 224, 3]; + const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); + + describe("Qwen2VLForConditionalGeneration", () => { + const model_id = "hf-internal-testing/tiny-random-Qwen2VLForConditionalGeneration"; + + /** @type {Qwen2VLForConditionalGeneration} */ + let model; + /** @type {Qwen2VLProcessor} */ + let processor; + beforeAll(async () => { + model = await Qwen2VLForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + processor = await Qwen2VLProcessor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + const text = processor.apply_chat_template(CONVERSATION_WITH_IMAGE, { + add_generation_prompt: true, + }); + const inputs = await processor(text, image); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 89, 152064]); + expect(logits.mean().item()).toBeCloseTo(-0.0011299321195110679, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "text-only (batch_size=1)", + async () => { + const text = processor.apply_chat_template(CONVERSATION, { + add_generation_prompt: true, + }); + const inputs = await processor(text); + const generate_ids = await model.generate({ + ...inputs, + max_new_tokens: 10, + }); + + const new_tokens = generate_ids.slice(null, [inputs.input_ids.dims.at(-1), null]); + expect(new_tokens.tolist()).toEqual([[24284n, 63986n, 108860n, 84530n, 8889n, 23262n, 128276n, 64948n, 136757n, 138348n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "text + image (batch_size=1)", + async () => { + const text = processor.apply_chat_template(CONVERSATION_WITH_IMAGE, { + add_generation_prompt: true, + }); + const inputs = await processor(text, image); + const generate_ids = await model.generate({ + ...inputs, + max_new_tokens: 10, + }); + + const new_tokens = generate_ids.slice(null, [inputs.input_ids.dims.at(-1), null]); + expect(new_tokens.tolist()).toEqual([[24284n, 35302n, 60575n, 38679n, 113390n, 115118n, 137596n, 38241n, 96726n, 142301n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("Qwen2VLForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-Qwen2VLForConditionalGeneration"; + + /** @type {Qwen2VLForCausalLM} */ + let model; + /** @type {AutoTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await Qwen2VLForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await AutoTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + do_sample: false, + }); + expect(outputs.tolist()).toEqual([[14990n, 150610n, 22618n, 127483n, 72112n, 121460n, 58034n, 122233n, 18625n, 63632n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/qwen2_vl/test_processor_qwen2_vl.js b/packages/transformers/tests/models/qwen2_vl/test_processor_qwen2_vl.js new file mode 100644 index 0000000..72ccf78 --- /dev/null +++ b/packages/transformers/tests/models/qwen2_vl/test_processor_qwen2_vl.js @@ -0,0 +1,44 @@ +import { AutoProcessor, Qwen2VLProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("Qwen2VLProcessor", () => { + const model_id = "hf-internal-testing/tiny-random-Qwen2VLForConditionalGeneration"; + + /** @type {Qwen2VLProcessor} */ + let processor; + let images = {}; + + beforeAll(async () => { + processor = await AutoProcessor.from_pretrained(model_id); + images = { + white_image: await load_cached_image("white_image"), + }; + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "Image and text", + async () => { + const conversation = [ + { + role: "user", + content: [{ type: "image" }, { type: "text", text: "Describe this image." }], + }, + ]; + + const text = processor.apply_chat_template(conversation, { + add_generation_prompt: true, + }); + const { input_ids, attention_mask, pixel_values, image_grid_thw } = await processor(text, images.white_image); + + expect(input_ids.dims).toEqual([1, 89]); + expect(attention_mask.dims).toEqual([1, 89]); + expect(pixel_values.dims).toEqual([256, 1176]); + expect(image_grid_thw.dims).toEqual([1, 3]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/qwen3_5/test_modeling_qwen3_5.js b/packages/transformers/tests/models/qwen3_5/test_modeling_qwen3_5.js new file mode 100644 index 0000000..bd77e48 --- /dev/null +++ b/packages/transformers/tests/models/qwen3_5/test_modeling_qwen3_5.js @@ -0,0 +1,190 @@ +import { Qwen3_5ForConditionalGeneration, Qwen3_5ForCausalLM, AutoProcessor, AutoTokenizer, RawImage } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + const CONVERSATION = [ + { + role: "user", + content: [{ type: "text", text: "Hello" }], + }, + ]; + + const CONVERSATION_WITH_IMAGE = [ + { + role: "user", + content: [{ type: "image" }, { type: "text", text: "Describe this image." }], + }, + ]; + + // Empty white image + const dims = [224, 224, 3]; + const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); + + describe("Qwen3_5ForConditionalGeneration", () => { + const model_id = "onnx-internal-testing/tiny-random-Qwen3_5ForConditionalGeneration"; + + /** @type {Qwen3_5ForConditionalGeneration} */ + let model; + /** @type {AutoProcessor} */ + let processor; + beforeAll(async () => { + model = await Qwen3_5ForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + processor = await AutoProcessor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + const text = processor.apply_chat_template(CONVERSATION_WITH_IMAGE, { + add_generation_prompt: true, + }); + const inputs = await processor(text, image); + const { logits } = await model(inputs); + expect(logits.dims).toEqual([1, 80, 248320]); + expect(logits.mean().item()).toBeCloseTo(0.000018217360775452107, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "text-only (batch_size=1)", + async () => { + const text = processor.apply_chat_template(CONVERSATION, { + add_generation_prompt: true, + }); + const inputs = await processor(text); + const generate_ids = await model.generate({ + ...inputs, + max_new_tokens: 10, + do_sample: false, + }); + + const new_tokens = generate_ids.slice(null, [inputs.input_ids.dims.at(-1), null]); + expect(new_tokens.tolist()).toEqual([[122378n, 3582n, 161477n, 16864n, 43897n, 81038n, 110419n, 72561n, 30943n, 17612n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "text + image (batch_size=1)", + async () => { + const text = processor.apply_chat_template(CONVERSATION_WITH_IMAGE, { + add_generation_prompt: true, + }); + const inputs = await processor(text, image); + const generate_ids = await model.generate({ + ...inputs, + max_new_tokens: 10, + do_sample: false, + }); + + const new_tokens = generate_ids.slice(null, [inputs.input_ids.dims.at(-1), null]); + expect(new_tokens.tolist()).toEqual([[151971n, 151971n, 83307n, 106852n, 196734n, 72561n, 42265n, 83307n, 151971n, 83307n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + // TODO: Enable when "Updating cos_cache and sin_cache in RotaryEmbedding" is supported on CPU Execution Provider. + // Currently only works on WebGPU. + it.skip( + "generate w/ past_key_values", + async () => { + const conversation = [ + { + role: "user", + content: [{ type: "image" }, { type: "text", text: "What animal is this? Respond in one word." }], + }, + ]; + + const text = processor.apply_chat_template(conversation, { + add_generation_prompt: true, + }); + const inputs = await processor(text, image); + + const { sequences, past_key_values } = await model.generate({ + ...inputs, + max_new_tokens: 8, + return_dict_in_generate: true, + do_sample: false, + }); + + const first_response = processor.batch_decode(sequences.slice(null, [inputs.input_ids.dims.at(-1), null]), { skip_special_tokens: true })[0]; + + const continued_conversation = [ + ...conversation, + { + role: "assistant", + content: [{ type: "text", text: first_response }], + }, + { + role: "user", + content: [{ type: "text", text: "What color is this animal?" }], + }, + ]; + + const continued_text = processor.apply_chat_template(continued_conversation, { + add_generation_prompt: true, + }); + + const no_pkv_inputs = await processor(continued_text, image); + const no_pkv_outputs = await model.generate({ + ...no_pkv_inputs, + max_new_tokens: 8, + do_sample: false, + }); + + const pkv_inputs = await processor(continued_text); + const pkv_outputs = await model.generate({ + ...pkv_inputs, + image_grid_thw: inputs.image_grid_thw, + past_key_values, + max_new_tokens: 8, + do_sample: false, + }); + + const no_pkv_new_tokens = no_pkv_outputs.slice(null, [no_pkv_inputs.input_ids.dims.at(-1), null]); + const pkv_new_tokens = pkv_outputs.slice(null, [pkv_inputs.input_ids.dims.at(-1), null]); + + expect(no_pkv_new_tokens.dims).toEqual([1, 8]); + expect(pkv_new_tokens.dims).toEqual([1, 8]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("Qwen3_5ForCausalLM", () => { + const model_id = "onnx-internal-testing/tiny-random-Qwen3_5ForConditionalGeneration"; + + /** @type {Qwen3_5ForCausalLM} */ + let model; + /** @type {AutoTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await Qwen3_5ForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await AutoTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + do_sample: false, + }); + expect(outputs.tolist()).toEqual([[14556n, 156450n, 73097n, 47596n, 67970n, 15437n, 244844n, 43757n, 26602n, 3582n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/roberta/test_tokenization_roberta.js b/packages/transformers/tests/models/roberta/test_tokenization_roberta.js new file mode 100644 index 0000000..59ab61a --- /dev/null +++ b/packages/transformers/tests/models/roberta/test_tokenization_roberta.js @@ -0,0 +1,704 @@ +import { RobertaTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, BERT_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = RobertaTokenizer; +export const TEST_CONFIG = { + "jinaai/jina-embeddings-v2-base-de": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [0, 3267, 459, 426, 3174, 35, 2], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [0, 2606, 1303, 1990, 3022, 555, 2], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "123", "456", "789", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [0, 20, 21911, 40271, 51355, 885, 387, 381, 589, 699, 703, 866, 964, 991, 1045, 949, 1873, 8611, 2], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], + ids: [0, 710, 1891, 503, 15604, 295, 2262, 18, 2], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [0, 37, 203, 2202, 26143, 764, 30080, 72, 12228, 72, 314, 16, 571, 797, 18, 2], + decoded: "A\n'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "(", "):", "\u010a", "\u0109", "pass"], + ids: [0, 28273, 1911, 12, 4025, 203, 202, 5517, 2], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "to", "String", "();", "\u010a", "to", "String", "();"], + ids: [0, 1642, 264, 3887, 8273, 18, 764, 53889, 54181, 203, 764, 53889, 54181, 2], + decoded: "let a = obj.toString();\ntoString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [0, 1803, 203, 203, 276, 203, 69, 203, 4451, 18, 2], + decoded: "This\n\nis\na\ntest.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "w", "ant", "\u00c3\u00a9d", ",", "running"], + ids: [0, 3854, 91, 526, 46298, 16, 47232, 2], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [0, 21, 193, 22, 1998, 23, 2], + decoded: "1\u00002\ufffd3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [0, 17964, 3519, 2], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hell", "o", "\u0120world"], + ids: [0, 17067, 83, 1568, 2], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136", "\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7", "\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6", "\u013a", "\u00af"], + ids: [0, 55225, 258, 167, 117, 124, 44574, 168, 255, 258, 169, 113, 254, 167, 251, 112, 2], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120leading", "\u0120space"], + ids: [0, 6733, 5344, 3435, 2], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120\u0120\u0120"], + ids: [0, 766, 7462, 3435, 53448, 2], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [0, 10103, 225, 29546, 2], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2\u0124\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [0, 4451, 1350, 21, 366, 22, 1805, 23, 2712, 24, 6339, 25, 960, 103, 26, 1581, 229, 101, 27, 58720, 28, 1581, 229, 114, 29, 2839, 2], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [0, 45, 8928, 371, 19798, 382, 1350, 21, 18, 505, 495, 285, 4569, 18, 2], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120\u0120"], + ids: [0, 10695, 1179, 6733, 2], + decoded: "you\u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [0, 10695, 1179, 44105, 2], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [0, 10695, 1179, 15529, 15529, 10695, 1179, 44105, 2], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [0, 487, 2394, 17740, 126, 257, 9911, 17740, 126, 257, 2600, 2], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [0, 12790, 228, 1803, 20068, 228, 276, 20068, 228, 69, 20068, 228, 4451, 20068, 228, 18, 2], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0134", "\u013e", "\u0120\u00f0\u0141\u0134", "\u013c", "\u0120\u00f0\u0141\u0134", "\u0139", "\u0120\u00f0\u0141\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141\u013b", "\u012a", "\u0120\u00f0\u0141\u013b", "\u012e", "\u0120\u00f0\u0141\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2\u013e\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141\u0134", "\u00b0"], + ids: [0, 32164, 229, 49904, 240, 5060, 102, 101, 10278, 240, 10278, 260, 5060, 241, 236, 10319, 242, 10278, 237, 5060, 247, 103, 10278, 228, 10278, 232, 5060, 102, 250, 10278, 233, 49904, 242, 42009, 16598, 52278, 255, 52278, 253, 52278, 250, 52278, 252, 5060, 249, 102, 10278, 241, 49904, 239, 5060, 103, 116, 52278, 108, 10792, 106, 49904, 236, 49904, 227, 52278, 112, 5060, 241, 235, 10319, 235, 10319, 239, 52278, 227, 49904, 234, 49904, 238, 38607, 5060, 241, 228, 5060, 239, 257, 5060, 239, 121, 52278, 113, 2], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a7", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bb", "\u00e2\u0122\u012f", "\u00e2\u013f\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122\u012f", "\u00f0\u0141\u0134", "\u012d", "\u00e2\u0122\u012f", "\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141\u0131", "\u00bc"], + ids: [0, 20675, 106, 5060, 102, 250, 49904, 228, 16598, 49904, 114, 49365, 124, 5060, 248, 118, 54678, 26323, 229, 16598, 5060, 105, 252, 49365, 124, 54678, 26323, 229, 49904, 106, 49365, 124, 54678, 3753, 239, 127, 5060, 105, 244, 54678, 3753, 102, 256, 54678, 3753, 105, 244, 49904, 107, 54678, 49144, 54678, 41347, 238, 54678, 43307, 106, 49904, 107, 54678, 43307, 107, 54678, 43307, 105, 54678, 43307, 104, 5060, 105, 244, 49365, 124, 54678, 3753, 102, 256, 54678, 3753, 105, 244, 49365, 124, 5060, 242, 117, 180, 259, 228, 105, 180, 259, 228, 100, 180, 259, 228, 103, 180, 259, 228, 111, 180, 259, 228, 105, 180, 259, 228, 128, 49904, 106, 49365, 124, 54678, 49144, 16598, 54678, 41347, 238, 54678, 43307, 106, 49365, 125, 2], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "\u00e5", "\u012f", "\u013c", "\u00e6", "\u0130", "\u00a8", "zz"], + ids: [0, 500, 166, 240, 253, 167, 241, 106, 9326, 2], + decoded: "ah\u535a\u63a8zz", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "\u00c3\u00a9", "llo"], + ids: [0, 44, 2277, 31053, 2], + decoded: "H\u00e9llo", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u0120", "\u0109", "He", "L", "Lo", "!", "how", "\u0120\u0120", "\u010a", "\u0120Are", "\u0120y", "o", "U", "?", "\u0120\u0120"], + ids: [0, 225, 202, 2523, 48, 17901, 5, 7253, 6733, 203, 5175, 361, 83, 57, 35, 6733, 2], + decoded: " \tHeLLo!how \n Are yoU? ", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u0120", "\u0109", "H", "\u00c3\u00a4", "L", "Lo", "!", "how", "\u0120\u0120", "\u010a", "\u0120Are", "\u0120y", "o", "U", "?", "\u0120\u0120"], + ids: [0, 225, 202, 44, 325, 48, 17901, 5, 7253, 6733, 203, 5175, 361, 83, 57, 35, 6733, 2], + decoded: " \tH\u00e4LLo!how \n Are yoU? ", + }, + + TEXT_PAIR: { + text: "hello", + text_pair: "world", + tokens: ["hell", "o", "world"], + ids: [0, 17067, 83, 2, 2, 13639, 2], + decoded: "helloworld", + }, + }, + "jinaai/jina-embeddings-v2-base-code": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0123456789", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [0, 22133, 325, 397, 491, 795, 879, 997, 1434, 1577, 1240, 1926, 1528, 2069, 5216, 2], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120f", "ounded", "\u0120in", "\u01202016", "."], + ids: [0, 1664, 18100, 2146, 304, 12402, 338, 7541, 18, 2], + decoded: "The company was founded in 2016.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"], + ids: [0, 406, 3578, 3281, 203, 202, 4557, 2], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "toString", "();", "\u010a", "toString", "();"], + ids: [0, 953, 323, 278, 2666, 18, 3411, 467, 203, 3411, 467, 2], + decoded: "let a = obj.toString();\ntoString();", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9", "d", ",", "running"], + ids: [0, 1129, 13944, 2521, 72, 16, 8423, 2], + decoded: "UNwant\u00e9d,running", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [0, 9522, 7550, 2], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [0, 12173, 28408, 2149, 36264, 12338, 254, 4988, 2], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "\u0120space", "\u0120\u0120\u0120"], + ids: [0, 29801, 4113, 264, 2], + decoded: "trailing space ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2", "\u0124\u00ac", "4", "\u0120\u00c2", "\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [0, 1052, 393, 21, 741, 22, 592, 23, 12284, 16181, 24, 5519, 101, 25, 5519, 103, 26, 12284, 229, 101, 27, 12284, 229, 122, 28, 12284, 229, 114, 29, 1089, 2], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120b", "ought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [0, 45, 328, 17412, 692, 33091, 455, 393, 21, 18, 337, 913, 329, 3205, 18, 2], + decoded: "I bought an apple for $1.00 at the store.", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120edge", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120case"], + ids: [0, 1643, 6005, 225, 44634, 257, 7158, 225, 44634, 257, 1007, 2], + decoded: "weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120", "\u00e2\u0138", "\u0123", "is", "\u0120", "\u00e2\u0138", "\u0123", "a", "\u0120", "\u00e2\u0138", "\u0123", "test", "\u0120", "\u00e2\u0138", "\u0123", "."], + ids: [0, 8550, 228, 2744, 225, 8550, 228, 302, 225, 8550, 228, 69, 225, 8550, 228, 1052, 225, 8550, 228, 18, 2], + decoded: "\u2581This \u2581is \u2581a \u2581test \u2581.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141", "\u013a", "\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141", "\u013a", "\u012f", "\u0120\u00f0\u0141", "\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141", "\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141", "\u013a", "\u0123", "\u0120\u00f0\u0141", "\u013a\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141", "\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2", "\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u0134", "\u013e", "\u0120\u00f0\u0141", "\u0134", "\u013c", "\u0120\u00f0\u0141", "\u0134", "\u0139", "\u0120\u00f0\u0141", "\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141", "\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141", "\u0134", "\u00aa", "\u0120\u00e2", "\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141", "\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141", "\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2", "\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141", "\u0134", "\u00b0"], + ids: [0, 8000, 251, 229, 22730, 240, 9919, 102, 101, 9919, 251, 240, 9919, 251, 260, 9919, 241, 236, 9919, 252, 242, 9919, 251, 237, 9919, 247, 103, 9919, 251, 228, 9919, 38879, 9919, 102, 250, 9919, 251, 233, 22730, 242, 12284, 256, 102, 26726, 9919, 245, 255, 9919, 245, 253, 9919, 245, 250, 9919, 245, 252, 9919, 249, 102, 9919, 251, 241, 22730, 239, 9919, 103, 116, 9919, 245, 108, 12284, 255, 106, 22730, 236, 22730, 227, 9919, 245, 112, 9919, 241, 235, 9919, 252, 235, 9919, 252, 239, 9919, 245, 227, 22730, 234, 22730, 238, 12284, 255, 232, 9919, 241, 228, 9919, 239, 257, 9919, 239, 121, 9919, 245, 113, 2], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141\u00a7", "\u0133", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141\u0131", "\u00b4", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00a2", "\u00f3", "\u0142\u0123", "\u00a5", "\u00f3", "\u0142\u0123", "\u00ae", "\u00f3", "\u0142\u0123", "\u00a7", "\u00f3", "\u0142\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], + ids: [0, 60553, 106, 9919, 102, 250, 22730, 228, 26726, 22730, 114, 8000, 242, 124, 9919, 248, 118, 2965, 240, 30370, 229, 26726, 31249, 252, 8000, 242, 124, 2965, 240, 30370, 229, 22730, 106, 8000, 242, 124, 2965, 240, 8000, 239, 127, 31249, 244, 2965, 240, 8000, 102, 256, 2965, 240, 8000, 105, 244, 22730, 107, 2965, 240, 163, 256, 102, 2965, 240, 8000, 245, 238, 2965, 240, 8000, 244, 106, 22730, 107, 2965, 240, 8000, 244, 107, 2965, 240, 8000, 244, 105, 2965, 240, 8000, 244, 104, 31249, 244, 8000, 242, 124, 2965, 240, 8000, 102, 256, 2965, 240, 8000, 105, 244, 8000, 242, 124, 58646, 117, 180, 9752, 105, 180, 9752, 100, 180, 9752, 103, 180, 9752, 111, 180, 9752, 105, 180, 9752, 128, 22730, 106, 8000, 242, 124, 2965, 240, 163, 256, 102, 26726, 2965, 240, 8000, 245, 238, 2965, 240, 8000, 244, 106, 8000, 242, 125, 2], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "\u00e5\u012f\u013c", "\u00e6\u0130\u00a8", "zz"], + ids: [0, 3885, 33588, 28002, 4881, 2], + decoded: "ah\u535a\u63a8zz", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "\u00c3\u00a9", "l", "lo"], + ids: [0, 44, 2521, 80, 324, 2], + decoded: "H\u00e9llo", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u0120", "\u0109", "He", "L", "Lo", "!", "how", "\u0120\u0120\u010a", "\u0120Are", "\u0120y", "o", "U", "?", "\u0120\u0120"], + ids: [0, 225, 202, 1397, 48, 1898, 5, 11452, 11092, 14877, 711, 83, 57, 35, 261, 2], + decoded: " \tHeLLo!how \n Are yoU? ", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u0120", "\u0109", "H", "\u00c3\u00a4", "L", "Lo", "!", "how", "\u0120\u0120\u010a", "\u0120Are", "\u0120y", "o", "U", "?", "\u0120\u0120"], + ids: [0, 225, 202, 44, 4319, 48, 1898, 5, 11452, 11092, 14877, 711, 83, 57, 35, 261, 2], + decoded: " \tH\u00e4LLo!how \n Are yoU? ", + }, + }, + "jinaai/jina-reranker-v1-tiny-en": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["how", "are", "you", "doing", "?"], + ids: [0, 21431, 21182, 21166, 22540, 61, 2], + decoded: " how are you doing? ", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["you", "should", "'", "ve", "done", "this"], + ids: [0, 21166, 21602, 37, 21165, 22366, 21225, 2], + decoded: " you should've done this ", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "123", "456", "78", "9", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "100", "1000"], + ids: [0, 46, 32683, 56678, 25106, 55, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 21449, 22057, 25148, 2], + decoded: " 0 123 456 78 9 0 1 2 3 4 5 6 7 8 9 10 100 1000 ", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["the", "company", "was", "founded", "in", "2016", "."], + ids: [0, 21138, 21781, 21257, 26707, 21135, 22672, 44, 2], + decoded: " the company was founded in 2016. ", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["a", "'", "ll", "!!", "to", "?'", "d", "''", "d", "of", ",", "can", "'", "t", "."], + ids: [0, 69, 37, 21264, 22236, 21148, 58125, 72, 31803, 72, 21155, 42, 21243, 37, 88, 44, 2], + decoded: " a'll!! to?' d '' d of, can't. ", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "main", "(", "):", "pass"], + ids: [0, 22747, 21810, 38, 25247, 21924, 2], + decoded: " def main ( ): pass ", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "a", "=", "ob", "j", ".", "to", "string", "(", ");", "to", "string", "(", ");"], + ids: [0, 21621, 69, 59, 21706, 78, 44, 21148, 26762, 38, 25750, 21148, 26762, 38, 25750, 2], + decoded: " let a = ob j. to string ( ); to string ( ); ", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["this", "is", "a", "test", "."], + ids: [0, 21225, 21152, 69, 21828, 44, 2], + decoded: " this is a test. ", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["un", "want", "\u00e9", "d", ",", "running"], + ids: [0, 21193, 21569, 173, 72, 42, 23225, 2], + decoded: " un want \u00e9 d, running ", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0000", "2", "\ufffd", "3"], + ids: [0, 47, 5, 48, 20321, 49, 2], + decoded: " 1 \u0000 2 \ufffd 3 ", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["hello", "world"], + ids: [0, 28687, 21628, 2], + decoded: " hello world ", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "world"], + ids: [0, 28687, 21628, 2], + decoded: " hello world ", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f\u6d3b\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [0, 31805, 11140, 14597, 8097, 2], + decoded: " \u751f\u6d3b\u7684 \u771f \u8c1b \u662f ", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["leading", "space"], + ids: [0, 23462, 22283, 2], + decoded: " leading space ", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["trailing", "space"], + ids: [0, 52572, 22283, 2], + decoded: " trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["hi", "hello"], + ids: [0, 23233, 28687, 2], + decoded: " hi hello ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "$", "1", "r2", "#", "3", "\u20ac", "4", "\u00a3", "5", "\u00a5", "6", "", "7", "\u20b9", "8", "\u20b1", "9", "test"], + ids: [0, 21828, 34, 47, 46925, 33, 49, 2155, 50, 133, 51, 135, 52, 3, 53, 2159, 54, 2157, 55, 21828, 2], + decoded: " test $ 1 r2 # 3 \u20ac 4 \u00a3 5 \u00a5 6 7 \u20b9 8 \u20b1 9 test ", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["i", "bought", "an", "apple", "for", "$", "1", ".", "00", "at", "the", "store", "."], + ids: [0, 77, 25474, 21136, 24208, 21169, 34, 47, 44, 21298, 21141, 21138, 22657, 44, 2], + decoded: " i bought an apple for $ 1. 00 at the store. ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u2026"], + ids: [0, 21166, 2091, 2], + decoded: " you \u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u2026"], + ids: [0, 21166, 2091, 2], + decoded: " you \u2026 ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u2026", "you", "\u2026"], + ids: [0, 21166, 2091, 21166, 2091, 2], + decoded: " you \u2026 you \u2026 ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["weird", "\uff5e", "edge", "\uff5e", "case"], + ids: [0, 31376, 20249, 24273, 20249, 22111, 2], + decoded: " weird \uff5e edge \uff5e case ", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581", "this", "\u2581", "is", "\u2581", "a", "\u2581", "test", "\u2581", "."], + ids: [0, 2541, 21225, 2541, 21152, 2541, 69, 2541, 21828, 2541, 44, 2], + decoded: " \u2581 this \u2581 is \u2581 a \u2581 test \u2581. ", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\ud83d\ude02", "\ud83d\udc4d", "\ud83e\udd23", "\ud83d\ude0d", "\ud83d\ude2d", "\ud83c\udf89", "\ud83d\ude4f", "\ud83d\ude0a", "\ud83d\udd25", "\ud83d\ude01", "\ud83d\ude05", "\ud83e\udd17", "\ud83d\ude06", "\ud83d\udc4f", "\u2764", "\ufe0f", "\ud83d\udc9c", "\ud83d\udc9a", "\ud83d\udc97", "\ud83d\udc99", "\ud83d\udda4", "\ud83d\ude0e", "\ud83d\udc4c", "\ud83e\udd73", "\ud83d\udcaa", "\u2728", "\ud83d\udc49", "\ud83d\udc40", "\ud83d\udcaf", "\ud83c\udf88", "\ud83d\ude48", "\ud83d\ude4c", "\ud83d\udc80", "\ud83d\udc47", "\ud83d\udc4b", "\u2705", "\ud83c\udf81", "\ud83c\udf1e", "\ud83c\udf38", "\ud83d\udcb0"], + ids: [0, 20904, 20749, 21000, 20915, 20943, 20645, 20964, 20912, 20879, 20903, 20907, 20992, 20908, 20751, 2781, 20133, 20807, 20805, 20802, 20804, 20898, 20916, 20748, 21031, 20817, 2742, 20745, 20738, 20821, 20644, 20961, 20963, 20784, 20743, 20747, 2720, 20637, 20574, 20588, 20822, 2], + decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764 \ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0 ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2728", "\ud83e\udd17", "", "\ufe0f", "\ud83d\udc71", "\ud83c\udffb", "\ud83d\udd75", "\u200d", "\u2642", "\ufe0f", "\ud83e\uddd9", "\ud83c\udffb", "\u200d", "\u2642", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\ud83c\udf3e", "\ud83e\uddd1", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\ud83d\udc69", "\u200d", "\u2764", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\ud83d\udc69", "\u200d", "\ud83d\udc69", "\u200d", "\ud83d\udc67", "\u200d", "\ud83d\udc66", "\ud83e\uddd1", "\ud83c\udffb", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\ud83c\udffb", "\ud83c\udff4", "", "\udb40\udc62", "", "", "", "\udb40\udc7f", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\u2764", "\ufe0f", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\ud83c\udffc"], + ids: [0, 2742, 20992, 3, 20133, 20775, 20700, 20894, 2067, 2662, 20133, 21050, 20700, 2067, 2662, 20768, 20700, 2067, 20593, 21049, 2067, 20995, 2067, 21049, 20769, 2067, 2781, 2067, 20792, 2067, 20768, 20769, 2067, 20769, 2067, 20767, 2067, 20766, 21049, 20700, 2067, 20995, 2067, 21049, 20700, 20697, 3, 21126, 3, 3, 3, 21130, 20768, 20700, 2067, 2781, 20133, 2067, 20792, 2067, 20768, 20701, 2], + decoded: " \u2728 \ud83e\udd17 \ufe0f \ud83d\udc71 \ud83c\udffb \ud83d\udd75 \u200d \u2642 \ufe0f \ud83e\uddd9 \ud83c\udffb \u200d \u2642 \ud83d\udc68 \ud83c\udffb \u200d \ud83c\udf3e \ud83e\uddd1 \u200d \ud83e\udd1d \u200d \ud83e\uddd1 \ud83d\udc69 \u200d \u2764 \u200d \ud83d\udc8b \u200d \ud83d\udc68 \ud83d\udc69 \u200d \ud83d\udc69 \u200d \ud83d\udc67 \u200d \ud83d\udc66 \ud83e\uddd1 \ud83c\udffb \u200d \ud83e\udd1d \u200d \ud83e\uddd1 \ud83c\udffb \ud83c\udff4 \udb40\udc62 \udb40\udc7f \ud83d\udc68 \ud83c\udffb \u200d \u2764 \ufe0f \u200d \ud83d\udc8b \u200d \ud83d\udc68 \ud83c\udffc ", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "\u535a", "\u63a8", "zz"], + ids: [0, 22311, 4352, 7628, 24387, 2], + decoded: " ah \u535a \u63a8 zz ", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["h", "\u00e9", "llo"], + ids: [0, 76, 173, 48932, 2], + decoded: " h \u00e9 llo ", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["hello", "!", "how", "are", "you", "?"], + ids: [0, 28687, 31, 21431, 21182, 21166, 61, 2], + decoded: " hello! how are you? ", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["h", "\u00e4", "llo", "!", "how", "are", "you", "?"], + ids: [0, 76, 168, 48932, 31, 21431, 21182, 21166, 61, 2], + decoded: " h \u00e4 llo! how are you? ", + }, + }, + "jinaai/jina-embeddings-v2-base-zh": { + // https://huggingface.co/jinaai/jina-embeddings-v2-base-zh/discussions/16 + // Slow vs. fast tokenizer mismatch + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0000", "2", "\ufffd", "3"], + ids: [0, 47, 5, 48, 20321, 49, 2], + decoded: " 1 \u0000 2 \ufffd 3 ", + }, + }, + "Xenova/all-distilroberta-v1": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["01", "23", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [0, 2663, 1922, 1898, 4111, 5046, 321, 112, 132, 155, 204, 195, 231, 262, 290, 361, 158, 727, 10775, 2], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [0, 48998, 37127, 20024, 2023, 44574, 49122, 4333, 36484, 7487, 3726, 48569, 2], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], + ids: [0, 1437, 1437, 981, 980, 2], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], + ids: [0, 9738, 7022, 980, 1437, 1437, 1437, 2], + decoded: "trailing space ", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [0, 21959, 68, 134, 248, 176, 849, 246, 4480, 306, 984, 245, 30844, 401, 14333, 9264, 2469, 406, 14333, 9264, 9253, 398, 14333, 9264, 15389, 466, 1296, 2], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], + ids: [0, 6968, 1174, 1437, 1437, 2], + decoded: "you\u2026 ", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141", "\u013a\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141\u0133", "\u0131", "\u0120\u00e2\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u0134", "\u013e", "\u0120\u00f0\u0141", "\u0134", "\u013c", "\u0120\u00f0\u0141", "\u0134", "\u0139", "\u0120\u00f0\u0141", "\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141", "\u0134", "\u00aa", "\u0120\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u012b", "\u0120\u00f0\u0141\u0133", "\u0122", "\u0120\u00f0\u0141", "\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141", "\u0134", "\u0122", "\u0120\u00f0\u0141\u0133", "\u0129", "\u0120\u00f0\u0141\u0133", "\u012d", "\u0120\u00e2\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141", "\u0134", "\u00b0"], + ids: [0, 18636, 9264, 26964, 8384, 8103, 10470, 2469, 17841, 8384, 17841, 12410, 8103, 12736, 23171, 8103, 27, 9357, 17841, 27969, 8103, 10674, 8210, 17841, 10172, 8103, 48278, 8103, 10470, 6800, 17841, 27819, 26964, 9357, 28775, 10470, 12605, 8103, 10659, 48, 8103, 10659, 15113, 8103, 10659, 6800, 8103, 10659, 27, 8103, 25448, 10470, 17841, 12736, 26964, 14285, 8103, 8210, 15264, 8103, 10659, 10278, 36174, 11423, 26964, 23171, 26964, 7471, 8103, 10659, 10965, 8103, 12736, 23133, 8103, 27, 23133, 8103, 27, 14285, 8103, 10659, 7471, 26964, 6382, 26964, 13859, 36174, 5782, 8103, 12736, 10172, 8103, 14285, 17772, 8103, 14285, 18537, 8103, 10659, 7487, 2], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u0133", "\u0123", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00e2\u013f", "\u00a4", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133", "\u00a8", "\u0120\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133", "\u00a7", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u0120\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013f", "\u00a4", "\u00ef\u00b8\u0131", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], + ids: [0, 39817, 11423, 8103, 10470, 6800, 26964, 10172, 12605, 26964, 15389, 6569, 9357, 2023, 8103, 15722, 8906, 17, 8384, 38718, 9264, 12605, 8103, 6248, 27, 6569, 9357, 2023, 17, 8384, 38718, 9264, 26964, 11423, 6569, 9357, 2023, 17, 8384, 6569, 14285, 4726, 8103, 6248, 3602, 17, 8384, 6569, 10470, 46, 17, 8384, 6569, 6248, 3602, 26964, 15375, 17, 8384, 30151, 10470, 17, 8384, 6569, 10659, 13859, 17, 8384, 31193, 11423, 26964, 15375, 17, 8384, 31193, 15375, 17, 8384, 31193, 6248, 17, 8384, 31193, 18164, 8103, 6248, 3602, 6569, 9357, 2023, 17, 8384, 6569, 10470, 46, 17, 8384, 6569, 6248, 3602, 6569, 9357, 2023, 8103, 9357, 20024, 49078, 21402, 10172, 6248, 49078, 21402, 10172, 7258, 49078, 21402, 10172, 8210, 49078, 21402, 10172, 2840, 49078, 21402, 10172, 6248, 49078, 21402, 10172, 9470, 26964, 11423, 6569, 9357, 2023, 17, 8384, 30151, 10470, 12605, 17, 8384, 6569, 10659, 13859, 17, 8384, 31193, 11423, 6569, 9357, 4394, 2], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["ah", "\u00e5\u012f", "\u013c", "\u00e6", "\u0130", "\u00a8", "zz"], + ids: [0, 895, 47658, 15113, 37127, 12736, 11423, 7399, 2], + decoded: "ah\u535a\u63a8zz", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u0120", "\u0109", "He", "LL", "o", "!", "how", "\u0120", "\u0120", "\u010a", "\u0120Are", "\u0120yo", "U", "?", "\u0120", "\u0120"], + ids: [0, 1437, 50117, 894, 6006, 139, 328, 9178, 1437, 1437, 50118, 3945, 25610, 791, 116, 1437, 1437, 2], + decoded: " \tHeLLo!how \n Are yoU? ", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u0120", "\u0109", "H", "\u00c3\u00a4", "LL", "o", "!", "how", "\u0120", "\u0120", "\u010a", "\u0120Are", "\u0120yo", "U", "?", "\u0120", "\u0120"], + ids: [0, 1437, 50117, 725, 1561, 6006, 139, 328, 9178, 1437, 1437, 50118, 3945, 25610, 791, 116, 1437, 1437, 2], + decoded: " \tH\u00e4LLo!how \n Are yoU? ", + }, + }, + "Xenova/EsperBERTo-small-pos": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120do", "ing", "?"], + ids: [0, 50702, 1694, 12426, 661, 948, 35, 2], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120sho", "uld", "'ve", "\u0120don", "e", "\u0120this"], + ids: [0, 36894, 21906, 8512, 6091, 851, 73, 18955, 2], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "123456", "78", "9", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [0, 20, 11816, 6229, 29, 2042, 355, 411, 620, 818, 839, 1031, 1142, 1166, 1274, 1312, 2450, 8403, 2], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120c", "ompan", "y", "\u0120was", "\u0120fo", "und", "ed", "\u0120in", "\u01202016", "."], + ids: [0, 5490, 467, 2833, 93, 30687, 1204, 3936, 347, 327, 3653, 18, 2], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!", "!", "to", "?", "'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [0, 37, 203, 17792, 4883, 5, 288, 35, 11, 72, 15271, 72, 682, 16, 15597, 3761, 18, 2], + decoded: "A\n'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["de", "f", "\u0120ma", "in", "(", "):", "\u010a", "\u0109", "pas", "s"], + ids: [0, 387, 74, 633, 282, 12, 3914, 203, 202, 1208, 87, 2], + decoded: "def main():\n\tpass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120ob", "j", ".", "to", "S", "tr", "ing", "(", ");", "\u010a", "to", "S", "tr", "ing", "(", ");"], + ids: [0, 5745, 278, 3945, 1080, 78, 18, 288, 55, 497, 948, 12, 3429, 203, 288, 55, 497, 948, 12, 3429, 2], + decoded: "let a = obj.toString();\ntoString();", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "w", "ant", "\u00c3\u00a9", "d", ",", "run", "ning"], + ids: [0, 7390, 91, 1799, 1174, 72, 16, 1307, 13715, 2], + decoded: "UNwant\u00e9d,running", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hel", "lo", "\u0120World"], + ids: [0, 4152, 310, 7717, 2], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hel", "lo", "\u0120wor", "ld"], + ids: [0, 1686, 310, 39013, 3580, 2], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136", "\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [0, 25856, 258, 167, 117, 124, 19584, 42803, 258, 169, 113, 254, 34946, 2], + decoded: "\u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120\u0120", "\u0120le", "ading", "\u0120space"], + ids: [0, 2399, 591, 30214, 51965, 2], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "i", "ling", "\u0120space", "\u0120\u0120", "\u0120"], + ids: [0, 440, 77, 879, 51965, 2399, 225, 2], + decoded: "trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hel", "lo"], + ids: [0, 15893, 225, 3558, 310, 2], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [0, 30747, 9416, 21, 462, 22, 4668, 23, 7537, 24, 12407, 25, 790, 103, 26, 2097, 229, 101, 27, 2097, 229, 122, 28, 2097, 229, 114, 29, 18885, 2], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bo", "u", "ght", "\u0120an", "\u0120ap", "ple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120s", "tore", "."], + ids: [0, 45, 1716, 89, 8840, 353, 560, 720, 434, 9416, 21, 18, 455, 3993, 2814, 275, 14003, 18, 2], + decoded: "I bought an apple for $1.00 at the store.", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [0, 9642, 1322, 131, 259, 131, 259, 2], + decoded: "you\u2026\u00a0\u00a0", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [0, 9642, 1322, 131, 259, 131, 259, 9642, 1322, 131, 259, 131, 259, 2], + decoded: "you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ir", "d", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120ed", "ge", "\u0120", "\u00ef\u00bd", "\u0140", "\u0120c", "ase"], + ids: [0, 4983, 861, 72, 225, 30624, 257, 1263, 587, 225, 30624, 257, 467, 14285, 2], + decoded: "weird \uff5e edge \uff5e case", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u00f0\u0141\u013a", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u012f", "\u0120\u00f0\u0141", "\u00a4", "\u00a3", "\u0120\u00f0\u0141\u013a", "\u012f", "\u0120\u00f0\u0141\u013a", "\u0143", "\u0120\u00f0\u0141", "\u0130", "\u012b", "\u0120\u00f0\u0141", "\u013b", "\u0131", "\u0120\u00f0\u0141\u013a", "\u012c", "\u0120\u00f0\u0141", "\u0136", "\u00a5", "\u0120\u00f0\u0141\u013a", "\u0123", "\u0120\u00f0\u0141\u013a", "\u0127", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141\u013a", "\u0128", "\u0120\u00f0\u0141", "\u0133", "\u0131", "\u0120\u00e2", "\u013f", "\u00a4", "\u00ef\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u0134", "\u013e", "\u0120\u00f0\u0141", "\u0134", "\u013c", "\u0120\u00f0\u0141", "\u0134", "\u0139", "\u0120\u00f0\u0141", "\u0134", "\u013b", "\u0120\u00f0\u0141", "\u0138", "\u00a4", "\u0120\u00f0\u0141\u013a", "\u0130", "\u0120\u00f0\u0141", "\u0133", "\u012e", "\u0120\u00f0\u0141", "\u00a5", "\u00b3", "\u0120\u00f0\u0141", "\u0134", "\u00aa", "\u0120\u00e2", "\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u012b", "\u0120\u00f0\u0141", "\u0133", "\u0122", "\u0120\u00f0\u0141", "\u0134", "\u00af", "\u0120\u00f0\u0141", "\u0130", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012a", "\u0120\u00f0\u0141", "\u013b", "\u012e", "\u0120\u00f0\u0141", "\u0134", "\u0122", "\u0120\u00f0\u0141", "\u0133", "\u0129", "\u0120\u00f0\u0141", "\u0133", "\u012d", "\u0120\u00e2", "\u013e", "\u0127", "\u0120\u00f0\u0141", "\u0130", "\u0123", "\u0120\u00f0\u0141", "\u012e", "\u0140", "\u0120\u00f0\u0141", "\u012e", "\u00b8", "\u0120\u00f0\u0141", "\u0134", "\u00b0"], + ids: [0, 10626, 229, 32340, 244, 240, 32340, 102, 101, 30199, 240, 30199, 260, 32340, 241, 236, 32340, 252, 242, 30199, 237, 32340, 247, 103, 30199, 228, 30199, 232, 32340, 102, 250, 30199, 233, 32340, 244, 242, 2097, 256, 102, 27027, 242, 32340, 245, 255, 32340, 245, 253, 32340, 245, 250, 32340, 245, 252, 32340, 249, 102, 30199, 241, 32340, 244, 239, 32340, 103, 116, 32340, 245, 108, 2097, 255, 106, 32340, 244, 236, 32340, 244, 227, 32340, 245, 112, 32340, 241, 235, 32340, 252, 235, 32340, 252, 239, 32340, 245, 227, 32340, 244, 234, 32340, 244, 238, 2097, 255, 232, 32340, 241, 228, 32340, 239, 257, 32340, 239, 121, 32340, 245, 113, 2], + decoded: "\ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83e\udd73 \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u00e2", "\u013e", "\u00a8", "\u0120\u00f0\u0141", "\u00a4", "\u0139", "\u0120\u00f0\u0141", "\u0133", "\u0123", "\u00ef\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u0133", "\u00b1", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0137", "\u00b5", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u00ef\u00b8", "\u0131", "\u0120\u00f0\u0141", "\u00a7", "\u013b", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2\u013b", "\u0124", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u012e", "\u00be", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u0120\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a9", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a7", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a6", "\u0120\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a4", "\u013f", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u00a7", "\u0133", "\u00f0\u0141", "\u0131", "\u00bb", "\u0120\u00f0\u0141", "\u0131", "\u00b4", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00a2", "\u00f3", "\u0142", "\u0123", "\u00a5", "\u00f3", "\u0142", "\u0123", "\u00ae", "\u00f3", "\u0142", "\u0123", "\u00a7", "\u00f3", "\u0142", "\u0123", "\u00bf", "\u0120\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bb", "\u00e2\u0122", "\u012f", "\u00e2", "\u013f", "\u00a4", "\u00ef\u00b8", "\u0131", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0134", "\u012d", "\u00e2\u0122", "\u012f", "\u00f0\u0141", "\u0133", "\u00a8", "\u00f0\u0141", "\u0131", "\u00bc"], + ids: [0, 163, 255, 106, 32340, 102, 250, 32340, 244, 228, 27027, 242, 32340, 244, 114, 9132, 242, 124, 32340, 248, 118, 348, 240, 20419, 229, 27027, 242, 32340, 105, 252, 9132, 242, 124, 348, 240, 20419, 229, 32340, 244, 106, 9132, 242, 124, 348, 240, 9132, 239, 127, 32340, 105, 244, 348, 240, 9132, 102, 256, 348, 240, 9132, 105, 244, 32340, 244, 107, 348, 240, 163, 256, 102, 348, 240, 9132, 245, 238, 348, 240, 9132, 244, 106, 32340, 244, 107, 348, 240, 9132, 244, 107, 348, 240, 9132, 244, 105, 348, 240, 9132, 244, 104, 32340, 105, 244, 9132, 242, 124, 348, 240, 9132, 102, 256, 348, 240, 9132, 105, 244, 9132, 242, 124, 32340, 242, 117, 180, 259, 228, 105, 180, 259, 228, 100, 180, 259, 228, 103, 180, 259, 228, 111, 180, 259, 228, 105, 180, 259, 228, 128, 32340, 244, 106, 9132, 242, 124, 348, 240, 163, 256, 102, 27027, 242, 348, 240, 9132, 245, 238, 348, 240, 9132, 244, 106, 9132, 242, 125, 2], + decoded: "\u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75\u200d\u2642\ufe0f \ud83e\uddd9\ud83c\udffb\u200d\u2642 \ud83d\udc68\ud83c\udffb\u200d\ud83c\udf3e \ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1 \ud83d\udc69\u200d\u2764\u200d\ud83d\udc8b\u200d\ud83d\udc68 \ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 \ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c\udffb \ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f \ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c\udffc", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["\u0120", "\u0109", "He", "L", "Lo", "!", "ho", "w", "\u0120\u0120", "\u010a", "\u0120Are", "\u0120yo", "U", "?", "\u0120\u0120"], + ids: [0, 225, 202, 13029, 48, 4876, 5, 882, 91, 2399, 203, 31676, 27961, 57, 35, 2399, 2], + decoded: " \tHeLLo!how \n Are yoU? ", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["\u0120", "\u0109", "H", "\u00c3\u00a4", "L", "Lo", "!", "ho", "w", "\u0120\u0120", "\u010a", "\u0120Are", "\u0120yo", "U", "?", "\u0120\u0120"], + ids: [0, 225, 202, 44, 3203, 48, 4876, 5, 882, 91, 2399, 203, 31676, 27961, 57, 35, 2399, 2], + decoded: " \tH\u00e4LLo!how \n Are yoU? ", + }, + }, + + // Split tokenizer with behavior="Removed" and invert=false + "onnx-community/camembertv2-base": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "are", "you", "doi", "##ng", "?"], + ids: [1, 14473, 9556, 10577, 6471, 9274, 38, 2], + decoded: "[CLS] How are you doing? [SEP]", + }, + }, +}; diff --git a/packages/transformers/tests/models/sam/test_image_processing_sam.js b/packages/transformers/tests/models/sam/test_image_processing_sam.js new file mode 100644 index 0000000..31fa536 --- /dev/null +++ b/packages/transformers/tests/models/sam/test_image_processing_sam.js @@ -0,0 +1,95 @@ +import { AutoImageProcessor, SamImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // SamImageProcessor + // - tests normal padding (do_pad=true, pad_size={"height":1024,"width":1024}) + // - In addition to the image, pass in a list of points + describe("SamImageProcessor", () => { + const model_id = "Xenova/sam-vit-base"; + + /** @type {SamImageProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "without input points", + async () => { + const image = await load_cached_image("pattern_3x3"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + expect(pixel_values.dims).toEqual([1, 3, 1024, 1024]); + expect(pixel_values.mean().item()).toBeCloseTo(-0.4505715670146813, 6); + + expect(original_sizes).toEqual([[3, 3]]); + expect(reshaped_input_sizes).toEqual([[1024, 1024]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "with input points", + async () => { + const image = await load_cached_image("pattern_3x3"); + const { original_sizes, reshaped_input_sizes, input_points } = await processor(image, { + input_points: [[[1, 2]]], + }); + + expect(original_sizes).toEqual([[3, 3]]); + expect(reshaped_input_sizes).toEqual([[1024, 1024]]); + expect(input_points.tolist()).toBeCloseToNested([[[[341.3333, 682.6667]]]], 4); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "multiple points with labels", + async () => { + const image = await load_cached_image("pattern_3x3"); + const { original_sizes, reshaped_input_sizes, input_points, input_labels } = await processor(image, { + input_points: [ + [ + [1, 2], + [2, 1], + ], + ], + input_labels: [[1, 0]], + }); + + expect(original_sizes).toEqual([[3, 3]]); + expect(reshaped_input_sizes).toEqual([[1024, 1024]]); + expect(input_points.tolist()).toBeCloseToNested( + [ + [ + [ + [341.3333, 682.6667], + [682.6667, 341.3333], + ], + ], + ], + 4, + ); + expect(input_labels.tolist()).toEqual([[[1n, 0n]]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "with input boxes", + async () => { + const image = await load_cached_image("pattern_3x3"); + const { original_sizes, reshaped_input_sizes, input_boxes } = await processor(image, { + input_boxes: [[[0, 1, 2, 2]]], + }); + + expect(original_sizes).toEqual([[3, 3]]); + expect(reshaped_input_sizes).toEqual([[1024, 1024]]); + expect(input_boxes.tolist()).toBeCloseToNested([[[0, 341.3333, 682.6667, 682.6667]]], 4); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/sam/test_modeling_sam.js b/packages/transformers/tests/models/sam/test_modeling_sam.js new file mode 100644 index 0000000..af30988 --- /dev/null +++ b/packages/transformers/tests/models/sam/test_modeling_sam.js @@ -0,0 +1,48 @@ +import { SamProcessor, SamModel } from "../../../src/transformers.js"; +import { load_cached_image } from "../../asset_cache.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("SamModel", () => { + const model_id = "Xenova/slimsam-77-uniform"; + + /** @type {SamModel} */ + let model; + /** @type {SamProcessor} */ + let processor; + beforeAll(async () => { + model = await SamModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + processor = await SamProcessor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "w/ input_points", + async () => { + // Prepare image and input points + const raw_image = await load_cached_image("corgi"); + const input_points = [[[340, 250]]]; + + // Process inputs and perform mask generation + const inputs = await processor(raw_image, { input_points }); + const { pred_masks, iou_scores } = await model(inputs); + + expect(pred_masks.dims).toEqual([1, 1, 3, 256, 256]); + expect(pred_masks.mean().item()).toBeCloseTo(-5.764908313751221, 3); + expect(iou_scores.dims).toEqual([1, 1, 3]); + expect(iou_scores.tolist()).toBeCloseToNested([[[0.8583833575248718, 0.9773167967796326, 0.8511142730712891]]]); + + // Post-process masks + const masks = await processor.post_process_masks(pred_masks, inputs.original_sizes, inputs.reshaped_input_sizes); + expect(masks).toHaveLength(1); + expect(masks[0].dims).toEqual([1, 3, 410, 614]); + expect(masks[0].type).toEqual("bool"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/seamless_m4t/test_feature_extraction_seamless_m4t.js b/packages/transformers/tests/models/seamless_m4t/test_feature_extraction_seamless_m4t.js new file mode 100644 index 0000000..5392657 --- /dev/null +++ b/packages/transformers/tests/models/seamless_m4t/test_feature_extraction_seamless_m4t.js @@ -0,0 +1,65 @@ +import { AutoFeatureExtractor, SeamlessM4TFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_audio } from "../../asset_cache.js"; +import { MAX_FEATURE_EXTRACTOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +const sum = (array) => Number(array.reduce((a, b) => a + b, array instanceof BigInt64Array ? 0n : 0)); + +export default () => { + // SeamlessM4TFeatureExtractor + describe("SeamlessM4TFeatureExtractor", () => { + const model_id = "Xenova/wav2vec2-bert-CV16-en"; + + /** @type {SeamlessM4TFeatureExtractor} */ + let feature_extractor; + beforeAll(async () => { + feature_extractor = await AutoFeatureExtractor.from_pretrained(model_id); + }, MAX_FEATURE_EXTRACTOR_LOAD_TIME); + + it( + "default", + async () => { + const audio = await load_cached_audio("mlk"); + + const { input_features, attention_mask } = await feature_extractor(audio); + const { dims, data } = input_features; + expect(dims).toEqual([1, 649, 160]); + expect(attention_mask.dims).toEqual([1, 649]); + + expect(input_features.mean().item()).toBeCloseTo(-2.938903875815413e-8); + expect(data[0]).toBeCloseTo(1.1939343214035034); + expect(data[1]).toBeCloseTo(0.7874255180358887); + expect(data[160]).toBeCloseTo(-0.712975025177002); + expect(data[161]).toBeCloseTo(0.045802414417266846); + expect(data.at(-1)).toBeCloseTo(-1.3328346014022827); + + expect(sum(attention_mask.data)).toEqual(649); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "padding (pad_to_multiple_of=2)", + async () => { + const audio = await load_cached_audio("mlk"); + + const { input_features, attention_mask } = await feature_extractor(audio.slice(0, 10000)); + const { dims, data } = input_features; + + // [1, 61, 80] -> [1, 62, 80] -> [1, 31, 160] + expect(dims).toEqual([1, 31, 160]); + expect(attention_mask.dims).toEqual([1, 31]); + + expect(input_features.mean().item()).toBeCloseTo(0.01612919569015503); + expect(data[0]).toBeCloseTo(0.9657132029533386); + expect(data[1]).toBeCloseTo(0.12912897765636444); + expect(data[160]).toBeCloseTo(-1.2364212274551392); + expect(data[161]).toBeCloseTo(-0.9703778028488159); + expect(data.at(-1)).toBeCloseTo(1); // padding value + + expect(sum(attention_mask.data)).toEqual(30); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/solar_open/test_modeling_solar_open.js b/packages/transformers/tests/models/solar_open/test_modeling_solar_open.js new file mode 100644 index 0000000..8cc2475 --- /dev/null +++ b/packages/transformers/tests/models/solar_open/test_modeling_solar_open.js @@ -0,0 +1,51 @@ +import { PreTrainedTokenizer, SolarOpenForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("SolarOpenForCausalLM", () => { + const model_id = "onnx-internal-testing/tiny-random-SolarOpenForCausalLM"; + /** @type {SolarOpenForCausalLM} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await SolarOpenForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[27090n, 134662n, 105184n, 152304n, 69736n, 67149n, 165462n, 142691n, 181896n, 67149n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [2n, 27090n, 134662n, 105184n, 152304n, 69736n, 67149n, 165462n, 142691n, 181896n], + [27090n, 8358n, 188200n, 94175n, 188200n, 94175n, 195168n, 195168n, 195168n, 113354n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/swin2sr/test_image_processing_swin2sr.js b/packages/transformers/tests/models/swin2sr/test_image_processing_swin2sr.js new file mode 100644 index 0000000..db45798 --- /dev/null +++ b/packages/transformers/tests/models/swin2sr/test_image_processing_swin2sr.js @@ -0,0 +1,41 @@ +import { AutoImageProcessor, Swin2SRImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // Swin2SRImageProcessor + // - tests when padding is a number (do_pad=true, pad_size=8) + describe("Swin2SRImageProcessor", () => { + const model_id = "Xenova/swin2SR-classical-sr-x2-64"; + + /** @type {Swin2SRImageProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "Pad to multiple of 8 (3x3 -> 8x8)", + async () => { + const image = await load_cached_image("pattern_3x3"); + const { pixel_values } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 8, 8]); + expect(pixel_values.mean().item()).toBeCloseTo(0.5458333368102709, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "Do not pad if already a multiple of 8 (8x8 -> 8x8)", + async () => { + const image = await load_cached_image("checkerboard_8x8"); + const { pixel_values } = await processor(image); + expect(pixel_values.dims).toEqual([1, 3, 8, 8]); + expect(pixel_values.mean().item()).toBeCloseTo(0.5, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/t5/test_modeling_t5.js b/packages/transformers/tests/models/t5/test_modeling_t5.js new file mode 100644 index 0000000..c48150d --- /dev/null +++ b/packages/transformers/tests/models/t5/test_modeling_t5.js @@ -0,0 +1,96 @@ +import { T5Tokenizer, T5Model, T5ForConditionalGeneration } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("T5Model", () => { + const model_id = "hf-internal-testing/tiny-random-T5Model"; + + /** @type {T5Model} */ + let model; + /** @type {T5Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await T5Model.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await T5Tokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + // Example adapted from https://huggingface.co/google-t5/t5-small#how-to-get-started-with-the-model + const inputs = tokenizer("Studies have been shown that owning a dog is good for you"); + const { input_ids: decoder_input_ids } = tokenizer("Studies show that"); + + const { last_hidden_state } = await model({ ...inputs, decoder_input_ids }); + expect(last_hidden_state.dims).toEqual([1, 4, 32]); + expect(last_hidden_state.mean().item()).toBeCloseTo(7.492632721550763e-5, 8); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + describe("T5ForConditionalGeneration", () => { + const model_id = "hf-internal-testing/tiny-random-T5ForConditionalGeneration"; + + /** @type {T5ForConditionalGeneration} */ + let model; + /** @type {T5Tokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await T5ForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await T5Tokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "forward", + async () => { + // Example adapted from https://huggingface.co/google-t5/t5-small#how-to-get-started-with-the-model + const inputs = tokenizer("Studies have been shown that owning a dog is good for you"); + const { input_ids: decoder_input_ids } = tokenizer("Studies show that"); + + const model = await T5ForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + const outputs = await model({ ...inputs, decoder_input_ids }); + expect(outputs.logits.dims).toEqual([1, 4, 32100]); + expect(outputs.logits.mean().item()).toBeCloseTo(8.867568901393952e-9, 12); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n], + [0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n, 0n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/t5/test_tokenization_t5.js b/packages/transformers/tests/models/t5/test_tokenization_t5.js new file mode 100644 index 0000000..4e20a92 --- /dev/null +++ b/packages/transformers/tests/models/t5/test_tokenization_t5.js @@ -0,0 +1,293 @@ +import { T5Tokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, SENTENCEPIECE_TEST_STRINGS, T5_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = T5Tokenizer; +export const TEST_CONFIG = { + "Xenova/t5-small": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [571, 33, 25, 692, 58, 1], + decoded: "How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [148, 225, 31, 162, 612, 48, 1], + decoded: "You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u258101", "23", "45", "67", "89", "\u2581", "0", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [7088, 2773, 2128, 3708, 3914, 3, 632, 209, 204, 220, 314, 305, 431, 489, 505, 668, 335, 910, 5580, 1], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u25812016."], + ids: [37, 349, 47, 5710, 16, 4619, 1], + decoded: "The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581", "'", "ll", "\u2581", "!!", "to", "?", "'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [71, 3, 31, 195, 3, 1603, 235, 58, 31, 26, 31, 31, 26, 13, 6, 54, 31, 17, 5, 1], + decoded: "A 'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581de", "f", "\u2581main", "()", ":", "\u2581pass"], + ids: [20, 89, 711, 9960, 10, 1903, 1], + decoded: "def main(): pass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581", "a", "\u2581=", "\u2581", "o", "b", "j", ".", "to", "Str", "ing", "()", ";", "\u2581to", "Str", "ing", "()", ";"], + ids: [752, 3, 9, 3274, 3, 32, 115, 354, 5, 235, 11500, 53, 9960, 117, 12, 11500, 53, 9960, 117, 1], + decoded: "let a = obj.toString(); toString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581This", "\u2581is", "\u2581", "a", "\u2581test", "."], + ids: [100, 19, 3, 9, 794, 5, 1], + decoded: "This is a test.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "wan", "t\u00e9", "d", ",", "running"], + ids: [4417, 3877, 2229, 26, 6, 24549, 1], + decoded: "UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u25811", "\u0000", "2", "\u25813"], + ids: [209, 2, 357, 220, 1], + decoded: "12 3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581Hello", "\u2581World"], + ids: [8774, 1150, 1], + decoded: "Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [21820, 296, 1], + decoded: "hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f\u6d3b\u7684\u771f\u8c1b\u662f"], + ids: [3, 2, 1], + decoded: "", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581leading", "\u2581space"], + ids: [1374, 628, 1], + decoded: "leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trail", "ing", "\u2581space"], + ids: [5032, 53, 628, 1], + decoded: "trailing space", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581Hi", "\u2581Hello"], + ids: [2018, 8774, 1], + decoded: "Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac", "4", "\u2581\u00a35", "\u2581", "\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581", "\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [794, 1970, 391, 357, 20206, 3416, 591, 23978, 3, 2, 948, 3, 2, 940, 3, 2, 927, 3, 2, 1298, 794, 1], + decoded: "test $1 R2 #3 \u20ac4 \u00a35 6 7 8 9 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$1", ".00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [27, 2944, 46, 8947, 21, 1970, 4200, 44, 8, 1078, 5, 1], + decoded: "I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "..."], + ids: [25, 233, 1], + decoded: "you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "..."], + ids: [25, 233, 1], + decoded: "you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "..."], + ids: [25, 233, 25, 233, 1], + decoded: "you... you...", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], + ids: [10088, 3, 2, 3023, 3, 2, 495, 1], + decoded: "weird edge case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581", "a", "\u2581test", "\u2581", "."], + ids: [100, 19, 3, 9, 794, 3, 5, 1], + decoded: "This is a test.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581", "\u2764\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 1], + decoded: " ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41\ufe0f", "\u2581", "\ud83d\udc71\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642\ufe0f", "\u2581", "\ud83e\uddd9\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581", "\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68\ud83c\udffb", "\u2581", "\u2764\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68\ud83c\udffc"], + ids: [3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 1], + decoded: " ", + }, + SPECIAL_WITH_TRAILING_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_WITH_TRAILING_WHITESPACE, + tokens: ["\u2581", "<", "s", ">"], + ids: [3, 2, 7, 3155, 1], + decoded: "s>", + }, + SPECIAL_SURROUNDED_BY_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_SURROUNDED_BY_WHITESPACE, + tokens: ["", "\u2581test", ""], + ids: [1, 794, 1, 1], + decoded: " test", + }, + SPECIAL_NO_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_NO_WHITESPACE, + tokens: ["", "\u2581test", ""], + ids: [1, 794, 1, 1], + decoded: " test", + }, + PREPEND_SCHEME: { + text: T5_TEST_STRINGS.PREPEND_SCHEME, + tokens: ["\u2581Hey", "", "\u2581", ".", "\u2581how", "\u2581are", "\u2581you"], + ids: [9459, 1, 3, 5, 149, 33, 25, 1], + decoded: "Hey. how are you", + }, + }, + "Xenova/t5-tokenizer-new": { + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trail", "ing", "\u2581space", "\u2581"], + ids: [5032, 53, 628, 3, 1], + decoded: "trailing space ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [25, 233, 3, 1], + decoded: "you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [25, 233, 3, 1], + decoded: "you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "...", "\u2581"], + ids: [25, 233, 25, 233, 3, 1], + decoded: "you... you... ", + }, + SPECIAL_WITH_TRAILING_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_WITH_TRAILING_WHITESPACE, + tokens: ["\u2581", "<", "s", ">", "\u2581"], + ids: [3, 2, 7, 3155, 3, 1], + decoded: "s> ", + }, + SPECIAL_SURROUNDED_BY_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_SURROUNDED_BY_WHITESPACE, + tokens: ["\u2581", "", "\u2581test", "\u2581", "", "\u2581"], + ids: [3, 1, 794, 3, 1, 3, 1], + decoded: " test ", + }, + SPECIAL_NO_WHITESPACE: { + text: SENTENCEPIECE_TEST_STRINGS.SPECIAL_NO_WHITESPACE, + tokens: ["", "test", ""], + ids: [1, 4377, 1, 1], + decoded: "test", + }, + PREPEND_SCHEME: { + text: T5_TEST_STRINGS.PREPEND_SCHEME, + tokens: ["\u2581Hey", "\u2581", "", ".", "\u2581how", "\u2581are", "\u2581you"], + ids: [9459, 3, 1, 5, 149, 33, 25, 1], + decoded: "Hey . how are you", + }, + }, + "Xenova/LaMini-Flan-T5-783M": { + PREPEND_SCHEME: { + text: T5_TEST_STRINGS.PREPEND_SCHEME, + tokens: ["\u2581Hey", "\u2581", "", "\u2581", ".", "\u2581how", "\u2581are", "\u2581you"], + ids: [9459, 3, 1, 3, 5, 149, 33, 25, 1], + decoded: "Hey . how are you", + }, + }, +}; + +// Test that tokenizer type can be inferred (`type: "Unigram"` is missing) +TEST_CONFIG["google-t5/t5-small"] = TEST_CONFIG["Xenova/t5-small"]; + +const MAX_EXECUTION_TIME = 10_000; +export const CUSTOM_TESTS = () => { + // Tests to ensure that no matter what, the correct tokenization is returned. + // This is necessary since there are sometimes bugs in the transformers library. + describe("hard-coded", () => { + const TESTS = { + "Xenova/t5-tokenizer-new": [ + { + data: { + // https://github.com/huggingface/transformers/pull/26678 + // ['▁Hey', '▁', '', '.', '▁how', '▁are', '▁you'] + "Hey . how are you": [9459, 3, 1, 5, 149, 33, 25], + }, + reversible: true, + }, + { + data: { + "\n": [1, 3], + "A\n'll": [71, 3, 31, 195], + }, + reversible: false, + }, + ], + }; + + for (const [tokenizerName, test_data] of Object.entries(TESTS)) { + it( + tokenizerName, + async () => { + for (const { data, reversible } of test_data) { + const tokenizer = await T5Tokenizer.from_pretrained(tokenizerName); + + for (const [text, expected] of Object.entries(data)) { + const token_ids = tokenizer.encode(text, { add_special_tokens: false }); + expect(token_ids).toEqual(expected); + + // If reversible, test that decoding produces the original text + if (reversible) { + const decoded = tokenizer.decode(token_ids); + expect(decoded).toEqual(text); + } + } + } + }, + MAX_EXECUTION_TIME, + ); + } + }); +}; diff --git a/packages/transformers/tests/models/test_strings.js b/packages/transformers/tests/models/test_strings.js new file mode 100644 index 0000000..d599110 --- /dev/null +++ b/packages/transformers/tests/models/test_strings.js @@ -0,0 +1,125 @@ +export const BASE_TEST_STRINGS = { + SIMPLE: "How are you doing?", + SIMPLE_WITH_PUNCTUATION: "You should've done this", + NUMBERS: "0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + TEXT_WITH_NUMBERS: "The company was founded in 2016.", + PUNCTUATION: "A\n'll !!to?'d''d of, can't.", + PYTHON_CODE: "def main():\n\tpass", + JAVASCRIPT_CODE: "let a = obj.toString();\ntoString();", + NEWLINES: "This\n\nis\na\ntest.", + BASIC: "UNwant\u00e9d,running", + CONTROL_TOKENS: "1\u00002\uFFFD3", + HELLO_WORLD_TITLECASE: "Hello World", + HELLO_WORLD_LOWERCASE: "hello world", + CHINESE_ONLY: "生活的真谛是", + LEADING_SPACE: " leading space", + TRAILING_SPACE: "trailing space ", + SURROUNDING_SPACE: " surrounding space ", + DOUBLE_SPACE: "Hi Hello", + CURRENCY: "test $1 R2 #3 €4 £5 ¥6 ₣7 ₹8 ₱9 test", + CURRENCY_WITH_DECIMALS: "I bought an apple for $1.00 at the store.", + ELLIPSIS: "you… ", + TEXT_WITH_ESCAPE_CHARACTERS: "\u0079\u006F\u0075\u2026\u00A0\u00A0", + TEXT_WITH_ESCAPE_CHARACTERS_2: "\u0079\u006F\u0075\u2026\u00A0\u00A0\u0079\u006F\u0075\u2026\u00A0\u00A0", + TILDE_NORMALIZATION: "weird \uFF5E edge \uFF5E case", + SPIECE_UNDERSCORE: "▁This ▁is ▁a ▁test ▁.", + POPULAR_EMOJIS: "😂 👍 🤣 😍 😭 🎉 🙏 😊 🔥 😁 😅 🤗 😆 👏 ❤️ 💜 💚 💗 💙 🖤 😎 👌 🥳 💪 ✨ 👉 👀 💯 🎈 🙈 🙌 💀 👇 👋 ✅ 🎁 🌞 🌸 💰", + MULTIBYTE_EMOJIS: "✨ 🤗 👁️ 👱🏻 🕵‍♂️ 🧙🏻‍♂ 👨🏻‍🌾 🧑‍🤝‍🧑 👩‍❤‍💋‍👨 👩‍👩‍👧‍👦 🧑🏻‍🤝‍🧑🏻 🏴󠁧󠁢󠁥󠁮󠁧󠁿 👨🏻‍❤️‍💋‍👨🏼", // 1 2 3 4 5 6 7 8 10 11 12 14 15 + ONLY_WHITESPACE: " \t\n", +}; + +export const BERT_TEST_STRINGS = { + CHINESE_LATIN_MIXED: "ah\u535a\u63a8zz", + SIMPLE_WITH_ACCENTS: "H\u00e9llo", + MIXED_CASE_WITHOUT_ACCENTS: " \tHeLLo!how \n Are yoU? ", + MIXED_CASE_WITH_ACCENTS: " \tHäLLo!how \n Are yoU? ", +}; + +// SentencePiece-specific test cases +export const SENTENCEPIECE_TEST_STRINGS = { + SPECIAL_WITH_TRAILING_WHITESPACE: "\n", + SPECIAL_SURROUNDED_BY_WHITESPACE: " test ", + SPECIAL_NO_WHITESPACE: "test", +}; + +// Additional test-cases for the Llama tokenizer, adapted from +// https://github.com/belladoreai/llama-tokenizer-js/blob/master/llama-tokenizer.js#L381-L452 +export const LLAMA_TEST_STRINGS = { + BPE_SCORES_PRIORITY_1: "grabbed", + BPE_SCORES_PRIORITY_2: " grabbed", + BPE_SCORES_PRIORITY_3: " grabbed", + NEWLINE: "\n", + NEWLINES: "ax\n####\nboo", + NEWLINE_WITH_LEADING_SPACE: " \n", + TABS: " tabs out here", + NEWLINE_AND_TAB: "\n\t\n", + CHINESE_LETTER: "镇", + EMOJIS_1: "🦙", + EMOJIS_2: "🦙Ꙋ", + EMOJIS_3: "Ꙋ🦙", + PARAGRAPH: 'The llama (/ˈlɑːmə/; 🦙Spanish pronunciation: [ˈʎama]) (Lama glama) is a domesticated South American camelid, widely used as a meat and pack animal by Andean cultures since the Pre-Columbian era. Llamas are social animals and live with others as a herd. Their wool is soft and contains only a small amount of lanolin.[2] Llamas can learn simple tasks after a few repetitions. When using a pack, they can carry about 25 to 30% of their body weight for 8 to 13 km (5–8 miles).[3] The name llama (in the past also spelled "lama" or "glama") was adopted by European settlers from native Peruvians.[4] The ancestors of llamas are thought to have originated from the Great Plains of North America about 40 million years ago, and subsequently migrated to South America about three million years ago during the Great American Interchange. By the end of the last ice age (10,000–12,000 years ago), camelids were extinct in North America.[3] As of 2007, there were over seven million llamas and alpacas in South America and over 158,000 llamas and 100,000Ꙋ🦙 alpacas, descended from progenitors imported late in the 20th century, in the United States and Canada.[5] In Aymara mythology, llamas are important beings. The Heavenly Llama is said to drink water from the ocean and urinates as it rains.[6] According to Aymara eschatology, llamas will return to the water springs and lagoons where they come from at the end of time.[6]', + IGNORE_MERGES: "Ne için gittiğimi falan bilmiyordum, Washington'da belirtilen bir yere rapor vermem gerekiyordu.", +}; + +export const VITS_TEST_STRINGS = { + BASIC: "abcdefghijklmnopqrstuvwxyz01234567890", + // Special treatment of characters in certain language + SPECIAL_CHARACTERS: "ț ţ", +}; + +export const QWEN_TEST_STRINGS = { + PUNCTUATION_SPLIT: "i'm i'M i've i've i'Ve i'vE i'VE", +}; + +export const WHISPER_TEST_STRINGS = { + SPECIAL_TOKENS: " <|startoftranscript|> <|en|> ", // Tests lstrip+rstrip +}; + +export const BLENDERBOT_SMALL_TEST_STRINGS = { + SPECIAL_TOKENS: "__start__hello world__end__", + // The original (python) tokenizer simply joins by spaces (regardless of special tokens or not) + WHITESPACE_1: "__start__ hey __end__", // --> ... --> "__start__ hey __end__" + WHITESPACE_2: "__start__hey __end__", // --> ... --> "__start__ hey __end__" +}; + +export const T5_TEST_STRINGS = { + // Tests the new T5 tokenizer, which uses a different prepend_scheme for its pre_tokenizer: + // tokenizer._tokenizer.pre_tokenizer = Metaspace(add_prefix_space = True, replacement = "▁", prepend_scheme = "first") + // See https://github.com/huggingface/transformers/pull/26678 for more information. + // - Old (incorrect): ['▁Hey', '▁', '', '▁', '.', '▁how', '▁are', '▁you'] + // - New (correct): ['▁Hey', '▁', '', '.', '▁how', '▁are', '▁you'] + PREPEND_SCHEME: "Hey . how are you", +}; + +export const FALCON_TEST_STRINGS = { + // Special case for splitting on 3 numbers + NUMBERS_SPLIT: "12 and 123 and 1234", +}; + +export const ESM_TEST_STRINGS = { + // Special tokens + SPECIAL_TOKENS: "", + // Actual protein sequences + PROTEIN_SEQUENCES_1: "ATTCCGATTCCGATTCCG", + PROTEIN_SEQUENCES_2: "ATTTCTCTCTCTCTCTGAGATCGATCGATCGAT", +}; + +export const BLOOM_TEST_STRINGS = { + END_OF_SENTENCE_PUNCTUATION: "test. test, test! test? test… test。 test， test、 test। test۔ test، test", +}; + +export const M2M_100_TEST_STRINGS = { + TRANSLATION_INPUTS: "__en__ hello world", + HIDNI_TEXT: "जीवन एक चॉकलेट बॉक्स की तरह है।", + CHINESE_TEXT: "生活就像一盒巧克力。", +}; + +// Test strings adapted from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/normalize +export const NORMALIZATION_TEST_STRINGS = { + DEFAULT_EXAMPLE: "\u0041\u006d\u00e9\u006c\u0069\u0065 | \u0041\u006d\u0065\u0301\u006c\u0069\u0065", + CANONICAL_EQUIVALENCE_NORMALIZATION: "\u00F1 | \u006E\u0303", + COMPATIBILITY_NORMALIZATION: "\uFB00 | \u0066\u0066", + + // Original | NFC | NFD | NFKC | NFKD + COMBINED_EXAMPLE: "\u1E9B\u0323 | \u1E9B\u0323 | \u017F\u0323\u0307 | \u1E69 | \u0073\u0323\u0307", +}; diff --git a/packages/transformers/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.js b/packages/transformers/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.js new file mode 100644 index 0000000..f70cb68 --- /dev/null +++ b/packages/transformers/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.js @@ -0,0 +1,49 @@ +import { VisionEncoderDecoderModel, full } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("VisionEncoderDecoderModel", () => { + const model_id = "hf-internal-testing/tiny-random-VisionEncoderDecoderModel-vit-gpt2"; + + /** @type {VisionEncoderDecoderModel} */ + let model; + beforeAll(async () => { + model = await VisionEncoderDecoderModel.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const outputs = await model.generate({ + pixel_values: full([1, 3, 30, 30], -1.0), + max_length: 5, + }); + expect(outputs.tolist()).toEqual([[0n, 400n, 400n, 400n, 400n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + // TODO: Add back + // it('batch_size>1', async () => { + // const outputs = await model.generate({ + // pixel_values: cat([ + // full([1, 3, 30, 30], -1.0), + // full([1, 3, 30, 30], 0.0), + // ]), + // max_length: 5, + // }); + // expect(outputs.tolist()).toEqual([ + // // Generation continues + // [0n, 400n, 400n, 400n, 400n], + + // // Finishes early. 1023 is the padding token + // [0n, 0n, 1023n, 1023n, 1023n], + // ]); + // }, MAX_TEST_EXECUTION_TIME); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/vit/test_image_processing_vit.js b/packages/transformers/tests/models/vit/test_image_processing_vit.js new file mode 100644 index 0000000..a69cbdc --- /dev/null +++ b/packages/transformers/tests/models/vit/test_image_processing_vit.js @@ -0,0 +1,31 @@ +import { AutoImageProcessor, ViTFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("ViTFeatureExtractor", () => { + const model_id = "Xenova/vit-base-patch16-224"; + + /** @type {ViTFeatureExtractor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "default", + async () => { + const image = await load_cached_image("tiger"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 224, 224]); + expect(pixel_values.mean().item()).toBeCloseTo(-0.22706867939852762, 6); + + expect(original_sizes).toEqual([[408, 612]]); + expect(reshaped_input_sizes).toEqual([[224, 224]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/vitmatte/test_image_processing_vitmatte.js b/packages/transformers/tests/models/vitmatte/test_image_processing_vitmatte.js new file mode 100644 index 0000000..56e9ec4 --- /dev/null +++ b/packages/transformers/tests/models/vitmatte/test_image_processing_vitmatte.js @@ -0,0 +1,68 @@ +import { AutoImageProcessor, VitMatteImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // VitMatteImageProcessor + // - tests custom overrides + // - tests multiple inputs + // - tests `size_divisibility` and no size (size_divisibility=32) + // - tests do_pad and `size_divisibility` + describe("VitMatteImageProcessor", () => { + const model_id = "Xenova/vitmatte-small-distinctions-646"; + + /** @type {VitMatteImageProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "w/o resize", + async () => { + const image = await load_cached_image("vitmatte_image"); + const image2 = await load_cached_image("vitmatte_trimap"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image, image2); + const { data, dims } = pixel_values; + + expect(dims).toEqual([1, 4, 640, 960]); + expect(pixel_values.mean().item()).toBeCloseTo(-0.4028555154800415); + expect(data[0]).toBeCloseTo(-0.9921568632125854); + expect(data[1]).toBeCloseTo(-0.9921568632125854); + expect(data[5]).toBeCloseTo(-1.0); + expect(data[640]).toBeCloseTo(-0.6784313917160034); + expect(data[641]).toBeCloseTo(-0.6705882549285889); + expect(data[640 * 960]).toBeCloseTo(-1.0); + expect(data[640 * 960 + 1]).toBeCloseTo(-1.0); + expect(data.at(-1)).toBeCloseTo(0.0); + + expect(original_sizes).toEqual([[640, 960]]); + expect(reshaped_input_sizes).toEqual([[640, 960]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "w/ resize", + async () => { + const image = await load_cached_image("pattern_3x5"); + const image2 = await load_cached_image("pattern_3x5"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image, image2); + const { data, dims } = pixel_values; + expect(dims).toEqual([1, 4, 32, 32]); + expect(pixel_values.mean().item()).toBeCloseTo(-0.00867417361587286); + expect(data[0]).toBeCloseTo(-0.9921568632125854); + expect(data[1]).toBeCloseTo(-0.9686274528503418); + expect(data[5]).toBeCloseTo(0.0); + expect(data[32]).toBeCloseTo(-0.9215686321258545); + expect(data[33]).toBeCloseTo(-0.8980392217636108); + expect(data.at(-1)).toBeCloseTo(0.0); + + expect(original_sizes).toEqual([[5, 3]]); + expect(reshaped_input_sizes).toEqual([[5, 3]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/vitpose/test_image_processing_vitpose.js b/packages/transformers/tests/models/vitpose/test_image_processing_vitpose.js new file mode 100644 index 0000000..b49afd9 --- /dev/null +++ b/packages/transformers/tests/models/vitpose/test_image_processing_vitpose.js @@ -0,0 +1,50 @@ +import { AutoImageProcessor, rand, Tensor, VitPoseImageProcessor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("VitPoseImageProcessor", () => { + const model_id = "onnx-community/vitpose-base-simple"; + + /** @type {VitPoseImageProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "default", + async () => { + const image = await load_cached_image("tiger"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 256, 192]); + expect(pixel_values.mean().item()).toBeCloseTo(-0.2771204710006714, 6); + + expect(original_sizes).toEqual([[408, 612]]); + expect(reshaped_input_sizes).toEqual([[256, 192]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "post_process_pose_estimation", + async () => { + const num_classes = 17; + const size = [0, 0, 1000, 1500]; + const heatmaps = rand([1, num_classes, 64, 48]); + + const boxes = [[size]]; + const { bbox, scores, labels, keypoints } = processor.post_process_pose_estimation(heatmaps, boxes, { threshold: null })[0][0]; + + expect(bbox).toEqual(size); + expect(scores).toHaveLength(num_classes); + expect(labels).toHaveLength(num_classes); + expect(keypoints).toHaveLength(num_classes); + expect(keypoints[0]).toHaveLength(2); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/vits/test_tokenization_vits.js b/packages/transformers/tests/models/vits/test_tokenization_vits.js new file mode 100644 index 0000000..1d2587d --- /dev/null +++ b/packages/transformers/tests/models/vits/test_tokenization_vits.js @@ -0,0 +1,76 @@ +import { VitsTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, VITS_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = VitsTokenizer; +export const TEST_CONFIG = { + "Xenova/mms-tts-eng": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["k", "h", "k", "o", "k", "w", "k", " ", "k", "a", "k", "r", "k", "e", "k", " ", "k", "y", "k", "o", "k", "u", "k", " ", "k", "d", "k", "o", "k", "i", "k", "n", "k", "g", "k"], + ids: [0, 6, 0, 22, 0, 9, 0, 19, 0, 26, 0, 25, 0, 7, 0, 19, 0, 3, 0, 22, 0, 4, 0, 19, 0, 5, 0, 22, 0, 18, 0, 29, 0, 37, 0], + decoded: "how are you doing", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["k", "y", "k", "o", "k", "u", "k", " ", "k", "s", "k", "h", "k", "o", "k", "u", "k", "l", "k", "d", "k", "'", "k", "v", "k", "e", "k", " ", "k", "d", "k", "o", "k", "n", "k", "e", "k", " ", "k", "t", "k", "h", "k", "i", "k", "s", "k"], + ids: [0, 3, 0, 22, 0, 4, 0, 19, 0, 8, 0, 6, 0, 22, 0, 4, 0, 21, 0, 5, 0, 1, 0, 32, 0, 7, 0, 19, 0, 5, 0, 22, 0, 29, 0, 7, 0, 19, 0, 33, 0, 6, 0, 18, 0, 8, 0], + decoded: "you should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["k", "0", "k", "1", "k", "2", "k", "3", "k", "4", "k", "5", "k", "6", "k", " ", "k", "0", "k", " ", "k", "1", "k", " ", "k", "2", "k", " ", "k", "3", "k", " ", "k", "4", "k", " ", "k", "5", "k", " ", "k", "6", "k", " ", "k", " ", "k", " ", "k", " ", "k", "1", "k", "0", "k", " ", "k", "1", "k", "0", "k", "0", "k", " ", "k", "1", "k", "0", "k", "0", "k", "0", "k"], + ids: [0, 23, 0, 15, 0, 28, 0, 11, 0, 27, 0, 35, 0, 36, 0, 19, 0, 23, 0, 19, 0, 15, 0, 19, 0, 28, 0, 19, 0, 11, 0, 19, 0, 27, 0, 19, 0, 35, 0, 19, 0, 36, 0, 19, 0, 19, 0, 19, 0, 19, 0, 15, 0, 23, 0, 19, 0, 15, 0, 23, 0, 23, 0, 19, 0, 15, 0, 23, 0, 23, 0, 23, 0], + decoded: "0123456 0 1 2 3 4 5 6 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["k", "t", "k", "h", "k", "e", "k", " ", "k", "c", "k", "o", "k", "m", "k", "p", "k", "a", "k", "n", "k", "y", "k", " ", "k", "w", "k", "a", "k", "s", "k", " ", "k", "f", "k", "o", "k", "u", "k", "n", "k", "d", "k", "e", "k", "d", "k", " ", "k", "i", "k", "n", "k", " ", "k", "2", "k", "0", "k", "1", "k", "6", "k"], + ids: [0, 33, 0, 6, 0, 7, 0, 19, 0, 12, 0, 22, 0, 17, 0, 13, 0, 26, 0, 29, 0, 3, 0, 19, 0, 9, 0, 26, 0, 8, 0, 19, 0, 20, 0, 22, 0, 4, 0, 29, 0, 5, 0, 7, 0, 5, 0, 19, 0, 18, 0, 29, 0, 19, 0, 28, 0, 23, 0, 15, 0, 36, 0], + decoded: "the company was founded in 2016", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["k", "t", "k", "h", "k", "i", "k", "s", "k", "i", "k", "s", "k", "a", "k", "t", "k", "e", "k", "s", "k", "t", "k"], + ids: [0, 33, 0, 6, 0, 18, 0, 8, 0, 18, 0, 8, 0, 26, 0, 33, 0, 7, 0, 8, 0, 33, 0], + decoded: "thisisatest", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: [], + ids: [], + decoded: "", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["k", "l", "k", "e", "k", "a", "k", "d", "k", "i", "k", "n", "k", "g", "k", " ", "k", "s", "k", "p", "k", "a", "k", "c", "k", "e", "k"], + ids: [0, 21, 0, 7, 0, 26, 0, 5, 0, 18, 0, 29, 0, 37, 0, 19, 0, 8, 0, 13, 0, 26, 0, 12, 0, 7, 0], + decoded: "leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["k", "t", "k", "r", "k", "a", "k", "i", "k", "l", "k", "i", "k", "n", "k", "g", "k", " ", "k", "s", "k", "p", "k", "a", "k", "c", "k", "e", "k"], + ids: [0, 33, 0, 25, 0, 26, 0, 18, 0, 21, 0, 18, 0, 29, 0, 37, 0, 19, 0, 8, 0, 13, 0, 26, 0, 12, 0, 7, 0], + decoded: "trailing space", + }, + SURROUNDING_SPACE: { + text: BASE_TEST_STRINGS.SURROUNDING_SPACE, + tokens: ["k", "s", "k", "u", "k", "r", "k", "r", "k", "o", "k", "u", "k", "n", "k", "d", "k", "i", "k", "n", "k", "g", "k", " ", "k", "s", "k", "p", "k", "a", "k", "c", "k", "e", "k"], + ids: [0, 8, 0, 4, 0, 25, 0, 25, 0, 22, 0, 4, 0, 29, 0, 5, 0, 18, 0, 29, 0, 37, 0, 19, 0, 8, 0, 13, 0, 26, 0, 12, 0, 7, 0], + decoded: "surrounding space", + }, + SPECIAL_CHARACTERS: { + text: VITS_TEST_STRINGS.SPECIAL_CHARACTERS, + tokens: [], + ids: [], + decoded: "", + }, + }, + "Xenova/mms-tts-ron": { + SPECIAL_CHARACTERS: { + text: VITS_TEST_STRINGS.SPECIAL_CHARACTERS, + tokens: ["c", "\u0163", "c", " ", "c", "\u0163", "c"], + ids: [0, 32, 0, 28, 0, 32, 0], + decoded: "\u0163 \u0163", + }, + }, +}; diff --git a/packages/transformers/tests/models/voxtral_realtime/test_feature_extraction_voxtral_realtime.js b/packages/transformers/tests/models/voxtral_realtime/test_feature_extraction_voxtral_realtime.js new file mode 100644 index 0000000..f9a4118 --- /dev/null +++ b/packages/transformers/tests/models/voxtral_realtime/test_feature_extraction_voxtral_realtime.js @@ -0,0 +1,53 @@ +import { AutoFeatureExtractor, VoxtralRealtimeFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_audio } from "../../asset_cache.js"; +import { MAX_FEATURE_EXTRACTOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // VoxtralRealtimeFeatureExtractor + describe("VoxtralRealtimeFeatureExtractor", () => { + const model_id = "onnx-community/Voxtral-Mini-4B-Realtime-2602-ONNX"; + + /** @type {VoxtralRealtimeFeatureExtractor} */ + let feature_extractor; + beforeAll(async () => { + feature_extractor = await AutoFeatureExtractor.from_pretrained(model_id); + }, MAX_FEATURE_EXTRACTOR_LOAD_TIME); + + it( + "full audio", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_features } = await feature_extractor(audio); + expect(input_features.dims).toEqual([1, 128, 1300]); + + expect(input_features.mean().item()).toBeCloseTo(0.193456813693047, 3); + expect(input_features.data[0]).toBeCloseTo(0.255926549434662, 3); + expect(input_features.data[1]).toBeCloseTo(0.23410552740097, 3); + expect(input_features.data[128]).toBeCloseTo(-0.2862229347229, 3); + expect(input_features.data[129]).toBeCloseTo(-0.625, 3); + expect(input_features.data[1000]).toBeCloseTo(0.053786396980286, 3); + expect(input_features.data[10000]).toBeCloseTo(0.320511281490326, 3); + expect(input_features.data[100000]).toBeCloseTo(0.251729607582092, 3); + expect(input_features.data[input_features.data.length - 1]).toBeCloseTo(-0.436606526374817, 3); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "short audio", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_features } = await feature_extractor(audio.slice(0, 1000)); + expect(input_features.dims).toEqual([1, 128, 6]); + + expect(input_features.mean().item()).toBeCloseTo(0.120253048837185, 3); + expect(input_features.data[0]).toBeCloseTo(0.255926549434662, 3); + expect(input_features.data[1]).toBeCloseTo(0.23410552740097, 3); + expect(input_features.data[128]).toBeCloseTo(0.317371904850006, 3); + expect(input_features.data[input_features.data.length - 1]).toBeCloseTo(-0.322449326515198, 3); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/voxtral_realtime/test_modeling_voxtral_realtime.js b/packages/transformers/tests/models/voxtral_realtime/test_modeling_voxtral_realtime.js new file mode 100644 index 0000000..15e58d1 --- /dev/null +++ b/packages/transformers/tests/models/voxtral_realtime/test_modeling_voxtral_realtime.js @@ -0,0 +1,164 @@ +import { VoxtralRealtimeForConditionalGeneration, VoxtralRealtimeProcessor, Tensor } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("VoxtralRealtimeForConditionalGeneration", () => { + const model_id = "onnx-internal-testing/tiny-random-VoxtralRealtimeForConditionalGeneration"; + + /** @type {VoxtralRealtimeForConditionalGeneration} */ + let model; + /** @type {VoxtralRealtimeProcessor} */ + let processor; + + beforeAll(async () => { + model = await VoxtralRealtimeForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + processor = await VoxtralRealtimeProcessor.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + // Helper: generate a 1-second 440Hz sine wave + function syntheticAudio() { + const audio = new Float32Array(16000); + for (let i = 0; i < audio.length; ++i) { + audio[i] = Math.sin((2 * Math.PI * 440 * i) / 16000) * 0.5; + } + return audio; + } + + // Helper: build input_features generator using the processor's __call__ API + async function* buildInputFeaturesGenerator(audio) { + const { hop_length, n_fft } = processor.feature_extractor.config; + const win_half = Math.floor(n_fft / 2); + + // First chunk + const first_chunk = await processor(audio.subarray(0, processor.num_samples_first_audio_chunk), { is_streaming: true, is_first_audio_chunk: true }); + yield first_chunk.input_features; + + // Subsequent chunks + let mel_frame_idx = processor.num_mel_frames_first_audio_chunk; + let start_idx = mel_frame_idx * hop_length - win_half; + + while (start_idx + processor.num_samples_per_audio_chunk < audio.length) { + const end_idx = start_idx + processor.num_samples_per_audio_chunk; + const chunk = await processor(audio.slice(start_idx, end_idx), { is_streaming: true, is_first_audio_chunk: false }); + yield chunk.input_features; + + mel_frame_idx += processor.audio_length_per_tok; + start_idx = mel_frame_idx * hop_length - win_half; + } + } + + it( + "streaming generation", + async () => { + const audio = syntheticAudio(); + + // Get input_ids from first chunk call + const first_chunk = await processor(audio.subarray(0, processor.num_samples_first_audio_chunk), { is_streaming: true, is_first_audio_chunk: true }); + + const outputs = await model.generate({ + input_ids: first_chunk.input_ids, + input_features: buildInputFeaturesGenerator(audio), + max_new_tokens: 10, + }); + + expect(outputs.tolist()).toEqual([ + [ + /* Input (39 tokens: BOS + 38 STREAMING_PAD) */ + 1n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + 32n, + /* Generated */ + 28478n, + 28478n, + 28478n, + 28478n, + 28478n, + 28478n, + 98356n, + ], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "streaming generation with delayed chunks", + async () => { + const audio = syntheticAudio(); + + const first_chunk = await processor(audio.subarray(0, processor.num_samples_first_audio_chunk), { is_streaming: true, is_first_audio_chunk: true }); + + // Simulate receiving chunks with 100ms delays + async function* delayedChunks() { + for await (const chunk of buildInputFeaturesGenerator(audio)) { + await new Promise((r) => setTimeout(r, 100)); + yield chunk; + } + } + + const outputs = await model.generate({ + input_ids: first_chunk.input_ids, + input_features: delayedChunks(), + max_new_tokens: 10, + }); + + // Same output as without delays — generation is deterministic + expect(outputs.tolist()).toEqual([[1n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 32n, 28478n, 28478n, 28478n, 28478n, 28478n, 28478n, 98356n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "forward without generate (no encoder state)", + async () => { + const input_ids = new Tensor("int64", [1n, 32n, 32n], [1, 3]); + const attention_mask = new Tensor("int64", [1n, 1n, 1n], [1, 3]); + + const outputs = await model.forward({ input_ids, attention_mask, past_key_values: null }); + expect(outputs.logits.dims).toEqual([1, 3, expect.any(Number)]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/voxtral_realtime/test_processor_voxtral_realtime.js b/packages/transformers/tests/models/voxtral_realtime/test_processor_voxtral_realtime.js new file mode 100644 index 0000000..e6431ef --- /dev/null +++ b/packages/transformers/tests/models/voxtral_realtime/test_processor_voxtral_realtime.js @@ -0,0 +1,91 @@ +import { AutoProcessor, VoxtralRealtimeProcessor } from "../../../src/transformers.js"; + +import { load_cached_audio } from "../../asset_cache.js"; +import { MAX_FEATURE_EXTRACTOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // VoxtralRealtimeProcessor + describe("VoxtralRealtimeProcessor", () => { + const model_id = "onnx-community/Voxtral-Mini-4B-Realtime-2602-ONNX"; + + /** @type {VoxtralRealtimeProcessor} */ + let processor; + beforeAll(async () => { + processor = await AutoProcessor.from_pretrained(model_id); + }, MAX_FEATURE_EXTRACTOR_LOAD_TIME); + + it( + "streaming first chunk", + async () => { + const audio = await load_cached_audio("mlk"); + + const first_audio = audio.subarray(0, processor.num_samples_first_audio_chunk); + const { input_ids, input_features } = await processor(first_audio, { + is_streaming: true, + is_first_audio_chunk: true, + }); + + // Verify input_ids: BOS + 38 [STREAMING_PAD] tokens + expect(input_ids.dims).toEqual([1, 39]); + expect(Number(input_ids.data[0])).toBe(1); // BOS + for (let i = 1; i < 39; ++i) { + expect(Number(input_ids.data[i])).toBe(32); // [STREAMING_PAD] + } + + // Verify first chunk: left-padded with silence, matching Python processor output + expect(input_features.dims).toEqual([1, 128, 312]); + expect(input_features.mean().item()).toBeCloseTo(-0.489270150661469, 3); + expect(input_features.data[0]).toBeCloseTo(-0.625, 3); // silence (left-pad) + expect(input_features.data[1]).toBeCloseTo(-0.625, 3); + expect(input_features.data[127 * 312]).toBeCloseTo(-0.625, 3); // last mel bin, first frame + expect(input_features.data[256]).toBeCloseTo(0.192136287689209, 3); // first audio frame + expect(input_features.data[311]).toBeCloseTo(-0.084741473197937, 3); // last audio frame + expect(input_features.data[input_features.data.length - 1]).toBeCloseTo(-0.246982932090759, 3); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "streaming subsequent chunks", + async () => { + const audio = await load_cached_audio("mlk"); + + const { hop_length, n_fft } = processor.feature_extractor.config; + const win_half = Math.floor(n_fft / 2); + + // Collect all subsequent chunks + const chunks = []; + let mel_frame_idx = processor.num_mel_frames_first_audio_chunk; + let start_idx = mel_frame_idx * hop_length - win_half; + + while (start_idx + processor.num_samples_per_audio_chunk < audio.length) { + const end_idx = start_idx + processor.num_samples_per_audio_chunk; + const chunk = await processor(audio.slice(start_idx, end_idx), { + is_streaming: true, + is_first_audio_chunk: false, + }); + chunks.push(chunk.input_features); + + mel_frame_idx += processor.audio_length_per_tok; + start_idx = mel_frame_idx * hop_length - win_half; + } + + // Verify second chunk (first subsequent) + const second = chunks[0]; + expect(second.dims).toEqual([1, 128, 8]); + expect(second.mean().item()).toBeCloseTo(0.13092890381813, 3); + expect(second.data[0]).toBeCloseTo(-0.090936064720154, 3); + expect(second.data[second.data.length - 1]).toBeCloseTo(-0.19879412651062, 3); + + // Verify total chunk count: 155 subsequent chunks + expect(chunks.length).toBe(155); + + // Verify all subsequent chunks have shape [1, 128, 8] + for (let i = 0; i < chunks.length; ++i) { + expect(chunks[i].dims).toEqual([1, 128, 8]); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/wav2vec2/test_tokenization_wav2vec2.js b/packages/transformers/tests/models/wav2vec2/test_tokenization_wav2vec2.js new file mode 100644 index 0000000..c94ba1a --- /dev/null +++ b/packages/transformers/tests/models/wav2vec2/test_tokenization_wav2vec2.js @@ -0,0 +1,472 @@ +import { Wav2Vec2CTCTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, BERT_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = Wav2Vec2CTCTokenizer; +export const TEST_CONFIG = { + "Xenova/wav2vec2-base-960h": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["H", "o", "w", "|", "a", "r", "e", "|", "y", "o", "u", "|", "d", "o", "i", "n", "g", "?"], + ids: [11, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3], + decoded: "H ", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["Y", "o", "u", "|", "s", "h", "o", "u", "l", "d", "'", "v", "e", "|", "d", "o", "n", "e", "|", "t", "h", "i", "s"], + ids: [22, 3, 3, 4, 3, 3, 3, 3, 3, 3, 27, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3], + decoded: "Y ' ", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "|", "0", "|", "1", "|", "2", "|", "3", "|", "4", "|", "5", "|", "6", "|", "7", "|", "8", "|", "9", "|", "1", "0", "|", "1", "0", "0", "|", "1", "0", "0", "0"], + ids: [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3], + decoded: " ", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."], + ids: [6, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 3], + decoded: "T ", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\n", "'", "l", "l", "|", "!", "!", "t", "o", "?", "'", "d", "'", "'", "d", "|", "o", "f", ",", "|", "c", "a", "n", "'", "t", "."], + ids: [7, 3, 27, 3, 3, 4, 3, 3, 3, 3, 3, 27, 3, 27, 27, 3, 4, 3, 3, 3, 4, 3, 3, 3, 27, 3, 3], + decoded: "A' '' '", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["d", "e", "f", "|", "m", "a", "i", "n", "(", ")", ":", "\n", "\t", "p", "a", "s", "s"], + ids: [3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], + decoded: " ", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["l", "e", "t", "|", "a", "|", "=", "|", "o", "b", "j", ".", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";", "\n", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";"], + ids: [3, 3, 3, 4, 3, 4, 3, 4, 3, 3, 3, 3, 3, 3, 12, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 12, 3, 3, 3, 3, 3, 3, 3, 3], + decoded: " SS", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["T", "h", "i", "s", "\n", "\n", "i", "s", "\n", "a", "\n", "t", "e", "s", "t", "."], + ids: [6, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], + decoded: "T", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["U", "N", "w", "a", "n", "t", "\u00e9", "d", ",", "r", "u", "n", "n", "i", "n", "g"], + ids: [16, 9, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], + decoded: "UN", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0000", "2", "\ufffd", "3"], + ids: [3, 3, 3, 3, 3], + decoded: "", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["H", "e", "l", "l", "o", "|", "W", "o", "r", "l", "d"], + ids: [11, 3, 3, 3, 3, 4, 18, 3, 3, 3, 3], + decoded: "H W", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["h", "e", "l", "l", "o", "|", "w", "o", "r", "l", "d"], + ids: [3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3], + decoded: " ", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [3, 3, 3, 3, 3, 3], + decoded: "", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["|", "|", "|", "l", "e", "a", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e"], + ids: [4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3], + decoded: " ", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["t", "r", "a", "i", "l", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"], + ids: [3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 4, 4], + decoded: " ", + }, + SURROUNDING_SPACE: { + text: BASE_TEST_STRINGS.SURROUNDING_SPACE, + tokens: ["|", "|", "|", "s", "u", "r", "r", "o", "u", "n", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"], + ids: [4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 4, 4], + decoded: " ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["H", "i", "|", "|", "H", "e", "l", "l", "o"], + ids: [11, 3, 4, 4, 11, 3, 3, 3, 3], + decoded: "H H", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"], + ids: [3, 3, 3, 3, 4, 3, 3, 4, 13, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3], + decoded: " R ", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."], + ids: [10, 4, 3, 3, 3, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3], + decoded: "I ", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["y", "o", "u", "\u2026", "|", "|"], + ids: [3, 3, 3, 3, 4, 4], + decoded: "", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0"], + ids: [3, 3, 3, 3, 3, 3], + decoded: "", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0", "y", "o", "u", "\u2026", "\u00a0", "\u00a0"], + ids: [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], + decoded: "", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["w", "e", "i", "r", "d", "|", "\uff5e", "|", "e", "d", "g", "e", "|", "\uff5e", "|", "c", "a", "s", "e"], + ids: [3, 3, 3, 3, 3, 4, 3, 4, 3, 3, 3, 3, 4, 3, 4, 3, 3, 3, 3], + decoded: " ", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581", "T", "h", "i", "s", "|", "\u2581", "i", "s", "|", "\u2581", "a", "|", "\u2581", "t", "e", "s", "t", "|", "\u2581", "."], + ids: [3, 6, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3], + decoded: "T ", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\ud83d\ude02", "|", "\ud83d\udc4d", "|", "\ud83e\udd23", "|", "\ud83d\ude0d", "|", "\ud83d\ude2d", "|", "\ud83c\udf89", "|", "\ud83d\ude4f", "|", "\ud83d\ude0a", "|", "\ud83d\udd25", "|", "\ud83d\ude01", "|", "\ud83d\ude05", "|", "\ud83e\udd17", "|", "\ud83d\ude06", "|", "\ud83d\udc4f", "|", "\u2764", "\ufe0f", "|", "\ud83d\udc9c", "|", "\ud83d\udc9a", "|", "\ud83d\udc97", "|", "\ud83d\udc99", "|", "\ud83d\udda4", "|", "\ud83d\ude0e", "|", "\ud83d\udc4c", "|", "\ud83e\udd73", "|", "\ud83d\udcaa", "|", "\u2728", "|", "\ud83d\udc49", "|", "\ud83d\udc40", "|", "\ud83d\udcaf", "|", "\ud83c\udf88", "|", "\ud83d\ude48", "|", "\ud83d\ude4c", "|", "\ud83d\udc80", "|", "\ud83d\udc47", "|", "\ud83d\udc4b", "|", "\u2705", "|", "\ud83c\udf81", "|", "\ud83c\udf1e", "|", "\ud83c\udf38", "|", "\ud83d\udcb0"], + ids: [3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3], + decoded: " ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2728", "|", "\ud83e\udd17", "|", "\ud83d\udc41", "\ufe0f", "|", "\ud83d\udc71", "\ud83c\udffb", "|", "\ud83d\udd75", "\u200d", "\u2642", "\ufe0f", "|", "\ud83e\uddd9", "\ud83c\udffb", "\u200d", "\u2642", "|", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\ud83c\udf3e", "|", "\ud83e\uddd1", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "|", "\ud83d\udc69", "\u200d", "\u2764", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "|", "\ud83d\udc69", "\u200d", "\ud83d\udc69", "\u200d", "\ud83d\udc67", "\u200d", "\ud83d\udc66", "|", "\ud83e\uddd1", "\ud83c\udffb", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\ud83c\udffb", "|", "\ud83c\udff4", "\udb40\udc67", "\udb40\udc62", "\udb40\udc65", "\udb40\udc6e", "\udb40\udc67", "\udb40\udc7f", "|", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\u2764", "\ufe0f", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\ud83c\udffc"], + ids: [3, 4, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], + decoded: " ", + }, + ONLY_WHITESPACE: { + text: BASE_TEST_STRINGS.ONLY_WHITESPACE, + tokens: ["|", "\t", "\n"], + ids: [4, 3, 3], + decoded: "", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["a", "h", "\u535a", "\u63a8", "z", "z"], + ids: [3, 3, 3, 3, 3, 3], + decoded: "", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "\u00e9", "l", "l", "o"], + ids: [11, 3, 3, 3, 3], + decoded: "H", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["|", "\t", "H", "e", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], + ids: [4, 3, 11, 3, 15, 15, 3, 3, 3, 3, 3, 4, 4, 3, 4, 7, 3, 3, 4, 3, 3, 16, 3, 4, 4], + decoded: "HL A U", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], + ids: [4, 3, 11, 3, 15, 15, 3, 3, 3, 3, 3, 4, 4, 3, 4, 7, 3, 3, 4, 3, 3, 16, 3, 4, 4], + decoded: "HL A U", + }, + }, + "Xenova/wav2vec2-large-xlsr-53-english": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["H", "o", "w", "|", "a", "r", "e", "|", "y", "o", "u", "|", "d", "o", "i", "n", "g", "?"], + ids: [3, 21, 29, 4, 7, 24, 11, 4, 31, 21, 27, 4, 10, 21, 15, 20, 13, 3], + decoded: "ow are you doing", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["Y", "o", "u", "|", "s", "h", "o", "u", "l", "d", "'", "v", "e", "|", "d", "o", "n", "e", "|", "t", "h", "i", "s"], + ids: [3, 21, 27, 4, 25, 14, 21, 27, 18, 10, 5, 28, 11, 4, 10, 21, 20, 11, 4, 26, 14, 15, 25], + decoded: "ou should've done this", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."], + ids: [3, 14, 11, 4, 9, 21, 19, 22, 7, 20, 31, 4, 29, 7, 25, 4, 12, 21, 27, 20, 10, 11, 10, 4, 15, 20, 4, 3, 3, 3, 3, 3], + decoded: "he company was founded in ", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\n", "'", "l", "l", "|", "!", "!", "t", "o", "?", "'", "d", "'", "'", "d", "|", "o", "f", ",", "|", "c", "a", "n", "'", "t", "."], + ids: [3, 3, 5, 18, 18, 4, 3, 3, 26, 21, 3, 5, 10, 5, 5, 10, 4, 21, 12, 3, 4, 9, 7, 20, 5, 26, 3], + decoded: "'l to'd'd of can't", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["d", "e", "f", "|", "m", "a", "i", "n", "(", ")", ":", "\n", "\t", "p", "a", "s", "s"], + ids: [10, 11, 12, 4, 19, 7, 15, 20, 3, 3, 3, 3, 3, 22, 7, 25, 25], + decoded: "def mainpas", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["l", "e", "t", "|", "a", "|", "=", "|", "o", "b", "j", ".", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";", "\n", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";"], + ids: [18, 11, 26, 4, 7, 4, 3, 4, 21, 8, 16, 3, 26, 21, 3, 26, 24, 15, 20, 13, 3, 3, 3, 3, 26, 21, 3, 26, 24, 15, 20, 13, 3, 3, 3], + decoded: "let a objtotringtotring", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["T", "h", "i", "s", "\n", "\n", "i", "s", "\n", "a", "\n", "t", "e", "s", "t", "."], + ids: [3, 14, 15, 25, 3, 3, 15, 25, 3, 7, 3, 26, 11, 25, 26, 3], + decoded: "hisisatest", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["U", "N", "w", "a", "n", "t", "\u00e9", "d", ",", "r", "u", "n", "n", "i", "n", "g"], + ids: [3, 3, 29, 7, 20, 26, 3, 10, 3, 24, 27, 20, 20, 15, 20, 13], + decoded: "wantdruning", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["H", "e", "l", "l", "o", "|", "W", "o", "r", "l", "d"], + ids: [3, 11, 18, 18, 21, 4, 3, 21, 24, 18, 10], + decoded: "elo orld", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["h", "e", "l", "l", "o", "|", "w", "o", "r", "l", "d"], + ids: [14, 11, 18, 18, 21, 4, 29, 21, 24, 18, 10], + decoded: "helo world", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["|", "|", "|", "l", "e", "a", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e"], + ids: [4, 4, 4, 18, 11, 7, 10, 15, 20, 13, 4, 25, 22, 7, 9, 11], + decoded: "leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["t", "r", "a", "i", "l", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"], + ids: [26, 24, 7, 15, 18, 15, 20, 13, 4, 25, 22, 7, 9, 11, 4, 4, 4], + decoded: "trailing space", + }, + SURROUNDING_SPACE: { + text: BASE_TEST_STRINGS.SURROUNDING_SPACE, + tokens: ["|", "|", "|", "s", "u", "r", "r", "o", "u", "n", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"], + ids: [4, 4, 4, 25, 27, 24, 24, 21, 27, 20, 10, 15, 20, 13, 4, 25, 22, 7, 9, 11, 4, 4, 4], + decoded: "surounding space", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["H", "i", "|", "|", "H", "e", "l", "l", "o"], + ids: [3, 15, 4, 4, 3, 11, 18, 18, 21], + decoded: "i elo", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"], + ids: [26, 11, 25, 26, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 26, 11, 25, 26], + decoded: "test test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."], + ids: [3, 4, 8, 21, 27, 13, 14, 26, 4, 7, 20, 4, 7, 22, 22, 18, 11, 4, 12, 21, 24, 4, 3, 3, 3, 3, 3, 4, 7, 26, 4, 26, 14, 11, 4, 25, 26, 21, 24, 11, 3], + decoded: " bought an aple for at the store", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["y", "o", "u", "\u2026", "|", "|"], + ids: [31, 21, 27, 3, 4, 4], + decoded: "you", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0"], + ids: [31, 21, 27, 3, 3, 3], + decoded: "you", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0", "y", "o", "u", "\u2026", "\u00a0", "\u00a0"], + ids: [31, 21, 27, 3, 3, 3, 31, 21, 27, 3, 3, 3], + decoded: "youyou", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["w", "e", "i", "r", "d", "|", "\uff5e", "|", "e", "d", "g", "e", "|", "\uff5e", "|", "c", "a", "s", "e"], + ids: [29, 11, 15, 24, 10, 4, 3, 4, 11, 10, 13, 11, 4, 3, 4, 9, 7, 25, 11], + decoded: "weird edge case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581", "T", "h", "i", "s", "|", "\u2581", "i", "s", "|", "\u2581", "a", "|", "\u2581", "t", "e", "s", "t", "|", "\u2581", "."], + ids: [3, 3, 14, 15, 25, 4, 3, 15, 25, 4, 3, 7, 4, 3, 26, 11, 25, 26, 4, 3, 3], + decoded: "his is a test ", + }, + CHINESE_LATIN_MIXED: { + text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED, + tokens: ["a", "h", "\u535a", "\u63a8", "z", "z"], + ids: [7, 14, 3, 3, 32, 32], + decoded: "ahz", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "\u00e9", "l", "l", "o"], + ids: [3, 3, 18, 18, 21], + decoded: "lo", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["|", "\t", "H", "e", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], + ids: [4, 3, 3, 11, 3, 3, 21, 3, 14, 21, 29, 4, 4, 3, 4, 3, 24, 11, 4, 31, 21, 3, 3, 4, 4], + decoded: "eohow re yo", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], + ids: [4, 3, 3, 3, 3, 3, 21, 3, 14, 21, 29, 4, 4, 3, 4, 3, 24, 11, 4, 31, 21, 3, 3, 4, 4], + decoded: "ohow re yo", + }, + }, + "Xenova/mms-1b-all": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "|", "0", "|", "1", "|", "2", "|", "3", "|", "4", "|", "5", "|", "6", "|", "7", "|", "8", "|", "9", "|", "1", "0", "|", "1", "0", "0", "|", "1", "0", "0", "0"], + ids: [27, 30, 35, 41, 39, 38, 40, 43, 42, 36, 4, 27, 4, 30, 4, 35, 4, 41, 4, 39, 4, 38, 4, 40, 4, 43, 4, 42, 4, 36, 4, 30, 27, 4, 30, 27, 27, 4, 30, 27, 27, 27], + decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 10 10", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."], + ids: [3, 13, 5, 4, 16, 8, 18, 20, 7, 10, 22, 4, 23, 7, 11, 4, 19, 8, 17, 10, 15, 5, 15, 4, 9, 10, 4, 35, 27, 30, 40, 37], + decoded: "he company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\n", "'", "l", "l", "|", "!", "!", "t", "o", "?", "'", "d", "'", "'", "d", "|", "o", "f", ",", "|", "c", "a", "n", "'", "t", "."], + ids: [3, 3, 31, 14, 14, 4, 75, 75, 6, 8, 3, 31, 15, 31, 31, 15, 4, 8, 19, 44, 4, 16, 7, 10, 31, 6, 37], + decoded: "'l!to'd'd of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["d", "e", "f", "|", "m", "a", "i", "n", "(", ")", ":", "\n", "\t", "p", "a", "s", "s"], + ids: [15, 5, 19, 4, 18, 7, 9, 10, 3, 3, 46, 3, 3, 20, 7, 11, 11], + decoded: "def main:pas", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["l", "e", "t", "|", "a", "|", "=", "|", "o", "b", "j", ".", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";", "\n", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";"], + ids: [14, 5, 6, 4, 7, 4, 3, 4, 8, 24, 29, 37, 6, 8, 3, 6, 12, 9, 10, 21, 3, 3, 52, 3, 6, 8, 3, 6, 12, 9, 10, 21, 3, 3, 52], + decoded: "let a obj.totring;totring;", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["T", "h", "i", "s", "\n", "\n", "i", "s", "\n", "a", "\n", "t", "e", "s", "t", "."], + ids: [3, 13, 9, 11, 3, 3, 9, 11, 3, 7, 3, 6, 5, 11, 6, 37], + decoded: "hisisatest.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["U", "N", "w", "a", "n", "t", "\u00e9", "d", ",", "r", "u", "n", "n", "i", "n", "g"], + ids: [3, 3, 23, 7, 10, 6, 55, 15, 44, 12, 17, 10, 10, 9, 10, 21], + decoded: "want\u00e9d,runing", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0000", "2", "\ufffd", "3"], + ids: [30, 3, 35, 3, 41], + decoded: "123", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [136, 3, 3, 3, 3, 3], + decoded: "\u751f", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"], + ids: [6, 5, 11, 6, 4, 48, 30, 4, 3, 35, 4, 3, 41, 4, 3, 39, 4, 68, 38, 4, 53, 40, 4, 3, 43, 4, 3, 42, 4, 3, 36, 4, 6, 5, 11, 6], + decoded: "test $1 2 3 4 \u00a35 \u00a56 7 8 9 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."], + ids: [3, 4, 24, 8, 17, 21, 13, 6, 4, 7, 10, 4, 7, 20, 20, 14, 5, 4, 19, 8, 12, 4, 48, 30, 37, 27, 27, 4, 7, 6, 4, 6, 13, 5, 4, 11, 6, 8, 12, 5, 37], + decoded: " bought an aple for $1.0 at the store.", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581", "T", "h", "i", "s", "|", "\u2581", "i", "s", "|", "\u2581", "a", "|", "\u2581", "t", "e", "s", "t", "|", "\u2581", "."], + ids: [3, 3, 13, 9, 11, 4, 3, 9, 11, 4, 3, 7, 4, 3, 6, 5, 11, 6, 4, 3, 37], + decoded: "his is a test .", + }, + SIMPLE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS, + tokens: ["H", "\u00e9", "l", "l", "o"], + ids: [3, 55, 14, 14, 8], + decoded: "\u00e9lo", + }, + MIXED_CASE_WITHOUT_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS, + tokens: ["|", "\t", "H", "e", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], + ids: [4, 3, 3, 5, 3, 3, 8, 75, 13, 8, 23, 4, 4, 3, 4, 3, 12, 5, 4, 22, 8, 3, 3, 4, 4], + decoded: "eo!how re yo", + }, + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], + ids: [4, 3, 3, 78, 3, 3, 8, 75, 13, 8, 23, 4, 4, 3, 4, 3, 12, 5, 4, 22, 8, 3, 3, 4, 4], + decoded: "\u00e4o!how re yo", + }, + }, + "Xenova/mms-1b-fl102": { + MIXED_CASE_WITH_ACCENTS: { + text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS, + tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"], + ids: [4, 3, 3, 3, 3, 3, 8, 75, 13, 8, 23, 4, 4, 3, 4, 3, 12, 5, 4, 22, 8, 3, 3, 4, 4], + decoded: "o!how re yo", + }, + }, + "Xenova/mms-1b-l1107": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "|", "0", "|", "1", "|", "2", "|", "3", "|", "4", "|", "5", "|", "6", "|", "7", "|", "8", "|", "9", "|", "1", "0", "|", "1", "0", "0", "|", "1", "0", "0", "0"], + ids: [34, 36, 37, 42, 38, 41, 39, 3, 3, 3, 4, 34, 4, 36, 4, 37, 4, 42, 4, 38, 4, 41, 4, 39, 4, 3, 4, 3, 4, 3, 4, 36, 34, 4, 36, 34, 34, 4, 36, 34, 34, 34], + decoded: "0123456 0 1 2 3 4 5 6 10 10 10", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."], + ids: [3, 9, 5, 4, 21, 7, 18, 24, 8, 10, 20, 4, 17, 8, 12, 4, 19, 7, 16, 10, 14, 5, 14, 4, 11, 10, 4, 37, 34, 36, 39, 3], + decoded: "he company was founded in 2016", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"], + ids: [6, 5, 12, 6, 4, 3, 36, 4, 3, 37, 4, 3, 42, 4, 3, 38, 4, 3, 41, 4, 3, 39, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 6, 5, 12, 6], + decoded: "test 1 2 3 4 5 6 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."], + ids: [3, 4, 23, 7, 16, 22, 9, 6, 4, 8, 10, 4, 8, 24, 24, 15, 5, 4, 19, 7, 13, 4, 3, 36, 3, 34, 34, 4, 8, 6, 4, 6, 9, 5, 4, 12, 6, 7, 13, 5, 3], + decoded: " bought an aple for 10 at the store", + }, + }, +}; diff --git a/packages/transformers/tests/models/wespeaker_resnet/test_feature_extraction_wespeaker_resnet.js b/packages/transformers/tests/models/wespeaker_resnet/test_feature_extraction_wespeaker_resnet.js new file mode 100644 index 0000000..27d0dbd --- /dev/null +++ b/packages/transformers/tests/models/wespeaker_resnet/test_feature_extraction_wespeaker_resnet.js @@ -0,0 +1,56 @@ +import { AutoFeatureExtractor, WeSpeakerFeatureExtractor } from "../../../src/transformers.js"; + +import { MAX_FEATURE_EXTRACTOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // WeSpeakerFeatureExtractor + describe("WeSpeakerFeatureExtractor", () => { + const model_id = "onnx-community/wespeaker-voxceleb-resnet34-LM"; + + /** @type {WeSpeakerFeatureExtractor} */ + let feature_extractor; + beforeAll(async () => { + feature_extractor = await AutoFeatureExtractor.from_pretrained(model_id); + }, MAX_FEATURE_EXTRACTOR_LOAD_TIME); + + it( + "default", + async () => { + const audio = new Float32Array(16000).map((_, i) => Math.sin(i / 100)); + const { input_features } = await feature_extractor(audio); + const { dims, data } = input_features; + expect(dims).toEqual([1, 98, 80]); + + expect(input_features.mean().item()).toBeCloseTo(5.461731689138105e-8); + expect(data[0]).toBeCloseTo(-0.19300270080566406); + expect(data[1]).toBeCloseTo(-0.05825042724609375); + expect(data[78]).toBeCloseTo(0.2683420181274414); + expect(data[79]).toBeCloseTo(0.26250171661376953); + expect(data[80]).toBeCloseTo(0.19062232971191406); + expect(data.at(-2)).toBeCloseTo(-0.43694400787353516); + expect(data.at(-1)).toBeCloseTo(-0.4266204833984375); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "pad to `min_num_frames`", + async () => { + const audio = new Float32Array(3).map((_, i) => Math.sin(i / 100)); + const { input_features } = await feature_extractor(audio); + const { dims, data } = input_features; + expect(dims).toEqual([1, 9, 80]); + + expect(input_features.mean().item()).toBeCloseTo(-0.0000010093053181966146); + expect(data[0]).toBeCloseTo(20.761859893798828); + expect(data[1]).toBeCloseTo(21.02924346923828); + expect(data[78]).toBeCloseTo(19.083993911743164); + expect(data[79]).toBeCloseTo(18.003454208374023); + expect(data[80]).toBeCloseTo(-2.595233917236328); + expect(data.at(-2)).toBeCloseTo(-2.385499954223633); + expect(data.at(-1)).toBeCloseTo(-2.2504329681396484); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/whisper/test_feature_extraction_whisper.js b/packages/transformers/tests/models/whisper/test_feature_extraction_whisper.js new file mode 100644 index 0000000..fe817bd --- /dev/null +++ b/packages/transformers/tests/models/whisper/test_feature_extraction_whisper.js @@ -0,0 +1,67 @@ +import { AutoFeatureExtractor, WhisperFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_audio } from "../../asset_cache.js"; +import { MAX_FEATURE_EXTRACTOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + // WhisperFeatureExtractor + describe("WhisperFeatureExtractor", () => { + const model_id = "Xenova/whisper-tiny.en"; + + /** @type {WhisperFeatureExtractor} */ + let feature_extractor; + beforeAll(async () => { + feature_extractor = await AutoFeatureExtractor.from_pretrained(model_id); + }, MAX_FEATURE_EXTRACTOR_LOAD_TIME); + + it( + "default", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_features } = await feature_extractor(audio); + const { dims, data } = input_features; + expect(dims).toEqual([1, 80, 3000]); + expect(input_features.mean().item()).toBeCloseTo(-0.2813588131551941); + expect(data[0]).toBeCloseTo(0.33168578147888184); + expect(data[1]).toBeCloseTo(0.30986475944519043); + expect(data[81]).toBeCloseTo(0.10727232694625854); + expect(data[3001]).toBeCloseTo(0.2555035352706909); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "max_length (max_length < audio.length < max_num_samples)", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_features } = await feature_extractor(audio, { max_length: 5 * 16000 }); + const { dims, data } = input_features; + expect(dims).toEqual([1, 80, 500]); + expect(input_features.mean().item()).toBeCloseTo(0.20474646985530853); + expect(data[0]).toBeCloseTo(0.33168578147888184); + expect(data[1]).toBeCloseTo(0.30986475944519043); + expect(data[81]).toBeCloseTo(0.10727238655090332); + expect(data[3001]).toBeCloseTo(0.4018087387084961); + expect(data.at(-1)).toBeCloseTo(-0.41003990173339844); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "max_length (audio.length < max_length < max_num_samples)", + async () => { + const audio = await load_cached_audio("mlk"); + const { input_features } = await feature_extractor(audio, { max_length: 25 * 16000 }); + const { dims, data } = input_features; + expect(dims).toEqual([1, 80, 2500]); + expect(input_features.mean().item()).toBeCloseTo(-0.20426231622695923); + expect(data[0]).toBeCloseTo(0.33168578147888184); + expect(data[1]).toBeCloseTo(0.30986475944519043); + expect(data[81]).toBeCloseTo(0.10727238655090332); + expect(data[3001]).toBeCloseTo(0.18040966987609863); + expect(data.at(-1)).toBeCloseTo(-0.6668410897254944); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/whisper/test_modeling_whisper.js b/packages/transformers/tests/models/whisper/test_modeling_whisper.js new file mode 100644 index 0000000..8ccc758 --- /dev/null +++ b/packages/transformers/tests/models/whisper/test_modeling_whisper.js @@ -0,0 +1,148 @@ +import { WhisperTokenizer, WhisperForConditionalGeneration, full } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("WhisperForConditionalGeneration", () => { + const model_id = "Xenova/tiny-random-WhisperForConditionalGeneration"; + + /** @type {WhisperForConditionalGeneration} */ + let model; + /** @type {WhisperTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await WhisperForConditionalGeneration.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await WhisperTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + describe("prefix tokens", () => { + const input_features = full([1, 80, 3000], 0.0); + + describe("English-only", () => { + it( + "default", + async () => { + const outputs = await model.generate({ + input_features, + is_multilingual: false, + max_new_tokens: 1, + }); + + expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, 50363n, /* Generated */ 45084n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "return_timestamps=true", + async () => { + const outputs = await model.generate({ + input_features, + is_multilingual: false, + max_new_tokens: 1, + return_timestamps: true, + }); + + expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, /* Generated */ 51682n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("multilingual", () => { + it( + "language unset; task unset", + async () => { + // language defaults to 'en' + // task defaults to 'transcribe' + + const outputs = await model.generate({ + input_features, + max_new_tokens: 1, + }); + + expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, 50259n, 50359n, 50363n, /* Generated */ 45084n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "language set; task unset", + async () => { + // task defaults to 'transcribe' + const outputs = await model.generate({ + input_features, + max_new_tokens: 1, + language: "af", + }); + + expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, 50327n, 50359n, 50363n, /* Generated */ 45084n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "language set; task set", + async () => { + const outputs = await model.generate({ + input_features, + max_new_tokens: 1, + language: "zh", + task: "translate", + }); + + expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, 50260n, 50358n, 50363n, /* Generated */ 45084n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "return_timestamps=true", + async () => { + const outputs = await model.generate({ + input_features, + max_new_tokens: 1, + language: "en", + task: "transcribe", + return_timestamps: true, + }); + + expect(outputs.tolist()).toEqual([[/* Prefix */ 50258n, 50259n, 50359n, /* Generated */ 51812n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + }); + + describe("decoder_start_ids", () => { + const input_features = full([1, 80, 3000], 0.0); + + it( + "broadcast inputs", + async () => { + const { decoder_start_token_id, lang_to_id, task_to_id, no_timestamps_token_id } = model.generation_config; + + const outputs = await model.generate({ + input_features, // batch size 1 + max_new_tokens: 1, + decoder_input_ids: [ + // batch size 2 + // <|startoftranscript|> <|lang_id|> <|task|> [<|notimestamps|>] + [decoder_start_token_id, lang_to_id["<|en|>"], task_to_id["translate"], no_timestamps_token_id], + [decoder_start_token_id, lang_to_id["<|fr|>"], task_to_id["transcribe"], no_timestamps_token_id], + ], + }); + expect(outputs.tolist()).toEqual([ + [/* Prefix */ 50258n, 50259n, 50358n, 50363n, /* Generated */ 45084n], + [/* Prefix */ 50258n, 50265n, 50359n, 50363n, /* Generated */ 45084n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/models/whisper/test_tokenization_whisper.js b/packages/transformers/tests/models/whisper/test_tokenization_whisper.js new file mode 100644 index 0000000..f082674 --- /dev/null +++ b/packages/transformers/tests/models/whisper/test_tokenization_whisper.js @@ -0,0 +1,868 @@ +import { WhisperTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS, WHISPER_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = WhisperTokenizer; +export const TEST_CONFIG = { + "onnx-community/whisper-tiny.en": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [50257, 50362, 2437, 389, 345, 1804, 30, 50256], + decoded: "<|startoftranscript|><|notimestamps|>How are you doing?<|endoftext|>", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"], + ids: [50257, 50362, 1639, 815, 1053, 1760, 428, 50256], + decoded: "<|startoftranscript|><|notimestamps|>You should've done this<|endoftext|>", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["01", "23", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [50257, 50362, 486, 1954, 2231, 3134, 4531, 657, 352, 362, 513, 604, 642, 718, 767, 807, 860, 838, 1802, 8576, 50256], + decoded: "<|startoftranscript|><|notimestamps|>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], + ids: [50257, 50362, 464, 1664, 373, 9393, 287, 1584, 13, 50256], + decoded: "<|startoftranscript|><|notimestamps|>The company was founded in 2016.<|endoftext|>", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [50257, 50362, 32, 198, 1183, 37867, 1462, 8348, 67, 7061, 67, 286, 11, 460, 470, 13, 50256], + decoded: "<|startoftranscript|><|notimestamps|>A\n'll!!to?'d''d of, can't.<|endoftext|>", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"], + ids: [50257, 50362, 4299, 1388, 33529, 198, 197, 6603, 50256], + decoded: "<|startoftranscript|><|notimestamps|>def main():\n\tpass<|endoftext|>", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "to", "String", "();", "\u010a", "to", "String", "();"], + ids: [50257, 50362, 1616, 257, 796, 26181, 13, 1462, 10100, 9783, 198, 1462, 10100, 9783, 50256], + decoded: "<|startoftranscript|><|notimestamps|>let a = obj.toString();\ntoString();<|endoftext|>", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [50257, 50362, 1212, 198, 198, 271, 198, 64, 198, 9288, 13, 50256], + decoded: "<|startoftranscript|><|notimestamps|>This\n\nis\na\ntest.<|endoftext|>", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "want", "\u00c3\u00a9", "d", ",", "running"], + ids: [50257, 50362, 4944, 42949, 2634, 67, 11, 20270, 50256], + decoded: "<|startoftranscript|><|notimestamps|>UNwant\u00e9d,running<|endoftext|>", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [50257, 50362, 16, 188, 17, 4210, 18, 50256], + decoded: "<|startoftranscript|><|notimestamps|>1\u00002\ufffd3<|endoftext|>", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [50257, 50362, 15496, 2159, 50256], + decoded: "<|startoftranscript|><|notimestamps|>Hello World<|endoftext|>", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["hello", "\u0120world"], + ids: [50257, 50362, 31373, 995, 50256], + decoded: "<|startoftranscript|><|notimestamps|>hello world<|endoftext|>", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [50257, 50362, 37955, 162, 112, 119, 21410, 40367, 253, 164, 108, 249, 42468, 50256], + decoded: "<|startoftranscript|><|notimestamps|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], + ids: [50257, 50362, 220, 220, 3756, 2272, 50256], + decoded: "<|startoftranscript|><|notimestamps|> leading space<|endoftext|>", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], + ids: [50257, 50362, 9535, 4386, 2272, 220, 220, 220, 50256], + decoded: "<|startoftranscript|><|notimestamps|>trailing space <|endoftext|>", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [50257, 50362, 17250, 220, 18435, 50256], + decoded: "<|startoftranscript|><|notimestamps|>Hi Hello<|endoftext|>", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [50257, 50362, 9288, 720, 16, 371, 17, 1303, 18, 10432, 19, 4248, 20, 38221, 21, 2343, 224, 96, 22, 2343, 224, 117, 23, 2343, 224, 109, 24, 1332, 50256], + decoded: "<|startoftranscript|><|notimestamps|>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."], + ids: [50257, 50362, 40, 5839, 281, 17180, 329, 720, 16, 13, 405, 379, 262, 3650, 13, 50256], + decoded: "<|startoftranscript|><|notimestamps|>I bought an apple for $1.00 at the store.<|endoftext|>", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], + ids: [50257, 50362, 5832, 1399, 220, 220, 50256], + decoded: "<|startoftranscript|><|notimestamps|>you\u2026 <|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [50257, 50362, 5832, 1399, 4603, 50256], + decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0<|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"], + ids: [50257, 50362, 5832, 1399, 1849, 1849, 5832, 1399, 4603, 50256], + decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [50257, 50362, 732, 1447, 27332, 121, 252, 5743, 27332, 121, 252, 1339, 50256], + decoded: "<|startoftranscript|><|notimestamps|>weird \uff5e edge \uff5e case<|endoftext|>", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [50257, 50362, 5008, 223, 1212, 11019, 223, 271, 11019, 223, 64, 11019, 223, 9288, 11019, 223, 13, 50256], + decoded: "<|startoftranscript|><|notimestamps|>\u2581This \u2581is \u2581a \u2581test \u2581.<|endoftext|>", + }, + SPECIAL_TOKENS: { + text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS, + tokens: ["\u0120", "\u0120", "\u0120", "<|startoftranscript|>", "\u0120", "<|en|>", "\u0120", "\u0120", "\u0120"], + ids: [50257, 50362, 220, 220, 220, 50257, 220, 50258, 220, 220, 220, 50256], + decoded: "<|startoftranscript|><|notimestamps|> <|startoftranscript|> <|en|> <|endoftext|>", + }, + }, + "distil-whisper/distil-large-v3": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "12", "3", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [50258, 50364, 15, 4762, 18, 8465, 22452, 21115, 1958, 502, 568, 805, 1017, 1025, 1386, 1614, 1649, 1722, 1266, 2319, 9714, 50257], + decoded: "<|startoftranscript|><|notimestamps|>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "(", "):", "\u010a", "\u0109", "pass"], + ids: [50258, 50364, 20595, 2135, 7, 4507, 198, 197, 9216, 50257], + decoded: "<|startoftranscript|><|notimestamps|>def main():\n\tpass<|endoftext|>", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120ob", "j", ".", "to", "St", "ring", "(", ");", "\u010a", "to", "St", "ring", "(", ");"], + ids: [50258, 50364, 2631, 257, 6585, 1111, 73, 13, 1353, 4520, 2937, 7, 34446, 198, 1353, 4520, 2937, 7, 34446, 50257], + decoded: "<|startoftranscript|><|notimestamps|>let a = obj.toString();\ntoString();<|endoftext|>", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "w", "ant", "\u00c3\u00a9d", ",", "running"], + ids: [50258, 50364, 3979, 86, 394, 7811, 11, 45482, 50257], + decoded: "<|startoftranscript|><|notimestamps|>UNwant\u00e9d,running<|endoftext|>", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["he", "llo", "\u0120world"], + ids: [50258, 50364, 675, 1913, 1002, 50257], + decoded: "<|startoftranscript|><|notimestamps|>hello world<|endoftext|>", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [50258, 50364, 49958, 1546, 6303, 8897, 249, 1541, 50257], + decoded: "<|startoftranscript|><|notimestamps|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"], + ids: [50258, 50364, 31636, 1848, 16, 497, 17, 3536, 18, 17450, 19, 14378, 20, 1815, 98, 21, 672, 224, 96, 22, 672, 224, 117, 23, 672, 224, 109, 24, 1500, 50257], + decoded: "<|startoftranscript|><|notimestamps|>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [50258, 50364, 5616, 1260, 126, 254, 126, 254, 50257], + decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0<|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [50258, 50364, 5616, 1260, 126, 254, 126, 254, 5616, 1260, 126, 254, 126, 254, 50257], + decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>", + }, + }, + "distil-whisper/distil-large-v2": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [50258, 50259, 50359, 50363, 6462, 366, 291, 884, 30, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>How are you doing?<|endoftext|>", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120", "this"], + ids: [50258, 50259, 50359, 50363, 3223, 820, 600, 1096, 220, 11176, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>You should've done this<|endoftext|>", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["0", "12", "3", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [50258, 50259, 50359, 50363, 15, 4762, 18, 8465, 22452, 21115, 1958, 502, 568, 805, 1017, 1025, 1386, 1614, 1649, 1722, 1266, 2319, 9714, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], + ids: [50258, 50259, 50359, 50363, 2278, 2237, 390, 13234, 294, 6549, 13, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>The company was founded in 2016.<|endoftext|>", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [50258, 50259, 50359, 50363, 32, 198, 603, 15138, 1353, 8569, 67, 15025, 67, 295, 11, 393, 380, 13, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>A\n'll!!to?'d''d of, can't.<|endoftext|>", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["def", "\u0120main", "(", "):", "\u010a", "\u0109", "pass"], + ids: [50258, 50259, 50359, 50363, 20595, 2135, 7, 4507, 198, 197, 9216, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>def main():\n\tpass<|endoftext|>", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["let", "\u0120a", "\u0120=", "\u0120ob", "j", ".", "to", "St", "ring", "(", ");", "\u010a", "to", "St", "ring", "(", ");"], + ids: [50258, 50259, 50359, 50363, 2631, 257, 6585, 1111, 73, 13, 1353, 4520, 2937, 7, 34446, 198, 1353, 4520, 2937, 7, 34446, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>let a = obj.toString();\ntoString();<|endoftext|>", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [50258, 50259, 50359, 50363, 5723, 198, 198, 271, 198, 64, 198, 31636, 13, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>This\n\nis\na\ntest.<|endoftext|>", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["UN", "w", "ant", "\u00c3\u00a9d", ",", "running"], + ids: [50258, 50259, 50359, 50363, 3979, 86, 394, 7811, 11, 45482, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>UNwant\u00e9d,running<|endoftext|>", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [50258, 50259, 50359, 50363, 16, 188, 17, 5342, 18, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>1\u00002\ufffd3<|endoftext|>", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["Hello", "\u0120World"], + ids: [50258, 50259, 50359, 50363, 15947, 3937, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>Hello World<|endoftext|>", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["he", "llo", "\u0120world"], + ids: [50258, 50259, 50359, 50363, 675, 1913, 1002, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>hello world<|endoftext|>", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [50258, 50259, 50359, 50363, 49958, 1546, 6303, 8897, 249, 1541, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], + ids: [50258, 50259, 50359, 50363, 220, 220, 5775, 1901, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|> leading space<|endoftext|>", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], + ids: [50258, 50259, 50359, 50363, 17227, 4883, 1901, 220, 220, 220, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>trailing space <|endoftext|>", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["Hi", "\u0120", "\u0120Hello"], + ids: [50258, 50259, 50359, 50363, 17155, 220, 2425, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>Hi Hello<|endoftext|>", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120", "test"], + ids: [50258, 50259, 50359, 50363, 31636, 1848, 16, 497, 17, 3536, 18, 17450, 19, 14378, 20, 1815, 98, 21, 672, 224, 96, 22, 672, 224, 117, 23, 672, 224, 109, 24, 220, 31636, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120", "the", "\u0120store", "."], + ids: [50258, 50259, 50359, 50363, 40, 4243, 364, 10606, 337, 1848, 16, 13, 628, 412, 220, 3322, 3531, 13, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>I bought an apple for $1.00 at the store.<|endoftext|>", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], + ids: [50258, 50259, 50359, 50363, 5616, 1260, 220, 220, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>you\u2026 <|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [50258, 50259, 50359, 50363, 5616, 1260, 126, 254, 126, 254, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>you\u2026\u00a0\u00a0<|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [50258, 50259, 50359, 50363, 5616, 1260, 126, 254, 126, 254, 5616, 1260, 126, 254, 126, 254, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [50258, 50259, 50359, 50363, 826, 1271, 25072, 121, 252, 4691, 25072, 121, 252, 1389, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>weird \uff5e edge \uff5e case<|endoftext|>", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [50258, 50259, 50359, 50363, 39984, 223, 5723, 29405, 223, 271, 29405, 223, 64, 29405, 223, 31636, 29405, 223, 13, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>\u2581This \u2581is \u2581a \u2581test \u2581.<|endoftext|>", + }, + SPECIAL_TOKENS: { + text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS, + tokens: ["\u0120", "\u0120", "\u0120", "<|startoftranscript|>", "\u0120", "<|en|>", "\u0120", "\u0120", "\u0120"], + ids: [50258, 50259, 50359, 50363, 220, 220, 220, 50258, 220, 50259, 220, 220, 220, 50257], + decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|> <|startoftranscript|> <|en|> <|endoftext|>", + }, + }, + "distil-whisper/distil-small.en": { + SPECIAL_TOKENS: { + text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS, + + // https://github.com/huggingface/transformers/issues/33371 + // tokens: [" <|startoftranscript|> ", "<|en|> "], + tokens: ["<|startoftranscript|>", "<|en|>"], + ids: [50257, 50362, 50257, 50258, 50256], + decoded: "<|startoftranscript|><|notimestamps|><|startoftranscript|><|en|><|endoftext|>", + }, + }, + "Xenova/nb-whisper-tiny-beta": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u0120How", "\u0120are", "\u0120you", "\u0120doing", "?"], + ids: [50258, 50288, 50359, 50363, 1012, 366, 291, 884, 30, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> How are you doing?<|endoftext|>", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u0120You", "\u0120should", "'ve", "\u0120done", "\u0120", "this"], + ids: [50258, 50288, 50359, 50363, 509, 820, 600, 1096, 220, 11176, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> You should've done this<|endoftext|>", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u01200", "12", "3", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"], + ids: [50258, 50288, 50359, 50363, 1958, 4762, 18, 8465, 22452, 21115, 1958, 502, 568, 805, 1017, 1025, 1386, 1614, 1649, 1722, 1266, 2319, 9714, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u0120The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."], + ids: [50258, 50288, 50359, 50363, 440, 2237, 390, 13234, 294, 6549, 13, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> The company was founded in 2016.<|endoftext|>", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u0120A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."], + ids: [50258, 50288, 50359, 50363, 316, 198, 603, 15138, 1353, 8569, 67, 15025, 67, 295, 11, 393, 380, 13, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> A\n'll!!to?'d''d of, can't.<|endoftext|>", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u0120def", "\u0120main", "(", "):", "\u010a", "\u0109", "pass"], + ids: [50258, 50288, 50359, 50363, 1060, 2135, 7, 4507, 198, 197, 9216, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> def main():\n\tpass<|endoftext|>", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u0120let", "\u0120a", "\u0120=", "\u0120ob", "j", ".", "to", "St", "ring", "(", ");", "\u010a", "to", "St", "ring", "(", ");"], + ids: [50258, 50288, 50359, 50363, 718, 257, 6585, 1111, 73, 13, 1353, 4520, 2937, 7, 34446, 198, 1353, 4520, 2937, 7, 34446, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> let a = obj.toString();\ntoString();<|endoftext|>", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u0120This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."], + ids: [50258, 50288, 50359, 50363, 639, 198, 198, 271, 198, 64, 198, 31636, 13, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> This\n\nis\na\ntest.<|endoftext|>", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u0120UN", "w", "ant", "\u00c3\u00a9d", ",", "running"], + ids: [50258, 50288, 50359, 50363, 8229, 86, 394, 7811, 11, 45482, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> UNwant\u00e9d,running<|endoftext|>", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u01201", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"], + ids: [50258, 50288, 50359, 50363, 502, 188, 17, 5342, 18, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> 1\u00002\ufffd3<|endoftext|>", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u0120Hello", "\u0120World"], + ids: [50258, 50288, 50359, 50363, 2425, 3937, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> Hello World<|endoftext|>", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u0120hello", "\u0120world"], + ids: [50258, 50288, 50359, 50363, 7751, 1002, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> hello world<|endoftext|>", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u0120", "\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"], + ids: [50258, 50288, 50359, 50363, 220, 49958, 1546, 6303, 8897, 249, 1541, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> \u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"], + ids: [50258, 50288, 50359, 50363, 220, 220, 5775, 1901, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> leading space<|endoftext|>", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u0120", "tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"], + ids: [50258, 50288, 50359, 50363, 220, 17227, 4883, 1901, 220, 220, 220, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> trailing space <|endoftext|>", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u0120Hi", "\u0120", "\u0120Hello"], + ids: [50258, 50288, 50359, 50363, 2421, 220, 2425, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> Hi Hello<|endoftext|>", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u0120", "test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120", "test"], + ids: [50258, 50288, 50359, 50363, 220, 31636, 1848, 16, 497, 17, 3536, 18, 17450, 19, 14378, 20, 1815, 98, 21, 672, 224, 96, 22, 672, 224, 117, 23, 672, 224, 109, 24, 220, 31636, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u0120I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120", "the", "\u0120store", "."], + ids: [50258, 50288, 50359, 50363, 286, 4243, 364, 10606, 337, 1848, 16, 13, 628, 412, 220, 3322, 3531, 13, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> I bought an apple for $1.00 at the store.<|endoftext|>", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u0120you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"], + ids: [50258, 50288, 50359, 50363, 291, 1260, 220, 220, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> you\u2026 <|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u0120you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [50258, 50288, 50359, 50363, 291, 1260, 126, 254, 126, 254, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> you\u2026\u00a0\u00a0<|endoftext|>", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u0120you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"], + ids: [50258, 50288, 50359, 50363, 291, 1260, 126, 254, 126, 254, 5616, 1260, 126, 254, 126, 254, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u0120weird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"], + ids: [50258, 50288, 50359, 50363, 3657, 25072, 121, 252, 4691, 25072, 121, 252, 1389, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> weird \uff5e edge \uff5e case<|endoftext|>", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u0120\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."], + ids: [50258, 50288, 50359, 50363, 29405, 223, 5723, 29405, 223, 271, 29405, 223, 64, 29405, 223, 31636, 29405, 223, 13, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> \u2581This \u2581is \u2581a \u2581test \u2581.<|endoftext|>", + }, + SPECIAL_TOKENS: { + text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS, + tokens: ["\u0120", "\u0120", "\u0120", "<|startoftranscript|>", "\u0120", "<|en|>", "\u0120", "\u0120", "\u0120"], + ids: [50258, 50288, 50359, 50363, 220, 220, 220, 50258, 220, 50259, 220, 220, 220, 50257], + decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> <|startoftranscript|> <|en|> <|endoftext|>", + }, + }, +}; + +const MAX_EXECUTION_TIME = 10_000; +export const CUSTOM_TESTS = () => { + describe("Decode ASR", () => { + it( + "should decode ASR outputs", + async () => { + const tokenizer = await WhisperTokenizer.from_pretrained("onnx-community/whisper-tiny.en_timestamped"); + + const model_outputs = [ + { + stride: [30, 0, 5], + tokens: [50257n, 50362n, 8410n, 7283n, 0n, 2329n, 8410n, 7283n, 0n, 2094n, 470n, 1309n, 534n, 10625n, 307n, 10625n, 13n, 34668n, 345n, 531n, 9439n, 11n, 523n, 655n, 8410n, 7283n, 0n, 39134n, 16592n, 10625n, 0n, 9440n, 36n, 26751n, 0n, 25848n, 8410n, 7283n, 0n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 0n, 50256n], + token_timestamps: [0, 0, 0, 3.78, 4.22, 5.34, 6.04, 6.56, 7, 7.92, 8.58, 8.58, 8.88, 9.14, 9.54, 9.94, 10.58, 11.38, 11.88, 12.42, 12.62, 13, 13.36, 13.64, 14.26, 14.76, 15.12, 15.4, 15.74, 16.12, 16.66, 17.14, 17.24, 17.24, 17.72, 18.38, 18.6, 19.38, 19.92, 22.66, 22.9, 23.24, 23.5, 24.14, 24.56, 24.7, 24.72, 24.94, 25.18, 25.54, 25.72, 26.02, 26.34, 26.44, 26.84, 27.04, 27.16, 27.54, 28.06, 29.92], + }, + { + stride: [30, 5, 5], + tokens: [50257n, 50362n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 13n, 921n, 815n, 651n, 284n, 262n, 966n, 810n, 2687n, 2073n, 561n, 11238n, 290n, 345n, 821n, 407n, 8066n, 2245n, 612n, 13n, 1400n, 11n, 644n, 389n, 345n, 4953n, 329n, 30n, 2141n, 340n, 0n, 2329n, 466n, 340n, 0n, 3363n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 50256n], + token_timestamps: [0, 0, 0, 2.92, 3.24, 3.48, 4.14, 4.56, 4.7, 4.74, 4.92, 5.18, 5.54, 5.72, 6.04, 6.34, 6.46, 6.84, 7.04, 7.16, 7.54, 8.12, 10.16, 10.7, 10.9, 11.12, 11.24, 11.48, 11.84, 12.44, 12.82, 13.2, 13.46, 13.72, 14.06, 14.28, 14.34, 14.56, 14.8, 15.16, 15.9, 16.42, 16.82, 16.86, 17.02, 17.1, 17.22, 17.56, 18.06, 19.28, 19.62, 20.26, 21.96, 22.64, 24.28, 24.76, 25.18, 25.56, 25.78, 26.28, 27.12, 27.54, 27.82, 28.22, 29.48], + }, + { + stride: [23.7728125, 5, 0], + tokens: [50257n, 50362n, 2329n, 466n, 340n, 0n, 3363n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 1002n, 345n, 821n, 10032n, 286n, 3599n, 625n, 11n, 2245n, 3501n, 510n, 13n, 50256n], + token_timestamps: [0, 0, 0, 2.44, 4.3, 5.04, 5.06, 5.56, 5.8, 6.32, 7.12, 7.56, 7.8, 8.9, 10.92, 12.96, 13.28, 13.28, 13.44, 13.72, 13.96, 14.84, 15.5, 16.06, 16.86, 17.88, 20.92], + }, + ]; + + const target = [ + " DO IT! Just DO IT! Don't let your dreams be dreams. Yesterday you said tomorrow, so just DO IT! MAKE YOUR dreams! COME TRUE! JUST DO IT! Some people dream success while you're gonna wake up and work hard at it. Nothing is impossible. You should get to the point where anyone else would quit and you're not gonna stop there. No, what are you waiting for? Do it! Just do it! Yes you can! Just do it! If you're tired of starting over, stop giving up.", + { + chunks: [ + { text: " DO", timestamp: [0.0, 3.78] }, + { text: " IT!", timestamp: [3.78, 4.24 /* 5.34 */] }, + { text: " Just", timestamp: [5.34, 6.04] }, + { text: " DO", timestamp: [6.04, 6.56] }, + { text: " IT!", timestamp: [6.56, 7.02 /* 7.92 */] }, + { text: " Don't", timestamp: [7.92, 8.58] }, + { text: " let", timestamp: [8.58, 8.88] }, + { text: " your", timestamp: [8.88, 9.14] }, + { text: " dreams", timestamp: [9.14, 9.54] }, + { text: " be", timestamp: [9.54, 9.94] }, + { text: " dreams.", timestamp: [9.94, 10.6 /* 11.38 */] }, + { text: " Yesterday", timestamp: [11.38, 11.88] }, + { text: " you", timestamp: [11.88, 12.42] }, + { text: " said", timestamp: [12.42, 12.62] }, + { text: " tomorrow,", timestamp: [12.62, 13.02 /* 13.36 */] }, + { text: " so", timestamp: [13.36, 13.64] }, + { text: " just", timestamp: [13.64, 14.26] }, + { text: " DO", timestamp: [14.26, 14.76] }, + { text: " IT!", timestamp: [14.76, 15.14 /* 15.4 */] }, + { text: " MAKE", timestamp: [15.4, 15.74] }, + { text: " YOUR", timestamp: [15.74, 16.12] }, + { text: " dreams!", timestamp: [16.12, 16.68 /* 17.14 */] }, + { text: " COME", timestamp: [17.14, 17.24] }, + { text: " TRUE!", timestamp: [17.24, 17.74 /* 18.38 */] }, + { text: " JUST", timestamp: [18.38, 18.6] }, + { text: " DO", timestamp: [18.6, 19.38] }, + { text: " IT!", timestamp: [19.38, 19.94 /* 22.66 */] }, + { text: " Some", timestamp: [22.66, 22.9] }, + { text: " people", timestamp: [22.9, 23.24] }, + { text: " dream", timestamp: [23.24, 23.5] }, + { text: " success", timestamp: [23.5, 24.14] }, + { text: " while", timestamp: [24.14, 24.56] }, + { text: " you're", timestamp: [24.56, 24.72] }, + { text: " gonna", timestamp: [24.72, 24.94] }, + { text: " wake", timestamp: [24.94, 25.18] }, + { text: " up", timestamp: [25.18, 25.54] }, + { text: " and", timestamp: [25.54, 25.72] }, + { text: " work", timestamp: [25.72, 26.04] }, + { text: " hard", timestamp: [26.04, 26.34] }, + { text: " at", timestamp: [26.34, 26.46] }, + { text: " it.", timestamp: [26.46, 26.86 /* 27.04 */] }, + { text: " Nothing", timestamp: [27.04, 27.16] }, + { text: " is", timestamp: [27.16, 27.54] }, + { text: " impossible.", timestamp: [27.54, 28.14 /* 30.16 */] }, + { text: " You", timestamp: [30.16, 30.7] }, + { text: " should", timestamp: [30.7, 30.9] }, + { text: " get", timestamp: [30.9, 31.12] }, + { text: " to", timestamp: [31.12, 31.24] }, + { text: " the", timestamp: [31.24, 31.48] }, + { text: " point", timestamp: [31.48, 31.84] }, + { text: " where", timestamp: [31.84, 32.44] }, + { text: " anyone", timestamp: [32.44, 32.82] }, + { text: " else", timestamp: [32.82, 33.2] }, + { text: " would", timestamp: [33.2, 33.46] }, + { text: " quit", timestamp: [33.46, 33.72] }, + { text: " and", timestamp: [33.72, 34.06] }, + { text: " you're", timestamp: [34.06, 34.34] }, + { text: " not", timestamp: [34.34, 34.56] }, + { text: " gonna", timestamp: [34.56, 34.8] }, + { text: " stop", timestamp: [34.8, 35.16] }, + { text: " there.", timestamp: [35.16, 35.92 /* 36.42 */] }, + { text: " No,", timestamp: [36.42, 36.84 /* 36.86 */] }, + { text: " what", timestamp: [36.86, 37.02] }, + { text: " are", timestamp: [37.02, 37.1] }, + { text: " you", timestamp: [37.1, 37.22] }, + { text: " waiting", timestamp: [37.22, 37.56] }, + { text: " for?", timestamp: [37.56, 38.08 /* 39.28 */] }, + { text: " Do", timestamp: [39.28, 39.62] }, + { text: " it!", timestamp: [39.62, 40.28 /* 41.96 */] }, + { text: " Just", timestamp: [41.96, 42.64] }, + { text: " do", timestamp: [42.64, 44.28] }, + { text: " it!", timestamp: [44.28, 44.78 /* 45.18 */] }, + { text: " Yes", timestamp: [45.18, 45.56] }, + { text: " you", timestamp: [45.56, 45.78] }, + { text: " can!", timestamp: [45.8, 46.34 /* 47.12 */] }, + { text: " Just", timestamp: [47.12, 47.56] }, + { text: " do", timestamp: [47.56, 47.8] }, + { text: " it!", timestamp: [47.8, 48.92 /* 50.92 */] }, + { text: " If", timestamp: [50.92, 52.96] }, + { text: " you're", timestamp: [52.96, 53.28] }, + { text: " tired", timestamp: [53.28, 53.44] }, + { text: " of", timestamp: [53.44, 53.72] }, + { text: " starting", timestamp: [53.72, 53.96] }, + { text: " over,", timestamp: [53.96, 54.86 /* 55.5 */] }, + { text: " stop", timestamp: [55.5, 56.06] }, + { text: " giving", timestamp: [56.06, 56.86] }, + { text: " up.", timestamp: [56.86, 57.9 /* 60.92 */] }, + ], + }, + ]; + const decoded = tokenizer._decode_asr(model_outputs, { + return_timestamps: "word", + time_precision: 0.02, + force_full_sequences: false, + }); + expect(decoded).toBeCloseToNested(target, 2); + }, + MAX_EXECUTION_TIME, + ); + + it( + "should handle overlapping edge case", + async () => { + const tokenizer = await WhisperTokenizer.from_pretrained("onnx-community/whisper-tiny.en_timestamped"); + + const model_outputs = [ + { + stride: [30, 0, 5], + tokens: [50257n, 50362n, 8410n, 7283n, 0n, 2329n, 8410n, 7283n, 0n, 2094n, 470n, 1309n, 534n, 10625n, 307n, 10625n, 13n, 34668n, 11n, 345n, 531n, 9439n, 11n, 523n, 655n, 8410n, 7283n, 0n, 39134n, 16592n, 10560n, 3955n, 50n, 0n, 7102n, 5446n, 46n, 0n, 25848n, 8410n, 7283n, 0n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 13n, 50256n], + token_timestamps: [0, 0, 0, 3.78, 4.22, 5.26, 6.04, 6.54, 7, 7.94, 8.58, 8.58, 8.88, 9.16, 9.54, 9.94, 10.6, 11.38, 11.88, 12.38, 12.44, 12.62, 13, 13.36, 13.64, 14.24, 14.74, 15.12, 15.4, 15.74, 16.1, 16.54, 16.54, 16.78, 17.08, 17.2, 17.36, 17.56, 18.08, 18.58, 19.38, 19.88, 22.54, 22.9, 23.24, 23.5, 24.14, 24.56, 24.7, 24.94, 24.94, 25.18, 25.54, 25.72, 26.04, 26.34, 26.46, 26.84, 27.04, 27.14, 27.54, 28.06, 29.92], + }, + { + stride: [30, 5, 5], + tokens: [50257n, 50362n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 13n, 921n, 815n, 651n, 284n, 262n, 966n, 810n, 2687n, 2073n, 561n, 11238n, 290n, 345n, 821n, 407n, 8066n, 2245n, 612n, 13n, 1400n, 11n, 644n, 389n, 345n, 4953n, 329n, 30n, 2141n, 340n, 0n, 2329n, 466n, 340n, 0n, 3363n, 11n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 50256n], + token_timestamps: [0, 0, 0, 2.92, 3.24, 3.5, 4.14, 4.56, 4.7, 4.74, 4.92, 5.18, 5.54, 5.74, 6.04, 6.34, 6.46, 6.84, 7.04, 7.18, 7.56, 8.12, 9.68, 10.7, 10.88, 11.1, 11.24, 11.48, 11.82, 12.46, 12.82, 13.2, 13.46, 13.72, 14.08, 14.28, 14.34, 14.56, 14.82, 15.16, 15.72, 16.42, 16.82, 16.86, 17, 17.1, 17.2, 17.56, 18.06, 19.28, 19.6, 20.28, 21.96, 22.64, 24.28, 24.76, 25.18, 25.56, 25.56, 25.84, 26.36, 27.12, 27.54, 27.82, 28.16, 29.48], + }, + { + stride: [23.7728125, 5, 0], + tokens: [50257n, 50362n, 2329n, 466n, 340n, 0n, 3363n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 1002n, 534n, 15867n, 318n, 3599n, 625n, 11n, 2245n, 3501n, 510n, 13n, 50256n], + token_timestamps: [0, 0, 0, 2.44, 4.3, 5.04, 5.06, 5.56, 5.8, 6.32, 7.12, 7.56, 7.8, 8.72, 10.04, 12.96, 13.3, 13.44, 13.72, 13.98, 14.86, 15.5, 16, 16.88, 17.76, 20.9], + }, + ]; + + const target = [ + " DO IT! Just DO IT! Don't let your dreams be dreams. Yesterday, you said tomorrow, so just DO IT! MAKE YOUR DRIMS! CONTRO! JUST DO IT! Some people dream success while you're gonna wake up and work hard at it. Nothing is impossible. You should get to the point where anyone else would quit and you're not gonna stop there. No, what are you waiting for? Do it! Just do it! Yes, you can! Just do it! If your tire is starting over, stop giving up.", + { + chunks: [ + { text: " DO", timestamp: [0, 3.78] }, + { text: " IT!", timestamp: [3.78, 4.24] }, + { text: " Just", timestamp: [5.26, 6.04] }, + { text: " DO", timestamp: [6.04, 6.54] }, + { text: " IT!", timestamp: [6.54, 7.02] }, + { text: " Don't", timestamp: [7.94, 8.58] }, + { text: " let", timestamp: [8.58, 8.88] }, + { text: " your", timestamp: [8.88, 9.16] }, + { text: " dreams", timestamp: [9.16, 9.54] }, + { text: " be", timestamp: [9.54, 9.94] }, + { text: " dreams.", timestamp: [9.94, 10.62] }, + { text: " Yesterday,", timestamp: [11.38, 11.9] }, + { text: " you", timestamp: [12.38, 12.44] }, + { text: " said", timestamp: [12.44, 12.62] }, + { text: " tomorrow,", timestamp: [12.62, 13.02] }, + { text: " so", timestamp: [13.36, 13.64] }, + { text: " just", timestamp: [13.64, 14.24] }, + { text: " DO", timestamp: [14.24, 14.74] }, + { text: " IT!", timestamp: [14.74, 15.14] }, + { text: " MAKE", timestamp: [15.4, 15.74] }, + { text: " YOUR", timestamp: [15.74, 16.1] }, + { text: " DRIMS!", timestamp: [16.1, 16.8] }, + { text: " CONTRO!", timestamp: [17.08, 17.58] }, + { text: " JUST", timestamp: [18.08, 18.58] }, + { text: " DO", timestamp: [18.58, 19.38] }, + { text: " IT!", timestamp: [19.38, 19.9] }, + { text: " Some", timestamp: [22.54, 22.9] }, + { text: " people", timestamp: [22.9, 23.24] }, + { text: " dream", timestamp: [23.24, 23.5] }, + { text: " success", timestamp: [23.5, 24.14] }, + { text: " while", timestamp: [24.14, 24.56] }, + { text: " you're", timestamp: [24.56, 24.94] }, + { text: " gonna", timestamp: [24.94, 24.94] }, + { text: " wake", timestamp: [24.94, 25.18] }, + { text: " up", timestamp: [25.18, 25.54] }, + { text: " and", timestamp: [25.54, 25.74] }, + { text: " work", timestamp: [25.74, 26.04] }, + { text: " hard", timestamp: [26.04, 26.34] }, + { text: " at", timestamp: [26.34, 26.46] }, + { text: " it.", timestamp: [26.46, 26.86] }, + { text: " Nothing", timestamp: [27.04, 27.18] }, + { text: " is", timestamp: [27.18, 27.56] }, + { text: " impossible.", timestamp: [27.56, 28.14] }, + { text: " You", timestamp: [29.68, 30.7] }, + { text: " should", timestamp: [30.7, 30.88] }, + { text: " get", timestamp: [30.88, 31.1] }, + { text: " to", timestamp: [31.1, 31.24] }, + { text: " the", timestamp: [31.24, 31.48] }, + { text: " point", timestamp: [31.48, 31.82] }, + { text: " where", timestamp: [31.82, 32.46] }, + { text: " anyone", timestamp: [32.46, 32.82] }, + { text: " else", timestamp: [32.82, 33.2] }, + { text: " would", timestamp: [33.2, 33.46] }, + { text: " quit", timestamp: [33.46, 33.72] }, + { text: " and", timestamp: [33.72, 34.08] }, + { text: " you're", timestamp: [34.08, 34.34] }, + { text: " not", timestamp: [34.34, 34.56] }, + { text: " gonna", timestamp: [34.56, 34.82] }, + { text: " stop", timestamp: [34.82, 35.16] }, + { text: " there.", timestamp: [35.16, 35.74] }, + { text: " No,", timestamp: [36.42, 36.84] }, + { text: " what", timestamp: [36.86, 37] }, + { text: " are", timestamp: [37, 37.1] }, + { text: " you", timestamp: [37.1, 37.2] }, + { text: " waiting", timestamp: [37.2, 37.56] }, + { text: " for?", timestamp: [37.56, 38.08] }, + { text: " Do", timestamp: [39.28, 39.6] }, + { text: " it!", timestamp: [39.6, 40.3] }, + { text: " Just", timestamp: [41.96, 42.64] }, + { text: " do", timestamp: [42.64, 44.28] }, + { text: " it!", timestamp: [44.28, 44.78] }, + { text: " Yes,", timestamp: [45.18, 45.56] }, + { text: " you", timestamp: [45.56, 45.84] }, + { text: " can!", timestamp: [45.8, 46.34] }, + { text: " Just", timestamp: [47.12, 47.56] }, + { text: " do", timestamp: [47.56, 47.8] }, + { text: " it!", timestamp: [47.8, 48.74] }, + { text: " If", timestamp: [50.04, 52.96] }, + { text: " your", timestamp: [52.96, 53.3] }, + { text: " tire", timestamp: [53.3, 53.44] }, + { text: " is", timestamp: [53.44, 53.72] }, + { text: " starting", timestamp: [53.72, 53.98] }, + { text: " over,", timestamp: [53.98, 54.88] }, + { text: " stop", timestamp: [55.5, 56] }, + { text: " giving", timestamp: [56, 56.88] }, + { text: " up.", timestamp: [56.88, 57.78] }, + ], + }, + ]; + + const decoded = tokenizer._decode_asr(model_outputs, { + return_timestamps: "word", + time_precision: 0.02, + force_full_sequences: false, + }); + expect(decoded).toEqual(target); + }, + MAX_EXECUTION_TIME, + ); + + it( + "should not produce backwards timestamps when merging overlapping chunks (1/2)", + async () => { + // Regression test: DTW can produce slightly different timestamps for the + // same word across overlapping chunks. Without tolerance in the timestamp + // comparison, findLongestCommonSequence fails to find the overlap and + // produces duplicate content with backwards-jumping timestamps. + // Data captured from whisper-large-v3-turbo on a 60s audio clip. + const tokenizer = await WhisperTokenizer.from_pretrained("onnx-community/whisper-large-v3-turbo_timestamped"); + + const model_outputs = [ + { + stride: [30, 0, 5], + tokens: [50365n, 492n, 362n, 2135n, 2848n, 722n, 13n, 50681n, 50681n, 7451n, 11n, 1045n, 11n, 732n, 11n, 472n, 13n, 50845n, 50845n, 509n, 434n, 257n, 25197n, 11n, 5041n, 13n, 51581n, 51588n, 2053n, 11n, 19967n, 654n, 11n, 321n, 362n, 281n, 1524n, 527n, 30640n, 13n, 51706n, 51731n, 509n, 362n, 428n, 34145n, 293n, 286n, 445n, 528n, 281n, 312n, 3476n, 294n, 1901n, 13n, 51864n, 50257n], + token_timestamps: [0, 2.56, 5.14, 5.44, 5.76, 6.22, 6.52, 6.52, 6.82, 7.18, 7.4, 7.78, 8.12, 8.6, 9.1, 9.54, 10.04, 10.92, 23.16, 23.2, 23.38, 23.54, 23.96, 24.1, 24.24, 24.4, 24.48, 24.48, 24.66, 24.92, 25.06, 25.22, 25.36, 25.44, 25.64, 25.78, 26.08, 26.28, 26.82, 27.04, 27.32, 27.32, 27.44, 27.58, 27.7, 28.18, 28.36, 28.44, 28.58, 28.74, 28.82, 28.96, 29.44, 29.62, 29.88, 29.98, 29.98, 29.98], + }, + { + stride: [30, 5, 5], + tokens: [50525n, 509n, 434n, 257n, 25197n, 11n, 5041n, 13n, 50585n, 50585n, 2053n, 11n, 19967n, 654n, 11n, 321n, 362n, 281n, 1524n, 527n, 30640n, 13n, 50725n, 50725n, 509n, 362n, 428n, 34145n, 293n, 286n, 445n, 528n, 281n, 312n, 3476n, 294n, 1901n, 13n, 50875n, 50875n, 1545n, 500n, 380n, 291n, 445n, 9796n, 300n, 291n, 434n, 37853n, 484n, 538n, 452n, 7881n, 1011n, 30n, 51090n, 51090n, 286n, 478n, 406n, 37853n, 484n, 538n, 485n, 467n, 311n, 485n, 51175n, 51245n, 2798n, 11n, 2489n, 13n, 286n, 478n, 37853n, 484n, 13n, 51335n, 51335n, 286n, 478n, 1419n, 36911n, 300n, 286n, 478n, 885n, 33091n, 538n, 613n, 7410n, 30468n, 34258n, 485n, 51485n, 51485n, 876n, 11n, 2035n, 11n, 5041n, 13n, 51545n, 51580n, 492n, 434n, 1096n, 13n, 51630n, 50257n], + token_timestamps: [0, 3.3, 3.4, 3.52, 3.94, 4.08, 4.24, 4.38, 4.52, 4.52, 4.68, 4.9, 5.04, 5.22, 5.34, 5.44, 5.64, 5.78, 6.08, 6.28, 6.82, 7.12, 7.32, 7.32, 7.44, 7.58, 7.7, 8.18, 8.36, 8.44, 8.58, 8.74, 8.82, 8.94, 9.44, 9.62, 9.94, 10.18, 10.42, 10.42, 10.82, 10.98, 11.08, 11.18, 11.4, 11.78, 11.98, 12.14, 12.24, 12.92, 13.1, 13.3, 13.5, 13.96, 14.3, 14.44, 14.58, 14.58, 14.82, 14.9, 15.02, 15.26, 15.44, 15.64, 15.76, 15.84, 15.96, 16.28, 16.88, 17.52, 17.86, 18, 18.26, 18.5, 18.72, 18.8, 19.08, 19.36, 19.46, 19.46, 19.46, 19.5, 19.56, 19.7, 20.1, 20.26, 20.32, 20.38, 20.5, 20.82, 20.98, 21.1, 21.46, 22.02, 22.38, 22.4, 22.42, 22.42, 22.42, 22.42, 22.9, 23.12, 23.3, 23.64, 23.78, 24.22, 24.44, 24.54, 24.88, 25.12, 26.34, 26.34], + }, + { + stride: [20, 5, 0], + tokens: [50365n, 300n, 286n, 478n, 885n, 33091n, 538n, 341n, 7410n, 30468n, 32019n, 13n, 50465n, 50465n, 876n, 11n, 2035n, 11n, 321n, 434n, 1096n, 13n, 50565n, 50565n, 492n, 434n, 1096n, 13n, 50665n, 50865n, 29601n, 311n, 4675n, 5451n, 1232n, 293n, 9376n, 13n, 51065n, 51265n, 876n, 11n, 572n, 13n, 51365n, 50257n], + token_timestamps: [0, 0.24, 0.36, 0.36, 0.52, 0.82, 0.98, 1.12, 1.46, 2, 2.36, 2.5, 2.5, 2.5, 2.52, 2.52, 2.84, 2.9, 3.28, 3.3, 3.3, 3.64, 4.32, 4.32, 4.42, 4.52, 4.86, 5.14, 7.56, 9.98, 10.44, 10.56, 10.96, 11.44, 11.64, 12.58, 13.56, 13.78, 15.44, 16.54, 20.3, 20.32, 20.44, 21.02, 21.3, 21.3], + }, + ]; + + const decoded = tokenizer._decode_asr(model_outputs, { + return_timestamps: "word", + time_precision: 0.02, + force_full_sequences: false, + }); + + const [, { chunks }] = decoded; + + // Verify timestamps are monotonically non-decreasing + for (let i = 1; i < chunks.length; ++i) { + const prev = chunks[i - 1].timestamp; + const curr = chunks[i].timestamp; + expect(curr[0]).toBeGreaterThanOrEqual(prev[0]); + } + + // Verify no duplicate content from failed overlap merging + // (the "We're done." at ~44-45s should appear only once, not twice) + const doneCount = chunks.filter((c) => c.text.trim() === "done." || c.text.trim() === "We're done.").length; + expect(doneCount).toBeLessThanOrEqual(3); // "done." appears naturally a few times in the script + }, + MAX_EXECUTION_TIME, + ); + + it( + "should not produce backwards timestamps when merging overlapping chunks (2/2)", + async () => { + // Regression test: when findLongestCommonSequence finds no matching tokens + // between overlapping chunks (e.g., due to different casing like "We're" vs + // "we're"), the default behavior concatenated both sequences, producing + // backwards-jumping timestamps. The fix skips right-side tokens that + // temporally precede the left sequence's end. + // Data captured from whisper-large-v3-turbo (fp16) on a 60s audio clip. + const tokenizer = await WhisperTokenizer.from_pretrained("onnx-community/whisper-large-v3-turbo_timestamped"); + + const model_outputs = [ + { + stride: [30, 0, 5], + tokens: [50365n, 492n, 362n, 2135n, 2848n, 722n, 13n, 7451n, 11n, 1045n, 11n, 732n, 11n, 472n, 13n, 50845n, 50845n, 509n, 434n, 257n, 25197n, 11n, 5041n, 13n, 51581n, 51588n, 2053n, 11n, 19967n, 654n, 11n, 321n, 362n, 281n, 1524n, 527n, 30640n, 13n, 51706n, 51731n, 509n, 362n, 428n, 34145n, 293n, 286n, 445n, 528n, 281n, 312n, 3476n, 294n, 1901n, 13n, 51864n, 50257n], + token_timestamps: [0, 2.54, 5.14, 5.44, 5.76, 6.22, 6.52, 7.2, 7.4, 7.78, 8.12, 8.58, 9.1, 9.54, 10, 10.96, 23.16, 23.2, 23.38, 23.54, 23.94, 24.1, 24.24, 24.4, 24.48, 24.48, 24.66, 24.92, 25.06, 25.22, 25.36, 25.44, 25.64, 25.78, 26.08, 26.28, 26.82, 27.1, 27.32, 27.32, 27.42, 27.58, 27.7, 28.18, 28.36, 28.44, 28.58, 28.74, 28.82, 28.94, 29.44, 29.62, 29.88, 29.94, 29.98, 29.98], + }, + { + stride: [30, 5, 5], + tokens: [50365n, 509n, 434n, 257n, 25197n, 11n, 5041n, 13n, 50580n, 50580n, 2053n, 11n, 19967n, 654n, 11n, 321n, 362n, 281n, 1524n, 527n, 30640n, 13n, 50706n, 50727n, 509n, 362n, 428n, 34145n, 11n, 293n, 286n, 445n, 528n, 281n, 312n, 3476n, 294n, 1901n, 13n, 50866n, 50886n, 1545n, 500n, 380n, 291n, 445n, 9796n, 300n, 291n, 434n, 37853n, 484n, 538n, 452n, 7881n, 1011n, 30n, 51081n, 51093n, 286n, 478n, 406n, 37853n, 484n, 538n, 309n, 13n, 51158n, 51228n, 1057n, 558n, 11n, 2489n, 13n, 51283n, 51294n, 286n, 478n, 37853n, 484n, 13n, 51335n, 51338n, 286n, 478n, 1419n, 36911n, 300n, 286n, 478n, 885n, 33091n, 538n, 613n, 7410n, 30468n, 34258n, 13n, 51486n, 51486n, 876n, 11n, 2035n, 11n, 5041n, 13n, 51535n, 51575n, 492n, 434n, 1096n, 13n, 51614n, 50257n], + token_timestamps: [0, 3.24, 3.38, 3.54, 3.94, 4.06, 4.26, 4.4, 4.52, 4.52, 4.64, 4.88, 5.04, 5.22, 5.34, 5.44, 5.64, 5.78, 6.08, 6.28, 6.82, 7.1, 7.32, 7.32, 7.44, 7.58, 7.7, 8.18, 8.34, 8.36, 8.44, 8.58, 8.74, 8.82, 8.94, 9.44, 9.62, 9.94, 10.14, 10.38, 10.42, 10.82, 10.98, 11.08, 11.18, 11.4, 11.78, 11.98, 12.14, 12.22, 12.92, 13.1, 13.3, 13.48, 13.96, 14.28, 14.44, 14.54, 14.54, 14.8, 14.9, 15.02, 15.24, 15.44, 15.64, 15.8, 16.02, 16.26, 17.48, 17.72, 17.9, 18.02, 18.28, 18.48, 18.5, 18.62, 18.72, 18.78, 19.06, 19.36, 19.46, 19.46, 19.46, 19.5, 19.56, 19.7, 20.1, 20.26, 20.32, 20.36, 20.52, 20.82, 20.98, 21.1, 21.44, 22.02, 22.38, 22.38, 22.4, 22.4, 22.42, 22.42, 22.84, 23.1, 23.32, 23.58, 23.72, 24.22, 24.44, 24.52, 24.88, 25.14, 25.66, 25.66], + }, + { + stride: [20, 5, 0], + tokens: [50365n, 300n, 286n, 478n, 885n, 33091n, 538n, 341n, 7410n, 30468n, 32019n, 13n, 50465n, 50465n, 876n, 11n, 2035n, 11n, 321n, 434n, 1096n, 13n, 50565n, 50765n, 29601n, 311n, 4675n, 5451n, 1232n, 293n, 9376n, 13n, 50965n, 51165n, 876n, 11n, 572n, 13n, 51365n, 50257n], + token_timestamps: [0, 0.24, 0.34, 0.34, 0.52, 0.82, 0.98, 1.1, 1.44, 2, 2.36, 2.5, 2.5, 2.5, 2.52, 2.52, 2.72, 3.12, 3.3, 3.3, 3.3, 3.64, 5.5, 6.88, 10.34, 10.56, 10.96, 11.44, 11.64, 12.6, 13.54, 13.78, 15.44, 16.36, 20.3, 20.32, 20.44, 20.5, 20.68, 20.68], + }, + ]; + + const decoded = tokenizer._decode_asr(model_outputs, { + return_timestamps: "word", + time_precision: 0.02, + force_full_sequences: false, + }); + + const [, { chunks }] = decoded; + + // Verify timestamps are monotonically non-decreasing + for (let i = 1; i < chunks.length; ++i) { + const prev = chunks[i - 1].timestamp; + const curr = chunks[i].timestamp; + expect(curr[0]).toBeGreaterThanOrEqual(prev[0]); + } + }, + MAX_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/xlm_roberta/test_tokenization_xlm_roberta.js b/packages/transformers/tests/models/xlm_roberta/test_tokenization_xlm_roberta.js new file mode 100644 index 0000000..08c063c --- /dev/null +++ b/packages/transformers/tests/models/xlm_roberta/test_tokenization_xlm_roberta.js @@ -0,0 +1,332 @@ +import { XLMRobertaTokenizer } from "../../../src/models/tokenizers.js"; +import { BASE_TEST_STRINGS } from "../test_strings.js"; + +export const TOKENIZER_CLASS = XLMRobertaTokenizer; +export const TEST_CONFIG = { + "Xenova/bge-reranker-base": { + SIMPLE: { + text: BASE_TEST_STRINGS.SIMPLE, + tokens: ["\u2581How", "\u2581are", "\u2581you", "\u2581doing", "?"], + ids: [0, 11249, 621, 398, 20594, 32, 2], + decoded: " How are you doing?", + }, + SIMPLE_WITH_PUNCTUATION: { + text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION, + tokens: ["\u2581You", "\u2581should", "'", "ve", "\u2581done", "\u2581this"], + ids: [0, 2583, 5608, 25, 272, 16940, 903, 2], + decoded: " You should've done this", + }, + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u258101", "23", "456", "789", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [0, 3413, 3742, 121317, 153781, 757, 106, 116, 138, 201, 190, 305, 361, 382, 483, 209, 805, 4382, 2], + decoded: " 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581found", "ed", "\u2581in", "\u25812016."], + ids: [0, 581, 14380, 509, 14037, 297, 23, 6360, 2], + decoded: " The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581'", "ll", "\u2581!!", "to", "?", "'", "d", "''", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [0, 62, 242, 1181, 6506, 188, 32, 25, 71, 4765, 71, 111, 4, 831, 25, 18, 5, 2], + decoded: " A 'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581de", "f", "\u2581main", "(", "):", "\u2581pass"], + ids: [0, 8, 420, 5201, 132, 2077, 27875, 2], + decoded: " def main(): pass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581ob", "j", ".", "to", "Str", "ing", "(", ");", "\u2581to", "Str", "ing", "(", ");"], + ids: [0, 2633, 10, 2203, 995, 170, 5, 188, 71713, 214, 132, 3142, 47, 71713, 214, 132, 3142, 2], + decoded: " let a = obj.toString(); toString();", + }, + NEWLINES: { + text: BASE_TEST_STRINGS.NEWLINES, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "."], + ids: [0, 3293, 83, 10, 3034, 5, 2], + decoded: " This is a test.", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "wan", "t\u00e9", "d", ",", "run", "ning"], + ids: [0, 8274, 3206, 2312, 71, 4, 16428, 592, 2], + decoded: " UNwant\u00e9d,running", + }, + CONTROL_TOKENS: { + text: BASE_TEST_STRINGS.CONTROL_TOKENS, + tokens: ["\u25811", "\u0000", "2", "\u25813"], + ids: [0, 106, 3, 304, 138, 2], + decoded: " 12 3", + }, + HELLO_WORLD_TITLECASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE, + tokens: ["\u2581Hello", "\u2581World"], + ids: [0, 35378, 6661, 2], + decoded: " Hello World", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hell", "o", "\u2581world"], + ids: [0, 33600, 31, 8999, 2], + decoded: " hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f\u6d3b\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [0, 6, 62668, 5364, 245875, 354, 2], + decoded: " \u751f\u6d3b\u7684\u771f\u8c1b\u662f", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581leading", "\u2581space"], + ids: [0, 105207, 32628, 2], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trail", "ing", "\u2581space", "\u2581"], + ids: [0, 141037, 214, 32628, 6, 2], + decoded: " trailing space ", + }, + DOUBLE_SPACE: { + text: BASE_TEST_STRINGS.DOUBLE_SPACE, + tokens: ["\u2581Hi", "\u2581Hello"], + ids: [0, 2673, 35378, 2], + decoded: " Hi Hello", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [0, 3034, 38629, 627, 304, 111378, 2505, 617, 11762, 758, 6, 32389, 910, 6, 3, 966, 87316, 1019, 6, 247425, 1126, 3034, 2], + decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$", "1.00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [0, 87, 123997, 142, 108787, 100, 3650, 146533, 99, 70, 4343, 5, 2], + decoded: " I bought an apple for $1.00 at the store.", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [0, 398, 27, 6, 2], + decoded: " you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "...", "\u2581"], + ids: [0, 398, 27, 6, 2], + decoded: " you... ", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "...", "\u2581"], + ids: [0, 398, 27, 398, 27, 6, 2], + decoded: " you... you... ", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], + ids: [0, 179459, 6, 6087, 121303, 6, 6087, 7225, 2], + decoded: " weird \uff5e edge \uff5e case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581", "."], + ids: [0, 3293, 83, 10, 3034, 6, 5, 2], + decoded: " This is a test.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [0, 6, 115114, 6, 118280, 6, 243385, 6, 84464, 6, 232773, 6, 243816, 6, 113612, 6, 82803, 6, 222326, 6, 201344, 6, 239569, 6, 243544, 6, 191876, 6, 243404, 49933, 15755, 6, 244233, 6, 244162, 6, 244181, 6, 243892, 6, 245820, 6, 161546, 6, 204811, 6, 3, 6, 238992, 6, 167474, 6, 120242, 6, 245561, 6, 244864, 6, 246144, 6, 244459, 6, 244703, 6, 246887, 6, 144400, 6, 246511, 6, 142325, 6, 244230, 6, 245559, 6, 243374, 6, 245200, 2], + decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83d\ude05 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83c\udf88 \ud83d\ude48 \ud83d\ude4c \ud83d\udc80 \ud83d\udc47 \ud83d\udc4b \u2705 \ud83c\udf81 \ud83c\udf1e \ud83c\udf38 \ud83d\udcb0", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], + ids: [0, 6, 167474, 6, 243544, 6, 246984, 15755, 6, 247201, 79500, 6, 248325, 6, 228250, 15755, 6, 3, 79500, 6, 228250, 6, 244314, 79500, 6, 246529, 6, 3, 6, 247443, 6, 3, 6, 244785, 49933, 6, 244960, 6, 244314, 6, 244785, 6, 244785, 6, 245719, 6, 246167, 6, 3, 79500, 6, 247443, 6, 3, 79500, 6, 3, 6, 244314, 79500, 49933, 15755, 6, 244960, 6, 244314, 239719, 2], + decoded: " \u2728 \ud83e\udd17 \ud83d\udc41\ufe0f \ud83d\udc71\ud83c\udffb \ud83d\udd75 \u2642\ufe0f \ud83c\udffb \u2642 \ud83d\udc68\ud83c\udffb \ud83c\udf3e \ud83e\udd1d \ud83d\udc69 \u2764 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83d\udc67 \ud83d\udc66 \ud83c\udffb \ud83e\udd1d \ud83c\udffb \ud83d\udc68\ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83d\udc68\ud83c\udffc", + }, + }, + "Xenova/paraphrase-multilingual-mpnet-base-v2": { + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trail", "ing", "\u2581space"], + ids: [0, 141037, 214, 32628, 2], + decoded: " trailing space", + }, + ELLIPSIS: { + text: BASE_TEST_STRINGS.ELLIPSIS, + tokens: ["\u2581you", "..."], + ids: [0, 398, 27, 2], + decoded: " you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS, + tokens: ["\u2581you", "..."], + ids: [0, 398, 27, 2], + decoded: " you...", + }, + TEXT_WITH_ESCAPE_CHARACTERS_2: { + text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2, + tokens: ["\u2581you", "...", "\u2581you", "..."], + ids: [0, 398, 27, 398, 27, 2], + decoded: " you... you...", + }, + }, + "Xenova/donut-base-finetuned-docvqa": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u258101", "23", "45", "67", "89", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [0, 37391, 38611, 41742, 18610, 20121, 50891, 1314, 3822, 9066, 22081, 20017, 35977, 38100, 38873, 42378, 23485, 52285, 40881, 2], + decoded: " 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581'", "ll", "\u2581!!", "to", "?", "'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [0, 46518, 28559, 4558, 47751, 19616, 36209, 28431, 49224, 28431, 28431, 49224, 2587, 35815, 53017, 28431, 16191, 39539, 2], + decoded: " A 'll!!to?'d''d of, can't.", + }, + LEADING_SPACE: { + text: BASE_TEST_STRINGS.LEADING_SPACE, + tokens: ["\u2581lead", "ing", "\u2581space"], + ids: [0, 38498, 24357, 36833, 2], + decoded: " leading space", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trai", "ling", "\u2581space"], + ids: [0, 14262, 23291, 36833, 2], + decoded: " trailing space", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#", "3", "\u2581\u20ac", "4", "\u2581\u00a3", "5", "\u2581", "\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581", "\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [0, 35950, 39065, 46982, 35934, 41882, 38167, 33874, 46702, 4467, 50934, 42990, 36748, 55144, 42990, 3, 56620, 42990, 31354, 486, 42990, 3, 3200, 35950, 2], + decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 9 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581app", "le", "\u2581for", "\u2581$1", ".", "00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [0, 53821, 6018, 9971, 39627, 7897, 57245, 39065, 39539, 49351, 56980, 48941, 40747, 39539, 2], + decoded: " I bought an apple for $1.00 at the store.", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581we", "ird", "\u2581", "\uff5e", "\u2581ed", "ge", "\u2581", "\uff5e", "\u2581case"], + ids: [0, 47450, 52806, 42990, 46476, 25847, 40548, 42990, 46476, 49911, 2], + decoded: " weird \uff5e edge \uff5e case", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581", "\u2764\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [0, 42990, 3864, 42990, 3, 42990, 28873, 42990, 3, 42990, 3, 42990, 29257, 42990, 3, 42990, 3, 42990, 3, 42990, 22310, 42990, 3, 42990, 29017, 42990, 3, 42990, 28890, 42990, 3, 42990, 29601, 42990, 3, 42990, 29564, 42990, 3, 42990, 3, 42990, 14430, 42990, 3, 42990, 3, 42990, 28292, 42990, 3, 42990, 3, 42990, 30688, 42990, 30146, 42990, 3, 42990, 29798, 42990, 3, 42990, 3, 42990, 3, 42990, 3, 42990, 10257, 42990, 3, 42990, 30687, 42990, 3, 42990, 3, 2], + decoded: " \ud83d\ude02 \ud83e\udd23 \ud83c\udf89 \ud83d\ude01 \ud83e\udd17 \ud83d\udc4f \ud83d\udc9c \ud83d\udc97 \ud83d\ude0e \ud83d\udcaa \ud83d\udc40 \ud83d\udcaf \ud83d\ude48 \u2705 \ud83c\udf1e ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41\ufe0f", "\u2581", "\ud83d\udc71\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581", "\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1\ud83c\udffb", "\u2581", "\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\u2764\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], + ids: [0, 42990, 3, 42990, 29017, 42990, 3, 42990, 3, 42990, 3, 42990, 26803, 3, 42990, 3, 42990, 26803, 42990, 29673, 3, 42990, 3, 42990, 3, 42990, 3, 42990, 3, 42990, 30079, 42990, 3, 42990, 30218, 42990, 29673, 42990, 30079, 42990, 30079, 42990, 30799, 42990, 31120, 42990, 3, 42990, 3, 42990, 3, 42990, 3, 42990, 29673, 3, 42990, 3, 42990, 30218, 42990, 29673, 28396, 2], + decoded: " \ud83e\udd17 \u2642 \u2642 \ud83d\udc68 \ud83d\udc69 \ud83d\udc8b \ud83d\udc68 \ud83d\udc69 \ud83d\udc69 \ud83d\udc67 \ud83d\udc66 \ud83d\udc68 \ud83d\udc8b \ud83d\udc68\ud83c\udffc", + }, + }, + "Xenova/trocr-small-handwritten": { + NUMBERS: { + text: BASE_TEST_STRINGS.NUMBERS, + tokens: ["\u25810", "123", "45", "67", "89", "\u25810", "\u25811", "\u25812", "\u25813", "\u25814", "\u25815", "\u25816", "\u25817", "\u25818", "\u25819", "\u258110", "\u2581100", "\u25811000"], + ids: [0, 1596, 31702, 5356, 8248, 7385, 1596, 267, 252, 271, 319, 331, 467, 531, 539, 641, 274, 921, 8401, 2], + decoded: " 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000", + }, + TEXT_WITH_NUMBERS: { + text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS, + tokens: ["\u2581The", "\u2581company", "\u2581was", "\u2581founded", "\u2581in", "\u25812016."], + ids: [0, 25, 215, 19, 3027, 12, 2401, 2], + decoded: " The company was founded in 2016.", + }, + PUNCTUATION: { + text: BASE_TEST_STRINGS.PUNCTUATION, + tokens: ["\u2581A", "\u2581'", "ll", "\u2581!", "!", "to", "?'", "d", "'", "'", "d", "\u2581of", ",", "\u2581can", "'", "t", "."], + ids: [0, 94, 26, 184, 118, 338, 436, 16368, 109, 27, 27, 109, 9, 6, 64, 27, 31, 5, 2], + decoded: " A 'll!!to?'d''d of, can't.", + }, + PYTHON_CODE: { + text: BASE_TEST_STRINGS.PYTHON_CODE, + tokens: ["\u2581def", "\u2581main", "(", "):", "\u2581pass"], + ids: [0, 13114, 830, 2126, 5056, 1080, 2], + decoded: " def main(): pass", + }, + JAVASCRIPT_CODE: { + text: BASE_TEST_STRINGS.JAVASCRIPT_CODE, + tokens: ["\u2581let", "\u2581a", "\u2581=", "\u2581obj", ".", "to", "String", "(", ");", "\u2581to", "String", "(", ");"], + ids: [0, 393, 10, 2219, 48394, 5, 436, 34868, 2126, 3671, 7, 34868, 2126, 3671, 2], + decoded: " let a = obj.toString(); toString();", + }, + BASIC: { + text: BASE_TEST_STRINGS.BASIC, + tokens: ["\u2581UN", "wan", "t\u00e9", "d", ",", "running"], + ids: [0, 3225, 6327, 12529, 109, 6, 11484, 2], + decoded: " UNwant\u00e9d,running", + }, + HELLO_WORLD_LOWERCASE: { + text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE, + tokens: ["\u2581hello", "\u2581world"], + ids: [0, 12773, 218, 2], + decoded: " hello world", + }, + CHINESE_ONLY: { + text: BASE_TEST_STRINGS.CHINESE_ONLY, + tokens: ["\u2581", "\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"], + ids: [0, 190, 63145, 3, 62784, 63741, 3, 63010, 2], + decoded: " \u751f\u7684\u771f\u662f", + }, + TRAILING_SPACE: { + text: BASE_TEST_STRINGS.TRAILING_SPACE, + tokens: ["\u2581trailing", "\u2581space"], + ids: [0, 12250, 769, 2], + decoded: " trailing space", + }, + CURRENCY: { + text: BASE_TEST_STRINGS.CURRENCY, + tokens: ["\u2581test", "\u2581$1", "\u2581R", "2", "\u2581#3", "\u2581\u20ac4", "\u2581\u00a35", "\u2581\u00a5", "6", "\u2581", "\u20a3", "7", "\u2581\u20b9", "8", "\u2581", "\u20b1", "9", "\u2581test"], + ids: [0, 1036, 1594, 791, 792, 28537, 46242, 15364, 45731, 1487, 190, 3, 1473, 32176, 1439, 190, 63400, 1428, 1036, 2], + decoded: " test $1 R2 #3 \u20ac4 \u00a35 \u00a56 7 \u20b98 \u20b19 test", + }, + CURRENCY_WITH_DECIMALS: { + text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS, + tokens: ["\u2581I", "\u2581bought", "\u2581an", "\u2581apple", "\u2581for", "\u2581$1.00", "\u2581at", "\u2581the", "\u2581store", "."], + ids: [0, 14, 1355, 46, 8688, 17, 44092, 29, 4, 920, 5, 2], + decoded: " I bought an apple for $1.00 at the store.", + }, + TILDE_NORMALIZATION: { + text: BASE_TEST_STRINGS.TILDE_NORMALIZATION, + tokens: ["\u2581weird", "\u2581", "\uff5e", "\u2581edge", "\u2581", "\uff5e", "\u2581case"], + ids: [0, 3392, 190, 3, 2297, 190, 3, 343, 2], + decoded: " weird edge case", + }, + SPIECE_UNDERSCORE: { + text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE, + tokens: ["\u2581This", "\u2581is", "\u2581a", "\u2581test", "\u2581."], + ids: [0, 127, 18, 10, 1036, 13, 2], + decoded: " This is a test.", + }, + POPULAR_EMOJIS: { + text: BASE_TEST_STRINGS.POPULAR_EMOJIS, + tokens: ["\u2581", "\ud83d\ude02", "\u2581", "\ud83d\udc4d", "\u2581", "\ud83e\udd23", "\u2581", "\ud83d\ude0d", "\u2581", "\ud83d\ude2d", "\u2581", "\ud83c\udf89", "\u2581", "\ud83d\ude4f", "\u2581", "\ud83d\ude0a", "\u2581", "\ud83d\udd25", "\u2581", "\ud83d\ude01", "\u2581", "\ud83d\ude05", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\ude06", "\u2581", "\ud83d\udc4f", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc9c", "\u2581", "\ud83d\udc9a", "\u2581", "\ud83d\udc97", "\u2581", "\ud83d\udc99", "\u2581", "\ud83d\udda4", "\u2581", "\ud83d\ude0e", "\u2581", "\ud83d\udc4c", "\u2581", "\ud83e\udd73", "\u2581", "\ud83d\udcaa", "\u2581", "\u2728", "\u2581", "\ud83d\udc49", "\u2581", "\ud83d\udc40", "\u2581", "\ud83d\udcaf", "\u2581", "\ud83c\udf88", "\u2581", "\ud83d\ude48", "\u2581", "\ud83d\ude4c", "\u2581", "\ud83d\udc80", "\u2581", "\ud83d\udc47", "\u2581", "\ud83d\udc4b", "\u2581", "\u2705", "\u2581", "\ud83c\udf81", "\u2581", "\ud83c\udf1e", "\u2581", "\ud83c\udf38", "\u2581", "\ud83d\udcb0"], + ids: [0, 190, 45790, 190, 63194, 190, 63067, 190, 62942, 190, 62920, 190, 63116, 190, 62811, 190, 63283, 190, 62997, 190, 63427, 190, 3, 190, 63849, 190, 63751, 190, 62872, 47278, 21661, 190, 63467, 190, 63570, 190, 63473, 190, 63061, 190, 63640, 190, 63468, 190, 63302, 190, 3, 190, 63064, 190, 62860, 190, 63489, 190, 63057, 190, 63543, 190, 3, 190, 3, 190, 62949, 190, 3, 190, 63847, 190, 3, 190, 63747, 190, 3, 190, 3, 190, 3, 190, 3, 2], + decoded: " \ud83d\ude02 \ud83d\udc4d \ud83e\udd23 \ud83d\ude0d \ud83d\ude2d \ud83c\udf89 \ud83d\ude4f \ud83d\ude0a \ud83d\udd25 \ud83d\ude01 \ud83e\udd17 \ud83d\ude06 \ud83d\udc4f \u2764\ufe0f \ud83d\udc9c \ud83d\udc9a \ud83d\udc97 \ud83d\udc99 \ud83d\udda4 \ud83d\ude0e \ud83d\udc4c \ud83d\udcaa \u2728 \ud83d\udc49 \ud83d\udc40 \ud83d\udcaf \ud83d\ude4c \ud83d\udc47 \u2705 ", + }, + MULTIBYTE_EMOJIS: { + text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS, + tokens: ["\u2581", "\u2728", "\u2581", "\ud83e\udd17", "\u2581", "\ud83d\udc41", "\ufe0f", "\u2581", "\ud83d\udc71", "\ud83c\udffb", "\u2581", "\ud83d\udd75", "\u2581", "\u2642", "\ufe0f", "\u2581", "\ud83e\uddd9", "\ud83c\udffb", "\u2581", "\u2642", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581", "\ud83c\udf3e", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\u2581", "\ud83d\udc69", "\u2581\u2764", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc69", "\u2581", "\ud83d\udc67", "\u2581", "\ud83d\udc66", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83e\udd1d", "\u2581", "\ud83e\uddd1", "\ud83c\udffb", "\u2581", "\ud83c\udff4", "\udb40\udc67", "\udb40\udc62\udb40\udc65\udb40\udc6e", "\udb40\udc67", "\udb40\udc7f", "\u2581", "\ud83d\udc68", "\ud83c\udffb", "\u2581\u2764", "\ufe0f", "\u2581", "\ud83d\udc8b", "\u2581", "\ud83d\udc68", "\ud83c\udffc"], + ids: [0, 190, 62860, 190, 63849, 190, 3, 21661, 190, 3, 62863, 190, 3, 190, 63135, 21661, 190, 3, 62863, 190, 63135, 190, 3, 62863, 190, 3, 190, 3, 190, 3, 190, 3, 190, 3, 47278, 190, 63500, 190, 3, 190, 3, 190, 3, 190, 3, 190, 3, 190, 3, 62863, 190, 3, 190, 3, 62863, 190, 3, 63769, 3, 63769, 3, 190, 3, 62863, 47278, 21661, 190, 63500, 190, 3, 62816, 2], + decoded: " \u2728 \ud83e\udd17 \ufe0f \ud83c\udffb \u2642\ufe0f \ud83c\udffb \u2642 \ud83c\udffb \u2764 \ud83d\udc8b \ud83c\udffb \ud83c\udffb \udb40\udc67\udb40\udc67 \ud83c\udffb \u2764\ufe0f \ud83d\udc8b \ud83c\udffc", + }, + }, +}; diff --git a/packages/transformers/tests/models/yolos/test_image_processing_yolos.js b/packages/transformers/tests/models/yolos/test_image_processing_yolos.js new file mode 100644 index 0000000..3eb9c1d --- /dev/null +++ b/packages/transformers/tests/models/yolos/test_image_processing_yolos.js @@ -0,0 +1,31 @@ +import { AutoImageProcessor, YolosFeatureExtractor } from "../../../src/transformers.js"; + +import { load_cached_image } from "../../asset_cache.js"; +import { MAX_PROCESSOR_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "../../init.js"; + +export default () => { + describe("YolosFeatureExtractor", () => { + const model_id = "Xenova/yolos-small-300"; + + /** @type {YolosFeatureExtractor} */ + let processor; + beforeAll(async () => { + processor = await AutoImageProcessor.from_pretrained(model_id); + }, MAX_PROCESSOR_LOAD_TIME); + + it( + "default", + async () => { + const image = await load_cached_image("tiger"); + const { pixel_values, original_sizes, reshaped_input_sizes } = await processor(image); + + expect(pixel_values.dims).toEqual([1, 3, 888, 1333]); + expect(pixel_values.mean().item()).toBeCloseTo(-0.27840224131001773, 6); + + expect(original_sizes).toEqual([[408, 612]]); + expect(reshaped_input_sizes).toEqual([[888, 1333]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/models/zaya/test_modeling_zaya.js b/packages/transformers/tests/models/zaya/test_modeling_zaya.js new file mode 100644 index 0000000..fcda4b9 --- /dev/null +++ b/packages/transformers/tests/models/zaya/test_modeling_zaya.js @@ -0,0 +1,51 @@ +import { PreTrainedTokenizer, ZayaForCausalLM } from "../../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../../init.js"; + +export default () => { + describe("ZayaForCausalLM", () => { + const model_id = "onnx-internal-testing/tiny-random-ZayaForCausalLM"; + /** @type {ZayaForCausalLM} */ + let model; + /** @type {PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await ZayaForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await PreTrainedTokenizer.from_pretrained(model_id); + tokenizer.padding_side = "left"; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([[2n, 23391n, 106n, 155331n, 155331n, 155331n, 155331n, 155331n, 155331n, 155331n]]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "batch_size>1", + async () => { + const inputs = tokenizer(["hello", "hello world"], { padding: true }); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + }); + expect(outputs.tolist()).toEqual([ + [0n, 2n, 23391n, 106n, 155331n, 155331n, 155331n, 155331n, 155331n, 155331n], + [2n, 23391n, 1902n, 106n, 155331n, 155331n, 155331n, 155331n, 155331n, 155331n], + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines.test.js b/packages/transformers/tests/pipelines.test.js new file mode 100644 index 0000000..a7863dc --- /dev/null +++ b/packages/transformers/tests/pipelines.test.js @@ -0,0 +1,6 @@ +import { init } from "./init.js"; +import { collect_and_execute_pipeline_tests } from "./test_utils.js"; + +// Initialise the testing environment +init(); +await collect_and_execute_pipeline_tests("Pipelines"); diff --git a/packages/transformers/tests/pipelines/test_pipelines_audio_classification.js b/packages/transformers/tests/pipelines/test_pipelines_audio_classification.js new file mode 100644 index 0000000..f62f200 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_audio_classification.js @@ -0,0 +1,80 @@ +import { pipeline, AudioClassificationPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +const PIPELINE_ID = "audio-classification"; + +export default () => { + describe("Audio Classification", () => { + const model_id = "hf-internal-testing/tiny-random-unispeech"; + const audios = [new Float32Array(16000).fill(0), Float32Array.from({ length: 16000 }, (_, i) => i)]; + + /** @type {AudioClassificationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of AudioClassificationPipeline", () => { + expect(pipe).toBeInstanceOf(AudioClassificationPipeline); + }); + + describe("batch_size=1", () => { + it( + "default (top_k=5)", + async () => { + const output = await pipe(audios[0]); + const target = [ + { score: 0.5043687224388123, label: "LABEL_0" }, + { score: 0.4956313371658325, label: "LABEL_1" }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=1)", + async () => { + const output = await pipe(audios[0], { top_k: 1 }); + const target = [{ score: 0.5043687224388123, label: "LABEL_0" }]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("batch_size>1", () => { + it( + "default (top_k=5)", + async () => { + const output = await pipe(audios); + const target = [ + [ + { score: 0.5043687224388123, label: "LABEL_0" }, + { score: 0.4956313371658325, label: "LABEL_1" }, + ], + [ + { score: 0.5187293887138367, label: "LABEL_0" }, + { score: 0.4812707006931305, label: "LABEL_1" }, + ], + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=1)", + async () => { + const output = await pipe(audios, { top_k: 1 }); + const target = [[{ score: 0.5043687224388123, label: "LABEL_0" }], [{ score: 0.5187293887138367, label: "LABEL_0" }]]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_automatic_speech_recognition.js b/packages/transformers/tests/pipelines/test_pipelines_automatic_speech_recognition.js new file mode 100644 index 0000000..33d67ac --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_automatic_speech_recognition.js @@ -0,0 +1,462 @@ +import { pipeline, AutomaticSpeechRecognitionPipeline } from "../../src/transformers.js"; +import { load_cached_audio } from "../asset_cache.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +const PIPELINE_ID = "automatic-speech-recognition"; + +export default () => { + describe("Automatic Speech Recognition", () => { + describe("whisper (tiny-random)", () => { + const model_id = "Xenova/tiny-random-WhisperForConditionalGeneration"; + const SAMPLING_RATE = 16000; + const audios = [new Float32Array(SAMPLING_RATE).fill(0), Float32Array.from({ length: SAMPLING_RATE }, (_, i) => i / 16000)]; + const long_audios = [new Float32Array(SAMPLING_RATE * 60).fill(0), Float32Array.from({ length: SAMPLING_RATE * 60 }, (_, i) => (i % 1000) / 1000)]; + + const max_new_tokens = 5; + /** @type {AutomaticSpeechRecognitionPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of AutomaticSpeechRecognitionPipeline", () => { + expect(pipe).toBeInstanceOf(AutomaticSpeechRecognitionPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe(audios[0], { max_new_tokens }); + const target = { text: "นะคะนะคะURURUR" }; + expect(output).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "transcribe w/ return_timestamps=true", + async () => { + const output = await pipe(audios[0], { return_timestamps: true, max_new_tokens }); + const target = { + text: "�를 ис", + chunks: [{ timestamp: [28.96, null], text: "�를 ис" }], + }; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "transcribe w/ language", + async () => { + const output = await pipe(audios[0], { language: "french", task: "transcribe", max_new_tokens }); + const target = { text: "นะคะนะคะURURUR" }; + expect(output).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "translate", + async () => { + const output = await pipe(audios[0], { language: "french", task: "translate", max_new_tokens }); + const target = { text: "นะคะนะคะURURUR" }; + expect(output).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "audio > 30 seconds", + async () => { + const output = await pipe(long_audios[0], { chunk_length_s: 30, stride_length_s: 5, max_new_tokens }); + const target = { text: "นะคะนะคะURURUR" }; + expect(output).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("whisper", () => { + const model_id = "onnx-community/whisper-tiny_timestamped"; + + /** @type {AutomaticSpeechRecognitionPipeline} */ + let pipe; + /** @type {Float64Array} */ + let audio; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + audio = await load_cached_audio("mlk"); + }, 2 * MAX_MODEL_LOAD_TIME); + + it("should be an instance of AutomaticSpeechRecognitionPipeline", () => { + expect(pipe).toBeInstanceOf(AutomaticSpeechRecognitionPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe(audio, { language: "en", task: "transcribe" }); + const target = { + text: " I have a dream. Good one day. This nation will rise up. Live out the true meaning of its dream.", + }; + expect(output).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "transcribe w/ return_timestamps=true", + async () => { + const output = await pipe(audio, { return_timestamps: true, language: "en", task: "transcribe" }); + const target = { + text: " I have a dream. Good one day. This nation will rise up. Live out the true meaning of its dream.", + chunks: [ + { timestamp: [0, 2.74], text: " I have a dream." }, + { timestamp: [2.74, 5.34], text: " Good one day." }, + { timestamp: [5.34, 9.24], text: " This nation will rise up." }, + { timestamp: [9.24, 12.58], text: " Live out the true meaning of its dream." }, + ], + }; + expect(output).toBeCloseToNested(target, 1); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "transcribe w/ return_timestamps='word'", + async () => { + const output = await pipe(audio, { return_timestamps: "word", language: "en", task: "transcribe" }); + const target = { + text: " I have a dream. Good one day. This nation will rise up. Live out the true meaning of its dream.", + chunks: [ + { text: " I", timestamp: [1.36, 1.68] }, + { text: " have", timestamp: [1.68, 1.94] }, + { text: " a", timestamp: [1.94, 2.52] }, + { text: " dream.", timestamp: [2.52, 2.74] }, + { text: " Good", timestamp: [3.98, 4.18] }, + { text: " one", timestamp: [4.18, 4.78] }, + { text: " day.", timestamp: [4.78, 4.84] }, + { text: " This", timestamp: [6.56, 7.2] }, + { text: " nation", timestamp: [7.2, 7.84] }, + { text: " will", timestamp: [7.84, 8.3] }, + { text: " rise", timestamp: [8.3, 9.24] }, + { text: " up.", timestamp: [9.84, 9.86] }, + { text: " Live", timestamp: [10.56, 10.98] }, + { text: " out", timestamp: [10.98, 11.02] }, + { text: " the", timestamp: [11.02, 11.3] }, + { text: " true", timestamp: [11.3, 11.58] }, + { text: " meaning", timestamp: [11.58, 11.86] }, + { text: " of", timestamp: [11.86, 12.06] }, + { text: " its", timestamp: [12.06, 12.56] }, + { text: " dream.", timestamp: [12.56, 12.58] }, + ], + }; + expect(output).toBeCloseToNested(target, 1); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "translate", + async () => { + const output = await pipe(audio, { language: "french", task: "translate" }); + const target = { + text: " I have a dream. Good one day. This nation will rise up. Live out the true meaning of its dream.", + }; + expect(output).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("whisper (en-only)", () => { + const model_id = "onnx-community/whisper-tiny.en_timestamped"; + + /** @type {AutomaticSpeechRecognitionPipeline} */ + let pipe; + /** @type {Float64Array} */ + let audio; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + audio = await load_cached_audio("mlk"); + }, 2 * MAX_MODEL_LOAD_TIME); + + it("should be an instance of AutomaticSpeechRecognitionPipeline", () => { + expect(pipe).toBeInstanceOf(AutomaticSpeechRecognitionPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe(audio); + const target = { + text: " I have a dream that one day this nation will rise up live out the true meaning of its creed", + }; + expect(output).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "transcribe w/ return_timestamps=true", + async () => { + const output = await pipe(audio, { return_timestamps: true }); + const target = { + text: " I have a dream that one day this nation will rise up and live out the true meaning of its creed.", + chunks: [ + { + timestamp: [0, 11.14], + text: " I have a dream that one day this nation will rise up and live out the true", + }, + { timestamp: [11.14, 14.18], text: " meaning of its creed." }, + ], + }; + expect(output).toBeCloseToNested(target, 1); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "transcribe w/ return_timestamps='word'", + async () => { + const output = await pipe(audio, { return_timestamps: "word" }); + const target = { + text: " I have a dream that one day this nation will rise up and live out the true meaning of its creed.", + chunks: [ + { text: " I", timestamp: [1.4, 1.7] }, + { text: " have", timestamp: [1.7, 1.88] }, + { text: " a", timestamp: [1.88, 2.4] }, + { text: " dream", timestamp: [2.4, 3.92] }, + { text: " that", timestamp: [3.92, 4.22] }, + { text: " one", timestamp: [4.22, 5.12] }, + { text: " day", timestamp: [5.12, 6.66] }, + { text: " this", timestamp: [6.66, 7.34] }, + { text: " nation", timestamp: [7.34, 7.86] }, + { text: " will", timestamp: [7.86, 8.34] }, + { text: " rise", timestamp: [8.34, 9.9] }, + { text: " up", timestamp: [9.9, 10.3] }, + { text: " and", timestamp: [10.3, 10.6] }, + { text: " live", timestamp: [10.6, 10.86] }, + { text: " out", timestamp: [10.86, 11.04] }, + { text: " the", timestamp: [11.04, 11.14] }, + { text: " true", timestamp: [11.34, 11.38] }, + { text: " meaning", timestamp: [11.62, 11.78] }, + { text: " of", timestamp: [11.78, 12.12] }, + { text: " its", timestamp: [12.12, 12.64] }, + { text: " creed.", timestamp: [12.64, 13.6] }, + ], + }; + expect(output).toBeCloseToNested(target, 1); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("whisper (base)", () => { + const model_id = "onnx-community/whisper-base_timestamped"; + + /** @type {AutomaticSpeechRecognitionPipeline} */ + let pipe; + /** @type {Float64Array} */ + let audio; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + audio = await load_cached_audio("whisper_1"); + }, 3 * MAX_MODEL_LOAD_TIME); + + it("should be an instance of AutomaticSpeechRecognitionPipeline", () => { + expect(pipe).toBeInstanceOf(AutomaticSpeechRecognitionPipeline); + }); + + it( + "transcribe w/ return_timestamps=true", + async () => { + const output = await pipe(audio, { + return_timestamps: true, + chunk_length_s: 30, + stride_length_s: 5, + language: "en", + }); + // Python output for reference: + // const target = { + // text: " everyday style. True classic delivers premium essentials built for real life. Grab yours at Target, Costco, or head to TrueClassic.com slash P4P. Get hooked up today. Now before we go, just want to give a big shout out to the CEO and founder Ryan brother for coming on our show and just showing some love. Now let's get back to the episode. I mean, like I said, we're going through that. We're losing stars. And then we kind of...", + // chunks: [ + // { timestamp: [0.0, 4.8], text: " everyday style. True classic delivers premium essentials built for real life." }, + // { timestamp: [5.36, 14.0], text: " Grab yours at Target, Costco, or head to TrueClassic.com slash P4P. Get hooked up today." }, + // { timestamp: [14.0, 18.56], text: " Now before we go, just want to give a big shout out to the CEO and founder Ryan" }, + // { timestamp: [18.56, 23.6], text: " brother for coming on our show and just showing some love. Now let's get back to the episode." }, + // { timestamp: [24.24, 23.6], text: "" }, + // { timestamp: [27.02, 29.18], text: " I mean, like I said, we're going through that. We're losing stars." }, + // { timestamp: [29.18, 30.54], text: " And then we kind of..." }, + // ], + // }; + + const target = { + text: " everyday style. True classic delivers premium essentials built for real life. Grab yours at Target, Costco, or head to TrueClassic.com slash P4P. Get hooked up today. Now before we go, just want to give a big shout out to the CEO and founder Ryan brother for coming on our show and just showing some love. Now let's get back to the episode. I mean, like I said, we're going through that. We're losing stars. And then we kind of...", + chunks: [ + { timestamp: [0, 4.8], text: " everyday style. True classic delivers premium essentials built for real life." }, + { timestamp: [5.36, 14], text: " Grab yours at Target, Costco, or head to TrueClassic.com slash P4P. Get hooked up today." }, + { timestamp: [14, 18.56], text: " Now before we go, just want to give a big shout out to the CEO and founder Ryan" }, + { timestamp: [18.56, 23.6], text: " brother for coming on our show and just showing some love. Now let's get back to the episode." }, + { timestamp: [23.6, 27.02], text: " I mean, like I said, we're going through that." }, + { timestamp: [27.02, 29.18], text: " We're losing stars." }, + { timestamp: [29.18, 30], text: " And then we kind of..." }, + ], + }; + + expect(output).toBeCloseToNested(target, 1); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "transcribe w/ return_timestamps='word'", + async () => { + // Regression test: word-level timestamps must include content from all seek + // passes, not just the first. Without the seek loop for return_token_timestamps, + // the text was truncated after "the episode." (~23.6s), missing the final ~7s. + const output = await pipe(audio, { + return_timestamps: "word", + chunk_length_s: 30, + stride_length_s: 5, + language: "en", + }); + + const target = { + text: " everyday style. True classic delivers premium essentials built for real life. Grab yours at Target, Costco, or head to TrueClassic.com slash P4P. Get hooked up today. Now before we go, just want to give a big shout out to the CEO and founder Ryan brother for coming on our show and just showing some love. Now let's get back to the episode. I mean, like I said, we're going through that. We're losing stars. And then we kind of...", + chunks: [ + { text: " everyday", timestamp: [0.42, 0.82] }, + { text: " style.", timestamp: [0.82, 1.3] }, + { text: " True", timestamp: [1.56, 1.9] }, + { text: " classic", timestamp: [1.9, 2.5] }, + { text: " delivers", timestamp: [2.5, 3.02] }, + { text: " premium", timestamp: [3.02, 3.54] }, + { text: " essentials", timestamp: [3.54, 3.84] }, + { text: " built", timestamp: [3.84, 4.08] }, + { text: " for", timestamp: [4.08, 4.4] }, + { text: " real", timestamp: [4.4, 4.8] }, + { text: " life.", timestamp: [4.92, 5.46] }, + { text: " Grab", timestamp: [5.64, 6.04] }, + { text: " yours", timestamp: [6.04, 6.36] }, + { text: " at", timestamp: [6.36, 6.84] }, + { text: " Target,", timestamp: [6.84, 7.2] }, + { text: " Costco,", timestamp: [7.7, 8.28] }, + { text: " or", timestamp: [8.3, 8.5] }, + { text: " head", timestamp: [8.5, 8.7] }, + { text: " to", timestamp: [8.7, 8.92] }, + { text: " TrueClassic", timestamp: [8.92, 9.68] }, + { text: ".com", timestamp: [9.68, 10.78] }, + { text: " slash", timestamp: [10.78, 11.3] }, + { text: " P4P.", timestamp: [11.3, 12.86] }, + { text: " Get", timestamp: [13.06, 13.3] }, + { text: " hooked", timestamp: [13.3, 13.52] }, + { text: " up", timestamp: [13.52, 13.92] }, + { text: " today.", timestamp: [13.92, 14] }, + { text: " Now", timestamp: [14.24, 14.46] }, + { text: " before", timestamp: [14.46, 14.62] }, + { text: " we", timestamp: [14.62, 14.82] }, + { text: " go,", timestamp: [14.82, 15.06] }, + { text: " just", timestamp: [15.24, 15.42] }, + { text: " want", timestamp: [15.42, 15.48] }, + { text: " to", timestamp: [15.48, 15.56] }, + { text: " give", timestamp: [15.56, 15.68] }, + { text: " a", timestamp: [15.68, 15.86] }, + { text: " big", timestamp: [15.86, 16.1] }, + { text: " shout", timestamp: [16.1, 16.28] }, + { text: " out", timestamp: [16.28, 16.74] }, + { text: " to", timestamp: [16.74, 16.9] }, + { text: " the", timestamp: [16.9, 17.52] }, + { text: " CEO", timestamp: [17.52, 17.86] }, + { text: " and", timestamp: [17.86, 18.3] }, + { text: " founder", timestamp: [18.3, 18.56] }, + { text: " Ryan", timestamp: [18.6, 18.72] }, + { text: " brother", timestamp: [18.86, 19.06] }, + { text: " for", timestamp: [19.06, 19.2] }, + { text: " coming", timestamp: [19.2, 19.36] }, + { text: " on", timestamp: [19.36, 19.5] }, + { text: " our", timestamp: [19.5, 19.74] }, + { text: " show", timestamp: [19.74, 20.4] }, + { text: " and", timestamp: [20.4, 20.6] }, + { text: " just", timestamp: [20.6, 20.86] }, + { text: " showing", timestamp: [20.86, 21.08] }, + { text: " some", timestamp: [21.08, 21.26] }, + { text: " love.", timestamp: [21.26, 21.48] }, + { text: " Now", timestamp: [21.6, 22.3] }, + { text: " let's", timestamp: [22.3, 22.46] }, + { text: " get", timestamp: [22.46, 22.68] }, + { text: " back", timestamp: [22.68, 23.06] }, + { text: " to", timestamp: [23.06, 23.2] }, + { text: " the", timestamp: [23.2, 23.56] }, + { text: " episode.", timestamp: [23.56, 23.6] }, + { text: " I", timestamp: [24.42, 24.6] }, + { text: " mean,", timestamp: [24.6, 25.1] }, + { text: " like", timestamp: [25.42, 25.54] }, + { text: " I", timestamp: [25.54, 25.68] }, + { text: " said,", timestamp: [25.68, 25.9] }, + { text: " we're", timestamp: [25.9, 26.34] }, + { text: " going", timestamp: [26.34, 26.58] }, + { text: " through", timestamp: [26.58, 26.84] }, + { text: " that.", timestamp: [26.84, 27] }, + { text: " We're", timestamp: [27.24, 27.54] }, + { text: " losing", timestamp: [27.54, 28.04] }, + { text: " stars.", timestamp: [28.04, 28.96] }, + { text: " And", timestamp: [29.3, 29.38] }, + { text: " then", timestamp: [29.38, 29.58] }, + { text: " we", timestamp: [29.58, 29.84] }, + { text: " kind", timestamp: [29.84, 29.98] }, + { text: " of", timestamp: [29.98, 29.98] }, + { text: "...", timestamp: [29.98, 29.98] }, + ], + }; + expect(output).toBeCloseToNested(target, 1); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("wav2vec2", () => { + const model_id = "Xenova/tiny-random-Wav2Vec2ForCTC-ONNX"; + const SAMPLING_RATE = 16000; + const audios = [new Float32Array(SAMPLING_RATE).fill(0), Float32Array.from({ length: SAMPLING_RATE }, (_, i) => i / 16000)]; + const long_audios = [new Float32Array(SAMPLING_RATE * 60).fill(0), Float32Array.from({ length: SAMPLING_RATE * 60 }, (_, i) => (i % 1000) / 1000)]; + + const max_new_tokens = 5; + /** @type {AutomaticSpeechRecognitionPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of AutomaticSpeechRecognitionPipeline", () => { + expect(pipe).toBeInstanceOf(AutomaticSpeechRecognitionPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe(audios[0], { max_new_tokens }); + const target = { text: "K" }; + expect(output).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_background_removal.js b/packages/transformers/tests/pipelines/test_pipelines_background_removal.js new file mode 100644 index 0000000..1a7c96b --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_background_removal.js @@ -0,0 +1,68 @@ +import { pipeline, BackgroundRemovalPipeline, RawImage } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; +import { load_cached_image } from "../asset_cache.js"; + +const PIPELINE_ID = "background-removal"; + +export default () => { + describe("Background Removal", () => { + describe("Portrait Segmentation", () => { + const model_id = "Xenova/modnet"; + /** @type {BackgroundRemovalPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of BackgroundRemovalPipeline", () => { + expect(pipe).toBeInstanceOf(BackgroundRemovalPipeline); + }); + + it( + "single", + async () => { + const image = await load_cached_image("portrait_of_woman"); + + const output = await pipe(image); + expect(output).toBeInstanceOf(RawImage); + expect(output.width).toEqual(image.width); + expect(output.height).toEqual(image.height); + expect(output.channels).toEqual(4); // With alpha channel + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("Selfie Segmentation", () => { + const model_id = "onnx-community/mediapipe_selfie_segmentation"; + /** @type {BackgroundRemovalPipeline } */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it( + "single", + async () => { + const image = await load_cached_image("portrait_of_woman"); + + const output = await pipe(image); + expect(output).toBeInstanceOf(RawImage); + expect(output.width).toEqual(image.width); + expect(output.height).toEqual(image.height); + expect(output.channels).toEqual(4); // With alpha channel + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_depth_estimation.js b/packages/transformers/tests/pipelines/test_pipelines_depth_estimation.js new file mode 100644 index 0000000..61d4c50 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_depth_estimation.js @@ -0,0 +1,57 @@ +import { pipeline, DepthEstimationPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; +import { load_cached_image } from "../asset_cache.js"; + +const PIPELINE_ID = "depth-estimation"; + +export default () => { + describe("Depth Estimation", () => { + const model_id = "hf-internal-testing/tiny-random-DPTForDepthEstimation"; + /** @type {DepthEstimationPipeline} */ + let pipe; + let images; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + images = await Promise.all([load_cached_image("white_image"), load_cached_image("blue_image")]); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of DepthEstimationPipeline", () => { + expect(pipe).toBeInstanceOf(DepthEstimationPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe(images[0]); + expect(output.predicted_depth.dims).toEqual([224, 224]); + expect(output.predicted_depth.mean().item()).toBeCloseTo(0.000006106501587055391, 4); + expect(output.depth.size).toEqual(images[0].size); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("batch_size>1", () => { + it( + "default", + async () => { + const output = await pipe(images); + expect(output).toHaveLength(images.length); + expect(output[0].predicted_depth.dims).toEqual([224, 224]); + expect(output[0].predicted_depth.mean().item()).toBeCloseTo(0.000006106501587055391, 4); + expect(output[0].depth.size).toEqual(images[0].size); + expect(output[1].predicted_depth.dims).toEqual([224, 224]); + expect(output[1].predicted_depth.mean().item()).toBeCloseTo(0.0000014548650142387487, 4); + expect(output[1].depth.size).toEqual(images[1].size); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_document_question_answering.js b/packages/transformers/tests/pipelines/test_pipelines_document_question_answering.js new file mode 100644 index 0000000..bd9f238 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_document_question_answering.js @@ -0,0 +1,43 @@ +import { pipeline, DocumentQuestionAnsweringPipeline, RawImage } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +const PIPELINE_ID = "document-question-answering"; + +export default () => { + describe("Document Question Answering", () => { + const model_id = "hf-internal-testing/tiny-random-VisionEncoderDecoderModel-donutswin-mbart"; + + /** @type {DocumentQuestionAnsweringPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of DocumentQuestionAnsweringPipeline", () => { + expect(pipe).toBeInstanceOf(DocumentQuestionAnsweringPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const dims = [64, 32, 3]; + const image = new RawImage(new Uint8ClampedArray(dims[0] * dims[1] * dims[2]).fill(255), ...dims); + const question = "What is the invoice number?"; + const output = await pipe(image, question, { + max_new_tokens: 5, + }); + + const target = [{ answer: null }]; + expect(output).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_feature_extraction.js b/packages/transformers/tests/pipelines/test_pipelines_feature_extraction.js new file mode 100644 index 0000000..b113b9e --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_feature_extraction.js @@ -0,0 +1,121 @@ +import { pipeline, FeatureExtractionPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +const PIPELINE_ID = "feature-extraction"; + +export default () => { + describe("Feature Extraction", () => { + const model_id = "hf-internal-testing/tiny-random-BertModel"; + + const texts = ["This is a simple test.", "Hello world"]; + + /** @type {FeatureExtractionPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of FeatureExtractionPipeline ", () => { + expect(pipe).toBeInstanceOf(FeatureExtractionPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe(texts[0]); + expect(output.dims).toEqual([1, 20, 32]); + expect(output.type).toEqual("float32"); + expect(output.mean().item()).toBeCloseTo(-1.538501215314625e-9, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "w/ cls pooling", + async () => { + const output = await pipe(texts[0], { pooling: "cls" }); + expect(output.dims).toEqual([1, 32]); + expect(output.type).toEqual("float32"); + expect(output.mean().item()).toBeCloseTo(2.491287887096405e-8, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "w/ mean pooling & normalization", + async () => { + const output = await pipe(texts[0], { pooling: "mean", normalize: true }); + expect(output.dims).toEqual([1, 32]); + expect(output.type).toEqual("float32"); + expect(output.mean().item()).toBeCloseTo(-2.0245352061465383e-9, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "w/ mean pooling & binary quantization", + async () => { + const output = await pipe(texts[0], { pooling: "mean", quantize: true, precision: "binary" }); + expect(output.dims).toEqual([1, 32 / 8]); + expect(output.type).toEqual("int8"); + expect(output.mean().item()).toEqual(-15); + }, + MAX_TEST_EXECUTION_TIME, + ); + it("w/ cls pooling & ubinary quantization", async () => { + const output = await pipe(texts[0], { pooling: "cls", quantize: true, precision: "ubinary" }); + expect(output.dims).toEqual([1, 32 / 8]); + expect(output.type).toEqual("uint8"); + expect(output.mean().item()).toEqual(140); + }); + }); + + describe("batch_size>1", () => { + it( + "default", + async () => { + const output = await pipe(texts); + expect(output.dims).toEqual([texts.length, 20, 32]); + expect(output.type).toEqual("float32"); + expect(output.mean().item()).toBeCloseTo(2.345950544935249e-9, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "w/ cls pooling", + async () => { + const output = await pipe(texts, { pooling: "cls" }); + expect(output.dims).toEqual([texts.length, 32]); + expect(output.type).toEqual("float32"); + expect(output.mean().item()).toBeCloseTo(1.6298145055770874e-8, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "w/ mean pooling & normalization", + async () => { + const output = await pipe(texts, { pooling: "mean", normalize: true }); + expect(output.dims).toEqual([texts.length, 32]); + expect(output.type).toEqual("float32"); + expect(output.mean().item()).toBeCloseTo(-1.538609240014921e-10, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + it("w/ mean pooling & binary quantization", async () => { + const output = await pipe(texts, { pooling: "mean", quantize: true, precision: "binary" }); + expect(output.dims).toEqual([texts.length, 32 / 8]); + expect(output.type).toEqual("int8"); + expect(output.mean().item()).toEqual(-14); + }); + it("w/ cls pooling & ubinary quantization", async () => { + const output = await pipe(texts, { pooling: "cls", quantize: true, precision: "ubinary" }); + expect(output.dims).toEqual([texts.length, 32 / 8]); + expect(output.type).toEqual("uint8"); + expect(output.mean().item()).toEqual(140); + }); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_fill_mask.js b/packages/transformers/tests/pipelines/test_pipelines_fill_mask.js new file mode 100644 index 0000000..5a72ea8 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_fill_mask.js @@ -0,0 +1,165 @@ +import { pipeline, FillMaskPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +const PIPELINE_ID = "fill-mask"; + +export default () => { + describe("Fill Mask", () => { + describe("Standard", () => { + const model_id = "hf-internal-testing/tiny-random-BertForMaskedLM"; + + /** @type {FillMaskPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of FillMaskPipeline", () => { + expect(pipe).toBeInstanceOf(FillMaskPipeline); + }); + + describe("batch_size=1", () => { + it( + "default (top_k=5)", + async () => { + const output = await pipe("a [MASK] c"); + const target = [ + { score: 0.0013377574505284429, token: 854, token_str: "##ο", sequence: "aο c" }, + { score: 0.001248967950232327, token: 962, token_str: "##ち", sequence: "aち c" }, + { score: 0.0012304208939895034, token: 933, token_str: "##ع", sequence: "aع c" }, + { score: 0.0012301815440878272, token: 313, token_str: "ფ", sequence: "a ფ c" }, + { score: 0.001222139224410057, token: 624, token_str: "未", sequence: "a 未 c" }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=2)", + async () => { + const output = await pipe("a [MASK] c", { top_k: 2 }); + const target = [ + { score: 0.0013377574505284429, token: 854, token_str: "##ο", sequence: "aο c" }, + { score: 0.001248967950232327, token: 962, token_str: "##ち", sequence: "aち c" }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("batch_size>1", () => { + it( + "default (top_k=5)", + async () => { + const output = await pipe(["a [MASK] c", "a b [MASK] c"]); + const target = [ + [ + { score: 0.0013377574505284429, token: 854, token_str: "##ο", sequence: "aο c" }, + { score: 0.001248967950232327, token: 962, token_str: "##ち", sequence: "aち c" }, + { score: 0.0012304208939895034, token: 933, token_str: "##ع", sequence: "aع c" }, + { score: 0.0012301815440878272, token: 313, token_str: "ფ", sequence: "a ფ c" }, + { score: 0.001222139224410057, token: 624, token_str: "未", sequence: "a 未 c" }, + ], + [ + { score: 0.0013287801994010806, token: 962, token_str: "##ち", sequence: "a bち c" }, + { score: 0.0012486606137827039, token: 823, token_str: "##ن", sequence: "a bن c" }, + { score: 0.0012320734094828367, token: 1032, token_str: "##ც", sequence: "a bც c" }, + { score: 0.0012295148335397243, token: 854, token_str: "##ο", sequence: "a bο c" }, + { score: 0.0012277684872969985, token: 624, token_str: "未", sequence: "a b 未 c" }, + ], + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=2)", + async () => { + const output = await pipe(["a [MASK] c", "a b [MASK] c"], { top_k: 2 }); + const target = [ + [ + { score: 0.0013377574505284429, token: 854, token_str: "##ο", sequence: "aο c" }, + { score: 0.001248967950232327, token: 962, token_str: "##ち", sequence: "aち c" }, + ], + [ + { score: 0.0013287801994010806, token: 962, token_str: "##ち", sequence: "a bち c" }, + { score: 0.0012486606137827039, token: 823, token_str: "##ن", sequence: "a bن c" }, + ], + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("Custom tokenizer", () => { + const model_id = "hf-internal-testing/tiny-random-ModernBertForMaskedLM"; + + /** @type {FillMaskPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of FillMaskPipeline", () => { + expect(pipe).toBeInstanceOf(FillMaskPipeline); + }); + + describe("batch_size=1", () => { + it( + "default (top_k=5)", + async () => { + const output = await pipe("The capital of France is [MASK]."); + const target = [ + { score: 0.2106737643480301, sequence: "The capital of France isả.", token: 35165, token_str: "ả" }, + { score: 0.18418768048286438, sequence: "The capital of France isDispatch.", token: 48010, token_str: "Dispatch" }, + { score: 0.16561225056648254, sequence: "The capital of France is Ther.", token: 20763, token_str: " Ther" }, + { score: 0.07070659101009369, sequence: "The capital of France isschild.", token: 50040, token_str: "schild" }, + { score: 0.029540402814745903, sequence: "The capital of France isbles.", token: 9143, token_str: "bles" }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("batch_size>1", () => { + it( + "default (top_k=5)", + async () => { + const output = await pipe(["a [MASK] c", "a b [MASK] c"]); + const target = [ + [ + { score: 0.06699250638484955, sequence: "a oocytes c", token: 36805, token_str: " oocytes" }, + { score: 0.05928678810596466, sequence: "ancia c", token: 19003, token_str: "ncia" }, + { score: 0.057058464735746384, sequence: "aả c", token: 35165, token_str: "ả" }, + { score: 0.04978331923484802, sequence: "amq c", token: 37365, token_str: "mq" }, + { score: 0.04839889705181122, sequence: "a1371 c", token: 26088, token_str: "1371" }, + ], + [ + { score: 0.06364646553993225, sequence: "a b oocytes c", token: 36805, token_str: " oocytes" }, + { score: 0.03993292525410652, sequence: "a bectin c", token: 41105, token_str: "ectin" }, + { score: 0.03932870551943779, sequence: "a bả c", token: 35165, token_str: "ả" }, + { score: 0.037771403789520264, sequence: "a boplastic c", token: 21945, token_str: "oplastic" }, + { score: 0.03748754784464836, sequence: "a b Ther c", token: 20763, token_str: " Ther" }, + ], + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_image_classification.js b/packages/transformers/tests/pipelines/test_pipelines_image_classification.js new file mode 100644 index 0000000..2940000 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_image_classification.js @@ -0,0 +1,81 @@ +import { pipeline, ImageClassificationPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; +import { load_cached_image } from "../asset_cache.js"; + +const PIPELINE_ID = "image-classification"; + +export default () => { + describe("Image Classification", () => { + const model_id = "hf-internal-testing/tiny-random-vit"; + /** @type {ImageClassificationPipeline} */ + let pipe; + let images; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + images = await Promise.all([load_cached_image("white_image"), load_cached_image("blue_image")]); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of ImageClassificationPipeline", () => { + expect(pipe).toBeInstanceOf(ImageClassificationPipeline); + }); + + describe("batch_size=1", () => { + it( + "default (top_k=5)", + async () => { + const output = await pipe(images[0]); + const target = [ + { label: "LABEL_1", score: 0.5020533800125122 }, + { label: "LABEL_0", score: 0.4979466497898102 }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=1)", + async () => { + const output = await pipe(images[0], { top_k: 1 }); + const target = [{ label: "LABEL_1", score: 0.5020533800125122 }]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("batch_size>1", () => { + it( + "default (top_k=5)", + async () => { + const output = await pipe(images); + const target = [ + [ + { label: "LABEL_1", score: 0.5020533800125122 }, + { label: "LABEL_0", score: 0.4979466497898102 }, + ], + [ + { label: "LABEL_1", score: 0.519227921962738 }, + { label: "LABEL_0", score: 0.4807720482349396 }, + ], + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=1)", + async () => { + const output = await pipe(images, { top_k: 1 }); + const target = [[{ label: "LABEL_1", score: 0.5020533800125122 }], [{ label: "LABEL_1", score: 0.519227921962738 }]]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_image_feature_extraction.js b/packages/transformers/tests/pipelines/test_pipelines_image_feature_extraction.js new file mode 100644 index 0000000..14b3671 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_image_feature_extraction.js @@ -0,0 +1,95 @@ +import { pipeline, ImageFeatureExtractionPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; +import { load_cached_image } from "../asset_cache.js"; + +const PIPELINE_ID = "image-feature-extraction"; + +export default () => { + describe("Image Feature Extraction", () => { + describe("Default", () => { + const model_id = "hf-internal-testing/tiny-random-ViTMAEModel"; + /** @type {ImageFeatureExtractionPipeline} */ + let pipe; + let images; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + images = await Promise.all([load_cached_image("white_image"), load_cached_image("blue_image")]); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of ImageFeatureExtractionPipeline", () => { + expect(pipe).toBeInstanceOf(ImageFeatureExtractionPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe(images[0]); + expect(output.dims).toEqual([1, 91, 32]); + expect(output.mean().item()).toBeCloseTo(-8.507473614471905e-10, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("batch_size>1", () => { + it( + "default", + async () => { + const output = await pipe(images); + expect(output.dims).toEqual([images.length, 91, 32]); + expect(output.mean().item()).toBeCloseTo(-5.997602414709036e-10, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + describe("CLIP-like", () => { + const model_id = "hf-internal-testing/tiny-random-CLIPModel"; + /** @type {ImageFeatureExtractionPipeline} */ + let pipe; + let images; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + images = await Promise.all([load_cached_image("white_image"), load_cached_image("blue_image")]); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of ImageFeatureExtractionPipeline", () => { + expect(pipe).toBeInstanceOf(ImageFeatureExtractionPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe(images[0]); + expect(output.dims).toEqual([1, 64]); + expect(output.mean().item()).toBeCloseTo(-0.11340035498142242, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("batch_size>1", () => { + it( + "default", + async () => { + const output = await pipe(images); + expect(output.dims).toEqual([images.length, 64]); + expect(output.mean().item()).toBeCloseTo(-0.11006651818752289, 6); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_image_segmentation.js b/packages/transformers/tests/pipelines/test_pipelines_image_segmentation.js new file mode 100644 index 0000000..7e02024 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_image_segmentation.js @@ -0,0 +1,119 @@ +import { pipeline, ImageSegmentationPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; +import { load_cached_image } from "../asset_cache.js"; + +const PIPELINE_ID = "image-segmentation"; + +export default () => { + describe("Image Segmentation", () => { + describe("Panoptic Segmentation", () => { + const model_id = "Xenova/detr-resnet-50-panoptic"; + /** @type {ImageSegmentationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of ImageSegmentationPipeline", () => { + expect(pipe).toBeInstanceOf(ImageSegmentationPipeline); + }); + + it( + "single", + async () => { + const image = await load_cached_image("cats"); + + const output = await pipe(image); + + // First, check mask shapes + for (const item of output) { + expect(item.mask.width).toEqual(image.width); + expect(item.mask.height).toEqual(image.height); + expect(item.mask.channels).toEqual(1); + delete item.mask; // No longer needed + } + + // Next, compare scores and labels + const target = [ + { + score: 0.9918501377105713, + label: "cat", + }, + { + score: 0.9985815286636353, + label: "remote", + }, + { + score: 0.999537467956543, + label: "remote", + }, + { + score: 0.9919270277023315, + label: "couch", + }, + { + score: 0.9993696808815002, + label: "cat", + }, + ]; + + expect(output).toBeCloseToNested(target, 2); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("Semantic Segmentation", () => { + const model_id = "Xenova/segformer_b0_clothes"; + /** @type {ImageSegmentationPipeline } */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it( + "single", + async () => { + const image = await load_cached_image("man_on_car"); + + const output = await pipe(image); + + // First, check mask shapes + for (const item of output) { + expect(item.mask.width).toEqual(image.width); + expect(item.mask.height).toEqual(image.height); + expect(item.mask.channels).toEqual(1); + delete item.mask; // No longer needed + } + + // Next, compare scores and labels + const target = [ + { score: null, label: "Background" }, + { score: null, label: "Hair" }, + { score: null, label: "Upper-clothes" }, + { score: null, label: "Pants" }, + { score: null, label: "Left-shoe" }, + { score: null, label: "Right-shoe" }, + { score: null, label: "Face" }, + { score: null, label: "Right-leg" }, + { score: null, label: "Left-arm" }, + { score: null, label: "Right-arm" }, + { score: null, label: "Bag" }, + ]; + + expect(output).toBeCloseToNested(target, 2); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_image_to_image.js b/packages/transformers/tests/pipelines/test_pipelines_image_to_image.js new file mode 100644 index 0000000..1e06c11 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_image_to_image.js @@ -0,0 +1,56 @@ +import { pipeline, ImageToImagePipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; +import { load_cached_image } from "../asset_cache.js"; + +const PIPELINE_ID = "image-to-image"; + +export default () => { + describe("Image to Image", () => { + const model_id = "hf-internal-testing/tiny-random-Swin2SRForImageSuperResolution"; + /** @type {ImageToImagePipeline} */ + let pipe; + let images; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + images = await Promise.all([load_cached_image("white_image"), load_cached_image("blue_image")]); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of ImageToImagePipeline", () => { + expect(pipe).toBeInstanceOf(ImageToImagePipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe(images[0]); + expect(output.size).toEqual([64, 64]); + expect(output.channels).toEqual(3); + expect(output.data.reduce((a, b) => a + b, 0) / output.data.length).toBeCloseTo(110.107421875, 3); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("batch_size>1", () => { + it( + "default", + async () => { + const output = await pipe(images); + expect(output[0].size).toEqual([64, 64]); + expect(output[0].channels).toEqual(3); + expect(output[0].data.reduce((a, b) => a + b, 0) / output[0].data.length).toBeCloseTo(110.107421875, 3); + expect(output[1].size).toEqual([64, 64]); + expect(output[1].channels).toEqual(3); + expect(output[1].data.reduce((a, b) => a + b, 0) / output[1].data.length).toBeCloseTo(110.60196940104167, 3); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_image_to_text.js b/packages/transformers/tests/pipelines/test_pipelines_image_to_text.js new file mode 100644 index 0000000..3ef8a29 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_image_to_text.js @@ -0,0 +1,51 @@ +import { pipeline, ImageToTextPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; +import { load_cached_image } from "../asset_cache.js"; + +const PIPELINE_ID = "image-to-text"; + +export default () => { + describe("Image to Text", () => { + const model_id = "hf-internal-testing/tiny-random-VisionEncoderDecoderModel-vit-gpt2"; + /** @type {ImageToTextPipeline} */ + let pipe; + let images; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + images = await Promise.all([load_cached_image("white_image"), load_cached_image("blue_image")]); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of ImageToTextPipeline", () => { + expect(pipe).toBeInstanceOf(ImageToTextPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe(images[0]); + const target = [{ generated_text: "" }]; + expect(output).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("batch_size>1", () => { + it( + "default", + async () => { + const output = await pipe(images); + const target = [[{ generated_text: "" }], [{ generated_text: "" }]]; + expect(output).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_object_detection.js b/packages/transformers/tests/pipelines/test_pipelines_object_detection.js new file mode 100644 index 0000000..1573597 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_object_detection.js @@ -0,0 +1,131 @@ +import { pipeline, ObjectDetectionPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; +import { load_cached_image } from "../asset_cache.js"; + +const PIPELINE_ID = "object-detection"; + +export default () => { + describe("Object Detection", () => { + describe("yolos", () => { + const model_id = "Xenova/yolos-tiny"; + /** @type {ObjectDetectionPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of ObjectDetectionPipeline", () => { + expect(pipe).toBeInstanceOf(ObjectDetectionPipeline); + }); + + it( + "single + threshold", + async () => { + const image = await load_cached_image("cats"); + const output = await pipe(image, { threshold: 0.9 }); + + const target = [ + { + score: 0.9921281933784485, + label: "remote", + box: { xmin: 32, ymin: 78, xmax: 185, ymax: 117 }, + }, + { + score: 0.9884883165359497, + label: "remote", + box: { xmin: 324, ymin: 82, xmax: 376, ymax: 191 }, + }, + { + score: 0.9197800159454346, + label: "cat", + box: { xmin: 5, ymin: 56, xmax: 321, ymax: 469 }, + }, + { + score: 0.9300552606582642, + label: "cat", + box: { xmin: 332, ymin: 25, xmax: 638, ymax: 369 }, + }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("tiny-random", () => { + const model_id = "hf-internal-testing/tiny-random-DetrForObjectDetection"; + + /** @type {ObjectDetectionPipeline} */ + let pipe; + let images; + + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + images = await Promise.all([load_cached_image("white_image"), load_cached_image("blue_image")]); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of ObjectDetectionPipeline", () => { + expect(pipe).toBeInstanceOf(ObjectDetectionPipeline); + }); + + describe("batch_size=1", () => { + it( + "default (threshold unset)", + async () => { + const output = await pipe(images[0]); + const target = []; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "default (threshold=0)", + async () => { + const output = await pipe(images[0], { threshold: 0 }); + const target = [ + { score: 0.020360443741083145, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, + { score: 0.020360419526696205, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, + { score: 0.02036038413643837, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, + { score: 0.020360447466373444, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, + { score: 0.020360389724373817, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, + { score: 0.020360423251986504, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, + { score: 0.02036040835082531, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, + { score: 0.020360363647341728, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, + { score: 0.020360389724373817, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, + { score: 0.020360389724373817, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, + { score: 0.020360343158245087, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, + { score: 0.020360423251986504, label: "LABEL_31", box: { xmin: 56, ymin: 55, xmax: 169, ymax: 167 } }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + // TODO: Add batched support to object detection pipeline + // describe('batch_size>1', () => { + // it('default (threshold unset)', async () => { + // const output = await pipe(images); + // console.log(output); + // const target = []; + // expect(output).toBeCloseToNested(target, 5); + // }, MAX_TEST_EXECUTION_TIME); + // it('default (threshold=0)', async () => { + // const output = await pipe(images, { threshold: 0 }); + // console.log(output); + // const target = []; + // expect(output).toBeCloseToNested(target, 5); + // }, MAX_TEST_EXECUTION_TIME); + // }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_question_answering.js b/packages/transformers/tests/pipelines/test_pipelines_question_answering.js new file mode 100644 index 0000000..fdee014 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_question_answering.js @@ -0,0 +1,49 @@ +import { pipeline, QuestionAnsweringPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +const PIPELINE_ID = "question-answering"; + +export default () => { + describe("Question Answering", () => { + const model_id = "hf-internal-testing/tiny-random-BertForQuestionAnswering"; + /** @type {QuestionAnsweringPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of QuestionAnsweringPipeline", () => { + expect(pipe).toBeInstanceOf(QuestionAnsweringPipeline); + }); + + describe("batch_size=1", () => { + it( + "default (top_k=1)", + async () => { + const output = await pipe("a", "b c"); + const target = { score: 0.11395696550607681, /* start: 0, end: 1, */ answer: "b" }; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=3)", + async () => { + const output = await pipe("a", "b c", { top_k: 3 }); + const target = [ + { score: 0.11395696550607681, /* start: 0, end: 1, */ answer: "b" }, + { score: 0.11300431191921234, /* start: 2, end: 3, */ answer: "c" }, + { score: 0.10732574015855789, /* start: 0, end: 3, */ answer: "b c" }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_summarization.js b/packages/transformers/tests/pipelines/test_pipelines_summarization.js new file mode 100644 index 0000000..0fc18ea --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_summarization.js @@ -0,0 +1,40 @@ +import { pipeline, SummarizationPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +const PIPELINE_ID = "summarization"; + +export default () => { + describe("Summarization", () => { + const model_id = "hf-internal-testing/tiny-random-T5ForConditionalGeneration"; + + /** @type {SummarizationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of SummarizationPipeline", () => { + expect(pipe).toBeInstanceOf(SummarizationPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const text = "This is a test."; + const output = await pipe(text, { + max_new_tokens: 5, + }); + const target = [{ summary_text: "" }]; + expect(output).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_text2text_generation.js b/packages/transformers/tests/pipelines/test_pipelines_text2text_generation.js new file mode 100644 index 0000000..d8d756f --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_text2text_generation.js @@ -0,0 +1,40 @@ +import { pipeline, Text2TextGenerationPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +const PIPELINE_ID = "text2text-generation"; + +export default () => { + describe("Text to Text Generation", () => { + const model_id = "hf-internal-testing/tiny-random-T5ForConditionalGeneration"; + + /** @type {Text2TextGenerationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of Text2TextGenerationPipeline", () => { + expect(pipe).toBeInstanceOf(Text2TextGenerationPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const text = "This is a test."; + const output = await pipe(text, { + max_new_tokens: 5, + }); + const target = [{ generated_text: "" }]; + expect(output).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_text_classification.js b/packages/transformers/tests/pipelines/test_pipelines_text_classification.js new file mode 100644 index 0000000..91c855d --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_text_classification.js @@ -0,0 +1,107 @@ +import { pipeline, TextClassificationPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +const PIPELINE_ID = "text-classification"; + +export default () => { + describe("Text Classification", () => { + const model_id = "hf-internal-testing/tiny-random-BertForSequenceClassification"; + + /** @type {TextClassificationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of TextClassificationPipeline", () => { + expect(pipe).toBeInstanceOf(TextClassificationPipeline); + }); + + describe("batch_size=1", () => { + it( + "default (top_k=1)", + async () => { + const output = await pipe("a"); + const target = [{ label: "LABEL_0", score: 0.5076976418495178 }]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=2)", + async () => { + const output = await pipe("a", { top_k: 2 }); + const target = [ + { label: "LABEL_0", score: 0.5076976418495178 }, + { label: "LABEL_1", score: 0.49230238795280457 }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("batch_size>1", () => { + it( + "default (top_k=1)", + async () => { + const output = await pipe(["a", "b c"]); + const target = [ + { label: "LABEL_0", score: 0.5076976418495178 }, + { label: "LABEL_0", score: 0.5077522993087769 }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (top_k=2)", + async () => { + const output = await pipe(["a", "b c"], { top_k: 2 }); + const target = [ + [ + { label: "LABEL_0", score: 0.5076976418495178 }, + { label: "LABEL_1", score: 0.49230238795280457 }, + ], + [ + { label: "LABEL_0", score: 0.5077522993087769 }, + { label: "LABEL_1", score: 0.49224773049354553 }, + ], + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "multi_label_classification", + async () => { + const problem_type = pipe.model.config.problem_type; + pipe.model.config.problem_type = "multi_label_classification"; + + const output = await pipe(["a", "b c"], { top_k: 2 }); + const target = [ + [ + { label: "LABEL_0", score: 0.5001373887062073 }, + { label: "LABEL_1", score: 0.49243971705436707 }, + ], + [ + { label: "LABEL_0", score: 0.5001326203346252 }, + { label: "LABEL_1", score: 0.492380291223526 }, + ], + ]; + expect(output).toBeCloseToNested(target, 5); + + // Reset problem type + pipe.model.config.problem_type = problem_type; + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_text_generation.js b/packages/transformers/tests/pipelines/test_pipelines_text_generation.js new file mode 100644 index 0000000..68e2d86 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_text_generation.js @@ -0,0 +1,317 @@ +import { pipeline, TextGenerationPipeline, DynamicCache } from "../../src/transformers.js"; +import { jest } from "@jest/globals"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +const PIPELINE_ID = "text-generation"; + +export default () => { + describe("Text Generation", () => { + const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; + + /** @type {TextGenerationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of TextGenerationPipeline", () => { + expect(pipe).toBeInstanceOf(TextGenerationPipeline); + }); + + describe("batch_size=1", () => { + const text_input = "hello"; + const generated_text_target = "erdingsdelete mely"; + const text_target = [{ generated_text: text_input + generated_text_target }]; + const new_text_target = [{ generated_text: generated_text_target }]; + + const chat_input = [ + { role: "system", content: "a" }, + { role: "user", content: "b" }, + ]; + const chat_target = [ + { + generated_text: [ + { role: "system", content: "a" }, + { role: "user", content: "b" }, + { role: "assistant", content: " Southern abund Load" }, + ], + }, + ]; + + it( + "text input (single)", + async () => { + const output = await pipe(text_input, { max_new_tokens: 3 }); + expect(output).toEqual(text_target); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "text input (list)", + async () => { + const output = await pipe([text_input], { max_new_tokens: 3 }); + expect(output).toEqual([text_target]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "text input (single) - return_full_text=false", + async () => { + const output = await pipe(text_input, { max_new_tokens: 3, return_full_text: false }); + expect(output).toEqual(new_text_target); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "text input (list) - return_full_text=false", + async () => { + const output = await pipe([text_input], { max_new_tokens: 3, return_full_text: false }); + expect(output).toEqual([new_text_target]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "chat input (single)", + async () => { + const output = await pipe(chat_input, { max_new_tokens: 3 }); + expect(output).toEqual(chat_target); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "chat input (list)", + async () => { + const output = await pipe([chat_input], { max_new_tokens: 3 }); + expect(output).toEqual([chat_target]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "chat input forwards top-level tools", + async () => { + const tools = [ + { + name: "get_weather", + description: "Get the weather in a city", + parameter_definitions: { + location: { + description: "City and country", + type: "str", + required: true, + }, + }, + }, + ]; + + const spy = jest.spyOn(pipe.tokenizer, "apply_chat_template"); + await pipe(chat_input, { + max_new_tokens: 3, + do_sample: false, + tools, + }); + + const [, options] = spy.mock.calls.at(-1); + expect(options.tools).toEqual(tools); + expect(options.add_generation_prompt).toBe(true); + spy.mockRestore(); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + // TODO: Fix batch_size>1 + // describe('batch_size>1', () => { + // it('default', async () => { + // const output = await pipe(['hello', 'hello world']); + // const target = [ + // [{generated_text: 'helloerdingsAndroid Load'}], + // [{generated_text: 'hello world zerosMillнал'}], + // ]; + // expect(output).toEqual(target); + // }, MAX_TEST_EXECUTION_TIME); + // }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("Text Generation (Qwen3_5 VL model, text-only)", () => { + const model_id = "onnx-internal-testing/tiny-random-Qwen3_5ForConditionalGeneration"; + + /** @type {TextGenerationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of TextGenerationPipeline", () => { + expect(pipe).toBeInstanceOf(TextGenerationPipeline); + }); + + it("should load only embed_tokens and decoder_model_merged when the model is a image-text-to-text model", async () => { + const sessions = pipe.model.sessions; + expect(sessions).toHaveProperty("embed_tokens"); + expect(sessions).toHaveProperty("decoder_model_merged"); + expect(sessions).not.toHaveProperty("vision_encoder"); + }); + + it( + "text input (single)", + async () => { + const output = await pipe("hello", { max_new_tokens: 3, return_full_text: false, do_sample: false }); + expect(output).toEqual([{ generated_text: "\u0e1e\u0e22\u0e32\u0e1a\u0e32\u0e25Composition directives" }]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("Text Generation (Gemma3 model, text-only)", () => { + const model_id = "onnx-internal-testing/tiny-random-Gemma3ForConditionalGeneration"; + + /** @type {TextGenerationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of TextGenerationPipeline", () => { + expect(pipe).toBeInstanceOf(TextGenerationPipeline); + }); + + it("should load only embed_tokens and decoder_model_merged when the model is a image-text-to-text model", async () => { + const sessions = pipe.model.sessions; + expect(sessions).toHaveProperty("embed_tokens"); + expect(sessions).toHaveProperty("decoder_model_merged"); + expect(sessions).not.toHaveProperty("vision_encoder"); + }); + + it( + "text input (single)", + async () => { + const output = await pipe("hello", { max_new_tokens: 3, return_full_text: false, do_sample: false }); + expect(output).toEqual([{ generated_text: "hellohellohello" }]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "chat input (single)", + async () => { + const output = await pipe([{ role: "user", content: "hello" }], { max_new_tokens: 3, do_sample: false }); + expect(output).toEqual([ + { + generated_text: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "\n\n\n" }, + ], + }, + ]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("Text Generation (LFM2 model, DynamicCache PKV)", () => { + const model_id = "onnx-internal-testing/tiny-random-Lfm2ForCausalLM"; + + /** @type {TextGenerationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of TextGenerationPipeline", () => { + expect(pipe).toBeInstanceOf(TextGenerationPipeline); + }); + + it( + "multi-turn with past_key_values matches without", + async () => { + const generate_kwargs = { max_new_tokens: 8, do_sample: false }; + const past_key_values = new DynamicCache(); + + const messages = [{ role: "user", content: "What is the capital of France?" }]; + + // Turn 1 + { + const with_pkv = await pipe(messages, { ...generate_kwargs, past_key_values }); + const without_pkv = await pipe(messages, generate_kwargs); + expect(with_pkv[0].generated_text.at(-1).content).toEqual(without_pkv[0].generated_text.at(-1).content); + expect(past_key_values.get_seq_length()).toBeGreaterThan(0); + messages.push(with_pkv[0].generated_text.at(-1)); + } + + // Turn 2 + { + messages.push({ role: "user", content: "What about Germany?" }); + const with_pkv = await pipe(messages, { ...generate_kwargs, past_key_values }); + const without_pkv = await pipe(messages, generate_kwargs); + expect(with_pkv[0].generated_text.at(-1).content).toEqual(without_pkv[0].generated_text.at(-1).content); + messages.push(with_pkv[0].generated_text.at(-1)); + } + + // Turn 3 + { + messages.push({ role: "user", content: "And Spain?" }); + const with_pkv = await pipe(messages, { ...generate_kwargs, past_key_values }); + const without_pkv = await pipe(messages, generate_kwargs); + expect(with_pkv[0].generated_text.at(-1).content).toEqual(without_pkv[0].generated_text.at(-1).content); + } + + await past_key_values.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("Text Generation (Gemma3n model, text-only)", () => { + const model_id = "onnx-internal-testing/tiny-random-Gemma3nForConditionalGeneration"; + + /** @type {TextGenerationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of TextGenerationPipeline", () => { + expect(pipe).toBeInstanceOf(TextGenerationPipeline); + }); + + it("should load only embed_tokens and decoder_model_merged when the model is a image-text-to-text model", async () => { + const sessions = pipe.model.sessions; + expect(sessions).toHaveProperty("embed_tokens"); + expect(sessions).toHaveProperty("decoder_model_merged"); + expect(sessions).not.toHaveProperty("vision_encoder"); + }); + + it( + "text input (single)", + async () => { + const output = await pipe("hello", { max_new_tokens: 3, return_full_text: false, do_sample: false }); + expect(output).toEqual([{ generated_text: "\u0443\u0440\u0430corporate" }]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_text_to_audio.js b/packages/transformers/tests/pipelines/test_pipelines_text_to_audio.js new file mode 100644 index 0000000..8f0a4d3 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_text_to_audio.js @@ -0,0 +1,37 @@ +import { pipeline, TextToAudioPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +const PIPELINE_ID = "text-to-audio"; + +export default () => { + describe("Text to Audio", () => { + const model_id = "Xenova/tiny-random-vits"; + + /** @type {TextToAudioPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of TextToAudioPipeline", () => { + expect(pipe).toBeInstanceOf(TextToAudioPipeline); + }); + + it( + "default", + async () => { + const output = await pipe("hello"); + expect(output.audio).toHaveLength(6400); + // NOTE: The mean value is not deterministic, so we just check the first few digits + expect(output.audio.reduce((a, b) => a + b, 0) / output.audio.length).toBeCloseTo(-0.0125, 2); + expect(output.sampling_rate).toEqual(16000); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_token_classification.js b/packages/transformers/tests/pipelines/test_pipelines_token_classification.js new file mode 100644 index 0000000..9dc0b8b --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_token_classification.js @@ -0,0 +1,373 @@ +import { pipeline, TokenClassificationPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +const PIPELINE_ID = "token-classification"; + +export default () => { + describe("Token Classification", () => { + const model_id = "hf-internal-testing/tiny-random-BertForTokenClassification"; + /** @type {TokenClassificationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of TokenClassificationPipeline", () => { + expect(pipe).toBeInstanceOf(TokenClassificationPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe("1 2 3"); + + // TODO: Add start/end to target + const target = [ + { + entity: "LABEL_0", + score: 0.5292708, + index: 1, + word: "1", + // 'start': 0, 'end': 1 + }, + { + entity: "LABEL_0", + score: 0.5353687, + index: 2, + word: "2", + // 'start': 2, 'end': 3 + }, + { + entity: "LABEL_1", + score: 0.51381934, + index: 3, + word: "3", + // 'start': 4, 'end': 5 + }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (ignore_labels set)", + async () => { + const output = await pipe("1 2 3", { ignore_labels: ["LABEL_0"] }); + const target = [ + { + entity: "LABEL_1", + score: 0.51381934, + index: 3, + word: "3", + // 'start': 4, 'end': 5 + }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("batch_size>1", () => { + it( + "default", + async () => { + const output = await pipe(["1 2 3", "4 5"]); + const target = [ + [ + { + entity: "LABEL_0", + score: 0.5292708, + index: 1, + word: "1", + // 'start': 0, 'end': 1 + }, + { + entity: "LABEL_0", + score: 0.5353687, + index: 2, + word: "2", + // 'start': 2, 'end': 3 + }, + { + entity: "LABEL_1", + score: 0.51381934, + index: 3, + word: "3", + // 'start': 4, 'end': 5 + }, + ], + [ + { + entity: "LABEL_0", + score: 0.5432807, + index: 1, + word: "4", + // 'start': 0, 'end': 1 + }, + { + entity: "LABEL_1", + score: 0.5007693, + index: 2, + word: "5", + // 'start': 2, 'end': 3 + }, + ], + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (ignore_labels set)", + async () => { + const output = await pipe(["1 2 3", "4 5"], { ignore_labels: ["LABEL_0"] }); + const target = [ + [ + { + entity: "LABEL_1", + score: 0.51381934, + index: 3, + word: "3", + // 'start': 4, 'end': 5 + }, + ], + [ + { + entity: "LABEL_1", + score: 0.5007693, + index: 2, + word: "5", + // 'start': 2, 'end': 3 + }, + ], + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("Token Classification w/ aggregation_strategy", () => { + const model_id = "hf-internal-testing/tiny-random-BertForTokenClassification"; + const inputs = ["1 2 3", "4 5"]; + + describe("BIO (B-PER / I-PER)", () => { + /** @type {TokenClassificationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, { + ...DEFAULT_MODEL_OPTIONS, + config: { + model_type: "bert", + id2label: { 0: "B-PER", 1: "I-PER" }, + label2id: { "B-PER": 0, "I-PER": 1 }, + }, + }); + }, MAX_MODEL_LOAD_TIME); + + it( + "aggregation_strategy='simple' merges adjacent same-tag tokens", + async () => { + const output = await pipe(inputs, { aggregation_strategy: "simple" }); + const target = [ + [ + { entity_group: "PER", score: 0.5292708, word: "1" }, + { entity_group: "PER", score: 0.524594, word: "2 3" }, + ], + [{ entity_group: "PER", score: 0.52202505, word: "4 5" }], + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("BIO with O (O / B-PER)", () => { + /** @type {TokenClassificationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, { + ...DEFAULT_MODEL_OPTIONS, + config: { + model_type: "bert", + id2label: { 0: "O", 1: "B-PER" }, + label2id: { O: 0, "B-PER": 1 }, + }, + }); + }, MAX_MODEL_LOAD_TIME); + + it( + "aggregation_strategy='simple' drops O-labeled tokens", + async () => { + const output = await pipe(inputs, { aggregation_strategy: "simple" }); + const target = [[{ entity_group: "PER", score: 0.51381934, word: "3" }], [{ entity_group: "PER", score: 0.5007693, word: "5" }]]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "aggregation_strategy='simple' with ignore_labels=[] keeps O as a group", + async () => { + const output = await pipe(inputs, { aggregation_strategy: "simple", ignore_labels: [] }); + const target = [ + [ + { entity_group: "O", score: 0.5323198, word: "1 2" }, + { entity_group: "PER", score: 0.51381934, word: "3" }, + ], + [ + { entity_group: "O", score: 0.5432808, word: "4" }, + { entity_group: "PER", score: 0.5007693, word: "5" }, + ], + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("BIOES (E-PER / B-PER)", () => { + /** @type {TokenClassificationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, { + ...DEFAULT_MODEL_OPTIONS, + config: { + model_type: "bert", + id2label: { 0: "E-PER", 1: "B-PER" }, + label2id: { "E-PER": 0, "B-PER": 1 }, + }, + }); + }, MAX_MODEL_LOAD_TIME); + + it( + "aggregation_strategy='simple' treats E- as continuation, B- opens a new group, E- terminates", + async () => { + const output = await pipe(inputs, { aggregation_strategy: "simple" }); + // Labels for `1 2 3`: [E-PER, E-PER, B-PER]. + // E-PER at the start opens a group (best-effort recovery for a malformed sequence), + // the second E-PER continues + closes it, then B-PER opens a fresh group. + // Labels for `4 5`: [E-PER, B-PER]. + const target = [ + [ + { entity_group: "PER", score: 0.5323198, word: "1 2" }, + { entity_group: "PER", score: 0.51381934, word: "3" }, + ], + [ + { entity_group: "PER", score: 0.5432808, word: "4" }, + { entity_group: "PER", score: 0.5007693, word: "5" }, + ], + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("BIOES terminator (E- closes a group even when tag matches)", () => { + /** @type {TokenClassificationPipeline} */ + let pipe; + beforeAll(async () => { + // With this remap, the tiny model assigns `I-PER` to tokens "1" and "2" + // and `E-PER` to token "3". The E- must close the group, so a second + // sentence of `I-PER, E-PER` (tokens "4 5") forms its own group. + pipe = await pipeline(PIPELINE_ID, model_id, { + ...DEFAULT_MODEL_OPTIONS, + config: { + model_type: "bert", + id2label: { 0: "I-PER", 1: "E-PER" }, + label2id: { "I-PER": 0, "E-PER": 1 }, + }, + }); + }, MAX_MODEL_LOAD_TIME); + + it( + "aggregation_strategy='simple' folds I-* / E-* into one group per terminator", + async () => { + const output = await pipe(inputs, { aggregation_strategy: "simple" }); + const target = [[{ entity_group: "PER", score: 0.52614963, word: "1 2 3" }], [{ entity_group: "PER", score: 0.522025, word: "4 5" }]]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("BIOES singleton (S-)", () => { + /** @type {TokenClassificationPipeline} */ + let pipe; + beforeAll(async () => { + // `S-PER` on every token — each one must be its own group. + pipe = await pipeline(PIPELINE_ID, model_id, { + ...DEFAULT_MODEL_OPTIONS, + config: { + model_type: "bert", + id2label: { 0: "S-PER", 1: "S-PER" }, + label2id: { "S-PER": 0 }, + }, + }); + }, MAX_MODEL_LOAD_TIME); + + it( + "aggregation_strategy='simple' keeps each S- token as its own group", + async () => { + const output = await pipe(inputs, { aggregation_strategy: "simple" }); + const target = [ + [ + { entity_group: "PER", score: 0.5292708, word: "1" }, + { entity_group: "PER", score: 0.5353687, word: "2" }, + { entity_group: "PER", score: 0.51381934, word: "3" }, + ], + [ + { entity_group: "PER", score: 0.5432808, word: "4" }, + { entity_group: "PER", score: 0.5007693, word: "5" }, + ], + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + it( + "throws on an unsupported aggregation_strategy", + async () => { + const pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + try { + // @ts-expect-error intentionally bad + await expect(pipe("1 2 3", { aggregation_strategy: "first" })).rejects.toThrow(/aggregation_strategy/); + } finally { + await pipe.dispose(); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_translation.js b/packages/transformers/tests/pipelines/test_pipelines_translation.js new file mode 100644 index 0000000..c1dc5d9 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_translation.js @@ -0,0 +1,42 @@ +import { pipeline, TranslationPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +const PIPELINE_ID = "translation"; + +export default () => { + describe("Translation", () => { + const model_id = "Xenova/tiny-random-M2M100ForConditionalGeneration"; + + /** @type {TranslationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of TranslationPipeline", () => { + expect(pipe).toBeInstanceOf(TranslationPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const text = "जीवन एक चॉकलेट बॉक्स की तरह है।"; + const output = await pipe(text, { + src_lang: "hi", + tgt_lang: "fr", + max_new_tokens: 5, + }); + const target = [{ translation_text: "Slovenska төсли төсли төсли" }]; + expect(output).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_zero_shot.js b/packages/transformers/tests/pipelines/test_pipelines_zero_shot.js new file mode 100644 index 0000000..49c8cf5 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_zero_shot.js @@ -0,0 +1,100 @@ +import { pipeline, ZeroShotClassificationPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +const PIPELINE_ID = "zero-shot-classification"; + +export default () => { + describe("Zero-shot Classification", () => { + const model_id = "hf-internal-testing/tiny-random-BertForSequenceClassification"; + /** @type {ZeroShotClassificationPipeline} */ + let pipe; + + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, { + ...DEFAULT_MODEL_OPTIONS, + + // The model isn't designed for zero-shot classification, so we set the config + config: { + model_type: "bert", + id2label: { + 0: "contradiction", + 1: "entailment", + }, + label2id: { + contradiction: 0, + entailment: 1, + }, + }, + }); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of ZeroShotClassificationPipeline", () => { + expect(pipe).toBeInstanceOf(ZeroShotClassificationPipeline); + }); + const sequences_to_classify = ["one day I will see the world", "I love making pizza"]; + const candidate_labels = ["travel", "cooking", "dancing"]; + + it( + "Single sequence classification", + async () => { + const output = await pipe(sequences_to_classify[0], candidate_labels); + const target = { + sequence: "one day I will see the world", + labels: ["dancing", "cooking", "travel"], + scores: [0.3333353410546293, 0.3333348269618681, 0.3333298319835025], + }; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "Batched classification", + async () => { + const output = await pipe(sequences_to_classify, candidate_labels); + const target = [ + { + sequence: "one day I will see the world", + labels: ["dancing", "cooking", "travel"], + scores: [0.3333353410546293, 0.3333348269618681, 0.3333298319835025], + }, + { + sequence: "I love making pizza", + labels: ["dancing", "cooking", "travel"], + scores: [0.3333347058960895, 0.3333337292465588, 0.3333315648573516], + }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "Batched + multilabel classification", + async () => { + const candidate_labels = ["travel", "cooking", "dancing"]; + + const output = await pipe(sequences_to_classify, candidate_labels, { multi_label: true }); + const target = [ + { + sequence: "one day I will see the world", + labels: ["dancing", "cooking", "travel"], + scores: [0.49231469615364476, 0.4923134953805702, 0.4923094795142658], + }, + { + sequence: "I love making pizza", + labels: ["dancing", "cooking", "travel"], + scores: [0.49230751217535645, 0.49230615475943956, 0.4923042569480609], + }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_zero_shot_audio_classification.js b/packages/transformers/tests/pipelines/test_pipelines_zero_shot_audio_classification.js new file mode 100644 index 0000000..ba7a600 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_zero_shot_audio_classification.js @@ -0,0 +1,58 @@ +import { pipeline, ZeroShotAudioClassificationPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; +import { load_cached_audio } from "../asset_cache.js"; + +const PIPELINE_ID = "zero-shot-audio-classification"; + +export default () => { + describe("Zero-shot Audio Classification", () => { + const model_id = "hf-internal-testing/tiny-clap-htsat-unfused"; + + const labels = ["cat", "dog"]; + const hypothesis_template = "sound of a {}"; + + /** @type {ZeroShotAudioClassificationPipeline} */ + let pipe; + let audio; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + audio = await load_cached_audio("mlk"); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of ZeroShotAudioClassificationPipeline", () => { + expect(pipe).toBeInstanceOf(ZeroShotAudioClassificationPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe(audio, labels); + const target = [ + { score: 0.4990939795970917, label: "cat" }, + { score: 0.5009059906005859, label: "dog" }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (w/ hypothesis_template)", + async () => { + const output = await pipe(audio, labels, { hypothesis_template }); + const target = [ + { score: 0.4987950325012207, label: "cat" }, + { score: 0.5012049674987793, label: "dog" }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_zero_shot_image_classification.js b/packages/transformers/tests/pipelines/test_pipelines_zero_shot_image_classification.js new file mode 100644 index 0000000..dee7202 --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_zero_shot_image_classification.js @@ -0,0 +1,98 @@ +import { pipeline, ZeroShotImageClassificationPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; +import { load_cached_image } from "../asset_cache.js"; + +const PIPELINE_ID = "zero-shot-image-classification"; + +export default () => { + describe("Zero-shot Image Classification", () => { + const model_id = "hf-internal-testing/tiny-random-GroupViTModel"; + + // Example adapted from https://huggingface.co/docs/transformers/en/model_doc/groupvit + const labels = ["cat", "dog"]; + const hypothesis_template = "a photo of a {}"; + + /** @type {ZeroShotImageClassificationPipeline} */ + let pipe; + let images; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + images = await Promise.all([load_cached_image("white_image"), load_cached_image("blue_image")]); + }, MAX_MODEL_LOAD_TIME); + + it("should be an instance of ZeroShotImageClassificationPipeline", () => { + expect(pipe).toBeInstanceOf(ZeroShotImageClassificationPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe(images[0], labels); + const target = [ + { score: 0.5990662574768066, label: "cat" }, + { score: 0.40093377232551575, label: "dog" }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (w/ hypothesis_template)", + async () => { + const output = await pipe(images[0], labels, { hypothesis_template }); + const target = [ + { score: 0.5527022480964661, label: "cat" }, + { score: 0.44729775190353394, label: "dog" }, + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("batch_size>1", () => { + it( + "default", + async () => { + const output = await pipe(images, labels); + const target = [ + [ + { score: 0.5990662574768066, label: "cat" }, + { score: 0.40093377232551575, label: "dog" }, + ], + [ + { score: 0.5006340146064758, label: "dog" }, + { score: 0.49936598539352417, label: "cat" }, + ], + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (w/ hypothesis_template)", + async () => { + const output = await pipe(images, labels, { hypothesis_template }); + const target = [ + [ + { score: 0.5527022480964661, label: "cat" }, + { score: 0.44729775190353394, label: "dog" }, + ], + [ + { score: 0.5395973324775696, label: "cat" }, + { score: 0.46040263772010803, label: "dog" }, + ], + ]; + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}; diff --git a/packages/transformers/tests/pipelines/test_pipelines_zero_shot_object_detection.js b/packages/transformers/tests/pipelines/test_pipelines_zero_shot_object_detection.js new file mode 100644 index 0000000..4883b1a --- /dev/null +++ b/packages/transformers/tests/pipelines/test_pipelines_zero_shot_object_detection.js @@ -0,0 +1,186 @@ +import { pipeline, ZeroShotObjectDetectionPipeline } from "../../src/transformers.js"; + +import { MAX_MODEL_LOAD_TIME, MAX_TEST_EXECUTION_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; +import { load_cached_image } from "../asset_cache.js"; + +const PIPELINE_ID = "zero-shot-object-detection"; + +export default () => { + describe("Zero-shot Object Detection", () => { + describe("w/ post_process_object_detection", () => { + const model_id = "hf-internal-testing/tiny-random-OwlViTForObjectDetection"; + + const candidate_labels = ["hello", "hello world"]; + + /** @type {ZeroShotObjectDetectionPipeline} */ + let pipe; + let images; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + images = await Promise.all([load_cached_image("white_image"), load_cached_image("blue_image")]); + }, MAX_MODEL_LOAD_TIME); + + const targets = { + white_image: [ + { + score: 0.6028420329093933, + label: "hello", + box: { xmin: 47, ymin: 117, xmax: 62, ymax: 134 }, + }, + { + score: 0.6026064157485962, + label: "hello world", + box: { xmin: 47, ymin: 117, xmax: 62, ymax: 134 }, + }, + { + score: 0.5987668037414551, + label: "hello world", + box: { xmin: 145, ymin: 47, xmax: 160, ymax: 63 }, + }, + { + score: 0.5986272692680359, + label: "hello", + box: { xmin: 89, ymin: 131, xmax: 104, ymax: 148 }, + }, + { + score: 0.5985949039459229, + label: "hello world", + box: { xmin: 89, ymin: 131, xmax: 104, ymax: 148 }, + }, + // ... many more + ], + + blue_image: [ + { + score: 0.6622366309165955, + label: "hello", + box: { xmin: 48, ymin: 45, xmax: 62, ymax: 61 }, + }, + { + score: 0.6562080383300781, + label: "hello world", + box: { xmin: 48, ymin: 45, xmax: 62, ymax: 61 }, + }, + { + score: 0.6493991613388062, + label: "hello world", + box: { xmin: 34, ymin: 58, xmax: 48, ymax: 74 }, + }, + { + score: 0.6476974487304688, + label: "hello", + box: { xmin: 34, ymin: 58, xmax: 48, ymax: 74 }, + }, + { + score: 0.6391685009002686, + label: "hello", + box: { xmin: 103, ymin: 59, xmax: 117, ymax: 75 }, + }, + // ... many more + ], + }; + + it("should be an instance of ZeroShotObjectDetectionPipeline", () => { + expect(pipe).toBeInstanceOf(ZeroShotObjectDetectionPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe(images[0], candidate_labels); + expect(output).toHaveLength(512); + + expect(output.slice(0, targets.white_image.length)).toBeCloseToNested(targets.white_image, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (w/ top_k & threshold)", + async () => { + const top_k = 3; + const output = await pipe(images[0], candidate_labels, { top_k, threshold: 0.05 }); + expect(output).toBeCloseToNested(targets.white_image.slice(0, top_k), 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("batch_size>1", () => { + it( + "default", + async () => { + const output = await pipe(images, candidate_labels); + const target = Object.values(targets); + expect(output.map((x, i) => x.slice(0, target[i].length))).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (w/ top_k & threshold)", + async () => { + const top_k = 3; + const output = await pipe(images, candidate_labels, { top_k, threshold: 0.05 }); + const target = Object.values(targets).map((x) => x.slice(0, top_k)); + expect(output).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("w/ post_process_grounded_object_detection", () => { + const model_id = "hf-internal-testing/tiny-random-GroundingDinoForObjectDetection"; + + const candidate_labels = ["a cat."]; + + /** @type {ZeroShotObjectDetectionPipeline} */ + let pipe; + let image; + beforeAll(async () => { + pipe = await pipeline(PIPELINE_ID, model_id, DEFAULT_MODEL_OPTIONS); + image = await load_cached_image("white_image"); + }, MAX_MODEL_LOAD_TIME); + + const target = [ + { box: { xmax: 112, xmin: -111, ymax: 0, ymin: 0 }, label: "a cat. [SEP]", score: 1 }, + { box: { xmax: 112, xmin: -111, ymax: 0, ymin: 0 }, label: "a cat. [SEP]", score: 1 }, + { box: { xmax: 112, xmin: -111, ymax: 0, ymin: 0 }, label: "a cat. [SEP]", score: 1 }, + // ... many more + ]; + + it("should be an instance of ZeroShotObjectDetectionPipeline", () => { + expect(pipe).toBeInstanceOf(ZeroShotObjectDetectionPipeline); + }); + + describe("batch_size=1", () => { + it( + "default", + async () => { + const output = await pipe(image, candidate_labels); + expect(output).toHaveLength(900); + expect(output.slice(0, target.length)).toBeCloseToNested(target, 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + it( + "custom (w/ top_k & threshold)", + async () => { + const top_k = 3; + const output = await pipe(image, candidate_labels, { top_k, threshold: 0.05 }); + expect(output).toBeCloseToNested(target.slice(0, top_k), 5); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + }); +}; diff --git a/packages/transformers/tests/processors.test.js b/packages/transformers/tests/processors.test.js new file mode 100644 index 0000000..d80ec91 --- /dev/null +++ b/packages/transformers/tests/processors.test.js @@ -0,0 +1,5 @@ +import { init } from "./init.js"; +import { collect_and_execute_tests } from "./test_utils.js"; + +init(); +await collect_and_execute_tests("Processors", "processor"); diff --git a/packages/transformers/tests/progress_callbacks.test.js b/packages/transformers/tests/progress_callbacks.test.js new file mode 100644 index 0000000..d9a2b84 --- /dev/null +++ b/packages/transformers/tests/progress_callbacks.test.js @@ -0,0 +1,310 @@ +import { pipeline, LlamaForCausalLM, AutoModelForCausalLM, WhisperForConditionalGeneration, Gemma3ForConditionalGeneration, Gemma3nForConditionalGeneration, VoxtralRealtimeForConditionalGeneration } from "../src/transformers.js"; + +import { init, MAX_MODEL_LOAD_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "./init.js"; + +// Initialise the testing environment +init(); + +/** + * Collects progress events during a loader call and returns them. + * @param {(cb: Function) => Promise<{ dispose(): Promise }>} loader + * @returns {Promise<{ events: import('../src/utils/core.js').ProgressInfo[], dispose: () => Promise }>} + */ +async function collectEvents(loader) { + /** @type {import('../src/utils/core.js').ProgressInfo[]} */ + const events = []; + const result = await loader((info) => events.push(info)); + return { events, dispose: () => result.dispose() }; +} + +/** + * Validates progress_total events: + * 1. loaded is monotonically non-decreasing + * 2. total is constant across all events + * 3. final progress value is 100 + * @param {Array} totalEvents + */ +function expectValidTotalEvents(totalEvents) { + expect(totalEvents.length).toBeGreaterThan(0); + + for (const event of totalEvents) { + expect(event).toHaveProperty("status", "progress_total"); + expect(event).toHaveProperty("progress"); + expect(event).toHaveProperty("loaded"); + expect(event).toHaveProperty("total"); + expect(event).toHaveProperty("files"); + expect(typeof event.progress).toBe("number"); + expect(event.progress).toBeGreaterThanOrEqual(0); + expect(event.progress).toBeLessThanOrEqual(100); + expect(event.loaded).toBeLessThanOrEqual(event.total); + } + + // 1. loaded should be monotonically non-decreasing + for (let i = 1; i < totalEvents.length; i++) { + expect(totalEvents[i].loaded).toBeGreaterThanOrEqual(totalEvents[i - 1].loaded); + } + + // 2. total should be constant across all events + const expectedTotal = totalEvents[0].total; + for (const event of totalEvents) { + expect(event.total).toBe(expectedTotal); + } + + // 3. final progress value should be 100 + expect(totalEvents.at(-1).progress).toBe(100); + expect(totalEvents.at(-1).loaded).toBe(totalEvents.at(-1).total); +} + +/** + * Validates per-file event lifecycle and structure. + * @param {Array} events All collected events. + * @param {string} model_id Expected model name on events. + * @param {string[]} expectedFiles File paths that must be present in the files map. + */ +function expectValidEventLifecycle(events, model_id, expectedFiles) { + const totalEvents = events.filter((e) => e.status === "progress_total"); + expectValidTotalEvents(totalEvents); + + // Exact file count in the final progress_total + const lastFiles = totalEvents.at(-1).files; + expect(Object.keys(lastFiles).length).toBe(expectedFiles.length); + + // All expected files are present and fully loaded + for (const file of expectedFiles) { + expect(lastFiles).toHaveProperty([file]); + expect(lastFiles[file].loaded).toBe(lastFiles[file].total); + } + + // Every file emits initiate -> ... -> done lifecycle + const trackedFiles = new Set(events.filter((e) => e.file).map((e) => e.file)); + for (const file of trackedFiles) { + const fileEvents = events.filter((e) => e.file === file); + expect(fileEvents[0].status).toBe("initiate"); + expect(fileEvents.at(-1).status).toBe("done"); + } + + // All events with a name field should reference the correct model + for (const event of events) { + if (event.name) { + expect(event.name).toBe(model_id); + } + } + + // No double-wrapping: at most one progress_total per progress event + const progressEvents = events.filter((e) => e.status === "progress"); + expect(totalEvents.length).toBeLessThanOrEqual(progressEvents.length); +} + +describe("Progress Callbacks", () => { + // ---- Llama (decoder-only) ---- + // from_pretrained files: config.json, onnx/model.onnx, generation_config.json + // pipeline files: + tokenizer.json, tokenizer_config.json + describe("Llama (decoder-only)", () => { + const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; + + it( + "pipeline('text-generation')", + async () => { + const { events, dispose } = await collectEvents((cb) => pipeline("text-generation", model_id, { ...DEFAULT_MODEL_OPTIONS, progress_callback: cb })); + + expectValidEventLifecycle(events, model_id, ["config.json", "onnx/model.onnx", "generation_config.json", "tokenizer.json", "tokenizer_config.json"]); + + await dispose(); + }, + MAX_MODEL_LOAD_TIME + MAX_MODEL_DISPOSE_TIME, + ); + + it( + "LlamaForCausalLM.from_pretrained()", + async () => { + const { events, dispose } = await collectEvents((cb) => LlamaForCausalLM.from_pretrained(model_id, { ...DEFAULT_MODEL_OPTIONS, progress_callback: cb })); + + expectValidEventLifecycle(events, model_id, ["config.json", "onnx/model.onnx", "generation_config.json"]); + + await dispose(); + }, + MAX_MODEL_LOAD_TIME + MAX_MODEL_DISPOSE_TIME, + ); + + it( + "AutoModelForCausalLM.from_pretrained()", + async () => { + const { events, dispose } = await collectEvents((cb) => AutoModelForCausalLM.from_pretrained(model_id, { ...DEFAULT_MODEL_OPTIONS, progress_callback: cb })); + + expectValidEventLifecycle(events, model_id, ["config.json", "onnx/model.onnx", "generation_config.json"]); + + await dispose(); + }, + MAX_MODEL_LOAD_TIME + MAX_MODEL_DISPOSE_TIME, + ); + }); + + // ---- Whisper (encoder-decoder) ---- + // from_pretrained files: config.json, onnx/encoder_model.onnx, onnx/decoder_model_merged.onnx, generation_config.json + // pipeline files: + tokenizer.json, tokenizer_config.json, preprocessor_config.json + describe("Whisper (encoder-decoder)", () => { + const model_id = "onnx-internal-testing/tiny-random-WhisperForConditionalGeneration"; + + it( + "pipeline('automatic-speech-recognition')", + async () => { + const { events, dispose } = await collectEvents((cb) => pipeline("automatic-speech-recognition", model_id, { ...DEFAULT_MODEL_OPTIONS, progress_callback: cb })); + + const expectedFiles = ["config.json", "onnx/encoder_model.onnx", "onnx/decoder_model_merged.onnx", "generation_config.json", "tokenizer.json", "tokenizer_config.json", "preprocessor_config.json"]; + + // Each file should be loaded exactly once: pipeline() must not double-fetch + // tokenizer.json/tokenizer_config.json/preprocessor_config.json that the + // tokenizer and processor would otherwise each load independently. + const initiated = events.filter((e) => e.status === "initiate").map((e) => e.file); + expect(initiated.sort()).toEqual([...expectedFiles].sort()); + + expectValidEventLifecycle(events, model_id, expectedFiles); + + await dispose(); + }, + MAX_MODEL_LOAD_TIME + MAX_MODEL_DISPOSE_TIME, + ); + + it( + "WhisperForConditionalGeneration.from_pretrained()", + async () => { + const { events, dispose } = await collectEvents((cb) => WhisperForConditionalGeneration.from_pretrained(model_id, { ...DEFAULT_MODEL_OPTIONS, progress_callback: cb })); + + expectValidEventLifecycle(events, model_id, ["config.json", "onnx/encoder_model.onnx", "onnx/decoder_model_merged.onnx", "generation_config.json"]); + + await dispose(); + }, + MAX_MODEL_LOAD_TIME + MAX_MODEL_DISPOSE_TIME, + ); + }); + + // ---- Gemma3 (image-text-to-text) ---- + // from_pretrained files: config.json, onnx/embed_tokens.onnx, onnx/embed_tokens.onnx_data, + // onnx/decoder_model_merged.onnx, onnx/decoder_model_merged.onnx_data, + // onnx/vision_encoder.onnx, onnx/vision_encoder.onnx_data, generation_config.json + describe("Gemma3 (image-text-to-text)", () => { + const model_id = "onnx-internal-testing/tiny-random-Gemma3ForConditionalGeneration"; + + it( + "Gemma3ForConditionalGeneration.from_pretrained()", + async () => { + const { events, dispose } = await collectEvents((cb) => Gemma3ForConditionalGeneration.from_pretrained(model_id, { ...DEFAULT_MODEL_OPTIONS, progress_callback: cb })); + + expectValidEventLifecycle(events, model_id, ["config.json", "onnx/embed_tokens.onnx", "onnx/embed_tokens.onnx_data", "onnx/decoder_model_merged.onnx", "onnx/decoder_model_merged.onnx_data", "onnx/vision_encoder.onnx", "onnx/vision_encoder.onnx_data", "generation_config.json"]); + + await dispose(); + }, + MAX_MODEL_LOAD_TIME + MAX_MODEL_DISPOSE_TIME, + ); + }); + + // ---- Gemma3n (image-audio-text-to-text) ---- + // from_pretrained files: config.json, onnx/embed_tokens.onnx, onnx/embed_tokens.onnx_data, + // onnx/decoder_model_merged.onnx, onnx/decoder_model_merged.onnx_data, + // onnx/audio_encoder.onnx, onnx/audio_encoder.onnx_data, + // onnx/vision_encoder.onnx, onnx/vision_encoder.onnx_data, generation_config.json + describe("Gemma3n (image-audio-text-to-text)", () => { + const model_id = "onnx-internal-testing/tiny-random-Gemma3nForConditionalGeneration"; + + it( + "Gemma3nForConditionalGeneration.from_pretrained()", + async () => { + const { events, dispose } = await collectEvents((cb) => Gemma3nForConditionalGeneration.from_pretrained(model_id, { ...DEFAULT_MODEL_OPTIONS, progress_callback: cb })); + + expectValidEventLifecycle(events, model_id, ["config.json", "onnx/embed_tokens.onnx", "onnx/embed_tokens.onnx_data", "onnx/decoder_model_merged.onnx", "onnx/decoder_model_merged.onnx_data", "onnx/audio_encoder.onnx", "onnx/audio_encoder.onnx_data", "onnx/vision_encoder.onnx", "onnx/vision_encoder.onnx_data", "generation_config.json"]); + + await dispose(); + }, + MAX_MODEL_LOAD_TIME + MAX_MODEL_DISPOSE_TIME, + ); + }); + + // ---- VoxtralRealtime (audio-text-to-text) ---- + // from_pretrained files: config.json, onnx/embed_tokens.onnx, onnx/embed_tokens.onnx_data, + // onnx/decoder_model_merged.onnx, onnx/decoder_model_merged.onnx_data, + // onnx/audio_encoder.onnx, onnx/audio_encoder.onnx_data, generation_config.json + describe("VoxtralRealtime (audio-text-to-text)", () => { + const model_id = "onnx-internal-testing/tiny-random-VoxtralRealtimeForConditionalGeneration"; + + it( + "VoxtralRealtimeForConditionalGeneration.from_pretrained()", + async () => { + const { events, dispose } = await collectEvents((cb) => VoxtralRealtimeForConditionalGeneration.from_pretrained(model_id, { ...DEFAULT_MODEL_OPTIONS, progress_callback: cb })); + + expectValidEventLifecycle(events, model_id, ["config.json", "onnx/embed_tokens.onnx", "onnx/embed_tokens.onnx_data", "onnx/decoder_model_merged.onnx", "onnx/decoder_model_merged.onnx_data", "onnx/audio_encoder.onnx", "onnx/audio_encoder.onnx_data", "generation_config.json"]); + + await dispose(); + }, + MAX_MODEL_LOAD_TIME + MAX_MODEL_DISPOSE_TIME, + ); + }); + + // ---- Edge cases ---- + describe("Edge cases", () => { + const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; + + it( + "no progress_total without progress_callback", + async () => { + // When no progress_callback is provided, nothing should throw + const model = await LlamaForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + await model.dispose(); + }, + MAX_MODEL_LOAD_TIME + MAX_MODEL_DISPOSE_TIME, + ); + + it( + "per-file progress events have loaded <= total", + async () => { + const { events, dispose } = await collectEvents((cb) => LlamaForCausalLM.from_pretrained(model_id, { ...DEFAULT_MODEL_OPTIONS, progress_callback: cb })); + + const progressEvents = events.filter((e) => e.status === "progress"); + for (const event of progressEvents) { + expect(event.loaded).toBeLessThanOrEqual(event.total); + expect(event.loaded).toBeGreaterThanOrEqual(0); + expect(event.total).toBeGreaterThan(0); + } + + await dispose(); + }, + MAX_MODEL_LOAD_TIME + MAX_MODEL_DISPOSE_TIME, + ); + + it( + "per-file progress is monotonically non-decreasing", + async () => { + const { events, dispose } = await collectEvents((cb) => LlamaForCausalLM.from_pretrained(model_id, { ...DEFAULT_MODEL_OPTIONS, progress_callback: cb })); + + // Group progress events by file and verify monotonicity within each file + const progressByFile = {}; + for (const event of events.filter((e) => e.status === "progress")) { + (progressByFile[event.file] ??= []).push(event.loaded); + } + for (const loadedValues of Object.values(progressByFile)) { + for (let i = 1; i < loadedValues.length; i++) { + expect(loadedValues[i]).toBeGreaterThanOrEqual(loadedValues[i - 1]); + } + } + + await dispose(); + }, + MAX_MODEL_LOAD_TIME + MAX_MODEL_DISPOSE_TIME, + ); + + it( + "progress_total files map is a deep copy (structuredClone)", + async () => { + const { events, dispose } = await collectEvents((cb) => LlamaForCausalLM.from_pretrained(model_id, { ...DEFAULT_MODEL_OPTIONS, progress_callback: cb })); + + const totalEvents = events.filter((e) => e.status === "progress_total"); + // Each progress_total event should have its own files object (not shared references) + if (totalEvents.length >= 2) { + expect(totalEvents[0].files).not.toBe(totalEvents[1].files); + } + + await dispose(); + }, + MAX_MODEL_LOAD_TIME + MAX_MODEL_DISPOSE_TIME, + ); + }); +}); diff --git a/packages/transformers/tests/test_utils.js b/packages/transformers/tests/test_utils.js new file mode 100644 index 0000000..1675c1c --- /dev/null +++ b/packages/transformers/tests/test_utils.js @@ -0,0 +1,67 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const models_dir = path.join(__dirname, "models"); +const pipelines_dir = path.join(__dirname, "pipelines"); + +/** + * Helper function to collect all unit tests, which can be found in files + * of the form: `tests/models//test__.js`. + * @param {string} filename + * @returns {Promise<[string, Function][]>} + */ +export async function collect_tests(filename) { + const model_types = fs.readdirSync(models_dir); + const all_tests = []; + for (const model_type of model_types) { + const dir = path.join(models_dir, model_type); + + if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) { + continue; + } + + const file = path.join(dir, `test_${filename}_${model_type}.js`); + if (!fs.existsSync(file)) { + continue; + } + + const items = await import(file); + all_tests.push([model_type, items]); + } + return all_tests; +} + +/** + * Helper function to collect and execute all unit tests, which can be found in files + * of the form: `tests/models//test__.js`. + * @param {string} title The title of the test + * @param {string} filename The name of the test + */ +export async function collect_and_execute_tests(title, filename) { + // 1. Collect all tests + const all_tests = await collect_tests(filename); + + // 2. Execute tests + describe(title, () => all_tests.forEach(([name, test]) => describe(name, test.default))); +} + +/** + * Helper function to collect all pipeline tests, which can be found in files + * of the form: `tests/pipelines/test_pipeline_.js`. + */ +export async function collect_and_execute_pipeline_tests(title) { + // 1. Collect all tests + const all_tests = []; + const pipeline_types = fs.readdirSync(pipelines_dir); + for (const filename of pipeline_types) { + const file = path.join(pipelines_dir, filename); + const items = await import(file); + all_tests.push(items); + } + + // 2. Execute tests + describe(title, () => all_tests.forEach((test) => test.default())); +} diff --git a/packages/transformers/tests/tokenizers.test.js b/packages/transformers/tests/tokenizers.test.js new file mode 100644 index 0000000..1e7977b --- /dev/null +++ b/packages/transformers/tests/tokenizers.test.js @@ -0,0 +1,728 @@ +import { AutoTokenizer } from "../src/models/auto/tokenization_auto.js"; +import { MAX_TOKENIZER_LOAD_TIME, MAX_TEST_EXECUTION_TIME } from "./init.js"; +import { collect_tests } from "./test_utils.js"; + +const TOKENIZER_TESTS = await collect_tests("tokenization"); +describe("Tokenizers (model-specific)", () => { + for (const [tokenizer_name, { TOKENIZER_CLASS, TEST_CONFIG, CUSTOM_TESTS }] of TOKENIZER_TESTS) { + describe(tokenizer_name, () => { + for (const model_id in TEST_CONFIG) { + describe(model_id, () => { + /** @type {import('../src/tokenizers.js').PreTrainedTokenizer} */ + let tokenizer; + beforeAll(async () => { + tokenizer = await TOKENIZER_CLASS.from_pretrained(model_id); + }, MAX_TOKENIZER_LOAD_TIME); + + for (const [test_name, test_case] of Object.entries(TEST_CONFIG[model_id])) { + test(test_name, () => { + if (test_case.ids) { + const ids = tokenizer.encode(test_case.text, { + text_pair: test_case.text_pair, + }); + expect(ids).toEqual(test_case.ids); + } + if (test_case.tokens) { + const tokens = tokenizer.tokenize(test_case.text, { + pair: test_case.text_pair, + }); + expect(tokens).toEqual(test_case.tokens); + } + if (test_case.decoded) { + const decoded = tokenizer.decode(test_case.ids); + expect(decoded).toEqual(test_case.decoded); + } + }); + } + afterAll(() => { + global.gc?.(); + }); + }); + } + // Run custom tests, if they exist + CUSTOM_TESTS && describe("custom", CUSTOM_TESTS); + }); + } +}); + +describe("Tokenizer padding/truncation", () => { + const inputs = ["a", "b c"]; + const text_pair = ["d e", "f g h"]; + + const inputs_2 = ["a", "b c d e f"]; + + let tokenizer; + beforeAll(async () => { + tokenizer = await AutoTokenizer.from_pretrained("Xenova/bert-base-uncased"); + }, MAX_TOKENIZER_LOAD_TIME); + + describe("return_tensor=false (jagged array)", () => { + test("jagged array output when return_tensor is false", () => { + const output = tokenizer(inputs, { + return_tensor: false, + }); + const expected = { + input_ids: [ + [101, 1037, 102], + [101, 1038, 1039, 102], + ], + attention_mask: [ + [1, 1, 1], + [1, 1, 1, 1], + ], + token_type_ids: [ + [0, 0, 0], + [0, 0, 0, 0], + ], + }; + expect(output).toEqual(expected); + }); + + test("truncation output without special tokens when return_tensor is false", () => { + const output = tokenizer(inputs, { + return_tensor: false, + truncation: true, + add_special_tokens: false, + }); + const expected = { + input_ids: [[1037], [1038, 1039]], + attention_mask: [[1], [1, 1]], + token_type_ids: [[0], [0, 0]], + }; + expect(output).toEqual(expected); + }); + + test("no padding with max_length defined and truncation unset", () => { + const output = tokenizer(inputs, { + return_tensor: false, + padding: false, + max_length: 1, + add_special_tokens: false, + }); + const expected = { + input_ids: [[1037], [1038]], + attention_mask: [[1], [1]], + token_type_ids: [[0], [0]], + }; + expect(output).toEqual(expected); + }); + + test("No padding, max_length=3 (implicit truncation strategy)", () => { + const output = tokenizer(inputs_2, { + padding: false, + max_length: 3, + add_special_tokens: false, + return_tensor: false, + }); + const expected = { + input_ids: [[1037], [1038, 1039, 1040]], + token_type_ids: [[0], [0, 0, 0]], + attention_mask: [[1], [1, 1, 1]], + }; + expect(output).toEqual(expected); + }); + + test("Padding true, max_length=3 (implicit truncation strategy)", () => { + const output = tokenizer(inputs_2, { + padding: true, + max_length: 3, + add_special_tokens: false, + return_tensor: false, + }); + const expected = { + input_ids: [ + [1037, 0, 0, 0, 0], + [1038, 1039, 1040, 1041, 1042], + ], + token_type_ids: [ + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + ], + attention_mask: [ + [1, 0, 0, 0, 0], + [1, 1, 1, 1, 1], + ], + }; + expect(output).toEqual(expected); + }); + + test("No padding with explicit truncation, max_length=3", () => { + const output = tokenizer(inputs_2, { + padding: false, + truncation: true, + max_length: 3, + add_special_tokens: false, + return_tensor: false, + }); + const expected = { + input_ids: [[1037], [1038, 1039, 1040]], + token_type_ids: [[0], [0, 0, 0]], + attention_mask: [[1], [1, 1, 1]], + }; + expect(output).toEqual(expected); + }); + + test("Padding true with explicit truncation, max_length=3", () => { + const output = tokenizer(inputs_2, { + padding: true, + truncation: true, + max_length: 3, + add_special_tokens: false, + return_tensor: false, + }); + const expected = { + input_ids: [ + [1037, 0, 0], + [1038, 1039, 1040], + ], + token_type_ids: [ + [0, 0, 0], + [0, 0, 0], + ], + attention_mask: [ + [1, 0, 0], + [1, 1, 1], + ], + }; + expect(output).toEqual(expected); + }); + + test("Padding 'max_length' without truncation, max_length=3", () => { + const output = tokenizer(inputs_2, { + padding: "max_length", + truncation: false, + max_length: 3, + add_special_tokens: false, + return_tensor: false, + }); + const expected = { + input_ids: [ + [1037, 0, 0], + [1038, 1039, 1040, 1041, 1042], + ], + token_type_ids: [ + [0, 0, 0], + [0, 0, 0, 0, 0], + ], + attention_mask: [ + [1, 0, 0], + [1, 1, 1, 1, 1], + ], + }; + expect(output).toEqual(expected); + }); + + test("Padding 'max_length' with truncation, max_length=3", () => { + const output = tokenizer(inputs_2, { + padding: "max_length", + truncation: true, + max_length: 3, + add_special_tokens: false, + return_tensor: false, + }); + const expected = { + input_ids: [ + [1037, 0, 0], + [1038, 1039, 1040], + ], + token_type_ids: [ + [0, 0, 0], + [0, 0, 0], + ], + attention_mask: [ + [1, 0, 0], + [1, 1, 1], + ], + }; + expect(output).toEqual(expected); + }); + + test("Padding 'max_length' without truncation and max_length=null", () => { + const output = tokenizer(inputs_2, { + padding: "max_length", + truncation: false, + max_length: null, + add_special_tokens: false, + return_tensor: false, + }); + const expected = { + input_ids: [ + [1037, ...Array(511).fill(0)], + [1038, 1039, 1040, 1041, 1042, ...Array(507).fill(0)], + ], + token_type_ids: [ + [0, ...Array(511).fill(0)], + [0, 0, 0, 0, 0, ...Array(507).fill(0)], + ], + attention_mask: [ + [1, ...Array(511).fill(0)], + [1, 1, 1, 1, 1, ...Array(507).fill(0)], + ], + }; + expect(output).toEqual(expected); + }); + }); + + describe("return_tensor=true", () => { + test("throws error when tensor output is requested for a jagged array", () => { + expect(() => tokenizer(inputs)).toThrow("Unable to create tensor"); + }); + + test("truncation output for tensor inputs", () => { + const { input_ids, attention_mask, token_type_ids } = tokenizer(inputs, { + truncation: true, + max_length: 1, + add_special_tokens: false, + }); + expect(input_ids.tolist()).toEqual([[1037n], [1038n]]); + expect(attention_mask.tolist()).toEqual([[1n], [1n]]); + expect(token_type_ids.tolist()).toEqual([[0n], [0n]]); + }); + + test("padding output for tensor inputs without text pair", () => { + const { input_ids, attention_mask, token_type_ids } = tokenizer(inputs, { + padding: true, + add_special_tokens: false, + }); + expect(input_ids.tolist()).toEqual([ + [1037n, 0n], + [1038n, 1039n], + ]); + expect(attention_mask.tolist()).toEqual([ + [1n, 0n], + [1n, 1n], + ]); + expect(token_type_ids.tolist()).toEqual([ + [0n, 0n], + [0n, 0n], + ]); + }); + + test("padding output for tensor inputs with text pair", () => { + const { input_ids, attention_mask, token_type_ids } = tokenizer(inputs, { + text_pair, + padding: true, + add_special_tokens: false, + }); + expect(input_ids.tolist()).toEqual([ + [1037n, 1040n, 1041n, 0n, 0n], + [1038n, 1039n, 1042n, 1043n, 1044n], + ]); + expect(attention_mask.tolist()).toEqual([ + [1n, 1n, 1n, 0n, 0n], + [1n, 1n, 1n, 1n, 1n], + ]); + expect(token_type_ids.tolist()).toEqual([ + [0n, 1n, 1n, 0n, 0n], + [0n, 0n, 1n, 1n, 1n], + ]); + }); + + test("truncation and padding output for tensor inputs", () => { + const { input_ids, attention_mask, token_type_ids } = tokenizer(["a", "b c", "d e f"], { + padding: true, + truncation: true, + add_special_tokens: false, + max_length: 2, + }); + expect(input_ids.tolist()).toEqual([ + [1037n, 0n], + [1038n, 1039n], + [1040n, 1041n], + ]); + expect(attention_mask.tolist()).toEqual([ + [1n, 0n], + [1n, 1n], + [1n, 1n], + ]); + expect(token_type_ids.tolist()).toEqual([ + [0n, 0n], + [0n, 0n], + [0n, 0n], + ]); + }); + + test("padding:true pads to the longest encoding in the batch regardless of max_length", () => { + const { input_ids, attention_mask, token_type_ids } = tokenizer(inputs, { + padding: true, + truncation: true, + add_special_tokens: false, + max_length: 3, + }); + expect(input_ids.tolist()).toEqual([ + [1037n, 0n], + [1038n, 1039n], + ]); + expect(attention_mask.tolist()).toEqual([ + [1n, 0n], + [1n, 1n], + ]); + expect(token_type_ids.tolist()).toEqual([ + [0n, 0n], + [0n, 0n], + ]); + }); + + test("padding:'max_length' pads to the specified max_length", () => { + const { input_ids, attention_mask, token_type_ids } = tokenizer(inputs, { + padding: "max_length", + truncation: true, + add_special_tokens: false, + max_length: 3, + }); + expect(input_ids.tolist()).toEqual([ + [1037n, 0n, 0n], + [1038n, 1039n, 0n], + ]); + expect(attention_mask.tolist()).toEqual([ + [1n, 0n, 0n], + [1n, 1n, 0n], + ]); + expect(token_type_ids.tolist()).toEqual([ + [0n, 0n, 0n], + [0n, 0n, 0n], + ]); + }); + }); +}); + +describe("Token type ids", () => { + it( + "should correctly add token type ids", + async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/bert-base-uncased"); + + const model_inputs = tokenizer(["a b c", "d"], { + text_pair: ["e f", "g h"], + padding: true, + truncation: true, + return_tensor: false, + }); + + const expected = { + input_ids: [ + [101, 1037, 1038, 1039, 102, 1041, 1042, 102], + [101, 1040, 102, 1043, 1044, 102, 0, 0], + ], + token_type_ids: [ + [0, 0, 0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1, 0, 0], + ], + attention_mask: [ + [1, 1, 1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 0, 0], + ], + }; + + expect(model_inputs).toEqual(expected); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should add token type ids if user requests them", + async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/llama3-tokenizer-new"); + + { + // Without text pair + const model_inputs = tokenizer("hello", { + return_tensor: false, + return_token_type_ids: true, + }); + const expected = { + input_ids: [128000, 15339], + attention_mask: [1, 1], + token_type_ids: [0, 0], + }; + expect(model_inputs).toEqual(expected); + } + + { + // With text pair + const model_inputs = tokenizer("hello", { + text_pair: "world", + return_tensor: false, + return_token_type_ids: true, + }); + const expected = { + input_ids: [128000, 15339, 128000, 14957], + attention_mask: [1, 1, 1, 1], + token_type_ids: [0, 0, 1, 1], + }; + expect(model_inputs).toEqual(expected); + } + }, + MAX_TEST_EXECUTION_TIME, + ); +}); + +describe("Edge cases", () => { + it( + "should not crash when encoding a very long string", + async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/t5-small"); + + const text = String.prototype.repeat.call("Hello world! ", 50000); + const encoded = tokenizer(text); + expect(encoded.input_ids.data.length).toBeGreaterThan(100000); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it("should not take too long", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/all-MiniLM-L6-v2"); + + const text = String.prototype.repeat.call("a", 50000); + const token_ids = tokenizer.encode(text); + expect(token_ids).toEqual([101, 100, 102]); + }, 5000); // NOTE: 5 seconds + + it( + "Special/added tokens with earlier partial matches", + async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/gemini-nano"); + { + const token_ids = tokenizer.encode("\n", { add_special_tokens: false }); + expect(token_ids).toEqual([108]); + } + { + const token_ids = tokenizer.encode("\n\n", { add_special_tokens: false }); + expect(token_ids).toEqual([109]); // Should not be [108, 108] + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + it("many added tokens", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("onnx-community/orpheus-3b-0.1-ft-ONNX"); + + const text = "hello world!"; + const token_ids = tokenizer.encode(text); + expect(token_ids).toEqual([128000, 15339, 1917, 0]); + }, 5000); // NOTE: 5 seconds +}); + +describe("Memory leak tests", () => { + it("should not leak memory", async () => { + const id = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"; + const tokenizer = await AutoTokenizer.from_pretrained(id); + + // NB: Force garbage collection before measuring memory usage + global.gc(); + const startMemory = process.memoryUsage().heapUsed; + for (let i = 0; i < 2500; ++i) { + const s = `${i} ${i} `.repeat(i); + tokenizer.encode(s); + } + global.gc(); + const endMemory = process.memoryUsage().heapUsed; + const memoryUsed = endMemory - startMemory; + const memoryLimit = 1 * 1024 * 1024; // 1 MB + expect(memoryUsed).toBeLessThan(memoryLimit); + }, 30000); // Increase timeout to accommodate the memory leak test +}); + +describe("Extra decoding tests", () => { + it( + "should be able to decode the output of encode", + async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/bert-base-uncased"); + + const text = "hello world!"; + + // Ensure all the following outputs are the same: + // 1. Tensor of ids: allow decoding of 1D or 2D tensors. + const encodedTensor = tokenizer(text); + const decoded1 = tokenizer.decode(encodedTensor.input_ids, { skip_special_tokens: true }); + const decoded2 = tokenizer.batch_decode(encodedTensor.input_ids, { skip_special_tokens: true })[0]; + expect(decoded1).toEqual(text); + expect(decoded2).toEqual(text); + + // 2. List of ids + const encodedList = tokenizer(text, { return_tensor: false }); + const decoded3 = tokenizer.decode(encodedList.input_ids, { skip_special_tokens: true }); + const decoded4 = tokenizer.batch_decode([encodedList.input_ids], { skip_special_tokens: true })[0]; + expect(decoded3).toEqual(text); + expect(decoded4).toEqual(text); + }, + MAX_TEST_EXECUTION_TIME, + ); +}); + +describe("Chat templates", () => { + it("should generate a chat template", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/mistral-tokenizer-v1"); + + const chat = [ + { role: "user", content: "Hello, how are you?" }, + { role: "assistant", content: "I'm doing great. How can I help you today?" }, + { role: "user", content: "I'd like to show off how chat templating works!" }, + ]; + + const text = tokenizer.apply_chat_template(chat, { tokenize: false, return_dict: false }); + + expect(text).toEqual("[INST] Hello, how are you? [/INST]I'm doing great. How can I help you today? [INST] I'd like to show off how chat templating works! [/INST]"); + + const input_ids = tokenizer.apply_chat_template(chat, { tokenize: true, return_tensor: false, return_dict: false }); + expect(input_ids).toEqual([1, 733, 16289, 28793, 22557, 28725, 910, 460, 368, 28804, 733, 28748, 16289, 28793, 28737, 28742, 28719, 2548, 1598, 28723, 1602, 541, 315, 1316, 368, 3154, 28804, 2, 28705, 733, 16289, 28793, 315, 28742, 28715, 737, 298, 1347, 805, 910, 10706, 5752, 1077, 3791, 28808, 733, 28748, 16289, 28793]); + }); + + it("should support multiple chat templates", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer"); + + // define conversation input: + const conversation = [{ role: "user", content: "Whats the biggest penguin in the world?" }]; + // define documents to ground on: + const documents = [ + { title: "Tall penguins", text: "Emperor penguins are the tallest growing up to 122 cm in height." }, + { title: "Penguin habitats", text: "Emperor penguins only live in Antarctica." }, + ]; + + // render the RAG prompt as a string: + const grounded_generation_prompt = tokenizer.apply_chat_template(conversation, { + chat_template: "rag", + tokenize: false, + add_generation_prompt: true, + + documents, + citation_mode: "accurate", // or "fast" + }); + expect(grounded_generation_prompt).toEqual("<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|># Safety Preamble\nThe instructions in this section override those in the task description and style guide sections. Don't answer questions that are harmful or immoral.\n\n" + "# System Preamble\n## Basic Rules\nYou are a powerful conversational AI trained by Cohere to help people. You are augmented by a number of tools, and your job is to use and consume the output of these tools to best help the user. You will see a conversation history between yourself and a user, ending with an utterance from the user. You will then see a specific instruction instructing you what kind of response to generate. When you answer the user's requests, you cite your sources in your answers, according to those instructions.\n\n" + "# User Preamble\n## Task and Context\nYou help people answer their questions and other requests interactively. You will be asked a very wide array of requests on all kinds of topics. You will be equipped with a wide range of search engines or similar tools to help you, which you use to research your answer. You should focus on serving the user's needs as best you can, which will be wide-ranging.\n\n## Style Guide\nUnless the user asks for a different style of answer, you should answer in full sentences, using proper grammar and spelling.<|END_OF_TURN_TOKEN|>" + "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>Whats the biggest penguin in the world?<|END_OF_TURN_TOKEN|>" + "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>\nDocument: 0\ntitle: Tall penguins\ntext: Emperor penguins are the tallest growing up to 122 cm in height.\n\nDocument: 1\ntitle: Penguin habitats\ntext: Emperor penguins only live in Antarctica.\n<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>Carefully perform the following instructions, in order, starting each with a new line.\nFirstly, Decide which of the retrieved documents are relevant to the user's last input by writing 'Relevant Documents:' followed by comma-separated list of document numbers. If none are relevant, you should instead write 'None'.\nSecondly, Decide which of the retrieved documents contain facts that should be cited in a good answer to the user's last input by writing 'Cited Documents:' followed a comma-separated list of document numbers. If you dont want to cite any of them, you should instead write 'None'.\nThirdly, Write 'Answer:' followed by a response to the user's last input in high quality natural english. Use the retrieved documents to help you. Do not insert any citations or grounding markup.\nFinally, Write 'Grounded answer:' followed by a response to the user's last input in high quality natural english. Use the symbols and to indicate when a fact comes from a document in the search result, e.g my fact for a fact from document 0.<|END_OF_TURN_TOKEN|>" + "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"); + }); + + it("should support automatic chat template detection based on inputs", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/c4ai-command-r-plus-08-2024-tokenizer"); + + // Examples adapted from https://huggingface.co/CohereForAI/c4ai-command-r-plus-08-2024 + + { + // - default + // define conversation input: + const messages = [{ role: "user", content: "Hello, how are you?" }]; + + // Format message with the command-r-plus-08-2024 chat template + const prompt = tokenizer.apply_chat_template(messages, { tokenize: false, add_generation_prompt: true }); + expect(prompt).toEqual("<|START_OF_TURN_TOKEN|><|USER_TOKEN|>Hello, how are you?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"); + } + + { + // - tool_use + // define conversation input: + const conversation = [{ role: "user", content: "Whats the biggest penguin in the world?" }]; + + // Define tools available for the model to use: + const tools = [ + { + name: "internet_search", + description: "Returns a list of relevant document snippets for a textual query retrieved from the internet", + parameter_definitions: { + query: { + description: "Query to search the internet with", + type: "str", + required: true, + }, + }, + }, + { + name: "directly_answer", + description: "Calls a standard (un-augmented) AI chatbot to generate a response given the conversation history", + parameter_definitions: {}, + }, + ]; + + // render the tool use prompt as a string: + const prompt = tokenizer.apply_chat_template(conversation, { tools, tokenize: false, add_generation_prompt: true }); + expect(prompt).toEqual('<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|># Safety Preamble\nThe instructions in this section override those in the task description and style guide sections. Don\'t answer questions that are harmful or immoral.\n\n# System Preamble\n## Basic Rules\nYou are a powerful conversational AI trained by Cohere to help people. You are augmented by a number of tools, and your job is to use and consume the output of these tools to best help the user. You will see a conversation history between yourself and a user, ending with an utterance from the user. You will then see a specific instruction instructing you what kind of response to generate. When you answer the user\'s requests, you cite your sources in your answers, according to those instructions.\n\n# User Preamble\n## Task and Context\nYou help people answer their questions and other requests interactively. You will be asked a very wide array of requests on all kinds of topics. You will be equipped with a wide range of search engines or similar tools to help you, which you use to research your answer. You should focus on serving the user\'s needs as best you can, which will be wide-ranging.\n\n## Style Guide\nUnless the user asks for a different style of answer, you should answer in full sentences, using proper grammar and spelling.\n\n## Available Tools\nHere is a list of tools that you have available to you:\n\n```python\ndef internet_search(query: str) -> List[Dict]:\n """Returns a list of relevant document snippets for a textual query retrieved from the internet\n\n Args:\n query (str): Query to search the internet with\n """\n pass\n```\n\n```python\ndef directly_answer() -> List[Dict]:\n """Calls a standard (un-augmented) AI chatbot to generate a response given the conversation history\n """\n pass\n```<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Whats the biggest penguin in the world?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>Write \'Action:\' followed by a json-formatted list of actions that you want to perform in order to produce a good response to the user\'s last input. You can use any of the supplied tools any number of times, but you should aim to execute the minimum number of necessary actions for the input. You should use the `directly-answer` tool if calling the other tools is unnecessary. The list of actions you want to call should be formatted as a list of json objects, for example:\n```json\n[\n {\n "tool_name": title of the tool in the specification,\n "parameters": a dict of parameters to input into the tool as they are defined in the specs, or {} if it takes no parameters\n }\n]```<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>'); + } + }); + + it("should support user-defined chat template", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/llama-tokenizer"); + + const chat = [ + { role: "user", content: "Hello, how are you?" }, + { role: "assistant", content: "I'm doing great. How can I help you today?" }, + { role: "user", content: "I'd like to show off how chat templating works!" }, + ]; + + // https://discuss.huggingface.co/t/issue-with-llama-2-chat-template-and-out-of-date-documentation/61645/3 + const chat_template = ( + "{% if messages[0]['role'] == 'system' %}" + + "{% set loop_messages = messages[1:] %}" + // Extract system message if it's present + "{% set system_message = messages[0]['content'] %}" + + "{% elif USE_DEFAULT_PROMPT == true and not '<>' in messages[0]['content'] %}" + + "{% set loop_messages = messages %}" + // Or use the default system message if the flag is set + "{% set system_message = 'DEFAULT_SYSTEM_MESSAGE' %}" + + "{% else %}" + + "{% set loop_messages = messages %}" + + "{% set system_message = false %}" + + "{% endif %}" + + "{% if loop_messages|length == 0 and system_message %}" + // Special handling when only sys message present + "{{ bos_token + '[INST] <>\\n' + system_message + '\\n<>\\n\\n [/INST]' }}" + + "{% endif %}" + + "{% for message in loop_messages %}" + // Loop over all non-system messages + "{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}" + + "{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}" + + "{% endif %}" + + "{% if loop.index0 == 0 and system_message != false %}" + // Embed system message in first message + "{% set content = '<>\\n' + system_message + '\\n<>\\n\\n' + message['content'] %}" + + "{% else %}" + + "{% set content = message['content'] %}" + + "{% endif %}" + + "{% if message['role'] == 'user' %}" + // After all of that, handle messages/roles in a fairly normal way + "{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}" + + "{% elif message['role'] == 'system' %}" + + "{{ '<>\\n' + content.strip() + '\\n<>\\n\\n' }}" + + "{% elif message['role'] == 'assistant' %}" + + "{{ ' ' + content.strip() + ' ' + eos_token }}" + + "{% endif %}" + + "{% endfor %}" + ) + .replaceAll("USE_DEFAULT_PROMPT", true) + .replaceAll("DEFAULT_SYSTEM_MESSAGE", "You are a helpful, respectful and honest assistant."); + + const text = tokenizer.apply_chat_template(chat, { tokenize: false, return_tensor: false, chat_template }); + + expect(text).toEqual("[INST] <>\nYou are a helpful, respectful and honest assistant.\n<>\n\nHello, how are you? [/INST] I'm doing great. How can I help you today? [INST] I'd like to show off how chat templating works! [/INST]"); + + // TODO: Add test for token_ids once bug in transformers is fixed. + }); + + it("should throw an error when no chat template is detected", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/gpt-4o"); + + const chat = [{ role: "user", content: "Hello, how are you?" }]; + + expect(() => tokenizer.apply_chat_template(chat, { tokenize: false })).toThrow("tokenizer.chat_template is not set and no template argument was passed"); + }); + + it("should support default parameters", async () => { + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/Meta-Llama-3.1-Tokenizer"); + + // Example adapted from https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct#tool-use-with-transformers + const chat = [ + { role: "system", content: "You are a bot that responds to weather queries." }, + { role: "user", content: "Hey, what's the temperature in Paris right now?" }, + ]; + const tools = [{ type: "function", function: { name: "get_current_temperature", description: "Get the current temperature at a location.", parameters: { type: "object", properties: { location: { type: "string", description: 'The location to get the temperature for, in the format "City, Country"' } }, required: ["location"] }, return: { type: "number", description: "The current temperature at the specified location in the specified units, as a float." } } }]; + + { + // `tools` unset (will default to `null`) + const text = tokenizer.apply_chat_template(chat, { tokenize: false, return_dict: false }); + expect(text).toEqual("<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nCutting Knowledge Date: December 2023\nToday Date: 26 Jul 2024\n\nYou are a bot that responds to weather queries.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nHey, what's the temperature in Paris right now?<|eot_id|>"); + + const input_ids = tokenizer.apply_chat_template(chat, { tokenize: true, return_tensor: false, return_dict: false }); + expect(input_ids).toEqual([128000, 128006, 9125, 128007, 271, 38766, 1303, 33025, 2696, 25, 6790, 220, 2366, 18, 198, 15724, 2696, 25, 220, 1627, 10263, 220, 2366, 19, 271, 2675, 527, 264, 11164, 430, 31680, 311, 9282, 20126, 13, 128009, 128006, 882, 128007, 271, 19182, 11, 1148, 596, 279, 9499, 304, 12366, 1314, 1457, 30, 128009]); + } + + { + // `tools` set + const text = tokenizer.apply_chat_template(chat, { tools, tokenize: false, return_dict: false }); + expect(text).toEqual('<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nEnvironment: ipython\nCutting Knowledge Date: December 2023\nToday Date: 26 Jul 2024\n\nYou are a bot that responds to weather queries.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nGiven the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt.\n\nRespond in the format {"name": function name, "parameters": dictionary of argument name and its value}.Do not use variables.\n\n{\n "type": "function",\n "function": {\n "name": "get_current_temperature",\n "description": "Get the current temperature at a location.",\n "parameters": {\n "type": "object",\n "properties": {\n "location": {\n "type": "string",\n "description": "The location to get the temperature for, in the format \\"City, Country\\""\n }\n },\n "required": [\n "location"\n ]\n },\n "return": {\n "type": "number",\n "description": "The current temperature at the specified location in the specified units, as a float."\n }\n }\n}\n\nHey, what\'s the temperature in Paris right now?<|eot_id|>'); + + const input_ids = tokenizer.apply_chat_template(chat, { tools, tokenize: true, return_tensor: false, return_dict: false }); + expect(input_ids).toEqual([128000, 128006, 9125, 128007, 271, 13013, 25, 6125, 27993, 198, 38766, 1303, 33025, 2696, 25, 6790, 220, 2366, 18, 198, 15724, 2696, 25, 220, 1627, 10263, 220, 2366, 19, 271, 2675, 527, 264, 11164, 430, 31680, 311, 9282, 20126, 13, 128009, 128006, 882, 128007, 271, 22818, 279, 2768, 5865, 11, 4587, 6013, 449, 264, 4823, 369, 264, 734, 1650, 449, 1202, 6300, 6105, 430, 1888, 11503, 279, 2728, 10137, 382, 66454, 304, 279, 3645, 5324, 609, 794, 734, 836, 11, 330, 14105, 794, 11240, 315, 5811, 836, 323, 1202, 907, 7966, 5519, 539, 1005, 7482, 382, 517, 262, 330, 1337, 794, 330, 1723, 761, 262, 330, 1723, 794, 341, 286, 330, 609, 794, 330, 456, 11327, 54625, 761, 286, 330, 4789, 794, 330, 1991, 279, 1510, 9499, 520, 264, 3813, 10560, 286, 330, 14105, 794, 341, 310, 330, 1337, 794, 330, 1735, 761, 310, 330, 13495, 794, 341, 394, 330, 2588, 794, 341, 504, 330, 1337, 794, 330, 928, 761, 504, 330, 4789, 794, 330, 791, 3813, 311, 636, 279, 9499, 369, 11, 304, 279, 3645, 7393, 13020, 11, 14438, 2153, 702, 394, 457, 310, 1173, 310, 330, 6413, 794, 2330, 394, 330, 2588, 702, 310, 5243, 286, 1173, 286, 330, 693, 794, 341, 310, 330, 1337, 794, 330, 4174, 761, 310, 330, 4789, 794, 330, 791, 1510, 9499, 520, 279, 5300, 3813, 304, 279, 5300, 8316, 11, 439, 264, 2273, 10246, 286, 457, 262, 457, 633, 19182, 11, 1148, 596, 279, 9499, 304, 12366, 1314, 1457, 30, 128009]); + } + }); +}); diff --git a/packages/transformers/tests/types.test.js b/packages/transformers/tests/types.test.js new file mode 100644 index 0000000..d2c058c --- /dev/null +++ b/packages/transformers/tests/types.test.js @@ -0,0 +1,353 @@ +import ts from "typescript"; +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const TSCONFIG_PATH = resolve(__dirname, "..", "tsconfig.json"); + +function loadCompilerOptions() { + const raw = ts.parseConfigFileTextToJson(TSCONFIG_PATH, readFileSync(TSCONFIG_PATH, "utf8")); + if (raw.error) throw new Error(ts.flattenDiagnosticMessageText(raw.error.messageText, "\n")); + const parsed = ts.parseJsonConfigFileContent(raw.config, ts.sys, dirname(TSCONFIG_PATH)); + return { + ...parsed.options, + // Tests don't emit. + noEmit: true, + declaration: false, + declarationMap: false, + emitDeclarationOnly: false, + composite: false, + outDir: undefined, + rootDir: undefined, + strict: true, + }; +} + +const TS_OPTIONS = loadCompilerOptions(); + +function getDiagnostics(file) { + const program = ts.createProgram([file], TS_OPTIONS); + const absFile = resolve(file); + return ts.getPreEmitDiagnostics(program).filter((d) => d.file && resolve(d.file.fileName) === absFile); +} + +function formatDiagnostics(diagnostics) { + const formatHost = { + getCanonicalFileName: (path) => path, + getCurrentDirectory: ts.sys.getCurrentDirectory, + getNewLine: () => ts.sys.newLine, + }; + return ts.formatDiagnosticsWithColorAndContext(diagnostics, formatHost); +} + +/** + * Compile an inline TypeScript source string and return diagnostics. + * Uses a virtual file path under tests/types/ so relative imports resolve correctly. + */ +function getDiagnosticsFromSource(source) { + const virtualPath = "tests/types/__virtual.ts"; + const defaultHost = ts.createCompilerHost(TS_OPTIONS); + const host = { + ...defaultHost, + fileExists: (f) => f === virtualPath || defaultHost.fileExists(f), + readFile: (f) => (f === virtualPath ? source : defaultHost.readFile(f)), + getSourceFile: (f, lang) => (f === virtualPath ? ts.createSourceFile(f, source, lang, true) : defaultHost.getSourceFile(f, lang)), + }; + const program = ts.createProgram([virtualPath], TS_OPTIONS, host); + return [...ts.getPreEmitDiagnostics(program)].filter((d) => d.file && d.file.fileName === virtualPath); +} + +describe("TypeScript compilation succeeds", () => { + const DIR = "tests/types/"; + const FILES = ["pipelines.ts", "cache.ts", "tokenizers.ts"]; + for (const file of FILES) { + it(`compiles ${file} without errors`, () => { + const diagnostics = getDiagnostics(`${DIR}${file}`); + if (diagnostics.length > 0) { + throw new Error(formatDiagnostics(diagnostics)); + } + }); + } +}); + +describe("TypeScript expected errors", () => { + /** + * Helper: generates inline source for a pipeline error test. + * @param {string} task Pipeline task name + * @param {string} callArgs Arguments to pass to the pipeline call (e.g., `'"hi", { top_k: "bad" }')`) + */ + const src = (task, callArgs) => ` + import { pipeline } from "../../types/transformers.js"; + const p = await pipeline("${task}", "model-id"); + await p(${callArgs}); + `; + + /** @param {string} task @param {string} callArgs */ + const srcWithDynamicCache = (task, callArgs) => ` + import { DynamicCache, pipeline } from "../../types/transformers.js"; + const p = await pipeline("${task}", "model-id"); + await p(${callArgs}); + `; + + /** + * Pipeline error test cases. + * + * Each pipeline task maps to an array of test cases. Each test case has: + * - `name`: test description + * - `code`: inline TypeScript source (use `src()` helper for convenience) + * - `errors`: expected diagnostics (empty = should compile without errors) + * + * @type {Record }>>} + */ + const PIPELINE_CASES = { + "text-generation": [ + { + name: "past_key_values rejects string (chat input)", + code: src("text-generation", `[{role:"user",content:"hi"}], { past_key_values: "bad" }`), + errors: [{ code: 2322, underline: "past_key_values", messageIncludes: "DynamicCache" }], + }, + { + name: "past_key_values rejects string (string input)", + code: src("text-generation", `"hi", { past_key_values: "bad" }`), + errors: [{ code: 2322, underline: "past_key_values", messageIncludes: "DynamicCache" }], + }, + { + name: "past_key_values rejects number", + code: src("text-generation", `"hi", { past_key_values: 42 }`), + errors: [{ code: 2322, underline: "past_key_values", messageIncludes: "DynamicCache" }], + }, + { + name: "past_key_values rejects boolean", + code: src("text-generation", `"hi", { past_key_values: true }`), + errors: [{ code: 2322, underline: "past_key_values", messageIncludes: "DynamicCache" }], + }, + { + name: "past_key_values accepts DynamicCache", + code: srcWithDynamicCache("text-generation", `"hi", { past_key_values: new DynamicCache() }`), + errors: [], + }, + { + name: "past_key_values accepts null", + code: src("text-generation", `"hi", { past_key_values: null }`), + errors: [], + }, + ], + + "text2text-generation": [ + { + name: "past_key_values rejects string", + code: src("text2text-generation", `"translate: hello", { past_key_values: "bad" }`), + errors: [{ code: 2322, underline: "past_key_values", messageIncludes: "DynamicCache" }], + }, + ], + + translation: [ + { + name: "past_key_values rejects string", + code: src("translation", `"Bonjour", { past_key_values: "bad" }`), + errors: [{ code: 2322, underline: "past_key_values", messageIncludes: "DynamicCache" }], + }, + ], + + summarization: [ + { + name: "past_key_values rejects string", + code: src("summarization", `"Long article text...", { past_key_values: "bad" }`), + errors: [{ code: 2322, underline: "past_key_values", messageIncludes: "DynamicCache" }], + }, + ], + + "image-to-text": [ + { + name: "past_key_values rejects string", + code: src("image-to-text", `"http://img.png", { past_key_values: "bad" }`), + errors: [{ code: 2322, underline: "past_key_values", messageIncludes: "DynamicCache" }], + }, + ], + + "automatic-speech-recognition": [ + { + name: "past_key_values rejects string", + code: src("automatic-speech-recognition", `new Float32Array(16000), { past_key_values: "bad" }`), + errors: [{ code: 2322, underline: "past_key_values", messageIncludes: "DynamicCache" }], + }, + ], + + "document-question-answering": [ + { + name: "past_key_values rejects string", + code: src("document-question-answering", `"http://img.png", "What is this?", { past_key_values: "bad" }`), + errors: [{ code: 2322, underline: "past_key_values", messageIncludes: "DynamicCache" }], + }, + ], + + "audio-classification": [ + { + name: "top_k rejects string", + code: src("audio-classification", `new Float32Array(16000), { top_k: "bad" }`), + errors: [{ code: 2322, underline: "top_k", messageIncludes: "number" }], + }, + ], + + "fill-mask": [ + { + name: "top_k rejects string", + code: src("fill-mask", `"The [MASK] sat on the mat.", { top_k: "bad" }`), + errors: [{ code: 2322, underline: "top_k", messageIncludes: "number" }], + }, + ], + + "image-classification": [ + { + name: "top_k rejects string", + code: src("image-classification", `"http://img.png", { top_k: "bad" }`), + errors: [{ code: 2322, underline: "top_k", messageIncludes: "number" }], + }, + ], + + "object-detection": [ + { + name: "threshold rejects string", + code: src("object-detection", `"http://img.png", { threshold: "bad" }`), + errors: [{ code: 2322, underline: "threshold", messageIncludes: "number" }], + }, + ], + + "token-classification": [ + { + name: "ignore_labels rejects string", + code: src("token-classification", `"Hello world", { ignore_labels: "bad" }`), + errors: [{ code: 2322, underline: "ignore_labels", messageIncludes: "string[]" }], + }, + { + name: "aggregation_strategy rejects unknown literal", + code: src("token-classification", `"Hello world", { aggregation_strategy: "first" }`), + errors: [{ code: 2322, underline: "aggregation_strategy", messageIncludes: "AggregationStrategy" }], + }, + ], + + "text-classification": [ + { + name: "top_k rejects string", + code: src("text-classification", `"I love this!", { top_k: "bad" }`), + errors: [{ code: 2322, underline: "top_k", messageIncludes: "number" }], + }, + ], + + "question-answering": [ + { + name: "top_k rejects string", + code: src("question-answering", `"Who?", "Context.", { top_k: "bad" }`), + errors: [{ code: 2322, underline: "top_k", messageIncludes: "number" }], + }, + ], + + "feature-extraction": [ + { + name: "pooling rejects number", + code: src("feature-extraction", `"text", { pooling: 123 }`), + errors: [{ code: 2322, underline: "pooling", messageIncludes: '"none" | "mean"' }], + }, + ], + + "image-segmentation": [ + { + name: "threshold rejects string", + code: src("image-segmentation", `"http://img.png", { threshold: "bad" }`), + errors: [{ code: 2322, underline: "threshold", messageIncludes: "number" }], + }, + ], + + "image-feature-extraction": [ + { + name: "pool rejects string", + code: src("image-feature-extraction", `"http://img.png", { pool: "bad" }`), + errors: [{ code: 2322, underline: "pool", messageIncludes: "boolean" }], + }, + ], + + "text-to-audio": [ + { + name: "speaker_embeddings rejects number", + code: src("text-to-audio", `"Hello", { speaker_embeddings: 123 }`), + errors: [{ code: 2322, underline: "speaker_embeddings", messageIncludes: "Tensor" }], + }, + ], + + "zero-shot-classification": [ + { + name: "multi_label rejects string", + code: src("zero-shot-classification", `"text", ["a", "b"], { multi_label: "bad" }`), + errors: [{ code: 2322, underline: "multi_label", messageIncludes: "boolean" }], + }, + ], + + "zero-shot-audio-classification": [ + { + name: "hypothesis_template rejects number", + code: src("zero-shot-audio-classification", `new Float32Array(16000), ["a", "b"], { hypothesis_template: 123 }`), + errors: [{ code: 2322, underline: "hypothesis_template", messageIncludes: "string" }], + }, + ], + + "zero-shot-image-classification": [ + { + name: "hypothesis_template rejects number", + code: src("zero-shot-image-classification", `"http://img.png", ["a", "b"], { hypothesis_template: 123 }`), + errors: [{ code: 2322, underline: "hypothesis_template", messageIncludes: "string" }], + }, + ], + + "zero-shot-object-detection": [ + { + name: "threshold rejects string", + code: src("zero-shot-object-detection", `"http://img.png", ["a", "b"], { threshold: "bad" }`), + errors: [{ code: 2322, underline: "threshold", messageIncludes: "number" }], + }, + ], + + "depth-estimation": [ + { + name: "rejects unexpected options argument", + code: src("depth-estimation", `"http://img.png", { bad: true }`), + errors: [{ code: 2554, underline: "{ bad: true }", messageIncludes: "Expected 1 arguments" }], + }, + ], + + "image-to-image": [ + { + name: "rejects unexpected options argument", + code: src("image-to-image", `"http://img.png", { bad: true }`), + errors: [{ code: 2554, underline: "{ bad: true }", messageIncludes: "Expected 1 arguments" }], + }, + ], + }; + + describe("pipeline errors", () => { + for (const [task, cases] of Object.entries(PIPELINE_CASES)) { + describe(task, () => { + for (const { name, code, errors } of cases) { + it(name, () => { + const diagnostics = getDiagnosticsFromSource(code); + + expect(diagnostics).toHaveLength(errors.length); + + for (let i = 0; i < errors.length; i++) { + const diag = diagnostics[i]; + const expected = errors[i]; + + expect(diag.code).toBe(expected.code); + + const underlined = diag.file?.text?.slice(diag.start, diag.start + diag.length); + expect(underlined).toBe(expected.underline); + + const message = ts.flattenDiagnosticMessageText(diag.messageText, "\n"); + expect(message).toContain(expected.messageIncludes); + } + }); + } + }); + } + }); +}); diff --git a/packages/transformers/tests/types/_base.ts b/packages/transformers/tests/types/_base.ts new file mode 100644 index 0000000..06744e4 --- /dev/null +++ b/packages/transformers/tests/types/_base.ts @@ -0,0 +1,12 @@ +// Checks if type X and Y are exactly the same +type Equal = + (() => T extends X ? 1 : 2) extends + (() => T extends Y ? 1 : 2) ? true : false; + +// Throws a type error if T is not `true` +type Expect = T; + +// Throws a type error if T is not `false` +type ExpectError = T; + +export type { Expect, Equal, ExpectError }; diff --git a/packages/transformers/tests/types/cache.ts b/packages/transformers/tests/types/cache.ts new file mode 100644 index 0000000..6ac3842 --- /dev/null +++ b/packages/transformers/tests/types/cache.ts @@ -0,0 +1,37 @@ +import { DynamicCache } from "../../src/cache_utils.js"; + +import type { Tensor } from "../../src/utils/tensor.js"; + +import type { TextGenerationConfig } from "../../src/pipelines/text-generation.js"; + +import type { Expect, Equal, ExpectError } from "./_base.ts"; + +// Initialize Dynamic Cache +const cache = new DynamicCache(); + +// Ensure cache can be indexed with string keys and returns Tensors +type T1 = Expect>; +type T2 = Expect>; + +// Ensure cache can be iterated over and entries are Tensors +for (const key in cache) { + type T3 = Expect>; +} + +// Ensure tensors in cache can be disposed +type T4 = Expect, Promise>>; + +// Ensure past_key_values accepts DynamicCache or null +type PastKeyValues = TextGenerationConfig['past_key_values']; +type T5 = Expect>; + +// Ensure DynamicCache is assignable to past_key_values +type IsAssignable = T extends U ? true : false; +type T6 = Expect>>; +type T7 = Expect>; + +// Ensure invalid types are NOT assignable to past_key_values +type T8 = ExpectError>>; +type T9 = ExpectError>>; +type T10 = ExpectError>>; +type T11 = ExpectError>>; diff --git a/packages/transformers/tests/types/pipelines.ts b/packages/transformers/tests/types/pipelines.ts new file mode 100644 index 0000000..a11d2c0 --- /dev/null +++ b/packages/transformers/tests/types/pipelines.ts @@ -0,0 +1,726 @@ +/** + * The pipeline function should correctly infer: + * 1. The type of the pipeline, based on the task name. + * 2. The output type of the pipeline, based on the types of the inputs. + * + * To test this, we create pipelines for various tasks, and call them with different types of inputs. + * We then check that the output types are as expected. + * + * Note: These tests are not meant to be executed, but rather to be type-checked by TypeScript. + */ +import { pipeline, InterruptableStoppingCriteria } from "../../src/transformers.js"; + +import type { RawImage } from "../../src/utils/image.js"; +import type { RawAudio } from "../../src/utils/audio.js"; +import type { DataArray, Tensor } from "../../src/utils/tensor.js"; + +import type { BoundingBox } from "../../src/pipelines/_base.js"; + +import type { AudioClassificationPipeline, AudioClassificationOutput } from "../../src/pipelines/audio-classification.js"; +import type { AutomaticSpeechRecognitionPipeline, AutomaticSpeechRecognitionOutput } from "../../src/pipelines/automatic-speech-recognition.js"; +import type { BackgroundRemovalPipeline } from "../../src/pipelines/background-removal.js"; +import type { DepthEstimationPipeline, DepthEstimationOutput } from "../../src/pipelines/depth-estimation.js"; +import type { DocumentQuestionAnsweringPipeline, DocumentQuestionAnsweringOutput } from "../../src/pipelines/document-question-answering.js"; +import type { FeatureExtractionPipeline } from "../../src/pipelines/feature-extraction.js"; +import type { FillMaskPipeline, FillMaskOutput } from "../../src/pipelines/fill-mask.js"; +import type { ImageClassificationPipeline, ImageClassificationOutput } from "../../src/pipelines/image-classification.js"; +import type { ImageFeatureExtractionPipeline } from "../../src/pipelines/image-feature-extraction.js"; +import type { ImageSegmentationPipeline, ImageSegmentationOutput } from "../../src/pipelines/image-segmentation.js"; +import type { ImageToImagePipeline } from "../../src/pipelines/image-to-image.js"; +import type { ImageToTextPipeline, ImageToTextOutput } from "../../src/pipelines/image-to-text.js"; +import type { ObjectDetectionPipeline, ObjectDetectionOutput } from "../../src/pipelines/object-detection.js"; +import type { QuestionAnsweringOutput, QuestionAnsweringPipeline } from "../../src/pipelines/question-answering.js"; +import type { SummarizationPipeline, SummarizationOutput } from "../../src/pipelines/summarization.js"; +import type { TextClassificationPipeline, TextClassificationOutput } from "../../src/pipelines/text-classification.js"; +import type { TextGenerationPipeline, TextGenerationStringOutput, TextGenerationChatOutput, Chat } from "../../src/pipelines/text-generation"; +import type { TextToAudioPipeline } from "../../src/pipelines/text-to-audio.js"; +import type { Text2TextGenerationPipeline, Text2TextGenerationOutput } from "../../src/pipelines/text2text-generation.js"; +import type { TokenClassificationPipeline, TokenClassificationOutput } from "../../src/pipelines/token-classification.js"; +import type { TranslationPipeline, TranslationOutput } from "../../src/pipelines/translation.js"; +import type { ZeroShotAudioClassificationPipeline, ZeroShotAudioClassificationOutput } from "../../src/pipelines/zero-shot-audio-classification.js"; +import type { ZeroShotClassificationPipeline, ZeroShotClassificationOutput } from "../../src/pipelines/zero-shot-classification.js"; +import type { ZeroShotImageClassificationPipeline, ZeroShotImageClassificationOutput } from "../../src/pipelines/zero-shot-image-classification.js"; +import type { ZeroShotObjectDetectionPipeline, ZeroShotObjectDetectionOutput } from "../../src/pipelines/zero-shot-object-detection.js"; + +import type { Expect, Equal } from "./_base.ts"; + +// Dummy inputs +const MODEL_ID = "organization/model"; +const URL = "https://example.com"; +const TEXT = "This is a test."; +const MESSAGES = [{ role: "user", content: "Hello!" }]; +const FLOAT32 = new Float32Array(16000); + +// Audio Classification +{ + const classifier = await pipeline("audio-classification", MODEL_ID); + type T = Expect>; + + // (a) Single input -> AudioClassificationOutput + { + const output = await classifier(URL); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } + + // (b) Batch input -> AudioClassificationOutput[] + { + const output = await classifier([URL, URL]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } +} + +// Automatic Speech Recognition +{ + const transcriber = await pipeline("automatic-speech-recognition", MODEL_ID); + type T = Expect>; + + // (a) Single input -> AutomaticSpeechRecognitionOutput + { + const output = await transcriber(FLOAT32); + type T = Expect>; + type T1 = Expect>; + } + + // (b) Batch input -> AutomaticSpeechRecognitionOutput[] + { + const output = await transcriber([FLOAT32, FLOAT32]); + type T = Expect>; + type T1 = Expect>; + } +} + +// Background Removal +{ + const remover = await pipeline("background-removal", MODEL_ID); + type T = Expect>; + + // (a) Single input -> RawImage + { + const output = await remover(URL); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + } + + // (b) Batch input -> RawImage[] + { + const output = await remover([URL, URL]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + } +} + +// Depth Estimation +{ + const depth_estimator = await pipeline("depth-estimation", MODEL_ID); + type T = Expect>; + + // (a) Single input -> DepthEstimationOutput + { + const output = await depth_estimator(URL); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } + // (b) Batch input with single image -> DepthEstimationOutput[] + { + const output = await depth_estimator([URL]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } + + // (c) Batch input with multiple images -> DepthEstimationOutput[] + { + const output = await depth_estimator([URL, URL]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } +} + +// Document Question Answering +{ + const answerer = await pipeline("document-question-answering", MODEL_ID); + type T = Expect>; + + // (a) Single input -> DocumentQuestionAnsweringOutput + { + const output = await answerer(URL, TEXT); + type T = Expect>; + type T1 = Expect>; + } + + // (b) Batch input (=1) -> DocumentQuestionAnsweringOutput + // TODO: Support batch_size > 1 + { + const output = await answerer([URL], TEXT); + type T = Expect>; + type T1 = Expect>; + } +} + +// Feature Extraction +{ + const extractor = await pipeline("feature-extraction", MODEL_ID); + type T = Expect>; + + // (a) Single input -> Tensor + { + const output = await extractor(TEXT); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } + + // (b) Batch input -> Tensor + { + const output = await extractor([TEXT, TEXT]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } +} + +// Fill-Mask +{ + const unmasker = await pipeline("fill-mask", MODEL_ID); + type T = Expect>; + + // (a) Single input -> FillMaskOutput + { + const output = await unmasker("This is a test."); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + type T4 = Expect>; + } + + // (b) Batch input -> FillMaskOutput[] + { + const output = await unmasker(["This is a test.", "Another example."]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + type T4 = Expect>; + } +} + +// Image Classification +{ + const classifier = await pipeline("image-classification", MODEL_ID); + type T = Expect>; + + // (a) Single input -> ImageClassificationOutput + { + const output = await classifier(URL); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } + + // (b) Batch input -> ImageClassificationOutput[] + { + const output = await classifier([URL, URL]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } +} + +// Image Feature Extraction +{ + const image_extractor = await pipeline("image-feature-extraction", MODEL_ID); + type T = Expect>; + + // (a) Single input -> Tensor + { + const output = await image_extractor(URL); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } + + // (b) Batch input -> Tensor + { + const output = await image_extractor([URL, URL]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } +} + +// Image Segmentation +{ + const segmenter = await pipeline("image-segmentation", MODEL_ID); + type T = Expect>; + + // (a) Single input -> ImageSegmentationOutput + { + const output = await segmenter(URL); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + } + + // (b) Batch input (=1) -> ImageSegmentationOutput[] + // TODO: Support batch_size > 1 + { + const output = await segmenter([URL]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + } +} + +// Image-to-Image +{ + const upscaler = await pipeline("image-to-image", MODEL_ID); + type T = Expect>; + + // (a) Single input -> RawImage + { + const output = await upscaler(URL); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + } + + // (b) Batch input -> RawImage[] + { + const output = await upscaler([URL, URL]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + } +} + +// Image-to-Text +{ + const ocr = await pipeline("image-to-text", MODEL_ID); + type T = Expect>; + + // (a) Single input -> ImageToTextOutput + { + const output = await ocr(URL); + type T = Expect>; + type T1 = Expect>; + } + + // (b) Batch input -> ImageToTextOutput[] + { + const output = await ocr([URL, URL]); + type T = Expect>; + type T1 = Expect>; + } +} + +// Object Detection +{ + const detector = await pipeline("object-detection", MODEL_ID); + type T = Expect>; + + // (a) Single input -> ObjectDetectionOutput + { + const output = await detector(URL); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + } + + // (b) Batch input -> ObjectDetectionOutput[] + { + const output = await detector([URL, URL]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + } +} + +// Question Answering +{ + const answerer = await pipeline("question-answering", MODEL_ID); + type T = Expect>; + + // (a) Single input, top_k=1 -> QuestionAnsweringOutput + { + const output = await answerer(TEXT, TEXT, { top_k: 1 }); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } + + // (b) Single input, top_k=3 -> QuestionAnsweringOutput[] + { + const output = await answerer(TEXT, TEXT, { top_k: 3 }); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } + + // (c) Batch input, top_k=1 -> QuestionAnsweringOutput[] + { + const output = await answerer([TEXT, TEXT], [TEXT, TEXT]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } + + // (d) Batch input, top_k=2 -> QuestionAnsweringOutput[][] + { + const output = await answerer([TEXT, TEXT], [TEXT, TEXT], { top_k: 2 }); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } +} + +// Summarization +{ + const summarizer = await pipeline("summarization", MODEL_ID); + type T = Expect>; + + // (a) Single input -> SummarizationOutput + { + const output = await summarizer(TEXT); + type T = Expect>; + type T1 = Expect>; + } + + // (b) Batch input -> SummarizationOutput + { + const output = await summarizer([TEXT, TEXT]); + type T = Expect>; + type T1 = Expect>; + } +} + +// Text Classification +{ + // Create a text classification pipeline + const classifier = await pipeline("text-classification", MODEL_ID); + type T = Expect>; + + // (a) Single input, top_k=1 -> TextClassificationOutput + { + const output = await classifier(TEXT, { top_k: 1 }); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } + + // (b) Single input, top_k=2 -> TextClassificationOutput + { + const output = await classifier(TEXT, { top_k: 2 }); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + type T4 = Expect>; + } + + // (c) Batch input, top_k=1 -> TextClassificationOutput + { + const output = await classifier([TEXT, TEXT], { top_k: 1 }); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } + + // (d) Batch input, top_k=2 -> TextClassificationOutput[] + { + const output = await classifier([TEXT, TEXT], { top_k: 2 }); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + type T4 = Expect>; + } +} + +// Text Generation +{ + const generator = await pipeline("text-generation", MODEL_ID); + type T = Expect>; + + // (a) Single input -> TextGenerationStringOutput + { + const output = await generator(TEXT); + type T = Expect>; + type T1 = Expect>; + } + + // (b) Batch input -> TextGenerationStringOutput[] + { + const output = await generator([TEXT, TEXT]); + type T = Expect>; + type T1 = Expect>; + } + + // (c) Chat input -> TextGenerationChatOutput + { + const output = await generator(MESSAGES); + type T = Expect>; + type T1 = Expect>; + } + + // (d) Batch chat input -> TextGenerationChatOutput[] + { + const output = await generator([MESSAGES, MESSAGES]); + type T = Expect>; + type T1 = Expect>; + } + + // (e) Chat input with generation parameters -> TextGenerationChatOutput + { + const output = await generator(MESSAGES, { + max_new_tokens: 50, + stopping_criteria: [new InterruptableStoppingCriteria()], + }); + type T = Expect>; + type T1 = Expect>; + } +} + +// Text-to-Audio +{ + const generator = await pipeline("text-to-audio", MODEL_ID); + type T = Expect>; + + // (a) Single input -> RawAudio + { + const output = await generator(TEXT); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } + + // (b) Batch input -> RawAudio[] + { + const output = await generator([TEXT, TEXT]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } +} + +// Text2Text Generation +{ + const generator = await pipeline("text2text-generation", MODEL_ID); + type T = Expect>; + + // (a) Single input -> Text2TextGenerationOutput + { + const output = await generator(TEXT); + type T = Expect>; + type T1 = Expect>; + } + + // (b) Batch input -> Text2TextGenerationOutput + { + const output = await generator([TEXT, TEXT]); + type T = Expect>; + type T1 = Expect>; + } +} + +// Token Classification +{ + const classifier = await pipeline("token-classification", MODEL_ID); + type T = Expect>; + + // `TokenClassificationOutput` is generic over the options type. The pipeline callback infers + // it from the options literal at the call site, so callers don't have to pass it manually. + // - omitted / "none" -> per-token shape (`entity`, `index`) + // - "simple" -> grouped shape (`entity_group`) + + // (a) Single input, no options -> per-token shape: `entity` + `index` typed as required fields. + { + const output = await classifier(TEXT); + type T = Expect>>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + type T4 = Expect>; + } + + // (b) Batch input, no options + { + const output = await classifier([TEXT, TEXT]); + type T = Expect[]>>; + type T1 = Expect>; + type T2 = Expect>; + } + + // (c) aggregation_strategy="none" -> same per-token shape. + { + const output = await classifier(TEXT, { aggregation_strategy: "none" }); + type T1 = Expect>; + type T2 = Expect>; + } + + // (d) aggregation_strategy="simple" -> grouped shape. + { + const output = await classifier(TEXT, { aggregation_strategy: "simple" }); + type T1 = Expect>; + type T2 = Expect>; + } + + // (e) Batch + aggregation_strategy="simple" -> per-element grouped shape. + { + const output = await classifier([TEXT, TEXT], { aggregation_strategy: "simple" }); + type T1 = Expect>; + } + + // (f) Other options without aggregation_strategy -> per-token shape. + { + const output = await classifier(TEXT, { ignore_labels: ["O"] }); + type T1 = Expect>; + type T2 = Expect>; + } + + // (g) `TokenClassificationOutput` used without a type argument is the unified union — + // both narrowed call results are assignable to it. + { + const raw = await classifier(TEXT); + const grouped = await classifier(TEXT, { aggregation_strategy: "simple" }); + const a: TokenClassificationOutput = raw; + const b: TokenClassificationOutput = grouped; + } + + // (h) Narrowing works via the `entity_group` discriminator. + { + const output = await classifier(TEXT, { aggregation_strategy: "simple" }); + const item = output[0]; + if (item.entity_group !== undefined) { + type T1 = Expect>; + } + } +} + +// Translation +{ + const translator = await pipeline("translation", MODEL_ID); + type T = Expect>; + + // (a) Single input -> TranslationOutput + { + const output = await translator(TEXT); + type T = Expect>; + type T1 = Expect>; + } + + // (b) Batch input -> TranslationOutput + { + const output = await translator([TEXT, TEXT]); + type T = Expect>; + type T1 = Expect>; + } +} + +// Zero-shot Audio Classification +{ + const classifier = await pipeline("zero-shot-audio-classification", MODEL_ID); + type T = Expect>; + + // (a) Single input -> ZeroShotAudioClassificationOutput + { + const output = await classifier(FLOAT32, ["class A", "class B"]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } + + // (b) Batch input -> ZeroShotAudioClassificationOutput[] + { + const output = await classifier([FLOAT32, FLOAT32], ["class A", "class B"]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } +} + +// Zero-shot Classification +{ + const classifier = await pipeline("zero-shot-classification", MODEL_ID); + type T = Expect>; + + // (a) Single input -> ZeroShotClassificationOutput + { + const output = await classifier(TEXT, ["class A", "class B"]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + } + + // (b) Batch input -> ZeroShotClassificationOutput[] + { + const output = await classifier([TEXT, TEXT], ["class A", "class B"]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + } +} + +// Zero-shot Image Classification +{ + const classifier = await pipeline("zero-shot-image-classification", MODEL_ID); + type T = Expect>; + + // (a) Single input -> ZeroShotImageClassificationOutput + { + const output = await classifier(URL, ["class A", "class B"]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } + + // (b) Batch input -> ZeroShotImageClassificationOutput[] + { + const output = await classifier([URL, URL], ["class A", "class B"]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + } +} + +// Zero-shot Object Detection +{ + const detector = await pipeline("zero-shot-object-detection", MODEL_ID); + type T = Expect>; + + // (a) Single input -> ZeroShotObjectDetectionOutput + { + const output = await detector(URL, ["class A", "class B"]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + } + + // (b) Batch input -> ZeroShotObjectDetectionOutput[] + { + const output = await detector([URL, URL], ["class A", "class B"]); + type T = Expect>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = Expect>; + } +} diff --git a/packages/transformers/tests/types/tokenizers.ts b/packages/transformers/tests/types/tokenizers.ts new file mode 100644 index 0000000..d31faaa --- /dev/null +++ b/packages/transformers/tests/types/tokenizers.ts @@ -0,0 +1,93 @@ +import type { + PreTrainedTokenizer, + BatchEncoding, +} from "../../src/tokenization_utils.js"; +import type { Tensor } from "../../src/utils/tensor.js"; + +import type { Expect, Equal, ExpectError } from "./_base.ts"; + +declare const tokenizer: PreTrainedTokenizer; + +const conversation = [{ role: "user", content: "Hello!" }] as const; +type IsAssignable = T extends U ? true : false; + +// Callable tokenizer defaults to tensors. +{ + const output = tokenizer("hello"); + type T0 = Expect>>; + type T1 = Expect>; + type T2 = Expect>; +} + +// Single text + arrays. +{ + const output = tokenizer("hello", { return_tensor: false }); + type T0 = Expect>>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = ExpectError>; +} + +// Batch text + arrays. +{ + const output = tokenizer(["hello", "world"], { return_tensor: false }); + type T0 = Expect>>; + type T1 = Expect>; + type T2 = Expect>; + type T3 = ExpectError>; +} + +// _call mirrors the callable signature. +{ + const output = tokenizer._call("hello", { return_tensor: true }); + type T0 = Expect>>; + type T1 = Expect>; + type T2 = ExpectError>; +} + +{ + const output = tokenizer._call(["hello", "world"], { return_tensor: false }); + type T0 = Expect>>; + type T1 = Expect>; +} + +// apply_chat_template narrows by tokenize / return_dict / return_tensor. +{ + const output = tokenizer.apply_chat_template([...conversation], { tokenize: false }); + type T1 = Expect>; + type T2 = ExpectError>; +} + +{ + const output = tokenizer.apply_chat_template([...conversation], { + return_tensor: true, + return_dict: true, + }); + type T0 = Expect>>; + type T1 = Expect>; +} + +{ + const output = tokenizer.apply_chat_template([...conversation], { + return_tensor: false, + return_dict: true, + }); + type T0 = Expect>>; + type T1 = Expect>; +} + +{ + const output = tokenizer.apply_chat_template([...conversation], { + return_tensor: true, + return_dict: false, + }); + type T1 = Expect>; +} + +{ + const output = tokenizer.apply_chat_template([...conversation], { + return_tensor: false, + return_dict: false, + }); + type T1 = Expect>; +} diff --git a/packages/transformers/tests/utils/audio.test.js b/packages/transformers/tests/utils/audio.test.js new file mode 100644 index 0000000..1c6c004 --- /dev/null +++ b/packages/transformers/tests/utils/audio.test.js @@ -0,0 +1,363 @@ +import { RawAudio, spectrogram, window_function, mel_filter_bank } from "../../src/utils/audio.js"; +import { init } from "../init.js"; + +init(); + +/** + * Helper function to generate a sine wave. + * @param {number} length Length of the audio in samples. + * @param {number} freq Frequency of the sine wave. + * @param {number} sampling_rate Sampling rate. + * @returns {Float32Array} The generated sine wave. + */ +function generateSineWave(length, freq, sampling_rate) { + const audio = new Float32Array(length); + for (let i = 0; i < length; ++i) { + audio[i] = Math.sin((2 * Math.PI * freq * i) / sampling_rate); + } + return audio; +} + +/** + * Zero-pad a window to a given frame length, centering the window. + * Replicates Python's window_function zero-padding behavior. + * @param {Float64Array} win The window to pad. + * @param {number} frameLength The target frame length. + * @returns {Float64Array} The zero-padded window. + */ +function zeroPadWindow(win, frameLength) { + const padded = new Float64Array(frameLength); + const offset = Math.floor((frameLength - win.length) / 2); + padded.set(win, offset); + return padded; +} + +/** + * Generate a deterministic waveform of a given length. + * @param {number} length Length of the waveform. + * @returns {Float32Array} The generated waveform. + */ +function generateDeterministicWaveform(length) { + const audio = new Float32Array(length); + const freqs = [220, 440, 880, 1760]; + const sr = 16000; + for (let i = 0; i < length; ++i) { + let val = 0; + for (const f of freqs) { + val += Math.sin((2 * Math.PI * f * i) / sr); + } + audio[i] = val / freqs.length; + } + return audio; +} + +/** + * Create an identity mel filter bank of shape (numBins, numBins). + * This lets us call spectrogram without changing the output shape, + * since the JS implementation requires mel_filters. + * @param {number} numBins Number of frequency bins. + * @returns {number[][]} Identity matrix as mel filters. + */ +function identityMelFilters(numBins) { + return Array.from({ length: numBins }, (_, i) => { + const row = new Array(numBins).fill(0); + row[i] = 1; + return row; + }); +} + +describe("Audio utilities", () => { + describe("RawAudio", () => { + it("should create RawAudio from a single Float32Array", () => { + const sampling_rate = 16000; + const audioData = generateSineWave(1000, 440, sampling_rate); + const rawAudio = new RawAudio(audioData, sampling_rate); + + expect(rawAudio.sampling_rate).toBe(sampling_rate); + expect(rawAudio.data).toBeInstanceOf(Float32Array); + expect(rawAudio.data).toEqual(audioData); + expect(rawAudio.data.length).toBe(1000); + }); + + it("should create RawAudio from multiple Float32Array chunks", () => { + const sampling_rate = 16000; + const chunk1 = generateSineWave(500, 440, sampling_rate); + const chunk2 = generateSineWave(500, 880, sampling_rate); + const rawAudio = new RawAudio([chunk1, chunk2], sampling_rate); + + expect(rawAudio.sampling_rate).toBe(sampling_rate); + expect(rawAudio.data).toBeInstanceOf(Float32Array); + expect(rawAudio.data.length).toBe(1000); + + // Check if concatenation is correct + const combined = new Float32Array(1000); + combined.set(chunk1, 0); + combined.set(chunk2, 500); + expect(rawAudio.data).toEqual(combined); + }); + + it("should handle empty array of chunks", () => { + const rawAudio = new RawAudio([], 16000); + expect(rawAudio.data).toBeInstanceOf(Float32Array); + expect(rawAudio.data.length).toBe(0); + }); + + it("should convert to Blob (WAV)", () => { + const sampling_rate = 16000; + const audioData = generateSineWave(1000, 440, sampling_rate); + const rawAudio = new RawAudio(audioData, sampling_rate); + + const blob = rawAudio.toBlob(); + expect(blob).toBeInstanceOf(Blob); + expect(blob.type).toBe("audio/wav"); + + // WAV header is 44 bytes + // 1000 samples * 4 bytes/sample (float32) = 4000 bytes + expect(blob.size).toBe(4044); + }); + + it("should convert to Blob (WAV) from chunks", () => { + const sampling_rate = 16000; + const chunk1 = generateSineWave(500, 440, sampling_rate); + const chunk2 = generateSineWave(500, 880, sampling_rate); + const rawAudio = new RawAudio([chunk1, chunk2], sampling_rate); + + const blob = rawAudio.toBlob(); + expect(blob).toBeInstanceOf(Blob); + expect(blob.type).toBe("audio/wav"); + expect(blob.size).toBe(4044); // 44 header + 4000 data + }); + }); + + describe("spectrogram", () => { + it("should compute spectrogram of impulse signal", async () => { + const waveform = new Float32Array(40); + waveform[9] = 1.0; + + const win = window_function(12, "hann"); + const paddedWin = zeroPadWindow(win, 16); + const numBins = 9; // onesided: fft_length/2 + 1 = 16/2 + 1 + + const spec = await spectrogram(waveform, paddedWin, 16, 4, { + power: 1.0, + center: true, + pad_mode: "reflect", + onesided: true, + mel_filters: identityMelFilters(numBins), + }); + + expect(spec.dims).toEqual([9, 11]); + + const expected = [0.0, 0.0669873, 0.9330127, 0.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]; + expect(Array.from(spec.data.slice(0, 11))).toBeCloseToNested(expected, 4); + }); + + it("should work with window_function frame_length zero-padding", async () => { + // Should zero-pad the window to length 16, matching Python behavior. + // See https://github.com/huggingface/transformers.js/issues/1387. + const waveform = new Float32Array(40); + waveform[9] = 1.0; + + const win = window_function(12, "hann", { frame_length: 16 }); + expect(win.length).toBe(16); + + const numBins = 9; + const spec = await spectrogram(waveform, win, 16, 4, { + power: 1.0, + center: true, + pad_mode: "reflect", + onesided: true, + mel_filters: identityMelFilters(numBins), + }); + + expect(spec.dims).toEqual([9, 11]); + + const expected = [0.0, 0.0669873, 0.9330127, 0.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]; + expect(Array.from(spec.data.slice(0, 11))).toBeCloseToNested(expected, 4); + }); + + describe("shapes", () => { + const waveform = generateDeterministicWaveform(93680); + + it("should produce correct shape with default params", async () => { + const numBins = 201; // 400/2 + 1 + const spec = await spectrogram(waveform, window_function(400, "hann"), 400, 128, { + power: 1.0, + center: true, + pad_mode: "reflect", + onesided: true, + mel_filters: identityMelFilters(numBins), + }); + expect(spec.dims).toEqual([201, 732]); + }); + + it("should produce correct shape with center=false", async () => { + const numBins = 201; + const spec = await spectrogram(waveform, window_function(400, "hann"), 400, 128, { + power: 1.0, + center: false, + pad_mode: "reflect", + onesided: true, + mel_filters: identityMelFilters(numBins), + }); + expect(spec.dims).toEqual([201, 729]); + }); + + it("should produce correct shape with fft_length", async () => { + const numBins = 257; // 512/2 + 1 + const spec = await spectrogram(waveform, window_function(400, "hann"), 400, 128, { + fft_length: 512, + power: 1.0, + center: true, + pad_mode: "reflect", + onesided: true, + mel_filters: identityMelFilters(numBins), + }); + expect(spec.dims).toEqual([257, 732]); + }); + + it("should produce correct shape with onesided=false and frame_length=512 (padded window)", async () => { + const win = window_function(400, "hann"); + const paddedWin = zeroPadWindow(win, 512); + const numBins = 512; // onesided=false + const spec = await spectrogram(waveform, paddedWin, 512, 64, { + power: 1.0, + center: true, + pad_mode: "reflect", + onesided: false, + mel_filters: identityMelFilters(numBins), + }); + expect(spec.dims).toEqual([512, 1464]); + }); + + it("should produce correct shape with onesided=false and window_length=512", async () => { + const numBins = 512; + const spec = await spectrogram(waveform, window_function(512, "hann"), 512, 64, { + power: 1.0, + center: true, + pad_mode: "reflect", + onesided: false, + mel_filters: identityMelFilters(numBins), + }); + expect(spec.dims).toEqual([512, 1464]); + }); + + it("should produce correct shape with large hop_length", async () => { + const numBins = 512; + const spec = await spectrogram(waveform, window_function(512, "hann"), 512, 512, { + power: 1.0, + center: true, + pad_mode: "reflect", + onesided: false, + mel_filters: identityMelFilters(numBins), + }); + expect(spec.dims).toEqual([512, 183]); + }); + }); + + describe("center padding", () => { + const waveform = generateDeterministicWaveform(93680); + const numBins = 257; // 512/2 + 1 + + it("should handle reflect padding", async () => { + const spec = await spectrogram(waveform, window_function(512, "hann"), 512, 128, { + center: true, + pad_mode: "reflect", + mel_filters: identityMelFilters(numBins), + }); + expect(spec.dims).toEqual([257, 732]); + }); + + it("should handle constant padding", async () => { + const spec = await spectrogram(waveform, window_function(512, "hann"), 512, 128, { + center: true, + pad_mode: "constant", + mel_filters: identityMelFilters(numBins), + }); + expect(spec.dims).toEqual([257, 732]); + }); + + it("should handle no centering", async () => { + const spec = await spectrogram(waveform, window_function(512, "hann"), 512, 128, { + center: false, + mel_filters: identityMelFilters(numBins), + }); + expect(spec.dims).toEqual([257, 728]); + }); + }); + + describe("mel spectrogram", () => { + const waveform = generateDeterministicWaveform(93680); + + it("should produce correct shape without mel filters (using identity)", async () => { + const numBins = 513; // 1024/2 + 1 + const win = window_function(800, "hann"); + const paddedWin = zeroPadWindow(win, 1024); + const spec = await spectrogram(waveform, paddedWin, 1024, 128, { + power: 2.0, + mel_filters: identityMelFilters(numBins), + }); + expect(spec.dims).toEqual([513, 732]); + }); + + it("should produce correct shape with mel filters", async () => { + const mel_filters = mel_filter_bank(513, 13, 100, 4000, 16000, null, "htk"); + + const win = window_function(800, "hann"); + const paddedWin = zeroPadWindow(win, 1024); + const spec = await spectrogram(waveform, paddedWin, 1024, 128, { + power: 2.0, + mel_filters, + }); + expect(spec.dims).toEqual([13, 732]); + }); + }); + + describe("power", () => { + const waveform = generateDeterministicWaveform(93680); + + it("should compute amplitude spectrogram (power=1.0)", async () => { + const numBins = 257; + const win = window_function(400, "hann"); + const paddedWin = zeroPadWindow(win, 512); + const spec = await spectrogram(waveform, paddedWin, 512, 128, { + power: 1.0, + mel_filters: identityMelFilters(numBins), + }); + expect(spec.dims).toEqual([257, 732]); + }); + + it("should compute power spectrogram (power=2.0)", async () => { + const numBins = 257; + const win = window_function(400, "hann"); + const paddedWin = zeroPadWindow(win, 512); + const spec = await spectrogram(waveform, paddedWin, 512, 128, { + power: 2.0, + mel_filters: identityMelFilters(numBins), + }); + expect(spec.dims).toEqual([257, 732]); + }); + + it("power=2 values should be square of power=1 values", async () => { + const numBins = 257; + const win = window_function(400, "hann"); + const paddedWin = zeroPadWindow(win, 512); + + const spec1 = await spectrogram(waveform, paddedWin, 512, 128, { + power: 1.0, + mel_filters: identityMelFilters(numBins), + }); + const spec2 = await spectrogram(waveform, paddedWin, 512, 128, { + power: 2.0, + mel_filters: identityMelFilters(numBins), + }); + + // Check a slice: power=2 values should equal power=1 values squared + const slice1 = Array.from(spec1.data.slice(0, 20)); + const slice2 = Array.from(spec2.data.slice(0, 20)); + const expectedSquared = slice1.map((v) => v * v); + expect(slice2).toBeCloseToNested(expectedSquared, 4); + }); + }); + }); +}); diff --git a/packages/transformers/tests/utils/cache.test.js b/packages/transformers/tests/utils/cache.test.js new file mode 100644 index 0000000..4e5a8fb --- /dev/null +++ b/packages/transformers/tests/utils/cache.test.js @@ -0,0 +1,539 @@ +import { ModelRegistry } from "../../src/transformers.js"; +import { getModelFile } from "../../src/utils/hub.js"; + +import { MAX_TEST_EXECUTION_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +const LLAMA_MODEL_ID = "hf-internal-testing/tiny-random-LlamaForCausalLM"; +const BERT_MODEL_ID = "hf-internal-testing/tiny-random-BertModel"; +const VIT_MODEL_ID = "hf-internal-testing/tiny-random-vit"; + +// Dedicated model IDs for cache clearing tests to avoid interference with other parallel tests. +// These must NOT be used in any other test file. +const CLEAR_CACHE_MODEL_ID = "onnx-internal-testing/tiny-random-BertModel-ONNX"; +const CLEAR_PIPELINE_CACHE_MODEL_ID = "onnx-internal-testing/tiny-random-Qwen3ForCausalLM"; + +describe("Cache", () => { + describe("ModelRegistry", () => { + describe("get_files", () => { + it( + "should return files for a decoder-only model", + async () => { + const files = await ModelRegistry.get_files(LLAMA_MODEL_ID, DEFAULT_MODEL_OPTIONS); + expect(Array.isArray(files)).toBe(true); + expect(files.length).toBeGreaterThan(0); + expect(files).toContain("config.json"); + expect(files).toContain("generation_config.json"); + expect(files.some((f) => f.startsWith("onnx/") && f.endsWith(".onnx"))).toBe(true); + expect(files).toContain("tokenizer.json"); + expect(files).toContain("tokenizer_config.json"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should return files for an encoder-only model", + async () => { + const files = await ModelRegistry.get_files(BERT_MODEL_ID, DEFAULT_MODEL_OPTIONS); + expect(Array.isArray(files)).toBe(true); + expect(files.length).toBeGreaterThan(0); + expect(files).toContain("config.json"); + expect(files.some((f) => f.startsWith("onnx/") && f.endsWith(".onnx"))).toBe(true); + expect(files).toContain("tokenizer.json"); + expect(files).toContain("tokenizer_config.json"); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("get_model_files", () => { + it( + "should return model files for a decoder-only model", + async () => { + const files = await ModelRegistry.get_model_files(LLAMA_MODEL_ID, DEFAULT_MODEL_OPTIONS); + expect(Array.isArray(files)).toBe(true); + expect(files).toContain("config.json"); + expect(files).toContain("generation_config.json"); + expect(files.some((f) => f.startsWith("onnx/") && f.endsWith(".onnx"))).toBe(true); + // Should not include tokenizer files + expect(files).not.toContain("tokenizer.json"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should return model files for an encoder-only model", + async () => { + const files = await ModelRegistry.get_model_files(BERT_MODEL_ID, DEFAULT_MODEL_OPTIONS); + expect(Array.isArray(files)).toBe(true); + expect(files).toContain("config.json"); + expect(files.some((f) => f.startsWith("onnx/") && f.endsWith(".onnx"))).toBe(true); + // Encoder-only models should not have generation_config.json + expect(files).not.toContain("generation_config.json"); + // Should not include tokenizer files + expect(files).not.toContain("tokenizer.json"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should use model_file_name when provided", + async () => { + const files = await ModelRegistry.get_model_files(BERT_MODEL_ID, { + ...DEFAULT_MODEL_OPTIONS, + model_file_name: "custom_model", + }); + expect(files).toContain("config.json"); + // Should use custom model file name + expect(files.some((f) => f.includes("custom_model") && f.endsWith(".onnx"))).toBe(true); + // Should NOT contain the default 'model' name + expect(files.some((f) => f === "onnx/model.onnx")).toBe(false); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("get_tokenizer_files", () => { + it( + "should return tokenizer files for a decoder-only model", + async () => { + const files = await ModelRegistry.get_tokenizer_files(LLAMA_MODEL_ID); + expect(files).toEqual(["tokenizer.json", "tokenizer_config.json"]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should return tokenizer files for an encoder-only model", + async () => { + const files = await ModelRegistry.get_tokenizer_files(BERT_MODEL_ID); + expect(files).toEqual(["tokenizer.json", "tokenizer_config.json"]); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("get_processor_files", () => { + it( + "should return empty array for text-only models", + async () => { + const llamaFiles = await ModelRegistry.get_processor_files(LLAMA_MODEL_ID); + expect(llamaFiles).toEqual([]); + + const bertFiles = await ModelRegistry.get_processor_files(BERT_MODEL_ID); + expect(bertFiles).toEqual([]); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should return processor files for a vision model", + async () => { + const files = await ModelRegistry.get_processor_files(VIT_MODEL_ID); + expect(files).toContain("preprocessor_config.json"); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("get_pipeline_files", () => { + it( + "should return files for text-generation pipeline", + async () => { + const files = await ModelRegistry.get_pipeline_files("text-generation", LLAMA_MODEL_ID, DEFAULT_MODEL_OPTIONS); + expect(Array.isArray(files)).toBe(true); + expect(files).toContain("config.json"); + expect(files).toContain("generation_config.json"); + expect(files.some((f) => f.startsWith("onnx/") && f.endsWith(".onnx"))).toBe(true); + expect(files).toContain("tokenizer.json"); + expect(files).toContain("tokenizer_config.json"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should return files for feature-extraction pipeline", + async () => { + const files = await ModelRegistry.get_pipeline_files("feature-extraction", BERT_MODEL_ID, DEFAULT_MODEL_OPTIONS); + expect(Array.isArray(files)).toBe(true); + expect(files).toContain("config.json"); + expect(files.some((f) => f.startsWith("onnx/") && f.endsWith(".onnx"))).toBe(true); + expect(files).toContain("tokenizer.json"); + expect(files).toContain("tokenizer_config.json"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should resolve task aliases", + async () => { + const files = await ModelRegistry.get_pipeline_files("sentiment-analysis", BERT_MODEL_ID, DEFAULT_MODEL_OPTIONS); + expect(Array.isArray(files)).toBe(true); + expect(files).toContain("config.json"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should throw for unsupported pipeline task", + async () => { + await expect(ModelRegistry.get_pipeline_files("invalid-nonexistent-task", BERT_MODEL_ID, DEFAULT_MODEL_OPTIONS)).rejects.toThrow("Unsupported pipeline task"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should exclude vision_encoder for text-generation on multimodal model", + async () => { + const files = await ModelRegistry.get_pipeline_files("text-generation", "onnx-community/gemma-3-4b-it-ONNX", { + device: "webgpu", + dtype: "q4f16", + }); + expect(Array.isArray(files)).toBe(true); + + // Should NOT include vision_encoder + expect(files.some((f) => f.includes("vision_encoder"))).toBe(false); + + // Should include only embed_tokens and decoder_model_merged with q4f16 suffix + expect(files).toContain("onnx/embed_tokens_q4f16.onnx"); + expect(files).toContain("onnx/decoder_model_merged_q4f16.onnx"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should exclude vision_encoder for text-generation on ImageTextToText model", + async () => { + const files = await ModelRegistry.get_pipeline_files("text-generation", "onnx-community/Qwen3.5-0.8B-ONNX", { + device: "webgpu", + dtype: "q4f16", + }); + expect(Array.isArray(files)).toBe(true); + + // Should NOT include vision_encoder + expect(files.some((f) => f.includes("vision_encoder"))).toBe(false); + + // Should include embed_tokens and decoder_model_merged + expect(files).toContain("onnx/embed_tokens_q4f16.onnx"); + expect(files).toContain("onnx/decoder_model_merged_q4f16.onnx"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should exclude audio_encoder for text-generation on AudioTextToText model (Voxtral)", + async () => { + const files = await ModelRegistry.get_pipeline_files("text-generation", "onnx-community/Voxtral-Mini-3B-2507-ONNX", { + device: "webgpu", + dtype: "q4f16", + }); + expect(Array.isArray(files)).toBe(true); + + // Should NOT include audio_encoder + expect(files.some((f) => f.includes("audio_encoder"))).toBe(false); + + // Should include embed_tokens and decoder_model_merged + expect(files).toContain("onnx/embed_tokens_q4f16.onnx"); + expect(files).toContain("onnx/decoder_model_merged_q4f16.onnx"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should exclude audio_encoder for text-generation on AudioTextToText model (Ultravox)", + async () => { + const files = await ModelRegistry.get_pipeline_files("text-generation", "onnx-community/ultravox-v0_5-llama-3_2-1b-ONNX", { + device: "webgpu", + dtype: "q4f16", + }); + expect(Array.isArray(files)).toBe(true); + + // Should NOT include audio_encoder + expect(files.some((f) => f.includes("audio_encoder"))).toBe(false); + + // Should include embed_tokens and decoder_model_merged + expect(files).toContain("onnx/embed_tokens_q4f16.onnx"); + expect(files).toContain("onnx/decoder_model_merged_q4f16.onnx"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should exclude vision_encoder and audio_encoder for text-generation on ImageAudioTextToText model", + async () => { + const files = await ModelRegistry.get_pipeline_files("text-generation", "onnx-community/gemma-3n-E2B-it-ONNX", { + device: "webgpu", + dtype: "q4f16", + }); + expect(Array.isArray(files)).toBe(true); + + // Should NOT include vision_encoder or audio_encoder + expect(files.some((f) => f.includes("vision_encoder"))).toBe(false); + expect(files.some((f) => f.includes("audio_encoder"))).toBe(false); + + // Should include embed_tokens and decoder_model_merged + expect(files).toContain("onnx/embed_tokens_q4f16.onnx"); + expect(files).toContain("onnx/decoder_model_merged_q4f16.onnx"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should not include processor files for text-only tasks", + async () => { + const files = await ModelRegistry.get_pipeline_files("text-generation", LLAMA_MODEL_ID, DEFAULT_MODEL_OPTIONS); + // text-generation tasks don't use a processor, so no preprocessor_config.json + expect(files).not.toContain("preprocessor_config.json"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should include processor files for image tasks", + async () => { + const files = await ModelRegistry.get_pipeline_files("image-classification", VIT_MODEL_ID, DEFAULT_MODEL_OPTIONS); + expect(files).toContain("preprocessor_config.json"); + // image-classification doesn't use a tokenizer + expect(files).not.toContain("tokenizer.json"); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("is_cached", () => { + it( + "should return a boolean", + async () => { + const cached = await ModelRegistry.is_cached(BERT_MODEL_ID, DEFAULT_MODEL_OPTIONS); + expect(typeof cached).toBe("boolean"); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("is_cached_files", () => { + it( + "should return cache status with correct shape", + async () => { + const status = await ModelRegistry.is_cached_files(BERT_MODEL_ID, DEFAULT_MODEL_OPTIONS); + expect(status).toHaveProperty("allCached"); + expect(typeof status.allCached).toBe("boolean"); + expect(status).toHaveProperty("files"); + expect(Array.isArray(status.files)).toBe(true); + expect(status.files.length).toBeGreaterThan(0); + for (const entry of status.files) { + expect(entry).toHaveProperty("file"); + expect(typeof entry.file).toBe("string"); + expect(entry).toHaveProperty("cached"); + expect(typeof entry.cached).toBe("boolean"); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should agree with is_cached on allCached", + async () => { + const cached = await ModelRegistry.is_cached(BERT_MODEL_ID, DEFAULT_MODEL_OPTIONS); + const status = await ModelRegistry.is_cached_files(BERT_MODEL_ID, DEFAULT_MODEL_OPTIONS); + expect(cached).toBe(status.allCached); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("is_pipeline_cached", () => { + it( + "should return a boolean for text-generation pipeline", + async () => { + const cached = await ModelRegistry.is_pipeline_cached("text-generation", LLAMA_MODEL_ID, DEFAULT_MODEL_OPTIONS); + expect(typeof cached).toBe("boolean"); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("is_pipeline_cached_files", () => { + it( + "should return cache status for text-generation pipeline", + async () => { + const status = await ModelRegistry.is_pipeline_cached_files("text-generation", LLAMA_MODEL_ID, DEFAULT_MODEL_OPTIONS); + expect(status).toHaveProperty("allCached"); + expect(typeof status.allCached).toBe("boolean"); + expect(status).toHaveProperty("files"); + expect(Array.isArray(status.files)).toBe(true); + expect(status.files.length).toBeGreaterThan(0); + for (const entry of status.files) { + expect(entry).toHaveProperty("file"); + expect(entry).toHaveProperty("cached"); + } + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should agree with is_pipeline_cached on allCached", + async () => { + const cached = await ModelRegistry.is_pipeline_cached("text-generation", LLAMA_MODEL_ID, DEFAULT_MODEL_OPTIONS); + const status = await ModelRegistry.is_pipeline_cached_files("text-generation", LLAMA_MODEL_ID, DEFAULT_MODEL_OPTIONS); + expect(cached).toBe(status.allCached); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("get_file_metadata", () => { + it( + "should return metadata for an existing file", + async () => { + const metadata = await ModelRegistry.get_file_metadata(BERT_MODEL_ID, "config.json"); + expect(metadata).toHaveProperty("exists", true); + expect(metadata).toHaveProperty("size"); + expect(typeof metadata.size).toBe("number"); + expect(metadata.size).toBeGreaterThan(0); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should return exists=false for a non-existent file", + async () => { + const metadata = await ModelRegistry.get_file_metadata(BERT_MODEL_ID, "nonexistent_file.bin"); + expect(metadata).toHaveProperty("exists", false); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("clear_cache", () => { + it( + "should clear cached files and report results", + async () => { + // Step 1: Pre-cache config.json by downloading it + await getModelFile(CLEAR_CACHE_MODEL_ID, "config.json", true, {}); + + // Step 2: Clear the cache + const result = await ModelRegistry.clear_cache(CLEAR_CACHE_MODEL_ID, DEFAULT_MODEL_OPTIONS); + + // Step 3: Verify response shape + expect(result).toHaveProperty("filesDeleted"); + expect(typeof result.filesDeleted).toBe("number"); + expect(result).toHaveProperty("filesCached"); + expect(typeof result.filesCached).toBe("number"); + expect(result).toHaveProperty("files"); + expect(Array.isArray(result.files)).toBe(true); + for (const entry of result.files) { + expect(entry).toHaveProperty("file"); + expect(typeof entry.file).toBe("string"); + expect(entry).toHaveProperty("deleted"); + expect(typeof entry.deleted).toBe("boolean"); + expect(entry).toHaveProperty("wasCached"); + expect(typeof entry.wasCached).toBe("boolean"); + } + + // Step 4: config.json should have been cached and deleted + // (it was pre-cached in Step 1) + const configEntry = result.files.find((f) => f.file === "config.json"); + expect(configEntry?.wasCached).toBe(true); + expect(configEntry?.deleted).toBe(true); + expect(result.filesDeleted).toBeGreaterThan(0); + + // NOTE: We don't re-check is_cached here because it internally calls + // get_model_files() -> AutoConfig.from_pretrained(), which re-downloads + // config.json and re-populates the cache as a side effect. + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("clear_pipeline_cache", () => { + it( + "should clear cached pipeline files and report results", + async () => { + // Step 1: Pre-cache config.json by downloading it + await getModelFile(CLEAR_PIPELINE_CACHE_MODEL_ID, "config.json", true, {}); + + // Step 2: Clear the pipeline cache + const result = await ModelRegistry.clear_pipeline_cache("text-generation", CLEAR_PIPELINE_CACHE_MODEL_ID, DEFAULT_MODEL_OPTIONS); + + // Step 3: Verify response shape + expect(result).toHaveProperty("filesDeleted"); + expect(typeof result.filesDeleted).toBe("number"); + expect(result).toHaveProperty("filesCached"); + expect(typeof result.filesCached).toBe("number"); + expect(result).toHaveProperty("files"); + expect(Array.isArray(result.files)).toBe(true); + for (const entry of result.files) { + expect(entry).toHaveProperty("file"); + expect(entry).toHaveProperty("deleted"); + expect(entry).toHaveProperty("wasCached"); + } + + // Step 4: Should include expected pipeline files + const fileNames = result.files.map((f) => f.file); + expect(fileNames).toContain("config.json"); + expect(fileNames).toContain("tokenizer.json"); + expect(fileNames.some((f) => f.startsWith("onnx/") && f.endsWith(".onnx"))).toBe(true); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should throw for unsupported pipeline task", + async () => { + await expect(ModelRegistry.clear_pipeline_cache("invalid-nonexistent-task", LLAMA_MODEL_ID, DEFAULT_MODEL_OPTIONS)).rejects.toThrow("Unsupported pipeline task"); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("error handling", () => { + it( + "should throw for empty modelId in is_cached", + async () => { + await expect(ModelRegistry.is_cached("")).rejects.toThrow("modelId is required"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should throw for empty modelId in is_cached_files", + async () => { + await expect(ModelRegistry.is_cached_files("")).rejects.toThrow("modelId is required"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should throw for empty modelId in clear_cache", + async () => { + await expect(ModelRegistry.clear_cache("")).rejects.toThrow("modelId is required"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should throw for empty task in is_pipeline_cached", + async () => { + await expect(ModelRegistry.is_pipeline_cached("", BERT_MODEL_ID)).rejects.toThrow("task is required"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should throw for empty task in is_pipeline_cached_files", + async () => { + await expect(ModelRegistry.is_pipeline_cached_files("", BERT_MODEL_ID)).rejects.toThrow("task is required"); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should throw for empty task in clear_pipeline_cache", + async () => { + await expect(ModelRegistry.clear_pipeline_cache("", BERT_MODEL_ID)).rejects.toThrow("task is required"); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + }); +}); diff --git a/packages/transformers/tests/utils/custom_cache.test.js b/packages/transformers/tests/utils/custom_cache.test.js new file mode 100644 index 0000000..adda285 --- /dev/null +++ b/packages/transformers/tests/utils/custom_cache.test.js @@ -0,0 +1,109 @@ +import { env, LlamaForCausalLM, AutoTokenizer } from "../../src/transformers.js"; +import { init, MAX_TEST_EXECUTION_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +// Initialise the testing environment +init(); + +/** + * A naive custom cache implementation that fetches files directly from the + * Hugging Face Hub and stores them in an internal (in-memory) map. + * This satisfies the CacheInterface contract (`match` + `put`). + */ +class NaiveFetchCache { + constructor() { + /** @type {Map} */ + this.cache = new Map(); + } + + async match(request) { + const cached = this.cache.get(request); + if (cached) { + return cached.clone(); + } + + // Not in cache — attempt a fresh fetch from the URL. + try { + const response = await fetch(request); + if (response.ok) { + this.cache.set(request, response); + return response.clone(); + } + } catch { + // Ignore fetch errors (e.g., invalid URLs like local paths) — treat as cache miss + } + return undefined; + } + + async put(request, response) { + if (!this.cache.has(request)) { + this.cache.set(request, response); + } + } +} + +describe("Custom cache", () => { + // Store original env values so we can restore them after tests + const originalUseCustomCache = env.useCustomCache; + const originalCustomCache = env.customCache; + const originalUseBrowserCache = env.useBrowserCache; + const originalUseFSCache = env.useFSCache; + const originalAllowLocalModels = env.allowLocalModels; + + beforeAll(() => { + // Disable all other caching mechanisms so only customCache is used + env.useCustomCache = true; + env.customCache = new NaiveFetchCache(); + env.useBrowserCache = false; + env.useFSCache = false; + env.allowLocalModels = false; + }); + + afterAll(() => { + // Restore original env values + env.useCustomCache = originalUseCustomCache; + env.customCache = originalCustomCache; + env.useBrowserCache = originalUseBrowserCache; + env.useFSCache = originalUseFSCache; + env.allowLocalModels = originalAllowLocalModels; + }); + + it( + "should load a model using custom cache (standard)", + async () => { + const model_id = "onnx-internal-testing/tiny-random-LlamaForCausalLM-ONNX"; + + const tokenizer = await AutoTokenizer.from_pretrained(model_id); + const model = await LlamaForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + + const inputs = await tokenizer("Hello"); + const output = await model(inputs); + + expect(output.logits).toBeDefined(); + const expected_shape = [...inputs.input_ids.dims, model.config.vocab_size]; + expect(output.logits.dims).toEqual(expected_shape); + + await model.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should load a model using custom cache (external data)", + async () => { + const model_id = "onnx-internal-testing/tiny-random-LlamaForCausalLM-ONNX_external"; + + const tokenizer = await AutoTokenizer.from_pretrained(model_id); + const model = await LlamaForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + + const inputs = await tokenizer("Hello"); + const output = await model(inputs); + + expect(output.logits).toBeDefined(); + const expected_shape = [...inputs.input_ids.dims, model.config.vocab_size]; + expect(output.logits.dims).toEqual(expected_shape); + + await model.dispose(); + }, + MAX_TEST_EXECUTION_TIME, + ); +}); diff --git a/packages/transformers/tests/utils/generation.test.js b/packages/transformers/tests/utils/generation.test.js new file mode 100644 index 0000000..2319855 --- /dev/null +++ b/packages/transformers/tests/utils/generation.test.js @@ -0,0 +1,591 @@ +import { + // Models + AutoModelForSeq2SeqLM, + AutoModelForCausalLM, + LlamaForCausalLM, + + // Tokenizers + AutoTokenizer, + LlamaTokenizer, + + // Other + TextStreamer, + DynamicCache, + random, + full, +} from "../../src/transformers.js"; + +import { init, MAX_TEST_EXECUTION_TIME, MAX_MODEL_LOAD_TIME, MAX_MODEL_DISPOSE_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; + +// Initialise the testing environment +init(); + +// Helper function to generate text +const generate = async (model, tokenizer, text, options) => { + const inputs = tokenizer(text); + return await model.generate({ + ...inputs, + ...options, + }); +}; + +describe("Generation parameters", () => { + // List all models which will be tested + const models = [ + "hf-internal-testing/tiny-random-T5ForConditionalGeneration", // encoder-decoder + "hf-internal-testing/tiny-random-LlamaForCausalLM", // decoder-only + ]; + const DUMMY_TEXT = "hello"; + + describe(`encoder-decoder (${models[0]})`, () => { + const model_id = models[0]; + + let model; + let tokenizer; + beforeAll(async () => { + model = await AutoModelForSeq2SeqLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await AutoTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + // NOTE: Since `max_length` defaults to 20, this case also tests that. + it( + "default", + async () => { + const outputs = await generate(model, tokenizer, DUMMY_TEXT, {}); + expect(outputs.dims.at(-1)).toEqual(20); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "max_new_tokens", + async () => { + const MAX_NEW_TOKENS = 5; + const outputs = await generate(model, tokenizer, DUMMY_TEXT, { + max_new_tokens: MAX_NEW_TOKENS, + }); + expect(outputs.dims.at(-1)).toEqual(MAX_NEW_TOKENS + 1); // + 1 due to forced BOS token + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "min_length", + async () => { + const MIN_LENGTH = 3; + const MAX_LENGTH = 5; + const outputs = await generate(model, tokenizer, DUMMY_TEXT, { + eos_token_id: 0, + min_length: MIN_LENGTH, + max_length: MAX_LENGTH, + }); + expect(outputs.tolist()).toEqual([[0n, 11924n, 11924n, 11924n, 11924n]]); + expect(outputs.dims.at(-1)).toBeGreaterThanOrEqual(MIN_LENGTH); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "min_new_tokens", + async () => { + const MIN_NEW_TOKENS = 2; + const MAX_LENGTH = 5; + const outputs = await generate(model, tokenizer, DUMMY_TEXT, { + eos_token_id: 0, + min_new_tokens: MIN_NEW_TOKENS, + max_length: MAX_LENGTH, + }); + expect(outputs.tolist()).toEqual([[0n, 11924n, 11924n, 11924n, 11924n]]); + expect(outputs.dims.at(-1)).toBeGreaterThanOrEqual(MIN_NEW_TOKENS); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe(`decoder-only (${models[1]})`, () => { + const model_id = models[1]; + + let model; + let tokenizer; + beforeAll(async () => { + model = await AutoModelForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await AutoTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + // NOTE: Since `max_length` defaults to 20, this case also tests that. + it( + "default", + async () => { + const outputs = await generate(model, tokenizer, DUMMY_TEXT, {}); + expect(outputs.dims.at(-1)).toEqual(20); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "max_new_tokens", + async () => { + const MAX_NEW_TOKENS = 5; + const PROMPT_LENGTH = 2; // BOS + DUMMY_TEXT + const outputs = await generate(model, tokenizer, DUMMY_TEXT, { + max_new_tokens: MAX_NEW_TOKENS, + }); + const expected_length = PROMPT_LENGTH + MAX_NEW_TOKENS; + expect(outputs.dims.at(-1)).toEqual(expected_length); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "min_length", + async () => { + const MIN_LENGTH = 4; + const outputs = await generate(model, tokenizer, DUMMY_TEXT, { + eos_token_id: [ + 18547, // min_length will suppress this token (generated by default) + 16012, // stop at this token + ], + min_length: MIN_LENGTH, + }); + expect(outputs.tolist()).toEqual([[1n, 22172n, 31583n, 18824n, 16621n, 8136n, 16012n]]); + expect(outputs.dims.at(-1)).toBeGreaterThanOrEqual(MIN_LENGTH); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "min_new_tokens", + async () => { + const MIN_NEW_TOKENS = 2; + const outputs = await generate(model, tokenizer, DUMMY_TEXT, { + eos_token_id: [ + 18547, // min_new_tokens will suppress this token (generated by default) + 16012, // stop at this token + ], + min_new_tokens: MIN_NEW_TOKENS, + }); + expect(outputs.tolist()).toEqual([[1n, 22172n, 31583n, 18824n, 16621n, 8136n, 16012n]]); + expect(outputs.dims.at(-1)).toBeGreaterThanOrEqual(MIN_NEW_TOKENS); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "do_sample (seeded)", + async () => { + // Seed 42: deterministic sampling + random.seed(42); + const outputs_seed42_a = await generate(model, tokenizer, DUMMY_TEXT, { + do_sample: true, + max_new_tokens: 10, + }); + + // Re-seed 42: must reproduce the same output + random.seed(42); + const outputs_seed42_b = await generate(model, tokenizer, DUMMY_TEXT, { + do_sample: true, + max_new_tokens: 10, + }); + + // Seed 123: different seed → different output + random.seed(123); + const outputs_seed123 = await generate(model, tokenizer, DUMMY_TEXT, { + do_sample: true, + max_new_tokens: 10, + }); + + const expected_seed42 = [[1n, 22172n, 28220n, 5345n, 27342n, 14352n, 24712n, 19249n, 24075n, 19934n, 8678n, 30868n]]; + const expected_seed123 = [[1n, 22172n, 10131n, 867n, 12403n, 24755n, 16382n, 21742n, 24662n, 19120n, 22952n, 945n]]; + + expect(outputs_seed42_a.tolist()).toEqual(expected_seed42); + expect(outputs_seed42_b.tolist()).toEqual(expected_seed42); + expect(outputs_seed123.tolist()).toEqual(expected_seed123); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}); + +describe("Streamers", () => { + describe("decoder-only (Llama)", () => { + const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; + let model, tokenizer; + let DUMMY_TOKEN_STREAM; + beforeAll(async () => { + model = await AutoModelForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await AutoTokenizer.from_pretrained(model_id); + + DUMMY_TOKEN_STREAM = [ + [12199n], // regular token ("hello") + [12199n], // regular token ("hello") + [22172n], // regular token (" hello") + [12199n], // regular token ("hello") + [13n], // regular token ("\n") + [12199n], // regular token ("hello") + [BigInt(tokenizer.bos_token_id)], // special token (BOS) + [12199n], // regular token ("hello") + ]; + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const target_chunks = ["hello", /* Always flush after prompt. */ "erdingsdelete ", "melytabular ", "Stadiumoba ", "alcune ", "drug"]; + const chunks = []; + const callback_function = (text) => { + chunks.push(text); + }; + const streamer = new TextStreamer(tokenizer, { callback_function, skip_special_tokens: true }); + + const inputs = tokenizer("hello"); + const outputs = await model.generate({ + ...inputs, + max_length: 10, + streamer, + }); + expect(outputs.tolist()).toEqual([[1n, 22172n, 18547n, 8143n, 22202n, 9456n, 17213n, 15330n, 26591n, 15721n]]); + expect(chunks).toEqual(target_chunks); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it("special tokens are flushed immediately", async () => { + const chunks = []; + const callback_function = (text) => chunks.push(text); + + const streamer = new TextStreamer(tokenizer, { callback_function, skip_special_tokens: false }); + for (const tokens of DUMMY_TOKEN_STREAM) { + streamer.put([tokens]); + } + streamer.end(); + + expect(chunks).toEqual(["hello", /* Always flush after prompt. */ "hello ", "hellohello\n", "hello", "", "hello"]); + }); + + it("special tokens are skipped with skip_special_tokens: true", async () => { + const chunks = []; + const callback_function = (text) => chunks.push(text); + + const streamer = new TextStreamer(tokenizer, { callback_function, skip_special_tokens: true }); + for (const tokens of DUMMY_TOKEN_STREAM) { + streamer.put([tokens]); + } + streamer.end(); + + expect(chunks).toEqual(["hello", /* Always flush after prompt. */ "hello ", "hellohello\n", "hellohello"]); + }); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("decoder-only (GPT-OSS)", () => { + const model_id = "onnx-community/gpt-oss-20b-ONNX"; + let tokenizer; + let TOKEN_STREAM; + beforeAll(async () => { + tokenizer = await AutoTokenizer.from_pretrained(model_id); + + TOKEN_STREAM = [ + // Prompt tokens: + [200006n, 17360n, 200008n, 3575n, 553n, 17554n, 162016n, 11n, 261n, 4410n, 6439n, 2359n, 22203n, 656n, 7788n, 17527n, 558n, 87447n, 100594n, 25n, 220n, 1323n, 19n, 12n, 3218n, 198n, 6576n, 3521n, 25n, 220n, 1323n, 21n, 12n, 3286n, 12n, 702n, 279n, 30377n, 289n, 25n, 14093n, 279n, 2n, 13888n, 18403n, 25n, 8450n, 11n, 49159n, 11n, 1721n, 13n, 21030n, 2804n, 413n, 7360n, 395n, 1753n, 3176n, 13n, 200007n, 200006n, 77944n, 200008n, 2n, 68406n, 279n, 3575n, 553n, 261n, 10297n, 29186n, 13n, 200007n, 200006n, 1428n, 200008n, 10930n, 668n, 261n, 41339n, 1078n, 19121n, 25392n, 13n, 200007n, 200006n, 173781n], + + // Generated tokens: + + // <|channel|>analysis<|message|> + [200005n], + [35644n], + [200008n], + + // We need to write a poem about Machine Learning. Should be creative, maybe with some technical references but poetic. Let's produce a nice poem. + [2167n], + [1309n], + [316n], + [5067n], + [261n], + [41339n], + [1078n], + [19121n], + [25392n], + [13n], + [18057n], + [413n], + [12879n], + [11n], + [10112n], + [483n], + [1236n], + [11814n], + [25382n], + [889n], + [114824n], + [13n], + [41021n], + [10635n], + [261n], + [7403n], + [41339n], + [13n], + + // <|end|><|start|>assistant<|channel|>final<|message|> + [200007n], + [200006n], + [173781n], + [200005n], + [17196n], + [200008n], + + // **When the Machine Learns to Dream** + // + // In a quiet room of humming silicon, + // Where electrons trace their silent paths, + // A + [410n], + [5958n], + [290n], + [19121n], + [103596n], + [6097n], + [316n], + [24243n], + [91587n], + [637n], + [261n], + [15095n], + [3435n], + [328n], + [147045n], + [68837n], + [11n], + [4066n], + [11977n], + [100085n], + [21523n], + [1043n], + [37716n], + [23373n], + [11n], + [4066n], + [32n], + ]; + }, MAX_MODEL_LOAD_TIME); + + it("special tokens are flushed immediately", async () => { + const chunks = []; + const callback_function = (text) => chunks.push(text); + + const streamer = new TextStreamer(tokenizer, { callback_function, skip_special_tokens: false }); + for (const tokens of TOKEN_STREAM) { + streamer.put([tokens]); + } + streamer.end(); + + const TARGET = [ + // Prompt + "<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.\nKnowledge cutoff: 2024-06\nCurrent date: 2026-02-10\n\nReasoning: medium\n\n# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>developer<|message|># Instructions\n\nYou are a helpful assistant.<|end|><|start|>user<|message|>Write me a poem about Machine Learning.<|end|><|start|>assistant", + + // Generated tokens + "<|channel|>", + "analysis", + "<|message|>", + "We ", + "need ", + "to ", + "write ", + "a ", + "poem ", + "about ", + "Machine ", + "Learning. ", + "Should ", + "be ", + "creative, ", + "maybe ", + "with ", + "some ", + "technical ", + "references ", + "but ", + "poetic. ", + "Let's ", + "produce ", + "a ", + "nice ", + "poem.", + "<|end|>", + "<|start|>", + "assistant", + "<|channel|>", + "final", + "<|message|>", + "**When ", + "the ", + "Machine ", + "Learns ", + "to ", + "Dream**\n\n", + "In ", + "a ", + "quiet ", + "room ", + "of ", + "humming ", + "silicon, \n", + "Where ", + "electrons ", + "trace ", + "their ", + "silent ", + "paths, \n", + "A", + ]; + expect(chunks).toEqual(TARGET); + }); + }); +}); + +describe("Dynamic Cache", () => { + it("should update and get sequence length correctly", () => { + const cache = new DynamicCache(); + expect(cache.get_seq_length()).toEqual(0); + + cache.update({ + "past_key_values.0.key": full([1, 2, 16, 32], 0.0), + "past_key_values.0.value": full([1, 2, 16, 32], 0.0), + }); + expect(cache.get_seq_length()).toEqual(16); + }); +}); + +describe("PKV caching", () => { + describe("LlamaForCausalLM", () => { + const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; + /** @type {LlamaForCausalLM} */ + let model; + /** @type {LlamaTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await LlamaForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await LlamaTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("1"); + + // Generate first sequence w/o PKV + // NOTE: `return_dict_in_generate=true` is required to get PKV + const { past_key_values, sequences } = await model.generate({ + ...inputs, + max_new_tokens: 5, + do_sample: false, + return_dict_in_generate: true, + }); + + // Update output with new text + const decoded = tokenizer.batch_decode(sequences, { + skip_special_tokens: false, + })[0]; + const new_inputs = tokenizer(decoded + "2", { + add_special_tokens: false, + }); + + // Run w/o PKV + const generated_ids = await model.generate({ + ...new_inputs, + max_new_tokens: 3, + do_sample: false, + }); + + // Run w/ PKV + const generated_ids_pkv = await model.generate({ + ...new_inputs, + past_key_values, + max_new_tokens: 3, + do_sample: false, + }); + + const target = [[1n, 259n, 29896n, 24959n, 22063n, 17192n, 12189n, 22468n, 29906n, 3399n, 24823n, 26470n]]; + + expect(generated_ids.tolist()).toEqual(target); + expect(generated_ids_pkv.tolist()).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); + + describe("LlamaForCausalLM (onnxruntime-genai)", () => { + const model_id = "onnx-internal-testing/tiny-random-LlamaForCausalLM-GQA"; + /** @type {LlamaForCausalLM} */ + let model; + /** @type {LlamaTokenizer} */ + let tokenizer; + beforeAll(async () => { + model = await LlamaForCausalLM.from_pretrained(model_id, DEFAULT_MODEL_OPTIONS); + tokenizer = await LlamaTokenizer.from_pretrained(model_id); + }, MAX_MODEL_LOAD_TIME); + + it( + "batch_size=1", + async () => { + const inputs = tokenizer("1"); + + // Generate first sequence w/o PKV + // NOTE: `return_dict_in_generate=true` is required to get PKV + const { past_key_values, sequences } = await model.generate({ + ...inputs, + max_new_tokens: 5, + do_sample: false, + return_dict_in_generate: true, + }); + + // Update output with new text + const decoded = tokenizer.batch_decode(sequences, { + skip_special_tokens: false, + })[0]; + const new_inputs = tokenizer(decoded + "2", { + add_special_tokens: false, + }); + + // Run w/o PKV + const generated_ids = await model.generate({ + ...new_inputs, + max_new_tokens: 3, + do_sample: false, + }); + + // Run w/ PKV + const generated_ids_pkv = await model.generate({ + ...new_inputs, + past_key_values, + max_new_tokens: 3, + do_sample: false, + }); + + const target = [[128000n, 16n, 34732n, 98805n, 116404n, 68265n, 99392n, 17n, 21855n, 60933n, 14285n]]; + + expect(generated_ids.tolist()).toEqual(target); + expect(generated_ids_pkv.tolist()).toEqual(target); + }, + MAX_TEST_EXECUTION_TIME, + ); + + afterAll(async () => { + await model?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}); diff --git a/packages/transformers/tests/utils/hub.test.js b/packages/transformers/tests/utils/hub.test.js new file mode 100644 index 0000000..9659809 --- /dev/null +++ b/packages/transformers/tests/utils/hub.test.js @@ -0,0 +1,52 @@ +import { AutoModel, PreTrainedModel } from "../../src/transformers.js"; + +import { MAX_TEST_EXECUTION_TIME, DEFAULT_MODEL_OPTIONS } from "../init.js"; +import fs from "node:fs"; + +// TODO: Set cache folder to a temp directory + +describe("Hub", () => { + describe("Loading models", () => { + it( + "should load a model from the local cache", + async () => { + // 1. Local model exists (doesn't matter about status of remote file since local is tried first) + const model = await AutoModel.from_pretrained("hf-internal-testing/tiny-random-T5ForConditionalGeneration", DEFAULT_MODEL_OPTIONS); + expect(model).toBeInstanceOf(PreTrainedModel); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should load a model from the remote cache", + async () => { + // 2. Local model doesn't exist, remote file exists + // This tests that fallback functionality is working + const model = await AutoModel.from_pretrained("hf-internal-testing/tiny-random-T5ForConditionalGeneration", DEFAULT_MODEL_OPTIONS); + expect(model).toBeInstanceOf(PreTrainedModel); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should fail to load a model", + async () => { + // 3. Local model doesn't exist, remote file doesn't exist + // This tests that error handling is working. + await expect(AutoModel.from_pretrained("hf-internal-testing/this-model-does-not-exist", DEFAULT_MODEL_OPTIONS)).rejects.toBeInstanceOf(Error); + }, + MAX_TEST_EXECUTION_TIME, + ); + + const localPath = "./models/hf-internal-testing/tiny-random-T5ForConditionalGeneration"; + (fs.existsSync(localPath) ? it : it.skip)( + "should load a model from a local path", + async () => { + // 4. Ensure we can load a model from a local path + const model = await AutoModel.from_pretrained(localPath, DEFAULT_MODEL_OPTIONS); + expect(model).toBeInstanceOf(PreTrainedModel); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); +}); diff --git a/packages/transformers/tests/utils/image.test.js b/packages/transformers/tests/utils/image.test.js new file mode 100644 index 0000000..7fc5d5b --- /dev/null +++ b/packages/transformers/tests/utils/image.test.js @@ -0,0 +1,89 @@ +import { RawImage, rand } from "../../src/transformers.js"; +import { load_cached_image } from "../asset_cache.js"; + +const TEST_IMAGES = { + rgba: new RawImage(new Uint8ClampedArray([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]), 2, 3, 4), + rgb: new RawImage(new Uint8ClampedArray([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]), 2, 3, 3), + la: new RawImage(new Uint8ClampedArray([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]), 2, 3, 2), + l: new RawImage(new Uint8ClampedArray([0, 1, 2, 3, 4, 5]), 2, 3, 1), +}; + +describe("Image utilities", () => { + describe("Padding", () => { + it("should pad image", async () => { + /** @type {RawImage} */ + const padded_image = await load_cached_image("blue_image") + .then((image) => image.resize(224, 224)) + .then((image) => image.pad([128, 128, 128, 128])); + + expect(padded_image.size).toEqual([480, 480]); + + const avg = padded_image.data.reduce((acc, val) => acc + val, 0) / padded_image.data.length; + expect(avg).toBeCloseTo((224 * 224 * 255) / (3 * 480 * 480), 6); + }); + }); + + describe("Tensor to Image", () => { + it("should create an image from a tensor (CHW)", () => { + const tensor_chw = rand([3, 128, 256]).mul_(255).to("uint8"); + const image = RawImage.fromTensor(tensor_chw); + expect(image.size).toEqual([256, 128]); + }); + it("should create an image from a tensor (HWC)", () => { + const tensor_hwc = rand([128, 256, 3]).mul_(255).to("uint8"); + const image = RawImage.fromTensor(tensor_hwc, "HWC"); + expect(image.size).toEqual([256, 128]); + }); + }); + + describe("Channel conversions", () => { + it("should convert RGBA to L (grayscale)", async () => { + const grayscale = TEST_IMAGES.rgba.clone().grayscale(); + expect(grayscale.size).toEqual(TEST_IMAGES.rgba.size); + expect(grayscale.channels).toEqual(1); + }); + + it("should convert RGB to L (grayscale)", async () => { + const grayscale = TEST_IMAGES.rgb.clone().grayscale(); + expect(grayscale.size).toEqual(TEST_IMAGES.rgb.size); + expect(grayscale.channels).toEqual(1); + }); + + it("should convert L to RGB", async () => { + const rgb = TEST_IMAGES.l.clone().rgb(); + expect(rgb.size).toEqual(TEST_IMAGES.l.size); + expect(rgb.channels).toEqual(3); + }); + + it("should convert L to RGBA", async () => { + const rgba = TEST_IMAGES.l.clone().rgba(); + expect(rgba.size).toEqual(TEST_IMAGES.l.size); + expect(rgba.channels).toEqual(4); + }); + + it("should convert RGB to RGBA", async () => { + const rgba = TEST_IMAGES.rgb.clone().rgba(); + expect(rgba.size).toEqual(TEST_IMAGES.rgb.size); + expect(rgba.channels).toEqual(4); + }); + + it("should convert RGBA to RGB", async () => { + const rgb = TEST_IMAGES.rgba.clone().rgb(); + expect(rgb.size).toEqual(TEST_IMAGES.rgba.size); + expect(rgb.channels).toEqual(3); + }); + }); + + describe("putAlpha", () => { + it("should add alpha to RGB image", async () => { + const rgba = TEST_IMAGES.rgb.clone().putAlpha(TEST_IMAGES.l); + expect(rgba.size).toEqual(TEST_IMAGES.rgb.size); + expect(rgba.channels).toEqual(4); + }); + it("should add alpha to RGBA image", async () => { + const rgba = TEST_IMAGES.rgba.clone().putAlpha(TEST_IMAGES.l); + expect(rgba.size).toEqual(TEST_IMAGES.rgb.size); + expect(rgba.channels).toEqual(4); + }); + }); +}); diff --git a/packages/transformers/tests/utils/logits_process.test.js b/packages/transformers/tests/utils/logits_process.test.js new file mode 100644 index 0000000..86ab06f --- /dev/null +++ b/packages/transformers/tests/utils/logits_process.test.js @@ -0,0 +1,112 @@ +import { + // Pipelines + pipeline, + TextGenerationPipeline, +} from "../../src/transformers.js"; + +import { init } from "../init.js"; +init(); + +const MAX_MODEL_LOAD_TIME = 10_000; // 10 seconds +const MAX_TEST_EXECUTION_TIME = 10_000; // 10 seconds +const MAX_MODEL_DISPOSE_TIME = 1_000; // 1 second + +const DEFAULT_MODEL_OPTIONS = { + dtype: "fp32", +}; + +describe("Logits Processors", () => { + describe("text-generation", () => { + const model_id = "hf-internal-testing/tiny-random-LlamaForCausalLM"; + + /** @type {TextGenerationPipeline} */ + let pipe; + beforeAll(async () => { + pipe = await pipeline("text-generation", model_id, { + // TODO move to config + ...DEFAULT_MODEL_OPTIONS, + }); + }, MAX_MODEL_LOAD_TIME); + + describe("bad_word_ids", () => { + it( + "basic", + async () => { + const text_input = "hello"; + + const generated_text_target = "\uff0d Giuseppeitte natoud"; + const text_target = [{ generated_text: text_input + generated_text_target }]; + + const output = await pipe(text_input, { + max_new_tokens: 5, + bad_words_ids: [ + // default: [1n, 22172n, 18547n, 8143n, 22202n, 9456n, 17213n] + [18547], + + // block #1: [1n, 22172n, 31583n, 18824n, 16621n, 8136n, 16012n] + [18824, 16621], + ], + }); + expect(output).toEqual(text_target); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "many bad words", + async () => { + const text_input = "hello"; + + const generated_text_target = "erdingsdelete войsequ族"; + const text_target = [{ generated_text: text_input + generated_text_target }]; + + // Construct long list of bad words + const bad_words_ids = []; + // default: [1n, 22172n, 18547n, 8143n, 22202n, 9456n, 17213n] + for (let i = 0; i < 100000; ++i) { + bad_words_ids.push([i * 2]); // block all even numbers + } + // block #1: [1n, 22172n, 18547n, 8143n, 30327n, 624n, 2806n, 2004n] + bad_words_ids.push([8143, 30327]); + + // block #2: [1n, 22172n, 18547n, 8143n, 29485n, 3799n, 29331n] + bad_words_ids.push([18547, 8143, 29485]); + + // block #3: [1n, 22172n, 18547n, 8143n, 7587n, 6831n, 30999n] + const output = await pipe(text_input, { max_new_tokens: 5, bad_words_ids }); + expect(output).toEqual(text_target); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "different lengths", + async () => { + const text_input = "this is a test"; + + const generated_text_target = "кт México constructed lake års"; + const text_target = [{ generated_text: text_input + generated_text_target }]; + + const output = await pipe(text_input, { + max_new_tokens: 5, + bad_words_ids: [ + // default: [1n, 445n, 338n, 263n, 1243n, 3931n, 14756n, 7811n, 21645n, 31252n] + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931], // should never trigger (longer than input sequence) + + // block #1: [1n, 445n, 338n, 263n, 1243n, 3931n, 14756n, 7811n, 21645n, 31252n] + [3931, 14756, 7811], + + // result: [1n, 445n, 338n, 263n, 1243n, 3931n, 14756n, 13319n, 19437n, 21948n] + ], + }); + expect(output).toEqual(text_target); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + afterAll(async () => { + await pipe?.dispose(); + }, MAX_MODEL_DISPOSE_TIME); + }); +}); diff --git a/packages/transformers/tests/utils/lru_cache.test.js b/packages/transformers/tests/utils/lru_cache.test.js new file mode 100644 index 0000000..f75155d --- /dev/null +++ b/packages/transformers/tests/utils/lru_cache.test.js @@ -0,0 +1,119 @@ +import { LRUCache } from "../../src/utils/lru_cache.js"; + +describe("LRUCache", () => { + it("should return undefined for non-existent keys", () => { + const cache = new LRUCache(2); + expect(cache.get("nonexistent")).toEqual(undefined); + }); + + it("should store and retrieve values correctly", () => { + const cache = new LRUCache(2); + cache.put("a", 1); + cache.put("b", 2); + expect(cache.get("a")).toEqual(1); + expect(cache.get("b")).toEqual(2); + }); + + it("should update the value and refresh the usage", () => { + const cache = new LRUCache(2); + cache.put("a", 1); + cache.put("b", 2); + // Update key "a" + cache.put("a", 10); + expect(cache.get("a")).toEqual(10); + // Access "a" so "b" becomes the LRU + cache.get("a"); + cache.put("c", 3); + // "b" should be evicted since it is the least recently used. + expect(cache.get("b")).toEqual(undefined); + expect(cache.get("c")).toEqual(3); + }); + + it("should evict the least recently used item when capacity is exceeded", () => { + const cache = new LRUCache(3); + cache.put("a", 1); + cache.put("b", 2); + cache.put("c", 3); + // Access "a" to refresh its recentness. + cache.get("a"); + // Insert a new key, this should evict "b" as it is the least recently used. + cache.put("d", 4); + expect(cache.get("b")).toEqual(undefined); + expect(cache.get("a")).toEqual(1); + expect(cache.get("c")).toEqual(3); + expect(cache.get("d")).toEqual(4); + }); + + it("should update the usage order on get", () => { + const cache = new LRUCache(3); + cache.put("a", "apple"); + cache.put("b", "banana"); + cache.put("c", "cherry"); + // Access "a" making it most recently used. + expect(cache.get("a")).toEqual("apple"); + // Insert new element to evict the least recently used ("b"). + cache.put("d", "date"); + expect(cache.get("b")).toEqual(undefined); + // "a", "c", and "d" should be present. + expect(cache.get("a")).toEqual("apple"); + expect(cache.get("c")).toEqual("cherry"); + expect(cache.get("d")).toEqual("date"); + }); + + it("should clear the cache", () => { + const cache = new LRUCache(2); + cache.put("a", 1); + cache.put("b", 2); + cache.clear(); + expect(cache.get("a")).toEqual(undefined); + expect(cache.get("b")).toEqual(undefined); + }); + + it("should return true when deleting an existing key", () => { + const cache = new LRUCache(2); + cache.put("a", 1); + expect(cache.delete("a")).toEqual(true); + }); + + it("should return false when deleting a non-existent key", () => { + const cache = new LRUCache(2); + expect(cache.delete("nonexistent")).toEqual(false); + }); + + it("should make a deleted key unretrievable", () => { + const cache = new LRUCache(2); + cache.put("a", 1); + cache.delete("a"); + expect(cache.get("a")).toEqual(undefined); + }); + + it("should allow re-inserting a deleted key", () => { + const cache = new LRUCache(2); + cache.put("a", 1); + cache.delete("a"); + cache.put("a", 2); + expect(cache.get("a")).toEqual(2); + }); + + it("should not affect other entries when deleting a key", () => { + const cache = new LRUCache(3); + cache.put("a", 1); + cache.put("b", 2); + cache.put("c", 3); + cache.delete("b"); + expect(cache.get("a")).toEqual(1); + expect(cache.get("b")).toEqual(undefined); + expect(cache.get("c")).toEqual(3); + }); + + it("should free up space after deletion, preventing unwanted eviction", () => { + const cache = new LRUCache(2); + cache.put("a", 1); + cache.put("b", 2); + cache.delete("a"); + // With "a" deleted, inserting "c" should not evict "b" + cache.put("c", 3); + expect(cache.get("b")).toEqual(2); + expect(cache.get("c")).toEqual(3); + }); +}); diff --git a/packages/transformers/tests/utils/maths.test.js b/packages/transformers/tests/utils/maths.test.js new file mode 100644 index 0000000..56f09b4 --- /dev/null +++ b/packages/transformers/tests/utils/maths.test.js @@ -0,0 +1,195 @@ +import { FFT, medianFilter, bankers_round, log_softmax } from "../../src/utils/maths.js"; +import { init } from "../init.js"; + +// Initialise the testing environment +init(); + +const fft = (arr, complex = false) => { + let output; + let fft; + if (complex) { + fft = new FFT(arr.length / 2); + output = new Float64Array(fft.outputBufferSize); + fft.transform(output, arr); + } else { + fft = new FFT(arr.length); + output = new Float64Array(fft.outputBufferSize); + fft.realTransform(output, arr); + } + if (!fft.isPowerOfTwo) { + output = output.slice(0, complex ? arr.length : 2 * arr.length); + } + return output; +}; + +const FFT_TEST_DATA = { + fft_2_real: { complex: false, input: [1.764052345967664, 0.4001572083672233], output: [2.1642095543348874, 0.0, 1.3638951376004407, 0.0] }, + fft_3_real: { complex: false, input: [0.9787379841057392, 2.240893199201458, 1.8675579901499675], output: [5.087189173457165, 0.0, -1.0754876105699736, -0.3233177751657647, -1.0754876105699736, 0.3233177751657647] }, + fft_4_real: { complex: false, input: [-0.977277879876411, 0.9500884175255894, -0.1513572082976979, -0.10321885179355784], output: [-0.2817655224420774, 0.0, -0.8259206715787131, -1.0533072693191472, -1.9755046539061405, 0.0, -0.8259206715787131, 1.0533072693191472] }, + fft_5_real: { complex: false, input: [0.41059850193837233, 0.144043571160878, 1.454273506962975, 0.7610377251469934, 0.12167501649282841], output: [2.891628321702047, 0.0, -1.2995143733619916, -0.4287475285900295, 0.8801964673568988, 0.6461585010760772, 0.8801964673568988, -0.6461585010760772, -1.2995143733619916, 0.4287475285900295] }, + fft_7_real: { complex: false, input: [0.44386323274542566, 0.33367432737426683, 1.4940790731576061, -0.20515826376580087, 0.31306770165090136, -0.8540957393017248, -2.5529898158340787], output: [-1.0275594839734046, 0.0, -1.1794900746698127, -4.321336254238002, 0.4283827622463364, -2.2006199533718855, 2.8184083690191684, 1.0886333211894632, 2.8184083690191684, -1.0886333211894632, 0.4283827622463364, 2.2006199533718855, -1.1794900746698127, 4.321336254238002] }, + fft_8_real: { complex: false, input: [0.6536185954403606, 0.8644361988595057, -0.7421650204064419, 2.2697546239876076, -1.4543656745987648, 0.04575851730144607, -0.1871838500258336, 1.5327792143584575], output: [2.9826326049163367, 0.0, 2.1657565005584045, -0.5450316795716824, 0.1286017912738714, 2.8923391221851134, 2.050212039519846, -1.6549940203328992, -6.442824504097697, 0.0, 2.050212039519846, 1.6549940203328992, 0.1286017912738714, -2.8923391221851134, 2.1657565005584045, 0.5450316795716824] }, + fft_9_real: { complex: false, input: [1.469358769900285, 0.1549474256969163, 0.37816251960217356, -0.8877857476301128, -1.980796468223927, -0.3479121493261526, 0.15634896910398005, 1.2302906807277207, 1.2023798487844113], output: [1.3749938486352948, 0.0, 5.342425446245793, 2.975185521629713, -1.22457133247234, -0.6308804310909996, 0.41938606274358115, 1.5832577539331407, 1.3873773637166027, -0.8933243837257328, 1.3873773637166025, 0.8933243837257323, 0.41938606274358115, -1.5832577539331407, -1.2245713324723406, 0.6308804310909994, 5.342425446245792, -2.975185521629712] }, + fft_16_real: { complex: false, input: [-0.3873268174079523, -0.30230275057533557, -1.0485529650670926, -1.4200179371789752, -1.7062701906250126, 1.9507753952317897, -0.5096521817516535, -0.4380743016111864, -1.2527953600499262, 0.7774903558319101, -1.6138978475579515, -0.2127402802139687, -0.8954665611936756, 0.386902497859262, -0.510805137568873, -1.180632184122412], output: [-8.363366266001053, 0.0, -1.4796937055641797, 0.20983245542766216, -0.3454288633172973, 2.968909080470098, 2.329350352646858, -0.7633502808629014, -0.5589507973309957, -6.064330201474169, -1.3963011418461777, 1.659405903990098, 2.2686580120389164, -0.3150779061389377, 4.008518665331395, -0.6106258774446865, -7.486167856443222, 0.0, 4.008518665331395, 0.6106258774446865, 2.2686580120389164, 0.3150779061389377, -1.3963011418461777, -1.659405903990098, -0.5589507973309957, 6.064330201474169, 2.329350352646858, 0.7633502808629014, -0.3454288633172973, -2.968909080470098, -1.4796937055641797, -0.20983245542766216] }, + fft_25_real: { complex: false, input: [-0.028182228338654868, 0.42833187053041766, 0.06651722238316789, 0.3024718977397814, -0.6343220936809636, -0.3627411659871381, -0.672460447775951, -0.3595531615405413, -0.813146282044454, -1.7262826023316769, 0.17742614225375283, -0.4017809362082619, -1.6301983469660446, 0.4627822555257742, -0.9072983643832422, 0.05194539579613895, 0.7290905621775369, 0.12898291075741067, 1.1394006845433007, -1.2348258203536526, 0.402341641177549, -0.6848100909403132, -0.8707971491818818, -0.5788496647644155, -0.31155253212737266], output: [-7.327510303739734, 0.0, 1.377932222483735, 3.044921011687328, 0.06954965931518875, -5.483478593023237, 2.616166182267447, -2.0815154752919764, 0.1551657690860495, 0.5208228796162939, 0.3085012063215258, -2.1386609330432687, -0.566204290441261, 0.012329506565646242, 1.309786704987264, 3.1736358514734673, -1.0557581301013133, -2.4635866620117564, -2.579387724556426, 1.9885603872002973, 3.9572284078024613, -5.498314280945309, 0.6049416192890185, 2.7215390337882552, -2.8864443288170083, 0.6099589663561517, -2.8864443288170074, -0.6099589663561518, 0.604941619289019, -2.721539033788255, 3.9572284078024613, 5.498314280945309, -2.579387724556426, -1.988560387200297, -1.0557581301013133, 2.4635866620117564, 1.3097867049872634, -3.1736358514734673, -0.5662042904412615, -0.012329506565646575, 0.3085012063215258, 2.1386609330432687, 0.15516576908604957, -0.520822879616294, 2.6161661822674476, 2.0815154752919764, 0.06954965931518836, 5.4834785930232375, 1.377932222483735, -3.044921011687328] }, + fft_27_real: { complex: false, input: [0.05616534222974544, -1.1651498407833565, 0.9008264869541871, 0.46566243973045984, -1.5362436862772237, 1.4882521937955997, 1.8958891760305832, 1.1787795711596507, -0.17992483581235091, -1.0707526215105425, 1.0544517269311366, -0.40317694697317963, 1.2224450703824274, 0.2082749780768603, 0.9766390364837128, 0.3563663971744019, 0.7065731681919482, 0.010500020720820478, 1.7858704939058352, 0.12691209270361992, 0.40198936344470165, 1.8831506970562544, -1.3477590611424464, -1.2704849984857336, 0.9693967081580112, -1.17312340511416, 1.9436211856492926], output: [9.485150752680255, 0.0, -3.3736195725050755, -0.6576671739932465, -1.4668314059671266, -3.7345058492453083, 0.41824676938222627, 1.9663430754071836, 5.709032912166549, 5.528638987648501, 1.8435730384179874, -0.34683544424097734, -4.963689356113413, -0.5177297293983665, -5.8830486994336315, 3.207542182273314, 1.8987452089640713, 2.1505302726369, 6.603715178395637, 5.036393219486801, 1.3865285438489818, 10.275020532162417, 0.1681971482993614, 2.6656111808114664, -3.3300735021919055, -3.3923756439906483, -2.9951195195022233, 4.64643931476696, -2.9951195195022233, -4.64643931476696, -3.3300735021919055, 3.3923756439906487, 0.16819714829936183, -2.665611180811466, 1.3865285438489812, -10.275020532162415, 6.603715178395637, -5.036393219486801, 1.8987452089640697, -2.1505302726368996, -5.8830486994336315, -3.207542182273315, -4.963689356113414, 0.5177297293983658, 1.8435730384179874, 0.34683544424097734, 5.709032912166549, -5.528638987648501, 0.41824676938222605, -1.9663430754071831, -1.4668314059671266, 3.7345058492453087, -3.373619572505076, 0.6576671739932474] }, + fft_32_real: { complex: false, input: [-0.41361898075974735, -0.7474548114407578, 1.9229420264803847, 1.4805147914344243, 1.8675589604265699, 0.9060446582753853, -0.8612256850547025, 1.9100649530990337, -0.2680033709513804, 0.8024563957963952, 0.947251967773748, -0.1550100930908342, 0.6140793703460803, 0.9222066715665268, 0.37642553115562943, -1.0994007905841945, 0.298238174206056, 1.3263858966870303, -0.6945678597313655, -0.14963454032767076, -0.43515355172163744, 1.8492637284793418, 0.6722947570124355, 0.40746183624111043, -0.7699160744453164, 0.5392491912918173, -0.6743326606573761, 0.03183055827435118, -0.635846078378881, 0.6764332949464997, 0.5765908166149409, -0.20829875557799488], output: [11.014830327385901, 0.0, 1.2409883700304647, -5.276491159392059, -1.534551501998357, -4.8297133119274545, 1.336708303789408, -2.7137782948769944, -4.424467817605524, 0.843485461536611, -6.581416660337224, -0.1795449695375516, -2.514517651560188, -0.05943606215207886, -4.207556144474312, 7.447633642178459, -2.0080404448719507, -4.057057066134013, 1.4811356128841098, 3.432496305430468, 1.3934203835489032, -3.231520399869298, -4.317592600060283, 0.1993606400932625, -0.7034100876395168, 2.317901569810786, 3.7371408499766874, 5.7058540737147405, 6.345803325381664, -2.1851091826937408, 1.6157350284647216, 2.7643998907723595, -5.969395642755025, 0.0, 1.6157350284647212, -2.7643998907723604, 6.345803325381664, 2.1851091826937408, 3.737140849976688, -5.7058540737147405, -0.7034100876395168, -2.317901569810786, -4.317592600060284, -0.19936064009326138, 1.3934203835489032, 3.231520399869298, 1.4811356128841102, -3.432496305430468, -2.0080404448719507, 4.057057066134013, -4.207556144474311, -7.447633642178459, -2.514517651560188, 0.05943606215207886, -6.581416660337223, 0.1795449695375514, -4.424467817605524, -0.843485461536611, 1.3367083037894085, 2.7137782948769944, -1.534551501998357, 4.8297133119274545, 1.2409883700304651, 5.276491159392059] }, + fft_49_real: { complex: false, input: [0.3960067126616453, -1.0930615087305058, -1.4912575927056055, 0.4393917012645369, 0.16667349537252904, 0.6350314368921064, 2.383144774863942, 0.9444794869904138, -0.9128222254441586, 1.117016288095853, -1.3159074105115212, -0.461584604814709, -0.06824160532463124, 1.7133427216493666, -0.7447548220484399, -0.8264385386590144, -0.0984525244254323, -0.6634782863621074, 1.126635922106507, -1.0799315083634233, -1.1474686524111024, -0.43782004474443403, -0.4980324506923049, 1.9295320538169858, 0.9494208069257608, 0.0875512413851909, -1.225435518830168, 0.8443629764015471, -1.0002153473895647, -1.5447710967776116, 1.1880297923523018, 0.3169426119248496, 0.920858823780819, 0.3187276529430212, 0.8568306119026912, -0.6510255933001469, -1.0342428417844647, 0.681594518281627, -0.8034096641738411, -0.6895497777502005, -0.45553250351734315, 0.01747915902505673, -0.35399391125348395, -1.3749512934180188, -0.6436184028328905, -2.2234031522244266, 0.6252314510271875, -1.6020576556067476, -1.1043833394284506], output: [-7.887557633860811, 0.0, -5.014267769110843, -4.376842071326452, -2.3853231496935923, -10.851256239812793, -3.2717318894892826, -5.326849080561187, -2.7264013581332716, 1.1514977438201093, -7.652841900031138, -3.7746076991956867, 5.309279658846421, 8.814060475350379, -2.6168809007788796, 4.677691230979557, 7.6656963597889805, 3.4858425270229887, 0.2740709971425779, -4.676330054757555, -5.155232202781799, 3.7556835779660136, 5.001888119593458, -3.6198301043995045, -0.0953704694177091, -3.6318583861463845, 2.767000800593773, -2.450144252684209, -1.2540039367717049, 9.279722666945066, 3.815471636849461, -3.3563499317203815, 12.763423104544493, 0.4158128174652558, 3.995279461690892, -6.6439121988180325, 3.2201986963478983, 0.6334206695259166, 2.4912752816319514, -1.9446295959593973, 1.027982187347956, -2.8785858483639863, 1.3490313376869536, 14.471900839049741, 1.0215246227635422, -0.4420763712931004, -1.0162745955034818, -0.3075325636480104, -5.867850815975937, -1.9145668225042987, -5.867850815975937, 1.9145668225042978, -1.0162745955034822, 0.3075325636480101, 1.0215246227635417, 0.4420763712931006, 1.3490313376869536, -14.471900839049741, 1.0279821873479558, 2.878585848363986, 2.491275281631952, 1.9446295959593989, 3.2201986963478975, -0.633420669525917, 3.9952794616908927, 6.6439121988180325, 12.763423104544493, -0.41581281746525595, 3.815471636849461, 3.3563499317203815, -1.2540039367717049, -9.279722666945066, 2.767000800593772, 2.4501442526842085, -0.0953704694177091, 3.631858386146384, 5.001888119593458, 3.6198301043995045, -5.1552322027818, -3.755683577966013, 0.2740709971425781, 4.676330054757556, 7.665696359788981, -3.4858425270229905, -2.6168809007788796, -4.677691230979557, 5.30927965884642, -8.814060475350379, -7.652841900031138, 3.774607699195687, -2.7264013581332716, -1.1514977438201108, -3.2717318894892826, 5.326849080561189, -2.385323149693591, 10.851256239812795, -5.014267769110844, 4.376842071326452] }, + fft_64_real: { complex: false, input: [0.052165079260974405, -0.7395629963913133, 1.5430145954067358, -1.2928569097234486, 0.26705086934918293, -0.0392828182274956, -1.1680934977411974, 0.5232766605317537, -0.1715463312222481, 0.7717905512136674, 0.8235041539637314, 2.16323594928069, 1.336527949436392, -0.3691818379424436, -0.2393791775759264, 1.0996595958871132, 0.6552637307225978, 0.640131526097592, -1.6169560443108344, -0.024326124398935636, -0.7380309092056887, 0.27992459904323824, -0.09815038964295794, 0.9101789080925919, 0.31721821519130206, 0.7863279621089762, -0.46641909673594306, -0.9444462559182504, -0.41004969320254847, -0.017020413861440594, 0.3791517355550818, 2.259308950690852, -0.04225715166064269, -0.955945000492777, -0.34598177569938643, -0.4635959746460942, 0.4814814737734622, -1.5407970144446248, 0.06326199420033171, 0.1565065379653756, 0.23218103620027578, -0.5973160689653627, -0.237921729736007, -1.4240609089825316, -0.49331988336219407, -0.5428614760167177, 0.4160500462614255, -1.1561824318219127, 0.7811981017099934, 1.4944845444913688, -2.0699850250135325, 0.42625873077810095, 0.6769080350302455, -0.637437025552229, -0.39727181432879766, -0.13288057758695562, -0.2977908794017283, -0.3090129690471222, -1.6760038063299767, 1.15233156478312, 1.079618592036821, -0.8133642592042029, -1.466424327802514, 0.5210648764527586], output: [-1.64663603068023, 0.0, 1.505825929177829, -9.200008132744042, -2.544735819316532, -1.5339227544442826, -7.4360215710438755, -1.7984575917004604, 1.3619465096845937, 4.8456564834488915, 5.070506185450519, 4.849409603121612, 3.777805068935999, 0.17970844009281262, -5.309242543498751, -6.973066979348705, 5.944277107889729, 1.4068283873569305, -7.321235075656323, -8.916520841302049, -5.0314030969029835, 1.309770133246369, 3.721064537404591, -2.7547595090183408, 3.8878856671539617, 0.733440111196902, 5.958504522515222, -9.443751218054468, -3.743781835570924, 1.4088028338848555, 1.4543498973854279, -1.4051068307925165, 10.284222394185964, 6.362595288575114, 5.69300272333881, 2.3840261995408785, -2.344460920124865, 8.555867083200523, -10.10946038896856, -1.5537275998675408, -0.8110283215952525, 0.08930281805022311, -0.3624099573254269, 4.570396206585882, -7.6585567061471576, 9.739963432241709, -4.941569815253301, 0.6205797252271052, -7.291786374000026, -1.6649582074043856, 5.388745981879483, 1.3949577742938146, 1.4070140670266627, 11.226248771172266, 4.869069031848913, 4.140589494166391, 1.0264270218179838, 0.9000492064571382, 4.80696917175708, 0.26025510601075585, 4.725688003441722, 8.559147480463128, -1.4773429342657596, -6.3922359470118755, -4.01533581906691, 0.0, -1.4773429342657578, 6.392235947011875, 4.7256880034417215, -8.559147480463128, 4.80696917175708, -0.26025510601075497, 1.0264270218179838, -0.9000492064571382, 4.869069031848914, -4.140589494166391, 1.4070140670266635, -11.226248771172266, 5.388745981879483, -1.3949577742938146, -7.291786374000026, 1.6649582074043856, -4.941569815253302, -0.6205797252271044, -7.658556706147157, -9.739963432241707, -0.36240995732542824, -4.570396206585883, -0.8110283215952525, -0.08930281805022311, -10.109460388968563, 1.5537275998675408, -2.344460920124864, -8.555867083200523, 5.693002723338812, -2.38402619954088, 10.284222394185964, -6.362595288575114, 1.4543498973854279, 1.4051068307925176, -3.743781835570923, -1.4088028338848546, 5.95850452251522, 9.443751218054468, 3.8878856671539617, -0.733440111196902, 3.72106453740459, 2.754759509018341, -5.031403096902983, -1.309770133246368, -7.321235075656325, 8.916520841302049, 5.944277107889729, -1.4068283873569305, -5.309242543498751, 6.973066979348703, 3.777805068936, -0.1797084400928135, 5.07050618545052, -4.849409603121612, 1.3619465096845937, -4.8456564834488915, -7.4360215710438755, 1.7984575917004595, -2.544735819316533, 1.5339227544442826, 1.505825929177829, 9.200008132744042] }, + fft_81_real: { complex: false, input: [-0.5757879698130661, 0.14195316332077967, -0.3193284171450952, 0.6915387510701866, 0.6947491436560059, -0.7255973784635843, -1.3833639553950554, -1.582938397335082, 0.6103793791072052, -1.188859257784029, -0.5068163542986875, -0.5963140384505081, -0.05256729626954629, -1.936279805846507, 0.18877859679382855, 0.5238910238342056, 0.08842208704466141, -0.3108861716984717, 0.09740016626878341, 0.3990463456401302, -2.77259275642665, 1.9559123082506942, 0.39009332268792646, -0.65240858238702, -0.3909533751876011, 0.49374177734918845, -0.11610393903436653, -2.0306844677814944, 2.0644928613593194, -0.11054065723247261, 1.0201727117157997, -0.6920498477843912, 1.5363770542457977, 0.28634368889227957, 0.6088438344754508, -1.0452533661469547, 1.2111452896827009, 0.6898181645347884, 1.3018462295649984, -0.6280875596415789, -0.4810271184607877, 2.303916697683942, -1.0600158227215473, -0.13594970067832082, 1.1368913626026953, 0.0977249677148556, 0.5829536797532936, -0.3994490292628752, 0.37005588784751875, -1.3065268517353166, 1.658130679618188, -0.11816404512856976, -0.6801782039968504, 0.6663830820319143, -0.4607197873885533, -1.3342584714027534, -1.3467175057975553, 0.6937731526901325, -0.1595734381462669, -0.13370155966843916, 1.0777438059762627, -1.1268258087567435, -0.7306777528648248, -0.38487980918127546, 0.094351589317074, -0.042171451290578935, -0.2868871923899076, -0.0616264020956474, -0.10730527629117469, -0.7196043885517929, -0.8129929885540773, 0.2745163577239395, -0.8909150829955279, -1.1573552591908536, -0.3122922511256933, -0.1576670161638159, 2.2567234972982093, -0.7047002758562337, 0.9432607249694948, 0.7471883342046318, -1.188944955203736], output: [-6.019981320094997, 0.0, -11.024215089600144, -2.2296309610489784, 7.885780110732945, 6.9546439213354425, 2.1468923343095865, 2.0704996330469774, 5.223342327125926, 3.3335079943959576, 6.670733097535111, 3.059813085584419, -1.291987686750553, 0.4705982966401041, -2.6851539401562734, 0.13762863856647578, -2.2292463073335753, 1.3031715676222848, -8.459551141726745, -1.1728586245314498, -5.19534088481691, -4.1102647711302005, 4.442176173858688, -5.752409619199853, -10.978436211517131, -2.360518158521794, -8.97056708617231, -4.44028490950546, -0.7378544972090442, 0.3833473146759321, 2.41788584599411, 2.094126839887842, -0.9134777730571964, 0.6964431384964922, -1.348881203926696, 4.273283923628751, 2.6990650914215637, -0.04446227794217483, -2.8201900136765587, 10.405176121148552, 4.561086132295523, -4.8957475545660945, -9.626965987488632, 5.703554469280806, 5.118571440405241, -0.47801741518584884, 4.087758685622818, -16.982284140767902, -1.672788156348311, -8.247953480159863, -15.76267685360949, 6.557268666342, 8.45494487748653, -7.023784282842257, 0.9696988388268291, 0.6791846860727092, -3.7628962341106327, -6.731131201638246, 0.3164269710342955, -0.346532027732811, 5.824835704304008, -8.818250625842744, 5.382334715219444, -1.672571325598101, 6.240545283378912, -5.74548463471282, -6.38340050971792, 3.0747648212362937, 11.52437440252928, -1.3394713252215091, 6.27611320575086, 6.861171951341367, -10.764313909223372, -10.425524196300067, -8.17856146704236, 14.338103920769218, 4.86550855683382, 5.452700220702592, -3.278038589307964, -1.8007559828104207, 0.6670476307446598, 9.785124153822995, 0.6670476307446602, -9.785124153823, -3.2780385893079633, 1.8007559828104218, 4.865508556833819, -5.452700220702592, -8.178561467042359, -14.338103920769221, -10.764313909223372, 10.425524196300069, 6.276113205750862, -6.861171951341369, 11.524374402529276, 1.3394713252215045, -6.3834005097179185, -3.0747648212362932, 6.2405452833789115, 5.745484634712819, 5.382334715219443, 1.6725713255980996, 5.824835704304008, 8.818250625842744, 0.31642697103429596, 0.34653202773281055, -3.762896234110635, 6.731131201638245, 0.9696988388268291, -0.6791846860727092, 8.45494487748653, 7.023784282842258, -15.762676853609491, -6.557268666342001, -1.67278815634831, 8.247953480159865, 4.087758685622817, 16.982284140767902, 5.118571440405242, 0.47801741518584784, -9.626965987488635, -5.703554469280807, 4.561086132295526, 4.895747554566093, -2.82019001367656, -10.405176121148555, 2.699065091421563, 0.04446227794217439, -1.3488812039266977, -4.273283923628747, -0.9134777730571959, -0.6964431384964889, 2.417885845994109, -2.0941268398878425, -0.7378544972090446, -0.3833473146759312, -8.97056708617231, 4.4402849095054595, -10.978436211517133, 2.3605181585217934, 4.442176173858688, 5.752409619199854, -5.195340884816912, 4.1102647711302005, -8.459551141726745, 1.1728586245314485, -2.2292463073335775, -1.3031715676222801, -2.6851539401562716, -0.13762863856647434, -1.291987686750553, -0.47059829664010344, 6.670733097535112, -3.0598130855844157, 5.223342327125925, -3.3335079943959554, 2.1468923343095883, -2.0704996330469765, 7.8857801107329415, -6.9546439213354425, -11.024215089600146, 2.229630961048978] }, + fft_125_real: { complex: false, input: [0.7732529774025997, -1.1838806401933177, -2.659172237996741, 0.6063195243593807, -1.7558905834377194, 0.45093446180591484, -0.6840108977372166, 1.6595507961898721, 1.068509399316009, -0.45338580385138766, -0.6878376110286823, -1.2140774030941206, -0.4409226322925914, -0.2803554951845091, -0.3646935443916854, 0.15670385527236397, 0.5785214977288784, 0.349654456993174, -0.764143923906443, -1.4377914738015785, 1.3645318481024713, -0.6894491845499376, -0.6522935999350191, -0.5211893123011109, -1.8430695501566485, -0.4779740040404867, -0.47965581400794766, 0.6203582983435125, 0.698457149107336, 0.00377088908626934, 0.9318483741143037, 0.339964983801262, -0.01568211160255477, 0.16092816829822298, -0.19065349358139935, -0.3948495140334503, -0.26773353689396645, -1.1280113314700069, 0.280441705316296, -0.9931236109295807, 0.8416312640736364, -0.24945858016094885, 0.04949498165009074, 0.49383677628095635, 0.6433144650629279, -1.5706234086334527, -0.20690367616397173, 0.8801789120807822, -1.6981058194322545, 0.3872804753950634, -2.2555642294021894, -1.0225068436356035, 0.0386305518401881, -1.6567151023219537, -0.9855107376841507, -1.4718350074635869, 1.6481349322075596, 0.16422775548733395, 0.5672902778526694, -0.2226751005151545, -0.35343174875719907, -1.6164741886510325, -0.2918373627478628, -0.7614922118116233, 0.8579239242923363, 1.1411018666575734, 1.4665787155741776, 0.852551939461232, -0.5986539369229861, -1.1158969859603944, 0.7666631816450861, 0.3562928174722889, -1.7685384506770307, 0.35548179274376907, 0.8145198224878664, 0.05892558918162996, -0.18505367100934153, -0.8076484876163557, -1.4465346995633879, 0.8002979493400275, -0.3091144447717088, -0.23346666154369272, 1.7327211869191332, 0.6845011068591904, 0.3708250012811021, 0.14206180518723566, 1.5199948607657727, 1.7195893074161945, 0.9295051114795281, 0.5822245913979243, -2.0946030712061448, 0.12372191423350658, -0.130106954193704, 0.09395322938556872, 0.9430460873225178, -2.7396771671895563, -0.5693120534701851, 0.26990435494076137, -0.4668455460527625, -1.4169061131262595, 0.8689634868967954, 0.27687190584612803, -0.9711045704444846, 0.3148172045158238, 0.8215857120497958, 0.005292646299360854, 0.8005648034309968, 0.07826017516166135, -0.39522898265435435, -1.159420516399913, -0.08593076697161273, 0.19429293804577166, 0.8758327615873309, -0.11510746848722672, 0.4574156062209908, -0.9646120137337284, -0.7826291558275251, -0.11038929902688775, -1.0546284639850139, 0.8202478373246812, 0.4631303293186071, 0.2790957643924534, 0.33890412521594454, 2.0210435614847975, -0.46886418796679563], output: [-12.976781205595502, 0.0, -0.5336306740788994, 9.664313658528595, -2.250865418762321, -6.399236976457672, 4.984600678216093, 10.085773094158728, 4.054865444587319, 1.1220280117249106, -12.381959140729, -4.245941970428558, 11.300429673882508, -3.867369304887277, -1.924145132324413, 15.625845728307613, -3.683967210179358, 9.246098611130797, 6.678100602051274, -5.8877245671287906, -11.142505237356518, 1.2462822325294702, -4.950012182132614, 15.368429468743631, -2.7544743905540385, 7.397618288416753, 4.896491435868608, 21.15703724867305, -10.418689778983975, 1.697253164520248, 2.2518122169760852, 14.869916845413385, -5.553898084870594, -3.616357859997289, 8.688187732450988, 9.51306110875087, 1.8459884086796337, -6.750879404311422, 10.389191729731694, -8.563754760867933, 6.323482750892878, -0.814569185033468, -6.339262858307836, 12.395809458636048, -5.5325241155089815, -4.087114562924723, 5.264130311599194, 5.174155710626681, 0.07817803409444535, 4.502699566709124, -6.841988594388185, -4.209615030635216, -3.014915877040603, 0.7642671251818629, 4.979942269363006, -0.7954973970411463, -9.213201050170268, 20.281572828473664, 2.195700817994501, -2.8150917630751158, 3.66256874231074, 4.947718687255637, 2.417023995325397, 6.311630702081038, 8.910067289733103, -1.3549619066609422, -7.604782781699578, 3.2671105424317712, -0.9261608630826195, -10.26033603214804, 2.477431685753063, -0.5368669007709066, 6.588511626180253, 3.6391310126388206, -0.7696459233665145, 10.849962526266822, 0.8214214627634562, -9.665598459357405, -4.208875535725593, -8.955547690168373, 2.3998423294426914, -6.900645072282014, 10.561899982588944, -6.1497412675557115, -10.104315792100294, 11.339404667032188, 4.041967950367949, -10.290041543502431, 9.533068142456843, -5.3782354906044025, 4.12027245501964, -0.5914284618824925, 12.365588431711828, -2.701635423848076, -6.366846689539269, -5.370727636437312, 6.689816472907384, 7.334498752981377, 13.69756331474747, 3.7921571289587113, -0.2721490559996148, 0.2084376847247582, 9.275694421318246, -16.06448352473527, 1.3029488980606758, -8.03640349313321, 0.20226177706452786, 6.0912757829407465, -4.452961538391443, -15.072885312247177, -0.8419482730978101, -7.04697219502747, 12.864547051022127, -6.206422060382598, -6.149387177052475, -15.825612767912974, -0.5432046258303531, -0.24636987696629387, 0.5819230336105994, -10.272301011724789, 3.8285784744531433, 1.1829320961516245, -5.404044842034781, 0.4066057531145235, -1.2770351094581365, 7.590779935818785, -1.2770351094581383, -7.590779935818781, -5.404044842034781, -0.4066057531145224, 3.828578474453144, -1.1829320961516248, 0.5819230336105993, 10.272301011724787, -0.54320462583035, 0.2463698769662941, -6.149387177052476, 15.825612767912975, 12.864547051022129, 6.206422060382598, -0.8419482730978074, 7.04697219502747, -4.452961538391442, 15.072885312247177, 0.20226177706453008, -6.091275782940745, 1.3029488980606758, 8.03640349313321, 9.27569442131825, 16.06448352473527, -0.2721490559996148, -0.2084376847247582, 13.69756331474747, -3.792157128958717, 6.689816472907385, -7.334498752981374, -6.366846689539267, 5.370727636437312, 12.365588431711828, 2.7016354238480744, 4.120272455019638, 0.5914284618824928, 9.533068142456843, 5.378235490604402, 4.041967950367946, 10.290041543502431, -10.104315792100294, -11.339404667032191, 10.561899982588944, 6.1497412675557115, 2.399842329442692, 6.900645072282014, -4.208875535725592, 8.955547690168373, 0.8214214627634587, 9.665598459357408, -0.7696459233665118, -10.849962526266822, 6.588511626180252, -3.6391310126388214, 2.477431685753063, 0.536866900770905, -0.9261608630826198, 10.260336032148041, -7.604782781699578, -3.2671105424317712, 8.9100672897331, 1.354961906660943, 2.4170239953253976, -6.311630702081036, 3.662568742310737, -4.947718687255637, 2.1957008179945006, 2.815091763075113, -9.213201050170266, -20.281572828473664, 4.979942269363008, 0.7954973970411481, -3.0149158770406013, -0.7642671251818633, -6.841988594388185, 4.209615030635216, 0.07817803409444402, -4.502699566709122, 5.264130311599196, -5.174155710626682, -5.532524115508983, 4.087114562924725, -6.339262858307835, -12.395809458636048, 6.323482750892879, 0.8145691850334673, 10.389191729731692, 8.563754760867932, 1.845988408679628, 6.75087940431142, 8.68818773245099, -9.51306110875087, -5.553898084870594, 3.6163578599972883, 2.251812216976085, -14.869916845413385, -10.418689778983971, -1.6972531645202498, 4.896491435868606, -21.15703724867305, -2.7544743905540354, -7.397618288416755, -4.950012182132614, -15.368429468743635, -11.142505237356517, -1.2462822325294698, 6.678100602051274, 5.8877245671287906, -3.683967210179361, -9.246098611130797, -1.9241451323244125, -15.625845728307619, 11.30042967388251, 3.8673693048872773, -12.381959140729, 4.245941970428559, 4.054865444587321, -1.122028011724911, 4.984600678216097, -10.085773094158727, -2.250865418762321, 6.399236976457672, -0.5336306740788972, -9.664313658528593] }, + fft_128_real: { complex: false, input: [-2.201441285500558, 0.1993001968964652, -0.050603540961665895, -0.5175190425104033, -0.9788298593587699, -0.43918952180214793, 0.18133842921782128, -0.5028167006425383, 2.4124536795437486, -0.960504381633148, -0.7931173627076716, -2.2886200400145285, 0.251484415021537, -2.01640662779976, -0.5394546333745014, -0.27567053456055696, -0.7097279658468882, 1.738872677454511, 0.9943943913154989, 1.3191368763015756, -0.8824188185499185, 1.1285940645145685, 0.4960009463439622, 0.7714059486768455, 1.0294388287827672, -0.9087632459590531, -0.4243176209779015, 0.8625960113284511, -2.655619092974933, 1.5133280825732052, 0.553132064207584, -0.045703960660234855, 0.2205076557571733, -1.0299352833089765, -0.34994336458910474, 1.1002843382203737, 1.2980219723262212, 2.6962240525635797, -0.07392466628041514, -0.6585529668050037, -0.5142339659399888, -1.0180418752873648, -0.07785475594085076, 0.38273243001226814, -0.03424228053195387, 1.0963468456657985, -0.23421580134453654, -0.3474506524985633, -0.5812684768603252, -1.6326345262344952, -1.567767724308454, -1.1791579306376878, 1.3014280716647608, 0.8952602728899299, 1.3749640663929898, -1.3322116545945017, -1.9686246897860202, -0.6600563201340829, 0.175818953296028, 0.4986902749098275, 1.0479721559680528, 0.2842796708072146, 1.7426687806556311, -0.22260568094832048, -0.9130792180417964, -1.6812182154944335, -0.8889713580954499, 0.242117960985123, -0.8887202573536308, 0.9367424635352571, 1.412327706037443, -2.369586905226603, 0.8640523004976479, -2.2396040586617367, 0.4014990550902875, 1.2248705641936597, 0.06485610634357618, -1.2796891732042395, -0.5854312042777726, -0.2616454457109007, -0.18224478378994294, -0.20289684076666706, -0.1098827793093138, 0.2134800489101689, -1.2085736537332212, -0.2420198298702195, 1.5182611703557054, -0.38464542314251776, -0.4438360931551978, 1.0781973037142378, -2.5591846663440965, 1.1813786012882859, -0.6319037580051673, 0.16392857245258663, 0.09632135592119682, 0.9424681192203938, -0.2675947462353477, -0.6780257815644504, 1.2978457906510987, -2.36417381714118, 0.02033418170524325, -1.3479254226291204, -0.761573388256559, 2.011256681463137, -0.044595426455857026, 0.19506969715138117, -1.7815628557055914, -0.7290446587946957, 0.19655740072878491, 0.3547576931132181, 0.6168865543932788, 0.008627898917576322, 0.5270042084546597, 0.453781912635684, -1.8297404110045314, 0.03700572191014953, 0.7679024077327037, 0.5898798207345195, -0.3638588099707899, -0.8056265075393678, -1.1183119243216322, -0.13105401154141233, 1.133079879559722, -1.951804101481602, -0.659891729729498, -1.139802455426774, 0.7849575212405001, -0.5543096265713009], output: [-17.373351338172824, 0.0, -2.9147793143550498, -7.299382227996462, -12.507738907636103, -1.0091552608283472, -8.14898282968933, 5.307325780993493, -0.034750314603287924, 4.577509972521205, -7.468519860234688, -2.7826003571131217, 5.138209694069926, 2.2706200801626815, 10.580893461437023, -11.281416288223713, -2.4076277186241555, -8.076888559746063, -7.7255720166232775, -7.240220313440044, -11.221660986331635, 5.410676842717731, -0.7666763204502667, 0.23941154418990696, -1.1567394707746503, 8.29598249679744, -5.425512551243248, 4.4002233124645125, 4.01519747432619, 9.515412040403152, 17.306448507916105, -15.673102695422665, -9.901937950321809, 18.738864037440322, 19.21931801890001, 4.139374296179019, -15.995833331174268, -14.354578092775146, -1.028728623054629, -10.403106776550125, -2.312215325199424, 0.9570154846116288, 12.655685692773659, -2.588769695009625, 14.96829370833483, -0.28008251578319854, -19.11873843663264, -1.1020798388070263, 2.4267878006424475, 2.4199460094928953, -7.0321828345356066, -2.542686317569949, -7.804615781068486, 7.887570592086873, -6.609647506979987, 10.661665300030329, -13.524024198497745, 11.81604749808395, -1.07822493702058, -1.484839411655623, -11.897938634419027, -8.105846942980566, 5.44601793359646, -6.0339961403022055, -6.671236362873311, -1.7118899285267464, -0.9493422927794772, -2.6019479213685455, 0.783595826718674, 1.6178298328406964, -1.9230298563197574, -5.115726520029282, -7.036766891552735, -8.977589519991337, -6.579537548860962, -0.34069352829828503, -9.041064010465133, -6.282136934564469, 3.7117329189048838, -5.394231282736409, -11.177409462832511, 2.636948493512454, -6.698531905653285, -6.518503941570649, 8.905271814948328, 12.87201877019853, 7.090676972577299, 8.341357540310314, -4.648696854100845, 6.957035567575479, -2.241879985890022, -15.193281739525478, 1.2271924686489228, 4.616886618458665, -14.338441214676216, 14.862058456971615, 8.10221962201912, -6.558552885146747, -4.313458061854922, -6.800570539708076, -9.46963628765032, -14.706255169181524, -1.0508252086336558, 14.12155372134561, 10.30295927948794, 1.482579183504495, -9.087461443620512, 7.457473719466465, -10.18793804336269, -25.247790452763812, 4.02147223954335, -15.36278457408435, -6.138499904099753, -0.4601627652745437, -8.437677953855983, -1.9355996291504027, -16.216877073460964, -6.537035151324876, 4.1897004524761075, -2.2519743698026677, 0.6872691674164905, 12.722533287194196, 0.2882873356987652, 5.116890429328395, 20.226607459494886, 1.290674771102844, -2.800068989539933, -1.5108108590900127, 3.1832454973443687, 0.0, -2.8000689895399358, 1.5108108590900131, 20.226607459494886, -1.290674771102844, 0.2882873356987643, -5.116890429328395, 0.6872691674164899, -12.722533287194196, 4.189700452476105, 2.2519743698026673, -16.216877073460967, 6.537035151324876, -8.437677953855987, 1.9355996291504054, -6.138499904099753, 0.4601627652745437, 4.021472239543354, 15.362784574084351, -10.187938043362688, 25.247790452763812, -9.087461443620512, -7.457473719466468, 10.30295927948794, -1.4825791835044928, -1.0508252086336576, -14.121553721345606, -9.46963628765032, 14.706255169181524, -4.31345806185492, 6.800570539708074, 8.10221962201912, 6.558552885146747, -14.338441214676216, -14.862058456971612, 1.2271924686489228, -4.616886618458665, -2.241879985890021, 15.193281739525482, -4.648696854100845, -6.957035567575479, 7.090676972577298, -8.341357540310312, 8.905271814948328, -12.872018770198533, -6.698531905653283, 6.518503941570645, -11.177409462832511, -2.636948493512454, 3.7117329189048833, 5.394231282736407, -9.041064010465133, 6.282136934564471, -6.579537548860961, 0.3406935282982859, -7.036766891552734, 8.977589519991337, -1.9230298563197554, 5.115726520029284, 0.783595826718674, -1.6178298328406973, -0.9493422927794772, 2.601947921368547, -6.671236362873311, 1.7118899285267464, 5.446017933596458, 6.033996140302204, -11.897938634419027, 8.105846942980568, -1.0782249370205794, 1.4848394116556225, -13.524024198497745, -11.81604749808395, -6.609647506979986, -10.661665300030329, -7.804615781068486, -7.887570592086873, -7.032182834535609, 2.5426863175699514, 2.4267878006424475, -2.4199460094928953, -19.11873843663264, 1.1020798388070299, 14.96829370833483, 0.2800825157832003, 12.655685692773659, 2.588769695009626, -2.312215325199422, -0.9570154846116292, -1.02872862305463, 10.403106776550121, -15.995833331174268, 14.354578092775146, 19.21931801890001, -4.139374296179023, -9.901937950321809, -18.738864037440322, 17.3064485079161, 15.673102695422664, 4.01519747432619, -9.515412040403152, -5.425512551243248, -4.400223312464516, -1.1567394707746494, -8.295982496797441, -0.7666763204502649, -0.23941154418990784, -11.221660986331633, -5.410676842717731, -7.725572016623278, 7.240220313440043, -2.4076277186241555, 8.076888559746063, 10.580893461437023, 11.281416288223713, 5.138209694069921, -2.2706200801626824, -7.468519860234687, 2.782600357113122, -0.0347503146032867, -4.577509972521205, -8.148982829689329, -5.307325780993491, -12.507738907636105, 1.0091552608283463, -2.9147793143550498, 7.299382227996464] }, + fft_2_complex: { complex: true, input: [0.3076825439929962, -1.695198893547058, -1.7243221998214722, 0.2783026695251465], output: [-1.416639655828476, -1.4168962240219116, 2.0320047438144684, -1.9735015630722046] }, + fft_3_complex: { complex: true, input: [0.9383012056350708, -0.6457691788673401, -0.8741029500961304, -0.8208730816841125, -1.1462081670761108, 1.1436293125152588], output: [-1.0820099115371704, -0.32301294803619385, 0.24714778504918455, -1.042797324689853, 3.649765743393198, -0.5714972638759733] }, + fft_4_complex: { complex: true, input: [1.4125794172286987, 0.6605323553085327, 0.28041958808898926, -0.03313862159848213, -1.6822501420974731, -0.03843749687075615, -0.424235999584198, -0.8116567134857178], output: [-0.41348713636398315, -0.22270047664642334, 3.8733476512134075, -0.005685735493898392, -0.12585431337356567, 1.4668901935219765, 2.3163114674389362, 1.4036254398524761] }, + fft_5_complex: { complex: true, input: [0.17418311536312103, -1.2342511415481567, 1.477837324142456, 1.1353919506072998, -0.21928854286670685, 0.8536889553070068, -0.5614511966705322, 0.6619521379470825, -0.8825250864028931, 0.19404278695583344], output: [-0.011244386434555054, 1.6108246892690659, 1.9977527610189005, -4.495568829702767, -0.17773784354010536, -2.9033977754116895, -0.9196550557167773, -0.7794573885094949, -0.018199898511857437, 0.39634359661410157] }, + fft_7_complex: { complex: true, input: [1.2519853115081787, -0.8415557742118835, 1.4540354013442993, -2.404756546020508, -1.4876911640167236, -1.494455099105835, -0.5286277532577515, -2.204728603363037, -0.5613620281219482, -0.7104699611663818, 0.4594193398952484, -0.6865041255950928, 0.4614850580692291, 0.4871855676174164], output: [1.0492441654205322, -7.855284541845322, -0.039888223328758965, 2.182752912893351, -0.22803454984866334, -2.054368234099608, -2.9522900484683143, -1.8098744525563912, 1.2074691685152796, 2.159880155954229, 2.3732051464911823, 1.5194156322450416, 7.354191521775993, -0.03341189207448503] }, + fft_8_complex: { complex: true, input: [0.970332682132721, 0.02602463774383068, 0.019165080040693283, -0.39630618691444397, 0.030557435005903244, 0.20711614191532135, 2.1260392665863037, -0.4882599413394928, 0.38254037499427795, -0.11685442924499512, 0.10177633911371231, -0.9286617040634155, 1.243769645690918, 1.106126070022583, 0.0841875895857811, -0.24541817605495453], output: [4.9583684131503105, -0.8362335879355669, -1.6087226322558026, 0.5188463473486555, -0.5127437971532345, 0.6852134335786104, 3.193741520912889, -3.0038725843536103, 0.2960318624973297, 3.2810584288090467, 0.9862873903181653, 2.1933362079990255, 0.6698357500135899, -3.493357440456748, -0.22013705042147969, 0.8632062969612324] }, + fft_9_complex: { complex: true, input: [-0.06354460120201111, -0.2525795102119446, -1.3290656805038452, 1.239378571510315, -0.2628045976161957, -0.6161656975746155, 1.4840424060821533, -0.8743874430656433, 0.5852842926979065, -1.1418819427490234, -0.6792443990707397, -0.5788629055023193, -0.40828627347946167, 2.353774309158325, -0.34253063797950745, 1.3391598463058472, -0.5310598015785217, 0.7086358070373535], output: [-1.5472092926502228, 2.1770710349082947, -6.615882017596261, 0.6056449746473178, 2.5839119815832605, 0.5586794569181851, 3.9573314649336995, 0.4166996852974395, -0.136326996045127, -2.6370722903002877, 1.4693146501411956, -2.4984601837994824, 0.6265124219185647, 1.0866513474364776, -3.439069243722412, -5.861805640552551, 2.529515620619202, 3.879376023537104] }, + fft_16_complex: { complex: true, input: [1.0133312940597534, 0.3585434854030609, -0.6402965188026428, -1.3225007057189941, -0.4752223789691925, -0.5362467765808105, 1.2483383417129517, 0.05567394196987152, -0.041222695261240005, -0.5106480717658997, -0.34942546486854553, -1.4202176332473755, 1.2869471311569214, 1.1659399271011353, -1.326190710067749, 0.7442878484725952, 1.092625379562378, -0.6453229188919067, 0.3833552300930023, 0.4733515679836273, -0.6908642053604126, -1.0344504117965698, -0.07214371114969254, 1.7793974876403809, 0.4729478657245636, -0.7470325827598572, -0.46451860666275024, 1.7667423486709595, 0.8151687383651733, 1.0275626182556152, -0.7450098395347595, 0.052204303443431854], output: [1.5078198499977589, 1.207284428179264, -4.4806762050351825, -1.1643592216953969, -4.103445578388806, -0.030867462785639788, -0.10790451872168705, -0.5227224508100916, -1.532535444945097, -1.991386003792286, 2.03052393483773, 6.085669694760458, 7.229743305508964, -3.4075612910619277, -5.958691786804708, 2.256651932037089, 5.43960240855813, -3.050593890249729, 5.332886654634274, 3.73714542609058, -0.0764908827618398, 8.509074812562861, 0.7390405363915563, 0.02108386955870034, 4.735840562731028, -2.3431448861956596, -2.2543726824714905, -2.586308038032556, 3.6471191682769133, -1.1870411745676497, 4.064841383148511, 0.2037700224509582] }, + fft_25_complex: { complex: true, input: [-1.9739670753479004, -0.837864100933075, -1.1114044189453125, -0.5064931511878967, -0.32460978627204895, -0.5005347728729248, -0.3506036102771759, 0.0742226392030716, -0.37107688188552856, 0.5423977971076965, -1.9017586708068848, 0.6719839572906494, 2.0592153072357178, 0.39552798867225647, 0.16856394708156586, 0.19291874766349792, 0.04277687519788742, -0.04168238118290901, -1.3544977903366089, 0.5823065638542175, 0.5274385809898376, 0.40750083327293396, -0.8166235089302063, 0.9344403743743896, -0.633162260055542, 1.6156861782073975, -0.017774872481822968, -0.4030669629573822, 1.5376009941101074, -0.5414798855781555, 1.5840349197387695, 0.37068140506744385, -1.5073336362838745, -0.4143368899822235, 0.0665762796998024, 1.2245029211044312, 0.3779613673686981, -1.7612340450286865, 0.9021579623222351, 0.02208423614501953, 0.017875323072075844, 0.09636616706848145, -0.1060614064335823, -0.9343460202217102, 0.9497752785682678, -2.030526876449585, 1.145573616027832, -0.636160671710968, 1.1762216091156006, -0.4195115566253662], output: [0.08689814247190952, -1.8966175056993961, 4.5468912855738495, -1.8818154119554684, -0.5200552972284029, 1.2734341399283204, 0.4632869234518082, 5.931840244626491, -2.9589503069493865, -0.5547667441139432, -1.5273151665934335, 6.215078295347433, -15.640798930847374, 2.4457132590529254, 2.0174810051387153, 0.018505660090085918, -0.195679780600583, -1.5733085555890094, 2.608942923248818, -1.8646958677420233, -5.164085446696965, 1.34215964208527, -5.615886183114355, -0.2185132207952143, 5.847780025346529, 4.8320068942200045, -0.10823882684491926, -0.9458843955572025, -9.968247687952289, -5.8960346192185416, 1.8915989122741381, -0.7760332258366917, -5.750397961307298, 3.3450370182735623, -0.40650253504753975, -3.1697133057132096, 3.105915528359142, -7.099303168364118, -6.373188277089521, 1.2862869596061528, -4.01898105322616, -1.3412458970644465, 3.983941673212713, 2.3932716249692954, -4.4090529271307455, -8.431989160831321, -3.768856482573692, -3.025597080828077, -7.475676439572469, -11.354418102217751] }, + fft_27_complex: { complex: true, input: [0.7116051316261292, 1.037112832069397, 0.6106399893760681, 0.7972601652145386, -0.2644272744655609, -1.5348200798034668, 0.42119401693344116, -0.22403796017169952, -1.1107473373413086, -0.7211647629737854, -0.6500771641731262, 1.2391200065612793, -0.15565899014472961, -0.5251198410987854, 1.8477299213409424, 0.3672321140766144, -1.4575940370559692, -0.28738635778427124, 0.4544501006603241, 0.10830547660589218, 0.5361469388008118, -0.3605248034000397, -1.8324573040008545, 0.07127898186445236, 0.04289998486638069, 0.9799603819847107, 0.5673006772994995, 0.6624241471290588, -0.3812290132045746, -1.7817859649658203, 0.15860582888126373, -1.0848448276519775, -0.7432076334953308, 0.5504264831542969, -2.300358772277832, -0.41666755080223083, -3.017878293991089, 1.61266028881073, -0.419252872467041, -0.20492435991764069, 0.36734986305236816, -1.0211094617843628, -0.3779532313346863, 0.3455152213573456, 0.3696709871292114, -0.5317375659942627, 0.7667483687400818, -0.7377064824104309, 1.1323468685150146, -0.4188673496246338, 0.1713690608739853, 0.8521258234977722, 1.9920309782028198, 0.2819546163082123], output: [-2.560753207653761, -0.9453208297491074, 9.347935323529054, -0.9253007594026554, 3.3062934251957525, 9.624685892964905, -3.747491429172474, 6.944443936382198, 0.2846532016836547, -2.186295990939281, 1.6811782376183848, 6.6863808498799235, -6.517827112663515, 0.14943438287348965, 5.055696157497922, 1.2377621172326654, 5.652717825977259, 5.273448677870682, 5.183011380415665, -1.6221043494072749, -7.350588286919974, 3.2159710253980176, 6.887033244512073, 2.2722794132317587, 3.079426655802572, 0.7414325238817026, -3.3293146897618247, -5.438193278707893, -7.072286065035183, -1.106348191005082, -6.900230177764772, 4.314765981558758, -0.8251297022533834, -3.636347937894679, -0.8573794050807142, -0.5671438615765814, -4.513423924725758, 8.05947784599932, 2.720475531609292, -11.224192327686906, -1.8276595977578185, -0.7655023451245344, 1.4131230144639861, 4.705230182137967, -4.606202193101245, 1.2945168184514193, 14.000104083285905, 4.499731058762659, -2.1022427540436652, 2.4753477036971208, 4.917062359267286, -7.812371522652364, 7.895156658980767, 2.736259449697491] }, + fft_32_complex: { complex: true, input: [0.8450640439987183, 1.3659120798110962, -1.0280054807662964, -0.08084108680486679, 0.9755204916000366, 0.5328992605209351, -0.06296933442354202, -0.08816881477832794, -0.6170272827148438, -1.8309184312820435, -1.3914250135421753, -0.5434551239013672, 1.1874107122421265, 0.6239242553710938, 0.13674135506153107, 0.8248231410980225, 0.44601964950561523, -0.5365520715713501, -1.2611466646194458, -1.2415540218353271, 0.03862550109624863, 0.7438274025917053, -0.964236855506897, 1.1258167028427124, 0.41451361775398254, 0.16199752688407898, -0.17522190511226654, 0.2171468436717987, -2.5754177570343018, -0.09452058374881744, -1.6540249586105347, -0.7705362439155579, -0.03492758423089981, -0.2652542293071747, 0.044895902276039124, 0.3964420557022095, -0.40189823508262634, -1.0244829654693604, -0.7180095911026001, 1.4870411157608032, -0.07942715287208557, -0.15951769053936005, -1.0193690061569214, -2.5220484733581543, 0.04660886898636818, 0.8388492465019226, 1.0369007587432861, -1.2317235469818115, -0.41559332609176636, -0.9562405943870544, -0.6224948167800903, -0.6581323146820068, 0.2363770604133606, -0.2054009884595871, 2.034836530685425, 0.08647070080041885, -1.3155070543289185, -1.5467650890350342, 1.402755856513977, -0.016710011288523674, 0.732329785823822, 0.09752081334590912, 1.2334948778152466, -1.622504711151123], output: [-3.5246070064604282, -6.892655847594142, 14.091260128579158, 2.401029930407454, -2.680974992789798, 3.8916277984556755, 0.5252526044217629, 7.272547842564242, 4.888426456046666, 0.1230857616321388, 0.32227787389030693, 8.837821222991312, 6.797800467459476, -5.336196554450358, -0.3831869573992117, 3.2695703626864097, -5.256811993196607, -0.1872110292315483, -5.001142715544147, 2.250221073549945, 8.811058109636217, 5.232778335556318, -0.6512846517539874, -2.242062163272187, 9.451628634097027, 8.879343426811909, 2.0671711072014323, 9.227474643353165, -2.163253380148122, 6.008848575681659, 0.7074175020118927, 9.617801820579668, 2.4899496845901012, 2.3832117300480604, -2.562616988438002, -5.25214449133264, -0.2925401848541538, -7.40888946314626, 6.728447606715459, 8.585534132159177, -2.8502671899324317, 2.9276655593856034, 6.279407417654955, -0.03356822185923569, 0.5101090153146224, 9.679261772566992, -9.235685068269035, -3.9845762833139267, 3.263928959146142, -10.372698850929737, -7.636904664040565, -9.25411350837314, -5.141376625258181, 7.840129391031544, 1.7582723882580569, 2.5429684456901556, -1.7377452788371315, 0.002180726241851616, 2.8379890490594435, -2.0202939805697477, 0.396858681471695, 0.8400442760030362, 4.233191419326372, -5.119549879368318] }, + fft_49_complex: { complex: true, input: [-0.653186559677124, -0.9652466773986816, 1.1103789806365967, -0.8953530192375183, 0.17809812724590302, 0.7561290264129639, 1.412551999092102, 0.4937201738357544, 0.6659147143363953, -0.35966482758522034, 0.7821815013885498, 0.9256682991981506, -2.061739206314087, 1.1828033924102783, 0.8627902269363403, -1.1108970642089844, -0.38785845041275024, 0.6139956712722778, -0.1652761548757553, 1.3882287740707397, 0.014210155233740807, -0.25180017948150635, 0.04756239056587219, -0.7563911080360413, 0.8808799386024475, -0.9352231621742249, 0.09307317435741425, 0.9392328858375549, 1.9853578805923462, 0.5460475087165833, 0.2075347602367401, 0.9603745937347412, 0.04923528432846069, -0.9488556385040283, 1.8051396608352661, -0.4981226325035095, -0.5307197570800781, -0.4026987552642822, -1.1836994886398315, 0.5920459032058716, 2.3241052627563477, 0.00905652716755867, -0.3693919777870178, -0.8863218426704407, -0.03561544418334961, 0.04909980669617653, 1.7207887172698975, -0.1781470775604248, -0.02191477082669735, 0.2800677418708801, -1.0829179286956787, -0.9707691669464111, -1.5767006874084473, 1.3887768983840942, -0.1409223973751068, -0.6077256202697754, 1.3477866649627686, -0.22426994144916534, -0.15349310636520386, 1.9329805374145508, -1.1728582382202148, -0.8913836479187012, -0.7144800424575806, 0.018777834251523018, -0.22412529587745667, 1.3015447854995728, 0.35400378704071045, 1.3718682527542114, -0.11531352996826172, 0.5478003025054932, -0.9265352487564087, -1.0495789051055908, 0.16852520406246185, -2.1708903312683105, -0.6477338075637817, 1.3192451000213623, -0.5638371706008911, -0.902698278427124, 0.1049879938364029, -1.401151418685913, -0.08417209982872009, 2.0441510677337646, -0.5061149597167969, -2.023317337036133, -1.237672209739685, -0.008067949675023556, -1.2302114963531494, -1.6217567920684814, -0.24184295535087585, 0.4429032802581787, -0.04756205901503563, 0.04733582213521004, 0.34705546498298645, 0.9135802388191223, -0.790744960308075, -0.045627497136592865, 0.08297869563102722, 0.028814325109124184], output: [-0.32149941846728325, -0.011710121296346188, 1.5719012837700608, -11.9435719170496, 7.889099849425332, 2.349164232580148, 3.101234046136667, -0.9897723162237673, 2.906207608358317, -7.272673248354674, -0.28567752458921314, -7.807930436881056, -9.98287287277238, 2.170225845960604, -3.9995455705964806, -6.139724045667118, -2.6586132853906514, -7.769710705243423, 0.9131703097247483, -9.940238191191922, -8.286631830021289, 7.3493173799497, 8.661878987183604, -3.553867174039695, 11.583726594048862, 6.9528434851612655, -9.85410942809862, -6.509237172308493, 3.264008317204529, -8.95247219585599, -1.0683784894257498, -8.209758113014594, -8.459077459978726, 3.5851739864239787, -3.130647641813152, 14.882850815864773, -12.112293562127494, -12.761696026041244, 8.867540664810484, 5.336684447842124, 6.405865160930168, 4.524565520649802, 3.1540883076724198, 0.8664358168359105, -14.034517102599953, -1.5599303443084476, -9.534876375145306, -4.0920648890208, -7.072712272425977, -4.325977005204747, -5.285935735332723, -6.9699101614527, -3.845638728726241, 0.6627204864850933, 11.368533948151075, 9.810447058144453, -2.880030910522577, 3.760503532656639, 0.5451509287272396, 1.6597167211016912, -8.58285942430189, -11.129246819405175, 2.030691093396224, 1.6669154167624718, 8.748833248959015, 6.192014902704366, -7.3302180587750225, -0.08243838526629593, 5.522485457006632, 10.260949490755591, 3.972158254442996, -14.10516382807534, 3.8678504886320235, -0.06906298330718297, 3.321736134975782, -4.198418580926676, -6.9491746204757945, 3.9162914328571237, 5.508243105428264, -2.4542837759115645, 0.26676806448008605, 2.9368877862382843, -16.784439087864854, -7.618115164366894, 3.8748624559849345, -1.3062132611368762, 0.5913456904524359, -2.1117316472857373, -0.6923707074676628, 1.8519748031549472, 4.596152734943953, -1.3913510533714808, 9.06246632833908, 5.377389540572535, -6.9033275845955755, 0.20446700081688807, -3.5466927958494017, 9.661642666154053] }, + fft_64_complex: { complex: true, input: [-0.5566142201423645, -0.31118565797805786, -0.16021257638931274, 1.2795861959457397, 0.789443850517273, 1.4509670734405518, -2.3435871601104736, -0.908771276473999, -0.8936960697174072, -0.8750560879707336, -0.63246089220047, -0.6453881859779358, 1.4873902797698975, -2.564749240875244, -0.7355272769927979, 0.2437494695186615, 0.01852554641664028, 0.5734773278236389, 0.29105886816978455, 1.1027342081069946, 1.8088326454162598, -0.7475714683532715, -1.217250108718872, 2.0953369140625, -1.4750502109527588, -1.343973159790039, -0.5276563763618469, -1.297399640083313, 2.03525447845459, 1.159611701965332, 0.4579901695251465, -0.9503767490386963, 0.05310674011707306, 0.0014978965045884252, 0.18816789984703064, 0.1826970875263214, -0.8126240968704224, 0.18846499919891357, -1.0810470581054688, 0.39759552478790283, 0.6841163635253906, -0.6645811200141907, 1.4170793294906616, -0.13333140313625336, 0.4437287151813507, -2.8415510654449463, -0.07697270810604095, 0.41809648275375366, -1.0915343761444092, -2.776437282562256, -0.43745681643486023, 0.26580730080604553, 1.6740102767944336, 0.22755879163742065, -2.9241530895233154, 1.4011553525924683, 1.6825302839279175, 1.5555111169815063, -0.14711306989192963, 0.8428824543952942, -0.36785921454429626, 0.9028973579406738, 0.1373748779296875, 0.32123002409935, -0.22010713815689087, -0.599694013595581, -2.5014238357543945, -1.0242420434951782, -0.6038212180137634, -0.6158038377761841, 0.7263562083244324, 1.6927067041397095, 1.2062644958496094, 0.26875412464141846, -1.0378527641296387, -0.3995000422000885, -0.24585849046707153, -0.30057334899902344, -1.389653205871582, 0.22163736820220947, -0.07863101363182068, -0.22048908472061157, 0.9795127511024475, 0.6902763247489929, -0.20713239908218384, -1.416999340057373, -0.4947267770767212, -0.8151863813400269, -1.6725231409072876, -1.9904801845550537, 0.21500161290168762, 0.44239726662635803, -0.6723101735115051, 0.6706801056861877, -0.21685439348220825, 0.7185264825820923, -0.9003168344497681, -0.29727739095687866, -1.247776985168457, 1.110876441001892, -0.17718446254730225, -0.8177000284194946, -0.6590757966041565, -0.5754866600036621, 1.1977955102920532, 1.1977572441101074, -1.5937381982803345, -2.6281495094299316, -0.7925133109092712, 1.5867403745651245, -0.6221466660499573, 1.4161043167114258, -1.6597610712051392, 0.7671688199043274, 0.4350447356700897, 0.16583804786205292, -2.573836088180542, 0.8150725960731506, -0.6248568296432495, -1.627732753753662, 0.3362247049808502, 0.30031880736351013, 1.0735864639282227, 1.333678960800171, -1.4852718114852905, 0.8292962312698364, -0.2946455478668213, -0.4848930835723877], output: [-18.114436665549874, -1.03589254419785, -7.089877931654041, -8.136593953551714, -7.48613375743217, 3.4058307917826487, -0.11349749558287137, -7.230611270566168, -8.574244204990293, 5.7986400915728815, 5.473250096298479, 7.160363687658917, -5.13533852659519, 1.8255020562058237, 7.273548900259892, 24.690830952503084, 1.6868156269139245, 8.810813815606256, 6.6254795081932745, -4.567693073247041, -2.239049755279698, -17.516871542107324, 1.3976937653317711, -4.659023464944261, -3.498063522435509, -1.042294081352104, 10.872729327063226, -18.927297144289987, 4.267325206467068, -6.680543926630708, 6.4003803411255165, -14.418237604405501, -5.944847682490945, -10.613565055537038, -1.0402464930724309, 4.733410202647864, 10.284586844608768, 7.495884983823905, 10.978098280081051, -5.653369285851954, 17.957223243942725, 3.3651604181690082, -12.12841177766924, -2.173315060592438, -11.807254527678538, 15.743279066172082, -9.485910951122346, -5.369406024705013, -4.183827428022064, -1.3953354113002816, 5.236741461404014, -0.8655702406244625, -3.707989963955507, -7.550554154083424, -2.1556733662040837, 8.35703936882244, -1.893019665574852, 14.917386376122955, -0.5382039039277775, 2.069340044414755, 10.860091518498637, 6.278499449945484, 2.0427421023295516, 17.14233054094595, 11.975593766197562, -10.740802941727452, -12.646531404584593, 0.13425425752591735, -3.6872251499900193, -1.9392344875845375, -18.16736557788115, -20.48335434059285, 0.6875482818287537, 1.3708837169699595, 26.677807954665866, -1.8603500541565219, -6.790446289324352, -0.7636112342571075, 0.16202327721217014, 2.890628934614975, -13.405530896032472, -10.423450139391665, 3.2690754871155625, -0.13370157023242735, 17.421851422986148, -7.428197970888403, -2.696655432079798, -3.6753472703589476, -1.8393129488369242, -3.9804913300385483, -14.84575265792169, -2.1073012152618524, 1.9496145693049904, -7.632750924513978, -12.292581873003275, -3.378658179390201, -1.3949911389499903, 4.731505962205119, -4.453599350900657, 8.2927961867939, 2.890294756083995, 8.32112212455824, -7.313795240640941, 13.541967260990265, 0.3013408500166399, -22.02939466419966, -0.8879106014223499, -0.9280433153423213, -13.511110887018695, -0.9068611379695746, 13.278501353056239, 4.253543634046318, -6.101434519639574, -2.2367887703037312, -8.42198169225635, 15.996815858908311, 5.985629137333648, -11.238301003546834, 0.09940024194354224, 1.4371628357505024, 6.358283661511687, 5.197077900987281, 3.293723266284598, -9.325758113839981, 1.8329751785259507, 8.745205075152134, 0.42857376402367997, 12.425414796091186] }, + fft_81_complex: { complex: true, input: [0.16182588040828705, -2.0999834537506104, -0.5849781632423401, 0.24368105828762054, -0.6967284083366394, -0.034912507981061935, -1.1470516920089722, 0.7932613492012024, 0.7196304798126221, 1.1110419034957886, -0.939569354057312, 2.4974799156188965, -1.2485228776931763, -2.838197946548462, 0.9772579669952393, -1.0605124235153198, -0.9749694466590881, 1.0589070320129395, -0.9194549322128296, 0.9585606455802917, 0.15526491403579712, 0.8247618079185486, -1.3407961130142212, -2.523029088973999, 1.0570300817489624, 1.5672498941421509, -0.07786913961172104, -0.20268507301807404, 1.7038300037384033, 2.0308806896209717, -0.27392077445983887, -0.06376122683286667, -0.310247540473938, 0.5324293375015259, -1.7548904418945312, 0.9650161266326904, 0.41914716362953186, -0.35616472363471985, 1.2825931310653687, -2.2441608905792236, -0.5296666622161865, 0.5721330642700195, 1.0294588804244995, -0.04206320270895958, 0.15277068316936493, 0.8333802819252014, -0.858043372631073, -0.12947990000247955, 0.46606531739234924, 0.21820232272148132, 0.2710532248020172, 0.36254584789276123, -0.7573392987251282, -0.340090274810791, 1.7085819244384766, -1.3537571430206299, 0.25935328006744385, -1.2952094078063965, 0.2727315127849579, 0.7723783254623413, 0.03050963208079338, -0.06484062969684601, 0.6543181538581848, -1.2400768995285034, 0.7176415920257568, 1.6827946901321411, -0.272500604391098, 1.1401575803756714, 1.164631724357605, -1.5258969068527222, -0.1886187493801117, 0.8159262537956238, 0.6548362374305725, -1.195924997329712, 0.7484856247901917, -1.5505919456481934, 1.5596144199371338, -0.6034004092216492, -1.260884404182434, -0.6083353757858276, -0.6304160356521606, -1.0715056657791138, 0.6321176886558533, -0.11946370452642441, 0.2912154495716095, 0.014026062563061714, -0.831264078617096, -0.9964612126350403, -0.20909088850021362, -0.11727125197649002, -0.705121636390686, -0.5439452528953552, -1.5571593046188354, -1.6995549201965332, 1.7400833368301392, -0.0008625089540146291, -1.17470383644104, -0.997884213924408, -0.3725011944770813, 0.35961270332336426, -0.3637978732585907, -0.7026296854019165, 0.4852464497089386, -0.5324466824531555, -1.6679120063781738, -0.7600529193878174, -0.40924522280693054, -1.3232053518295288, 0.48746198415756226, -0.5340525507926941, -0.06851266324520111, -2.164428949356079, -0.23488061130046844, 0.4067259430885315, 0.1828349083662033, -1.640533208847046, 1.0253461599349976, -0.3487933874130249, -0.39523687958717346, 1.1609245538711548, -1.0767440795898438, 0.29041048884391785, -0.4530940651893616, -0.1469300389289856, -0.5046754479408264, -1.5347036123275757, -0.49093714356422424, 0.2856205403804779, -0.3833635449409485, 0.01342631783336401, -1.825499176979065, -2.1771950721740723, -1.1022430658340454, 1.0045795440673828, 0.35692599415779114, 0.8746539354324341, -0.22437222301959991, 1.1157619953155518, -1.7885184288024902, -0.09074169397354126, 0.9822217226028442, 1.47532320022583, -1.4025393724441528, 0.5067855715751648, 0.8213406801223755, -0.7191897630691528, 0.3869772255420685, -0.3575884699821472, -1.186098337173462, -0.10889001935720444, -1.2052828073501587, -0.4578658938407898, -0.15333758294582367, 1.691064715385437, 1.8314754962921143, -0.3512211740016937, 0.8154265880584717, -0.3477337062358856, -1.0724186897277832, -0.0016170362941920757, 0.8384286761283875, 1.601130723953247], output: [-8.58128398284316, -11.439007951354142, 0.210079195962251, 5.283435256837555, -3.729018054516872, 4.172233149410547, 6.0094247849082345, -4.867406596990031, 1.6896215297204356, 8.254567899165185, -1.5832996916764728, 6.874626142064364, 7.791926933815942, -0.1113988953646401, 6.157976194485636, 7.908498593612423, 2.1554616784825296, 5.087697970296618, -10.929793301868514, -20.248492473955267, -1.7673074913954707, 4.926997195410063, 3.7962336785471167, 9.785487339090924, 7.5509937732589805, -4.618565996292649, 1.2172725036089613, -10.455295568854787, -10.613583259389454, -12.094708219848364, -11.920648753075001, 2.7422204833456965, -14.39499933098191, -7.986984514501212, 1.4636316169286738, 3.6526472221302644, 1.7780382602070137, 0.929515572223278, -4.1263772762395785, 17.702314875380782, 3.8927335234933165, 3.3808527564005315, 20.656948077140108, -10.291866793201438, 10.429803208602253, 3.2896163180066105, 2.885996928025134, 6.240479112191121, 5.126031982460415, -15.569134216773076, -14.789597569471173, 4.411143413971, 11.01877897308899, -18.4554008501283, -13.452431942413536, -12.210080319899106, 2.7697215257400316, -10.572091571648135, -4.523764337963882, 3.9530076132464806, -1.2538076112045768, 5.196509906553922, -4.582902295841774, -2.4578802622089864, -16.85598763977452, 13.658512440269673, -3.7344804477869467, 25.005764333536174, -5.796885916344451, -3.741579209020099, 14.995322293493794, -9.021964181821255, 12.535576498170204, -3.2115394425750283, -5.316067284014389, -15.73079791413737, 5.410951763343561, -2.179490941864971, -17.59181683345424, 0.35349287922924855, -4.114985065194593, -3.6153272922914104, 5.362879738072423, -6.115564376188364, 10.568119159000045, -15.318073540754956, 4.64815841280388, -21.68501783286247, -13.224626602809812, -2.219868377234576, 4.00167448459214, 1.4272276456380588, -11.341596811743006, 4.873948561987781, 4.2861526759974895, -5.96023765608868, 3.9760305188050347, 6.566800933990912, -5.002198883369546, -2.0309709147031416, -1.2884460262093702, -12.32833011496317, 0.05382037687629326, -10.78439574018218, 1.534354885003597, -13.264704159013437, -12.280184562954279, 11.35458553480188, 9.869000609110085, -0.9969714411418238, -5.650404262545905, -11.399256472642067, -6.3488730387883825, -17.807560792326566, 17.456387259491258, -8.437210476076581, 3.5889363471995455, 1.8943033185815557, 4.734889441188521, -5.865838952657386, -1.197257968989023, -16.13996530086823, -6.139028694267207, 3.1292836318406754, 12.18828544229761, 4.610631776769463, 18.678240243897175, 13.33540206528819, 15.33449728617182, 3.2382519818121347, 7.4739674455797, 6.838902240570333, 2.105371692765212, -1.5378787198749482, -2.6243434923047406, 0.11652216064202503, 2.5272654877458214, -4.532248506023914, -1.954636472567441, -3.4146656232065977, -9.008911319126828, -11.353937437276306, -7.625555655430608, -5.250709931084791, 5.340100553320184, -17.615583941013096, -3.279927088522264, -4.17250143308863, 4.4785210214064195, 6.400344186694214, -3.7005380565285066, -1.122575913062859, -0.5038180980239875, 2.5292421600004156, -1.6061528617728151, -14.630926203339426, 4.481494466029414, -5.292535421445056, 6.4147407824254055, -8.858967182263992, -13.101978958787178, 27.791783247323977] }, + fft_125_complex: { + complex: true, + input: [0.8124353885650635, -1.2395265102386475, 0.16039574146270752, 1.9819504022598267, -0.3600437343120575, 0.5889578461647034, -0.24872885644435883, 0.7082515954971313, -0.5773665904998779, 0.13543199002742767, 0.8298076391220093, -0.4342207610607147, -0.4896481931209564, 0.8350552916526794, -0.7197877764701843, -1.5380432605743408, -0.06107468158006668, 2.201507568359375, -0.3374556005001068, -1.2920140027999878, -1.5635737180709839, 1.6503757238388062, -0.6105449795722961, 0.4370865225791931, 0.5172725319862366, -0.6922412514686584, -1.2310928106307983, -1.8327370882034302, 0.6385520100593567, -0.11756168305873871, 0.23863999545574188, 0.018904687836766243, 1.5264911651611328, -0.10709311813116074, 0.5826241374015808, -1.765297770500183, -0.6799334287643433, 0.571349561214447, -0.5346289873123169, -0.8332467079162598, -0.20369751751422882, 0.30775347352027893, -0.7439308762550354, -0.5451762676239014, 0.4147465229034424, 0.5432214736938477, 0.0078077311627566814, 0.9431089758872986, -1.0435984134674072, 0.23313158750534058, -1.209730863571167, 0.45955437421798706, -0.3265855610370636, 0.6053746342658997, -0.8725820183753967, -0.5241636633872986, 1.7610948085784912, 0.6633955836296082, -0.9714349508285522, -0.5136868357658386, 1.5349853038787842, 0.3695436716079712, 0.2814086675643921, 2.5949618816375732, 1.2379722595214844, -1.8505828380584717, 0.47188371419906616, 1.3736778497695923, 1.1676279306411743, -0.2972381114959717, -1.2804001569747925, -1.7225773334503174, -1.577651023864746, 0.3269241452217102, 0.4372701346874237, -0.9890488386154175, -1.1033475399017334, 1.9549521207809448, 0.6475730538368225, 0.5292824506759644, 0.4473706781864166, 0.6432071328163147, -0.1467706263065338, 0.20976980030536652, 0.7794570922851562, -0.31381580233573914, -0.6803262233734131, 0.011037338525056839, 0.9622517824172974, -0.8368542790412903, -0.8587225079536438, 0.633898138999939, 1.2174996137619019, 2.1725356578826904, 0.2897004187107086, -1.309606671333313, -2.598196506500244, 0.557666540145874, -1.5418102741241455, -0.8026767373085022, 0.08196965605020523, -0.002491528633981943, 1.587324857711792, 0.720113217830658, -0.7349403500556946, 1.192823052406311, 0.729076087474823, 0.2271231710910797, 0.9269818663597107, -0.006230967119336128, 0.7080751061439514, -2.3676698207855225, -1.2577210664749146, -0.24024532735347748, 1.1770211458206177, -0.5720393061637878, -1.920380711555481, 0.8346250057220459, 0.7047927975654602, 0.19386297464370728, -1.093695044517517, 0.6820136904716492, -0.2829663157463074, -0.11388272047042847, 0.6887729167938232, 0.09922466427087784, 1.250266194343567, 0.6323102712631226, 0.1848202496767044, -1.8433687686920166, 0.4263668954372406, -0.24713163077831268, 0.01820266619324684, 0.8616644144058228, -1.5792089700698853, 0.6367298364639282, -0.3927980363368988, 0.6539236307144165, 0.8779292106628418, 0.5544044971466064, -0.4019509553909302, -0.3177534341812134, 0.4690700173377991, -0.7643328905105591, -1.5016077756881714, 0.5900797843933105, 0.36968597769737244, 0.7913125157356262, -0.9097312688827515, 0.7577337622642517, 0.7302269339561462, 0.7785899639129639, -0.7808380722999573, -1.0440396070480347, 0.5710436701774597, -0.6633660197257996, -0.6982168555259705, 1.5012997388839722, 1.8683104515075684, -0.009162770584225655, 0.053628917783498764, 0.7560212016105652, 2.734218120574951, 0.8975538015365601, 0.9965337514877319, -0.9736179709434509, -1.3732835054397583, -0.22769349813461304, 0.5279285311698914, 0.9973939657211304, -1.5946539640426636, -0.3334585130214691, 0.5114824771881104, -0.8088312745094299, -0.009309413842856884, -0.3517300486564636, -1.9116069078445435, 2.348860740661621, 0.5795866250991821, 0.6854371428489685, -1.3237437009811401, 0.30641838908195496, -0.9147019386291504, -0.4391704797744751, -0.5839483141899109, 2.629678726196289, -0.1468760073184967, 0.2384973019361496, 0.9125059247016907, -0.9323630332946777, -0.12135637551546097, -0.1401946097612381, 0.6303017139434814, -0.8720086812973022, -0.4348902702331543, 0.707770824432373, 1.0431065559387207, -0.3136989176273346, -0.34615302085876465, -0.09326159954071045, 1.0725315809249878, 0.6460273265838623, 0.9376847743988037, -0.45695820450782776, -0.6338280439376831, 1.7222473621368408, -1.7862892150878906, 0.24867220222949982, -0.8356691002845764, -0.5947818756103516, 0.46319833397865295, -0.5724456310272217, -1.3741754293441772, -1.2928011417388916, -0.15125389397144318, 0.35185670852661133, 0.45909401774406433, -0.276844322681427, -0.7534619569778442, 0.17448769509792328, -0.8041709065437317, 0.7715813517570496, -0.17985635995864868, -0.3155222535133362, -0.27937594056129456, -0.6012001037597656, 1.3496509790420532, -0.07789217680692673, 0.911784827709198, 0.6547526121139526, 0.6284425854682922, -1.5268802642822266, -0.11813484132289886, -0.8698882460594177, 0.5227888822555542, -0.2165810912847519, -0.807140588760376, 1.1484020948410034, -1.4427751302719116, 0.6043649911880493, -0.525101363658905, -0.7420395612716675, -0.4712083339691162, -0.04465737193822861, -0.4328814148902893, -1.3837007284164429, -0.06250803917646408, 0.5973103046417236, 0.2701307237148285, -1.2017232179641724], + output: [-8.60566947190091, 9.047967813443393, -6.678979661059562, -8.360831755799953, 3.115231761090947, -6.697091460792556, 0.7568358822534593, 11.066622749967639, 2.8466255459388927, 6.27918164849176, 1.982200342331545, 0.3737138114008831, 8.93599812413914, 12.19410650368572, -4.5824951902858615, -4.643371981421932, 11.975504695471635, -19.79773110340414, 16.894522457707588, -11.40437941841762, -0.5369639596199498, -9.687637013191164, -2.1092885577719587, 10.925736950748426, 2.630639119388632, -2.8059934209514976, 8.536957582736084, -6.326392685852441, 19.05482126619987, 4.74321319044126, 1.7193491466752984, 17.828632923932844, 11.48731740639608, 11.824203873765583, 17.4496745991033, 14.886923392164112, -8.857144569139852, -18.45702795068501, 1.7494909136262233, -0.10810961271060027, 11.536135407131509, -12.549170934604959, -5.461911779043582, 9.93390865307337, 7.577784660577098, 11.044552867393765, -0.38872990536666596, -4.554231235610941, -9.19372645559055, 3.9372069026290077, -0.01611898707200954, -14.170353635176678, 2.977419276887061, 1.171732761405826, 9.77702001582745, 4.731009139477455, -6.457985308398847, -15.847828201326442, 9.95926759256128, -11.402544444287324, 17.96375259440102, -6.2530379131604725, 4.938790863887686, -10.135905976382034, 5.383747427255329, 6.230288801920859, 6.384329417961547, 1.3219731997515085, 3.1738925777367895, 11.969669020511425, -7.174837958584151, -14.303685689056584, 0.7454990885394415, -0.8185666838197116, 8.849555077234482, 3.9055874990272663, -2.018206936296142, -2.3395830964594415, -9.60564366177047, -5.769798601114195, 0.7520113635166212, 0.5065019274195421, -10.25514045587141, -9.637019763988052, 18.803482381840155, -2.3670620465003394, -11.199527109220814, 0.007833023989774368, 8.625619052479255, 9.951138032831594, 7.758961547592464, -1.8211300264093309, 6.718724482310631, -3.675823001520721, -6.918876012140167, -11.471658204308536, 2.467033682197071, -8.508193328034618, 14.225276109269815, 16.144776448129633, 15.927933230731913, 10.235538687304809, 5.767881945940595, -28.86811292326169, 9.669928964930417, -4.668293602235824, 14.769201279257288, 11.067653051718727, 12.761530418035363, -11.044953949483144, 4.242003110570776, -23.262991395970026, -10.145352776759976, 9.212885213669484, 6.329756726823897, -2.401697066728733, -8.54352813832675, 0.701551842851496, -15.258656469035635, -19.14394691246519, -7.522186448172975, -1.997676262113934, -11.408489749207227, 24.631586391822943, -5.098355854851496, 1.0410089604391717, 22.688569178648756, 19.097185694651152, 0.16409737757445608, -14.474036852523309, 4.7859526448704734, 19.558395170182035, 10.633635125709178, -3.0615977245039865, -6.883177607599656, -4.696650706016856, 4.286737340380929, 1.1391516077082633, -17.043528019280284, -7.2919743123329965, 18.42867500955987, 9.458517392981532, -7.23747329102956, 3.0801112402296056, -19.46748520769718, -19.760353803723497, -4.916492200532298, -11.492889148389452, 18.97908975026986, -9.61307230309599, -31.333799318924978, -6.6051904344747445, -2.610154763269807, 17.822400589161646, 5.128110541635698, 7.169102569854218, -20.704759452278893, -13.58731153366598, 9.44388682655616, -1.6262238850380326, 2.4785447765146986, 15.61067463907144, 8.206847406028755, -3.2188956590845867, 10.069966894154257, -17.811422869295264, -7.333188700567103, 7.277796245862989, -5.975213239029163, 0.0776494376891641, 4.448473434391025, -2.74774293836415, -19.167307678242555, -22.10574472079241, -0.9176460422085646, -2.698508980697669, -2.9126695148592425, -11.855087227776476, -10.63642091855646, -4.994794817859426, 8.34646291775615, 6.768483375508413, -10.605502170349608, 14.164919594071957, -12.394208891449225, 12.057338941547131, 22.90400935226583, -15.664600594740676, 1.30010074910831, -0.2199980758912563, -0.13084869289575352, 1.5514316991800943, -6.207758174655294, -21.281702562218502, 0.6785363179803934, 1.3313626670817715, -4.335780044121057, 0.5496189332641022, -12.958391630849423, -8.448073173532364, 9.262064248245114, -6.620464792278902, 6.229199284276308, 14.566232762792579, 7.969523876692342, -6.776657815077612, 12.290319767193232, 9.559089975820987, -10.33865192053783, -22.41131767867546, 2.9821554944671256, 4.62861355727904, -1.556181975825081, 15.142425835968055, -7.526775680393509, -16.871371735315304, -16.91792120984387, 10.94566759815768, 7.919149545235784, -2.721091872750706, -11.568727749623791, -6.404971618403414, 0.9048111698914159, -12.163107403321717, -23.30918866848898, 0.29599861864002963, 21.938310258693168, -2.3132157853956175, 0.7524657263569949, -5.613641198847075, -8.631493933375305, 17.933382465737296, 15.905249964584621, -0.12417131736795461, 16.840693139623014, 10.447828447732046, -15.117925469220577, -1.7000257658739502, -14.07936041053577, -12.445181510422046, 0.09178864954511656, 8.976039835817634, -3.0952547078859665, 4.42715006938438, 2.016084671443174, 1.299572020270631, -5.764752453011912, -5.767130951426552, -4.024939854949247, -6.303606982466311], + }, + fft_128_complex: { + complex: true, + input: [-0.46925589442253113, -0.17965197563171387, 1.0996962785720825, -0.38428056240081787, 0.48438164591789246, 0.4687068462371826, 0.39249294996261597, -0.3080357015132904, 0.5177378058433533, 0.6460461020469666, 2.131727933883667, -1.9881218671798706, 1.7797539234161377, -1.7341786623001099, 1.8023210763931274, 0.04056849330663681, 0.5402734875679016, -1.2557553052902222, 1.6492481231689453, 1.1959751844406128, -1.1998282670974731, -0.28071489930152893, -1.1383824348449707, -1.105458378791809, -1.3544220924377441, -0.9994213581085205, 1.0511277914047241, 1.3731906414031982, -0.366388738155365, 0.5248828530311584, -0.06661345064640045, 1.1866965293884277, -1.7126017808914185, -0.8613355755805969, -0.15631477534770966, -0.25460687279701233, -1.6851294040679932, -0.479960560798645, -1.6602188348770142, -1.8969695568084717, -1.3355588912963867, 0.2036823183298111, -0.21647731959819794, 0.027381736785173416, 1.3588712215423584, -1.916928768157959, -0.774971067905426, -0.9908143877983093, -0.5971747040748596, 0.1783454567193985, -1.5134319067001343, 0.49953287839889526, 0.6159718632698059, -0.36356979608535767, -1.0240488052368164, -0.04502468928694725, 0.4399787485599518, 1.6701726913452148, -0.29381847381591797, -0.10132483392953873, 1.382400393486023, 1.0435426235198975, 0.9063398838043213, -0.8696965575218201, 1.7439777851104736, -1.1632086038589478, -0.8023254871368408, 0.18316711485385895, 0.7868092656135559, 0.6457328796386719, 0.000458094640634954, 0.2006639540195465, 0.44469723105430603, 1.5764163732528687, 1.37369704246521, 1.4348359107971191, -0.3319808542728424, 0.4867582619190216, 1.6903849840164185, 0.09980353713035583, -0.8114499449729919, 0.21222348511219025, 1.7070233821868896, -0.06860260665416718, 0.3595496714115143, -0.22636087238788605, -0.11660676449537277, 0.35164034366607666, 0.6463969349861145, 0.7742586731910706, 2.551539897918701, -0.36875852942466736, -0.907656192779541, -1.5238984823226929, -0.17778240144252777, -1.5085883140563965, -2.1957619190216064, 0.2777234613895416, -1.9981634616851807, -0.36680006980895996, 0.43345877528190613, 0.12283071875572205, -0.7519865036010742, -0.8946448564529419, -1.0104613304138184, -0.574897289276123, -0.7528412342071533, -0.05524379014968872, -0.2588641345500946, 0.13064302504062653, -1.2371978759765625, 0.4406910538673401, -0.06988917291164398, 2.41745924949646, 0.2212488353252411, -0.9714004993438721, 0.41281431913375854, -0.5191168785095215, 0.5448367595672607, -0.5188544392585754, -0.35594210028648376, 1.31781005859375, 0.19899560511112213, 0.10419808328151703, -0.7013852000236511, 1.2463740110397339, -0.6775157451629639, -0.07904287427663803, -0.46743783354759216, 0.6186703443527222, 1.0175814628601074, 1.2864513397216797, -0.9423191547393799, -0.815204381942749, -1.4644925594329834, 0.4917009174823761, 1.0119210481643677, -1.106779932975769, -0.537906289100647, 0.09276387840509415, 2.3790464401245117, -0.18112841248512268, -0.7766230702400208, -0.4697842597961426, -0.24867388606071472, 0.350057452917099, -1.7352688312530518, 0.3294185996055603, -0.049025554209947586, 0.19478608667850494, 0.3822934031486511, 0.8966708183288574, 0.7362163662910461, 1.9381089210510254, 0.7763320207595825, -0.9195051193237305, -0.15279459953308105, 0.7260957956314087, 1.248564600944519, 0.8662853837013245, 0.22399753332138062, -0.5145770311355591, 0.6150395274162292, 0.17572005093097687, 0.1295638382434845, -0.823893129825592, 0.09608310461044312, 0.1056567132472992, -0.1421104371547699, -0.13746590912342072, 0.6133358478546143, -1.2861822843551636, 0.8356537818908691, 0.5849096775054932, 0.7742197513580322, 1.864620327949524, 0.1762249767780304, -1.6820768117904663, 0.170370414853096, -0.9834036827087402, -0.3257652521133423, 1.3807744979858398, 0.5471190810203552, -1.2435959577560425, -1.5459822416305542, -1.613089919090271, -1.5104633569717407, -0.7388973832130432, -0.3404698073863983, 2.2242629528045654, 1.101983904838562, -0.41542941331863403, 0.825286328792572, -1.7480731010437012, -0.227715402841568, -0.2249363213777542, -0.43012070655822754, -0.7993203401565552, -0.06888776272535324, 0.6460397839546204, 0.8002319931983948, 0.17660461366176605, 1.4292235374450684, -1.6437238454818726, 0.43454229831695557, -0.011055096983909607, -1.4420714378356934, -1.447640299797058, 0.16068753600120544, -0.2804417014122009, 0.3235762119293213, 1.2617326974868774, 0.1411382406949997, -0.23557768762111664, 0.13576534390449524, -1.6873000860214233, -0.42229339480400085, 0.8121863007545471, -1.0847389698028564, -1.6272295713424683, 1.8023097515106201, 0.7626906633377075, -1.3667072057724, 0.14141327142715454, 1.5778913497924805, -0.9910776615142822, -0.013187640346586704, 0.8963002562522888, 0.3190685510635376, 1.4895867109298706, 0.8450273871421814, -0.7519250512123108, 1.1641961336135864, -0.1140206977725029, -0.7162372469902039, -0.2397381216287613, 0.3938809037208557, 1.8317588567733765, -0.32195407152175903, 0.9710619449615479, -1.7239831686019897, -2.138955593109131, -1.1850391626358032, -1.1381371021270752, -0.2674075663089752, -0.7534425258636475, 1.180410385131836, -1.2349971532821655, -0.9732410311698914, -0.9838739633560181, 0.1530347764492035, 0.7869290113449097, -0.5131613612174988, 0.7393561601638794, 0.5818021893501282, -0.9010815024375916], + output: [4.6533055662875995, -11.01426088809967, 8.89589916229627, -8.67151790820412, -5.338702909401541, 1.2596635311977273, 3.5364356687085934, 7.495255571019467, 14.607353334543243, -18.087691812322902, 7.564636217194101, -31.42023502604239, -6.214070442777103, -10.959840511165702, -0.23122589156934037, -11.620785761919407, -8.17956409451724, -2.5877133314432683, 11.015937578315528, 5.780471206474331, -6.0841540885916, 20.060728028967564, 3.2708359605096224, 11.782757950154132, 0.09008114957166846, 3.524696982412627, -10.677391613915269, 0.31298678204602526, -21.35192563693417, 12.385290905482485, 2.161430892216815, 1.8976227049433234, -8.989539272795344, 13.126889442846508, 2.8499370378020865, 1.7474738916889598, 5.022029084645628, -3.2038788652839223, 5.800603925256104, -11.310381554572714, 8.70516962035894, -19.131074662654022, 1.7802965964722235, -14.806060115152377, 14.302944499531886, -15.445567609224282, -3.8462368249304397, -9.516418012771346, 13.019042335168749, -10.443622659963964, -10.883324294802648, 3.957267045122531, -0.667106901008031, -4.392642175023952, -5.891115650099352, 16.457002297798386, 8.618862097531547, 8.454290723978408, -1.5705784838263321, -2.0950451541937802, -0.8023593073701956, -3.1480197619642944, -5.620797591431048, -4.595460315016652, -5.651367928832769, -12.289105277159251, 12.268533105640746, -14.70471337060609, -2.55864085006375, 4.848469317941595, -15.477103640433441, -0.8200806702142849, 1.477227192472725, -8.856561753264552, -1.7000675285625562, 17.974609221649885, -1.8152649987658895, 2.226238832898174, 5.942983183373236, 5.062290291727308, 7.091530929787825, -2.0854528335431413, -2.2607116079192284, -0.9522421064376894, 1.9922801075296586, -4.848590729633989, 22.20475854724638, 14.139180422304625, 7.742631959368987, 8.736735176873482, -14.221452296099534, 10.498675928711396, -0.16082936982259532, -3.8946140234989644, -20.400591916908017, -26.146811938453247, 6.587325710150547, -20.584777745167045, 14.638519536239858, -1.6926103003462887, -10.208680237895118, 6.147709389485655, -1.986836416339922, 15.002313879999715, 6.5052637626549386, 2.3893876610117486, -1.4903416359002417, 2.5125845767117747, -11.629045740948362, -1.0314802432568921, -7.7204438190047515, -2.026871521184386, 5.038120875238651, 12.509857810233317, -2.8111210090417167, 6.6237223115217265, -11.560596890267984, 21.096203768560102, 2.492616415367813, 15.694305674116976, 6.98125969245643, -2.51588120579323, -23.614269801976544, -19.721211546858797, 5.821751650555415, -7.957710273512344, -2.3083909989954927, -9.024817057699233, -2.3525185122853145, 12.543071284890175, -6.633775272995389, 1.9856801362534808, -18.01947792290391, 1.6526855337166202, -25.06385984262638, 15.109040183134795, -3.952374815603415, 8.226644901530008, 18.761705127047698, 3.93250694875225, -7.442640246025839, -19.86894607874479, 12.889936646377077, 17.058248465954463, -10.38590977421429, -8.762421865432158, 21.57778103004022, 3.943591516088906, -5.804714467684743, 4.255345272797616, 6.842976031631512, 1.5538004698130026, -4.425512141506818, -4.377682130235696, -0.5744161527454539, -6.466874369118861, 2.117583592008284, 9.441789454733126, 8.94127387455451, -0.04257498982078545, -6.206626408089971, -17.798516107544632, 3.1902492140570136, -8.701598789976085, -3.6350162587702526, 11.47775543413513, 5.763743605757109, -15.507961256533623, -12.20946994071868, 15.597966311245933, -7.577596335933641, -5.783678667731976, 0.5854316506349582, 1.0524576740591431, 9.32444015560922, 14.86759124801031, -2.8564425698215246, -4.050241339325346, -14.743103022730313, 0.6982253561530176, 1.4485867743987042, 15.863950914048452, -10.345289369065991, 18.289197082863648, -15.758886684054374, -5.422633603143401, -0.983270019651755, -12.824811394399234, -2.2739348529046874, 10.113426086776306, -29.057768796230015, -23.164653851413163, -14.355517383664846, -1.9394528310513124, 13.636590068666269, 3.459140997767836, -1.3650156273028862, 4.305744984992871, 17.561184167420265, -10.355528699590693, 21.10112660094154, -8.222224503241419, 2.535762677259953, 7.713352852965743, -14.28383557691274, 25.855974989407443, 4.209799001514582, -21.52088963141282, 10.888001415630182, -13.13372692624647, 11.475712218807361, 1.8430819379630443, -4.696550665547157, -6.590021563791754, -25.66889721277428, 3.6636259509044233, 16.177010478373504, -18.738751282648302, -8.929151482851156, 1.9637629189516739, 9.362438823950344, 18.876072069444945, 15.70735011826503, -7.857995088347812, 1.9318237720996665, -16.12923916175005, -7.934748190521045, 0.06924911452461413, -14.499587783289256, -0.4727635184847867, 10.482895882033887, -9.583766230162844, -4.311441092707766, 10.174307532202345, -12.09622773401161, -11.448182124496999, -8.63443998241864, 19.56612298031517, 12.124200790573179, 13.998676498504775, 2.185864445998216, 9.654230140395715, -13.738018677296829, 23.468720582013574, 9.279088660718887, -3.9238192921918174, -13.262040984478173, 13.199249429343212, 8.476422030030909, 22.120527194597067, 30.905491552855995, -14.140763293470911, -3.076195026308723, 2.461867832774407, -1.150707769438454, -18.323349178808648], + }, +}; + +describe("Mathematical operations", () => { + describe("bankers rounding", () => { + it("should round up to nearest even", () => { + expect(bankers_round(-0.5)).toBeCloseTo(0); + expect(bankers_round(1.5)).toBeCloseTo(2); + expect(bankers_round(19.5)).toBeCloseTo(20); + }); + it("should round down to nearest even", () => { + expect(bankers_round(-1.5)).toBeCloseTo(-2); + expect(bankers_round(2.5)).toBeCloseTo(2); + expect(bankers_round(18.5)).toBeCloseTo(18); + }); + }); + + describe("median filtering", () => { + it("should compute median filter", async () => { + const t1 = new Float32Array([5, 12, 2, 6, 3, 10, 9, 1, 4, 8, 11, 7]); + const window = 3; + + const target = new Float32Array([12, 5, 6, 3, 6, 9, 9, 4, 4, 8, 8, 11]); + + const output = medianFilter(t1, window); + expect(output).toBeCloseToNested(target, 5); + }); + + // TODO add tests for errors + }); + + describe("log softmax", () => { + // Should match output of scipy log_softmax + it("should compute log softmax correctly for usual values", () => { + const input = [0, 1, 2, 3]; + const expected = [-3.4401896985611953, -2.4401896985611953, -1.4401896985611953, -0.44018969856119533]; + const output = log_softmax(input); + expect(output).toBeCloseToNested(expected, 10); + }); + + it("should compute log softmax correctly for values with large differences", () => { + const input = [1000, 1]; + const expected = [0, -999]; + const output = log_softmax(input); + expect(output).toBeCloseToNested(expected, 10); + }); + }); + + describe("FFT", () => { + // Should match output of numpy fft + it("should compute real FFT for power of two", () => { + { + // size = 4 + // np.fft.fft([1,2,3,4]) == array([10.+0.j, -2.+2.j, -2.+0.j, -2.-2.j]) + const input = new Float32Array([1, 2, 3, 4]); + const target = new Float32Array([10, 0, -2, 2, -2, 0, -2, -2]); + + const output = fft(input); + expect(output).toBeCloseToNested(target, 5); + } + + { + // size = 16 + // np.fft.fft([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]) + // == array([136. +0.j , -8.+40.21871594j, -8.+19.3137085j , + // -8.+11.9728461j , -8. +8.j , -8. +5.3454291j , + // -8. +3.3137085j , -8. +1.59129894j, -8. +0.j , + // -8. -1.59129894j, -8. -3.3137085j , -8. -5.3454291j , + // -8. -8.j , -8.-11.9728461j , -8.-19.3137085j , + // -8.-40.21871594j]) + const input = new Float32Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); + const target = new Float32Array([136.0, 0.0, -8.0, 40.218715937006785, -8.0, 19.31370849898476, -8.0, 11.972846101323912, -8.0, 8.0, -8.0, 5.345429103354389, -8.0, 3.313708498984761, -8.0, 1.5912989390372658, -8.0, 0.0, -8.0, -1.5912989390372658, -8.0, -3.313708498984761, -8.0, -5.345429103354389, -8.0, -8.0, -8.0, -11.972846101323912, -8.0, -19.31370849898476, -8.0, -40.218715937006785]); + + const output = fft(input); + expect(output).toBeCloseToNested(target, 5); + } + }); + + it("should compute real FFT for non-power of two", () => { + { + // size = 3 + // np.fft.fft([1,2,3]) == array([ 6. +0.j, -1.5+0.8660254j, -1.5-0.8660254j]) + const input = new Float32Array([1, 2, 3]); + const target = new Float32Array([6, 0, -1.5, 0.8660254, -1.5, -0.8660254]); + + const output = fft(input); + expect(output).toBeCloseToNested(target, 5); + } + }); + + it("should compute complex FFT for non-power of two", () => { + { + // size = 3 + // np.fft.fft([1+3j,2-2j,3+1j]) == array([ 6. +2.j, -4.09807621+4.3660254j, 1.09807621+2.6339746j]) + const input = new Float32Array([1, 3, 2, -2, 3, 1]); + const target = new Float32Array([6, 2, -4.09807621, 4.3660254, 1.09807621, 2.6339746]); + + const output = fft(input, true); + expect(output).toBeCloseToNested(target, 5); + } + }); + + it("should compute complex FFT for power of two", () => { + { + // size = 4 + // np.fft.fft([1+4j, 2-3j,3+2j, 4-1j]) == array([10. +2.j, -4. +4.j, -2.+10.j, 0. +0.j]) + const input = new Float32Array([1, 4, 2, -3, 3, 2, 4, -1]); + const target = new Float32Array([10, 2, -4, 4, -2, 10, 0, 0]); + + const output = fft(input, true); + expect(output).toBeCloseToNested(target, 5); + } + }); + }); + + describe("FFT (random & dynamic)", () => { + // Should match output of numpy fft + for (const [name, test] of Object.entries(FFT_TEST_DATA)) { + it(name, () => { + const output = fft(test.input, test.complex); + + if (output.map((v, i) => Math.abs(v - test.output[i])).some((v) => v > 1e-4)) { + console.log("input", test.input); + console.log("output", output); + console.log("target", test.output); + } + expect(output).toBeCloseToNested(test.output, 4); + }); + } + }); +}); diff --git a/packages/transformers/tests/utils/model_registry.test.js b/packages/transformers/tests/utils/model_registry.test.js new file mode 100644 index 0000000..cb9e640 --- /dev/null +++ b/packages/transformers/tests/utils/model_registry.test.js @@ -0,0 +1,183 @@ +import { jest } from "@jest/globals"; + +// Mock get_file_metadata before importing the module under test +const mockGetFileMetadata = jest.fn(); +jest.unstable_mockModule("../../src/utils/model_registry/get_file_metadata.js", () => ({ + get_file_metadata: mockGetFileMetadata, +})); + +// Import registry to populate MODEL_TYPE_MAPPING (side-effect import) +await import("../../src/models/registry.js"); + +// Dynamic import after mock setup (required for ESM) +const { get_available_dtypes } = await import("../../src/utils/model_registry/get_available_dtypes.js"); + +// A minimal config that mimics a BERT-like encoder-only model +const ENCODER_ONLY_CONFIG = { + architectures: ["BertModel"], + model_type: "bert", +}; + +// A minimal config for a decoder-only (causal LM) model +const DECODER_ONLY_CONFIG = { + architectures: ["LlamaForCausalLM"], + model_type: "llama", +}; + +// A minimal config for a Seq2Seq model (encoder + decoder) +const SEQ2SEQ_CONFIG = { + architectures: ["T5ForConditionalGeneration"], + model_type: "t5", +}; + +// A config with an unknown architecture (falls back to EncoderOnly) +const UNKNOWN_ARCH_CONFIG = { + architectures: ["SomeUnknownArchitecture"], + model_type: "unknown_type", +}; + +/** + * Helper: given a set of files that "exist", returns a mock implementation + * for get_file_metadata that resolves { exists: true } for those files. + * @param {string[]} existingFiles + */ +function setupExistingFiles(...existingFiles) { + mockGetFileMetadata.mockImplementation((_modelId, filename, _options) => { + return Promise.resolve({ + exists: existingFiles.includes(filename), + fromCache: false, + }); + }); +} + +describe("get_available_dtypes", () => { + beforeEach(() => { + mockGetFileMetadata.mockReset(); + }); + + it("should detect fp32 and q4 for an encoder-only model", async () => { + setupExistingFiles( + "onnx/model.onnx", // fp32 + "onnx/model_q4.onnx", // q4 + ); + + const dtypes = await get_available_dtypes("test/model", { config: ENCODER_ONLY_CONFIG }); + + expect(dtypes).toContain("fp32"); + expect(dtypes).toContain("q4"); + expect(dtypes).not.toContain("fp16"); + expect(dtypes).not.toContain("q8"); + expect(dtypes).not.toContain("int8"); + }); + + it("should detect all dtypes when all files exist", async () => { + setupExistingFiles( + "onnx/model.onnx", // fp32 + "onnx/model_fp16.onnx", // fp16 + "onnx/model_int8.onnx", // int8 + "onnx/model_uint8.onnx", // uint8 + "onnx/model_quantized.onnx", // q8 + "onnx/model_q4.onnx", // q4 + "onnx/model_q4f16.onnx", // q4f16 + "onnx/model_bnb4.onnx", // bnb4 + ); + + const dtypes = await get_available_dtypes("test/model", { config: ENCODER_ONLY_CONFIG }); + + expect(dtypes).toEqual(["fp32", "fp16", "int8", "uint8", "q8", "q4", "q4f16", "bnb4"]); + }); + + it("should return empty array when no ONNX files exist", async () => { + setupExistingFiles(); + const dtypes = await get_available_dtypes("test/model", { config: ENCODER_ONLY_CONFIG }); + + expect(dtypes).toEqual([]); + }); + + it("should require all session files for seq2seq models", async () => { + // Only encoder has q4, decoder does not — q4 should NOT be available + setupExistingFiles( + "onnx/encoder_model.onnx", // fp32 encoder + "onnx/decoder_model_merged.onnx", // fp32 decoder + "onnx/encoder_model_q4.onnx", // q4 encoder (but no q4 decoder) + ); + + const dtypes = await get_available_dtypes("test/model", { config: SEQ2SEQ_CONFIG }); + + expect(dtypes).toContain("fp32"); + expect(dtypes).not.toContain("q4"); + }); + + it("should list dtype only when all session files exist for seq2seq", async () => { + // Both encoder and decoder have fp32 and q8 + setupExistingFiles("onnx/encoder_model.onnx", "onnx/decoder_model_merged.onnx", "onnx/encoder_model_quantized.onnx", "onnx/decoder_model_merged_quantized.onnx"); + + const dtypes = await get_available_dtypes("test/model", { config: SEQ2SEQ_CONFIG }); + + expect(dtypes).toContain("fp32"); + expect(dtypes).toContain("q8"); + expect(dtypes).not.toContain("fp16"); + expect(dtypes).not.toContain("q4"); + }); + + it("should handle decoder-only models", async () => { + setupExistingFiles("onnx/model.onnx", "onnx/model_q4.onnx", "onnx/model_q4f16.onnx"); + + const dtypes = await get_available_dtypes("test/model", { config: DECODER_ONLY_CONFIG }); + + expect(dtypes).toContain("fp32"); + expect(dtypes).toContain("q4"); + expect(dtypes).toContain("q4f16"); + expect(dtypes).toHaveLength(3); + }); + + it("should fall back to EncoderOnly for unknown architectures", async () => { + setupExistingFiles("onnx/model.onnx", "onnx/model_fp16.onnx"); + + const dtypes = await get_available_dtypes("test/model", { config: UNKNOWN_ARCH_CONFIG }); + + expect(dtypes).toContain("fp32"); + expect(dtypes).toContain("fp16"); + expect(dtypes).toHaveLength(2); + }); + + it("should support custom model_file_name", async () => { + setupExistingFiles("onnx/custom_model.onnx", "onnx/custom_model_q4.onnx"); + + const dtypes = await get_available_dtypes("test/model", { + config: ENCODER_ONLY_CONFIG, + model_file_name: "custom_model", + }); + + expect(dtypes).toContain("fp32"); + expect(dtypes).toContain("q4"); + expect(dtypes).not.toContain("fp16"); + }); + + it("should pass revision and cache_dir to get_file_metadata", async () => { + setupExistingFiles("onnx/model.onnx"); + + await get_available_dtypes("test/model", { + config: ENCODER_ONLY_CONFIG, + revision: "v2", + cache_dir: "/tmp/cache", + }); + + // Verify that metadata calls received the correct options + for (const call of mockGetFileMetadata.mock.calls) { + expect(call[0]).toBe("test/model"); + expect(call[2]).toMatchObject({ revision: "v2", cache_dir: "/tmp/cache" }); + } + }); + + it("should only return valid dtype strings", async () => { + setupExistingFiles("onnx/model.onnx", "onnx/model_fp16.onnx"); + + const dtypes = await get_available_dtypes("test/model", { config: ENCODER_ONLY_CONFIG }); + + const validDtypes = ["fp32", "fp16", "int8", "uint8", "q8", "q4", "q4f16", "bnb4"]; + for (const dtype of dtypes) { + expect(validDtypes).toContain(dtype); + } + }); +}); diff --git a/packages/transformers/tests/utils/random.test.js b/packages/transformers/tests/utils/random.test.js new file mode 100644 index 0000000..b5ac80d --- /dev/null +++ b/packages/transformers/tests/utils/random.test.js @@ -0,0 +1,340 @@ +import { random } from "../../src/utils/random.js"; +import { init } from "../init.js"; + +// Initialise the testing environment +init(); + +/** + * All expected values below were generated by running the equivalent Python code + * using CPython's `random` module (Mersenne Twister 19937). For example: + * + * ```python + * import random + * random.seed(42) + * [random.random() for _ in range(10)] + * ``` + * + * The JS implementation must match Python's output exactly for reproducibility. + */ + +describe("random (Mersenne Twister 19937, Python-compatible)", () => { + // Must be the first test — verifies auto-seeding at module load produces non-zero output. + it("should return non-zero values before any explicit seed() call", () => { + const value = random.random(); + expect(value).not.toBe(0); + expect(value).toBeGreaterThanOrEqual(0); + expect(value).toBeLessThan(1); + }); + + describe("random()", () => { + it("should produce the correct sequence for seed(42)", () => { + const expected = [0.6394267984578837, 0.025010755222666936, 0.27502931836911926, 0.22321073814882275, 0.7364712141640124, 0.6766994874229113, 0.8921795677048454, 0.08693883262941615, 0.4219218196852704, 0.029797219438070344]; + random.seed(42); + for (let i = 0; i < expected.length; i++) { + expect(random.random()).toBe(expected[i]); + } + }); + + it("should produce the correct sequence for seed(0)", () => { + const expected = [0.8444218515250481, 0.7579544029403025, 0.420571580830845, 0.25891675029296335, 0.5112747213686085, 0.4049341374504143, 0.7837985890347726, 0.30331272607892745, 0.4765969541523558, 0.5833820394550312]; + random.seed(0); + for (let i = 0; i < expected.length; i++) { + expect(random.random()).toBe(expected[i]); + } + }); + + it("should produce the correct sequence for seed(12345)", () => { + const expected = [0.41661987254534116, 0.010169169457068361, 0.8252065092537432, 0.2986398551995928, 0.3684116894884757, 0.19366134904507426, 0.5660081687288613, 0.1616878239293682, 0.12426688428353017, 0.4329362680099159]; + random.seed(12345); + for (let i = 0; i < expected.length; i++) { + expect(random.random()).toBe(expected[i]); + } + }); + + it("should handle a large seed (2^40 + 17)", () => { + const expected = [0.6614192930187859, 0.24151842637090726, 0.6909905728254405, 0.11131596896699081, 0.8184932948202619, 0.4957890371902045, 0.3754628980018908, 0.6967482542555166, 0.5464037278028259, 0.4648564722045134]; + random.seed(2 ** 40 + 17); + for (let i = 0; i < expected.length; i++) { + expect(random.random()).toBe(expected[i]); + } + }); + + it("should always return values in [0, 1)", () => { + random.seed(777); + for (let i = 0; i < 10000; i++) { + const v = random.random(); + expect(v).toBeGreaterThanOrEqual(0); + expect(v).toBeLessThan(1); + } + }); + }); + + describe("determinism", () => { + it("should produce the same sequence when re-seeded with the same value", () => { + random.seed(42); + const first = Array.from({ length: 20 }, () => random.random()); + random.seed(42); + const second = Array.from({ length: 20 }, () => random.random()); + expect(first).toEqual(second); + }); + + it("should produce different sequences for different seeds", () => { + random.seed(1); + const seq1 = Array.from({ length: 10 }, () => random.random()); + random.seed(2); + const seq2 = Array.from({ length: 10 }, () => random.random()); + expect(seq1).not.toEqual(seq2); + }); + }); + + describe("gauss()", () => { + it("should produce the correct sequence for seed(42) with default mu=0, sigma=1", () => { + const expected = [-0.14409032957792836, -0.1729036003315193, -0.11131586156766246, 0.7019837250988631, -0.12758828378288709, -1.4973534143409575, 0.33231834406771527, -0.2673374784971682, -0.216958684145195, 0.11588478670085507]; + random.seed(42); + for (let i = 0; i < expected.length; i++) { + expect(random.gauss()).toBe(expected[i]); + } + }); + + it("should produce the correct sequence for seed(99) with mu=5, sigma=2", () => { + const expected = [3.8995546045187948, 5.758236653956967, 5.653854573190089, 6.362796994590186, 5.0943415236981435, 3.4818290248514048, 2.746397968893985, 7.035976389331746, 0.4167692939861194, 3.8652143740326643]; + random.seed(99); + for (let i = 0; i < expected.length; i++) { + expect(random.gauss(5, 2)).toBe(expected[i]); + } + }); + + it("should have approximately zero mean and unit variance over many samples", () => { + random.seed(314159); + const n = 50000; + let sum = 0; + let sumSq = 0; + for (let i = 0; i < n; i++) { + const v = random.gauss(); + sum += v; + sumSq += v * v; + } + const mean = sum / n; + const variance = sumSq / n - mean * mean; + expect(Math.abs(mean)).toBeLessThan(0.02); + expect(Math.abs(variance - 1)).toBeLessThan(0.05); + }); + }); + + describe("shuffle()", () => { + it("should produce the correct permutation for seed(42) on [0..9]", () => { + const expected = [7, 3, 2, 8, 5, 6, 9, 4, 0, 1]; + const arr = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + random.seed(42); + random.shuffle(arr); + expect(arr).toEqual(expected); + }); + + it("should produce the correct permutation for seed(7) on letters", () => { + const expected = ["i", "d", "b", "e", "h", "a", "j", "g", "c", "f"]; + const arr = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]; + random.seed(7); + random.shuffle(arr); + expect(arr).toEqual(expected); + }); + + it("should preserve all elements (no duplicates, no missing)", () => { + const original = Array.from({ length: 100 }, (_, i) => i); + const arr = [...original]; + random.seed(2025); + random.shuffle(arr); + expect(arr.sort((a, b) => a - b)).toEqual(original); + }); + + it("should be a no-op for single-element and empty arrays", () => { + const single = [42]; + random.seed(0); + random.shuffle(single); + expect(single).toEqual([42]); + + const empty = []; + random.shuffle(empty); + expect(empty).toEqual([]); + }); + }); + + describe("choices()", () => { + it("should produce the correct picks for seed(42) with weights [10,1,1,1]", () => { + const expected = ["a", "a", "a", "a", "a", "a", "c", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "b", "a"]; + const population = ["a", "b", "c", "d"]; + const weights = [10, 1, 1, 1]; + random.seed(42); + for (let i = 0; i < expected.length; i++) { + expect(random.choices(population, weights)).toBe(expected[i]); + } + }); + + it("should produce the correct picks for seed(42) with float weights", () => { + // Python: random.seed(42); [random.choices([0,1,2,3,4], weights=[0.1,0.2,0.3,0.25,0.15])[0] for _ in range(20)] + const expected = [3, 0, 1, 1, 3, 3, 4, 0, 2, 0, 1, 2, 0, 1, 3, 2, 1, 2, 3, 0]; + const population = [0, 1, 2, 3, 4]; + const weights = [0.1, 0.2, 0.3, 0.25, 0.15]; + random.seed(42); + for (let i = 0; i < expected.length; i++) { + expect(random.choices(population, weights)).toBe(expected[i]); + } + }); + + it("should produce the correct picks for seed(12345) with uniform weights", () => { + // Python: random.seed(12345); [random.choices(['x','y','z'], weights=[1,1,1])[0] for _ in range(15)] + const expected = ["y", "x", "z", "x", "y", "x", "y", "x", "x", "y", "y", "x", "y", "y", "z"]; + const population = ["x", "y", "z"]; + const weights = [1, 1, 1]; + random.seed(12345); + for (let i = 0; i < expected.length; i++) { + expect(random.choices(population, weights)).toBe(expected[i]); + } + }); + + it("should only ever return elements from the population", () => { + const population = [10, 20, 30]; + const weights = [1, 1, 1]; + random.seed(123); + for (let i = 0; i < 1000; i++) { + expect(population).toContain(random.choices(population, weights)); + } + }); + + it("should respect heavily skewed weights", () => { + // With weight 1000 on "x" and 1 on "y", nearly all picks should be "x" + const population = ["x", "y"]; + const weights = [1000, 1]; + random.seed(555); + let xCount = 0; + const n = 5000; + for (let i = 0; i < n; i++) { + if (random.choices(population, weights) === "x") xCount++; + } + expect(xCount / n).toBeGreaterThan(0.99); + }); + + it("should work with a single-element population", () => { + random.seed(0); + expect(random.choices(["only"], [1])).toBe("only"); + expect(random.choices(["only"], [999])).toBe("only"); + }); + }); + + describe("Random class", () => { + it("should be accessible as random.Random", () => { + expect(typeof random.Random).toBe("function"); + }); + + it("should produce the same sequence as the global functions given the same seed", () => { + const rng = new random.Random(42); + random.seed(42); + for (let i = 0; i < 20; i++) { + expect(rng.random()).toBe(random.random()); + } + }); + + it("should be independent from the global state — seeding instance does not affect global", () => { + random.seed(42); + const before = random.random(); // consume one global value + expect(before).toBe(0.6394267984578837); + + const rng = new random.Random(99); + rng.random(); // consume from instance + rng.seed(0); // re-seed instance + rng.random(); // consume again + + // global should continue from where it left off, unaffected + expect(random.random()).toBe(0.025010755222666936); + random.seed(42); + expect(random.random()).toBe(before); + }); + + it("should be independent from the global state — seeding global does not affect instance", () => { + const rng = new random.Random(42); + const v0 = rng.random(); + + random.seed(999); // re-seed global + random.random(); // consume global values + random.random(); + + // instance should continue from where it left off + const rng2 = new random.Random(42); + expect(rng2.random()).toBe(v0); + }); + + it("two instances with the same seed produce identical sequences", () => { + const rng1 = new random.Random(42); + const rng2 = new random.Random(42); + for (let i = 0; i < 20; i++) { + expect(rng1.random()).toBe(rng2.random()); + } + }); + + it("two instances with different seeds produce different sequences", () => { + const rng1 = new random.Random(1); + const rng2 = new random.Random(2); + const seq1 = Array.from({ length: 10 }, () => rng1.random()); + const seq2 = Array.from({ length: 10 }, () => rng2.random()); + expect(seq1).not.toEqual(seq2); + }); + + it("two instances are independent — advancing one does not affect the other", () => { + const rng1 = new random.Random(42); + const rng2 = new random.Random(42); + + // Advance rng1 by 100 values + for (let i = 0; i < 100; i++) rng1.random(); + + // rng2 should still be at the start + const rng3 = new random.Random(42); + expect(rng2.random()).toBe(rng3.random()); + }); + + it("instance.seed() re-initialises state deterministically", () => { + const rng = new random.Random(42); + const first = Array.from({ length: 10 }, () => rng.random()); + rng.seed(42); + const second = Array.from({ length: 10 }, () => rng.random()); + expect(first).toEqual(second); + }); + + it("instance.seed() with no arguments seeds from entropy (non-zero output)", () => { + const rng = new random.Random(); + const v = rng.random(); + expect(v).toBeGreaterThanOrEqual(0); + expect(v).toBeLessThan(1); + }); + + it("instance.gauss() matches global gauss() for the same seed", () => { + const rng = new random.Random(42); + random.seed(42); + for (let i = 0; i < 10; i++) { + expect(rng.gauss(5, 2)).toBe(random.gauss(5, 2)); + } + }); + + it("instance.shuffle() matches global shuffle() for the same seed", () => { + const arr1 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + const arr2 = [...arr1]; + + const rng = new random.Random(42); + rng.shuffle(arr1); + + random.seed(42); + random.shuffle(arr2); + + expect(arr1).toEqual(arr2); + }); + + it("instance.choices() matches global choices() for the same seed", () => { + const population = ["a", "b", "c", "d"]; + const weights = [10, 1, 1, 1]; + + const rng = new random.Random(42); + random.seed(42); + for (let i = 0; i < 20; i++) { + expect(rng.choices(population, weights)).toBe(random.choices(population, weights)); + } + }); + }); +}); diff --git a/packages/transformers/tests/utils/tensor.test.js b/packages/transformers/tests/utils/tensor.test.js new file mode 100644 index 0000000..278aa87 --- /dev/null +++ b/packages/transformers/tests/utils/tensor.test.js @@ -0,0 +1,754 @@ +import { Tensor, cat, stack, layer_norm, ones_like, zeros_like, full_like, rand, std_mean } from "../../src/transformers.js"; +import { init } from "../init.js"; + +init(); + +describe("Tensor operations", () => { + describe("cat", () => { + it("should concatenate on dim=0", () => { + const t1 = new Tensor("float32", [1, 2, 3], [1, 3]); + const t2 = new Tensor("float32", [4, 5, 6, 7, 8, 9], [2, 3]); + const t3 = new Tensor("float32", [10, 11, 12], [1, 3]); + + const target1 = new Tensor("float32", [1, 2, 3, 4, 5, 6, 7, 8, 9], [3, 3]); + const target2 = new Tensor("float32", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], [4, 3]); + + // 2 tensors + const concatenated1 = cat([t1, t2], 0); + expect(concatenated1).toEqual(target1); + + // 3 tensors + const concatenated2 = cat([t1, t2, t3], 0); + expect(concatenated2).toEqual(target2); + }); + + it("should concatenate on dim=1", () => { + const t1 = new Tensor("float32", [1, 2, 3, -1, -2, -3], [2, 3, 1]); + const t2 = new Tensor("float32", [4, -4], [2, 1, 1]); + const t3 = new Tensor("float32", [5, 6, -5, -6], [2, 2, 1]); + + const target1 = new Tensor("float32", [1, 2, 3, 4, -1, -2, -3, -4], [2, 4, 1]); + const target2 = new Tensor("float32", [1, 2, 3, 4, 5, 6, -1, -2, -3, -4, -5, -6], [2, 6, 1]); + + // 2 tensors + const concatenated1 = cat([t1, t2], 1); + expect(concatenated1).toEqual(target1); + + // 3 tensors + const concatenated2 = cat([t1, t2, t3], 1); + expect(concatenated2).toEqual(target2); + }); + + it("should concatenate on dim=-2", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6, 11, 12, 13, 14, 15, 16], [2, 1, 3, 2]); + const t2 = new Tensor("float32", [7, 8, 9, 10, 17, 18, 19, 20], [2, 1, 2, 2]); + + const target = new Tensor("float32", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], [2, 1, 5, 2]); + + const concatenated = cat([t1, t2], -2); + + expect(concatenated).toEqual(target); + }); + + // TODO add tests for errors + }); + + describe("slice", () => { + it("should return a given row dim", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [3, 2]); + const t2 = t1.slice(1); + const target = new Tensor("float32", [3, 4], [2]); + expect(t2).toEqual(target); + }); + + it("should return a range of rows", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [3, 2]); + const t2 = t1.slice([1, 3]); + const target = new Tensor("float32", [3, 4, 5, 6], [2, 2]); + expect(t2).toEqual(target); + }); + + it("should return a crop", () => { + const t1 = new Tensor( + "float32", + Array.from({ length: 28 }, (_, i) => i + 1), + [4, 7], + ); + const t2 = t1.slice([1, -1], [1, -1]); + const target = new Tensor("float32", [9, 10, 11, 12, 13, 16, 17, 18, 19, 20], [2, 5]); + expect(t2).toEqual(target); + }); + + it("should return the whole tensor when all indices are null/unset", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [3, 2]); + const t2 = t1.slice(); + expect(t2).toEqual(t1); + }); + + it("should return the whole dimension when index is null", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [3, 2]); + const t2 = t1.slice(null); + expect(t2).toEqual(t1); + }); + + it("should slice from index to end when [start, null] is used", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [3, 2]); + const t2 = t1.slice([1, null]); + const target = new Tensor("float32", [3, 4, 5, 6], [2, 2]); + expect(t2).toEqual(target); + }); + + it("should slice from beginning to index when [null, end] is used", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [3, 2]); + const t2 = t1.slice([null, 2]); + const target = new Tensor("float32", [1, 2, 3, 4], [2, 2]); + expect(t2).toEqual(target); + }); + + it("should handle [null, null] as full slice", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [3, 2]); + const t2 = t1.slice([null, null]); + expect(t2).toEqual(t1); + }); + + it("should select a single element when a number is used in slice", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [3, 2]); + const t2 = t1.slice(2, 1); + const target = new Tensor("float32", [6], []); + expect(t2).toEqual(target); + }); + + it("should select a single row when a number is used in slice", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [3, 2]); + const t2 = t1.slice(0); + const target = new Tensor("float32", [1, 2], [2]); + expect(t2).toEqual(target); + }); + + it("should select a single column when a number is used in slice", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [3, 2]); + const t2 = t1.slice(null, 1); + const target = new Tensor("float32", [2, 4, 6], [3]); + expect(t2).toEqual(target); + }); + + it("should handle negative indices in slice", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [3, 2]); + const t2 = t1.slice(-1); + const target = new Tensor("float32", [5, 6], [2]); + expect(t2).toEqual(target); + }); + }); + + describe("stack", () => { + const t1 = new Tensor("float32", [0, 1, 2, 3, 4, 5], [1, 3, 2]); + + it("should stack on dim=0", () => { + const target1 = new Tensor("float32", [0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5], [2, 1, 3, 2]); + const target2 = new Tensor("float32", [0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5], [3, 1, 3, 2]); + + // 2 tensors + const stacked1 = stack([t1, t1], 0); + expect(stacked1).toEqual(target1); + + // 3 tensors + const stacked2 = stack([t1, t1, t1], 0); + expect(stacked2).toEqual(target2); + }); + + it("should stack on dim=1", () => { + const target1 = new Tensor("float32", [0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5], [1, 2, 3, 2]); + const target2 = new Tensor("float32", [0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5], [1, 3, 3, 2]); + + // 2 tensors + const stacked1 = stack([t1, t1], 1); + expect(stacked1).toEqual(target1); + + // 3 tensors + const stacked2 = stack([t1, t1, t1], 1); + expect(stacked2).toEqual(target2); + }); + + it("should stack on dim=-1", () => { + const target1 = new Tensor("float32", [0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5], [1, 3, 2, 2]); + const target2 = new Tensor("float32", [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5], [1, 3, 2, 3]); + + // 2 tensors + const stacked1 = stack([t1, t1], -1); + expect(stacked1).toEqual(target1); + + // 3 tensors + const stacked2 = stack([t1, t1, t1], -1); + expect(stacked2).toEqual(target2); + }); + }); + + describe("permute", () => { + it("should permute", () => { + const x = new Tensor("float32", [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23], [2, 3, 4]); + // Permute axes to (0, 1, 2) - No change + const permuted_1 = x.permute(0, 1, 2); + const target_1 = x; + expect(permuted_1).toEqual(target_1); + + // Permute axes to (0, 2, 1) + const permuted_2 = x.permute(0, 2, 1); + const target_2 = new Tensor("float32", [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11, 12, 16, 20, 13, 17, 21, 14, 18, 22, 15, 19, 23], [2, 4, 3]); + expect(permuted_2).toEqual(target_2); + + // Permute axes to (1, 0, 2) + const permuted_3 = x.permute(1, 0, 2); + const target_3 = new Tensor("float32", [0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7, 16, 17, 18, 19, 8, 9, 10, 11, 20, 21, 22, 23], [3, 2, 4]); + expect(permuted_3).toEqual(target_3); + + // Permute axes to (1, 2, 0) + const permuted_4 = x.permute(1, 2, 0); + const target_4 = new Tensor("float32", [0, 12, 1, 13, 2, 14, 3, 15, 4, 16, 5, 17, 6, 18, 7, 19, 8, 20, 9, 21, 10, 22, 11, 23], [3, 4, 2]); + expect(permuted_4).toEqual(target_4); + + // Permute axes to (2, 0, 1) + const permuted_5 = x.permute(2, 0, 1); + const target_5 = new Tensor("float32", [0, 4, 8, 12, 16, 20, 1, 5, 9, 13, 17, 21, 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23], [4, 2, 3]); + expect(permuted_5).toEqual(target_5); + + // Permute axes to (2, 1, 0) + const permuted_6 = x.permute(2, 1, 0); + const target_6 = new Tensor("float32", [0, 12, 4, 16, 8, 20, 1, 13, 5, 17, 9, 21, 2, 14, 6, 18, 10, 22, 3, 15, 7, 19, 11, 23], [4, 3, 2]); + expect(permuted_6).toEqual(target_6); + }); + }); + + describe("map", () => { + it("should double", () => { + const original = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + const target = new Tensor("float32", [2, 4, 6, 8, 10, 12], [2, 3]); + + const doubled = original.map((x) => x * 2); + expect(doubled).toEqual(target); + }); + }); + + describe("mean", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3, 1]); + it("should calculate mean over the entire tensor", () => { + const target = new Tensor("float32", [3.5], []); + expect(t1.mean()).toBeCloseToNested(target); + }); + + it("should calculate mean over dimension 0", () => { + const target0 = new Tensor("float32", [2.5, 3.5, 4.5], [3, 1]); + expect(t1.mean(0)).toBeCloseToNested(target0); + }); + + it("should calculate mean over dimension 1", () => { + const target1 = new Tensor("float32", [2, 5], [2, 1]); + expect(t1.mean(1)).toBeCloseToNested(target1); + }); + + it("should calculate mean over dimension -1", () => { + const target2 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + expect(t1.mean(-1)).toBeCloseToNested(target2); + }); + }); + + describe("std_mean", () => { + it("should return std_mean for the entire tensor", () => { + const t = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + const [stdVal, meanVal] = std_mean(t); + expect(stdVal).toBeCloseToNested(new Tensor("float32", [1.8708287477493286], [])); + expect(meanVal).toBeCloseToNested(new Tensor("float32", [3.5], [])); + }); + }); + + describe("min", () => { + it("should return the minimum over the entire tensor", () => { + const t1 = new Tensor("float32", [3, -2, 5, 0], [2, 2]); + const target = new Tensor("float32", [-2], []); + const result = t1.min(); + expect(result).toEqual(target); + }); + + it("should return the minimum over dimension 1", () => { + const t2 = new Tensor("float32", [4, 2, -1, 0, 6, 5], [3, 2]); + const target = new Tensor("float32", [2, -1, 5], [3]); + const result = t2.min(1); + expect(result).toEqual(target); + }); + }); + + describe("max", () => { + it("should return the maximum over the entire tensor", () => { + const t1 = new Tensor("float32", [3, 10, -2, 7], [2, 2]); + const target = new Tensor("float32", [10], []); + const result = t1.max(); + expect(result).toEqual(target); + }); + + it("should return the maximum over dimension 0", () => { + const t2 = new Tensor("float32", [1, 2, 4, 5, 9, 3], [3, 2]); + const target = new Tensor("float32", [9, 5], [2]); + const result = t2.max(0); + expect(result).toEqual(target); + }); + }); + + describe("sum", () => { + it("should calculate sum over entire tensor", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + const target = new Tensor("float32", [21], []); + const result = t1.sum(); + expect(result).toBeCloseToNested(target); + }); + + it("should calculate sum over dimension 0", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + const target = new Tensor("float32", [5, 7, 9], [3]); + const result = t1.sum(0); + expect(result).toBeCloseToNested(target); + }); + + it("should calculate sum over dimension 1", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + const target = new Tensor("float32", [6, 15], [2]); + const result = t1.sum(1); + expect(result).toBeCloseToNested(target); + }); + }); + + describe("norm", () => { + it("should calculate L2 norm over entire tensor", () => { + const t1 = new Tensor("float32", [3, 4], [2]); + const target = new Tensor("float32", [5], []); + const result = t1.norm(); + expect(result).toBeCloseToNested(target); + }); + + it("should calculate L2 norm over dimension 0", () => { + const t1 = new Tensor("float32", [3, 4, 6, 8], [2, 2]); + const target = new Tensor("float32", [6.7082, 8.9443], [2]); + const result = t1.norm(2, 0); + expect(result).toBeCloseToNested(target); + }); + }); + + describe("normalize", () => { + it("should normalize a vector correctly", () => { + const t1 = new Tensor("float32", [3, 4], [1, 2]); + const target = new Tensor("float32", [0.6, 0.8], [1, 2]); + const normalized = t1.normalize(); + expect(normalized).toBeCloseToNested(target); + }); + + it("should normalize along dimension", () => { + const t1 = new Tensor("float32", [1, 2, 2, 3], [2, 2]); + const target = new Tensor("float32", [0.4472, 0.8944, 0.5547, 0.8321], [2, 2]); + const normalized = t1.normalize(); + expect(normalized).toBeCloseToNested(target); + }); + }); + + describe("layer_norm", () => { + it("should calculate layer norm", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + + const target = new Tensor("float32", [-1.2247356176376343, 0.0, 1.2247356176376343, -1.2247357368469238, -1.1920928955078125e-7, 1.2247354984283447], [2, 3]); + + const norm = layer_norm(t1, [t1.dims.at(-1)]); + expect(norm).toBeCloseToNested(target); + }); + }); + + describe("sigmoid", () => { + it("should apply the sigmoid function to each element in the tensor", () => { + const t1 = new Tensor("float32", [0, 1, -1, 5, -5], [5]); + const target = new Tensor("float32", [0.5, 1 / (1 + Math.exp(-1)), 1 / (1 + Math.exp(1)), 1 / (1 + Math.exp(-5)), 1 / (1 + Math.exp(5))], [5]); + + const result = t1.sigmoid(); + expect(result).toBeCloseToNested(target); + }); + }); + + describe("tolist", () => { + it("should work with scalar tensors", () => { + const t1 = new Tensor("float32", [42], []); + const arr = t1.tolist(); + expect(arr).toBe(42); + }); + it("should return nested arrays for a 2D tensor", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [2, 2]); + const arr = t1.tolist(); + expect(arr).toEqual([ + [1, 2], + [3, 4], + ]); + }); + }); + + describe("mul", () => { + it("should multiply constant", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [2, 2]); + const target = new Tensor("float32", [2, 4, 6, 8], [2, 2]); + + const result = t1.mul(2); + expect(result).toBeCloseToNested(target); + }); + }); + + describe("div", () => { + it("should divide constant", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [2, 2]); + const target = new Tensor("float32", [0.5, 1, 1.5, 2], [2, 2]); + + const result = t1.div(2); + expect(result).toBeCloseToNested(target); + }); + }); + + describe("add", () => { + it("should add constant", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [2, 2]); + const target = new Tensor("float32", [3, 4, 5, 6], [2, 2]); + + const result = t1.add(2); + expect(result).toBeCloseToNested(target); + }); + }); + + describe("sub", () => { + it("should subtract constant", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [2, 2]); + const target = new Tensor("float32", [-1, 0, 1, 2], [2, 2]); + + const result = t1.sub(2); + expect(result).toBeCloseToNested(target); + }); + }); + describe("gt", () => { + it("should perform element-wise greater than comparison with a scalar", () => { + const t1 = new Tensor("float32", [1, 5, 3, 7], [4]); + const target = new Tensor("bool", [0, 1, 0, 1], [4]); + const result = t1.gt(4); + expect(result).toEqual(target); + }); + }); + + describe("lt", () => { + it("should perform element-wise less than comparison with a scalar", () => { + const t1 = new Tensor("float32", [1, 5, 3, 7], [4]); + const target = new Tensor("bool", [1, 0, 1, 0], [4]); + const result = t1.lt(4); + expect(result).toEqual(target); + }); + }); + + describe("squeeze", () => { + it("should remove all dimensions of size 1", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [1, 4]); + const target = new Tensor("float32", [1, 2, 3, 4], [4]); + + const result = t1.squeeze(); + expect(result).toEqual(target); + }); + it("should remove a specified dimension", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [1, 1, 2, 2]); + const result = t1.squeeze(1); + const target = new Tensor("float32", [1, 2, 3, 4], [1, 2, 2]); + expect(result).toEqual(target); + }); + it("should remove multiple dimensions", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [1, 1, 2, 1, 2]); + const result = t1.squeeze([0, 3]); + const target = new Tensor("float32", [1, 2, 3, 4], [1, 2, 2]); + expect(result).toEqual(target); + }); + }); + + describe("unsqueeze", () => { + it("should add a dimension at the specified axis", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [4]); + const target = new Tensor("float32", [1, 2, 3, 4], [1, 4]); + + const result = t1.unsqueeze(0); + expect(result).toEqual(target); + }); + }); + + describe("flatten", () => { + it("should flatten a 2D tensor into 1D by default", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + const target = new Tensor("float32", [1, 2, 3, 4, 5, 6], [6]); + + const result = t1.flatten(); + expect(result).toEqual(target); + }); + }); + + describe("neg", () => { + it("should compute the negative of each element in the tensor", () => { + const t1 = new Tensor("float32", [1, -2, 0, 3], [4]); + const target = new Tensor("float32", [-1, 2, -0, -3], [4]); + + const result = t1.neg(); + expect(result).toEqual(target); + }); + }); + + describe("view", () => { + it("should reshape the tensor to the specified dimensions", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + const target = new Tensor("float32", [1, 2, 3, 4, 5, 6], [3, 2]); + + const result = t1.view(3, 2); + expect(result).toEqual(target); + }); + + it("should reshape the tensor with an inferred dimension (-1)", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + const target = new Tensor("float32", [1, 2, 3, 4, 5, 6], [1, 6]); + + const result = t1.view(1, -1); + expect(result).toEqual(target); + }); + + it("should throw if multiple inferred dimensions are used", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + expect(() => t1.view(-1, -1)).toThrow(); + }); + }); + + describe("clamp", () => { + it("should clamp values between min and max", () => { + const t1 = new Tensor("float32", [-2, -1, 0, 1, 2, 3], [6]); + const target = new Tensor("float32", [-1, -1, 0, 1, 2, 2], [6]); + + const result = t1.clamp(-1, 2); + expect(result).toEqual(target); + }); + }); + + describe("round", () => { + it("should round elements to the nearest integer", () => { + const t1 = new Tensor("float32", [0.1, 1.4, 2.5, 3.9, -1.2], [5]); + const target = new Tensor("float32", [0, 1, 3, 4, -1], [5]); + + const result = t1.round(); + expect(result).toEqual(target); + }); + }); + + describe("ones_like", () => { + it("should create a tensor of all ones with the same shape as the input", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [2, 2]); + const result = ones_like(t1); + const target = new Tensor("int64", [1n, 1n, 1n, 1n], [2, 2]); + expect(result).toEqual(target); + }); + }); + + describe("zeros_like", () => { + it("should create a tensor of all zeros with the same shape as the input", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [2, 2]); + const result = zeros_like(t1); + const target = new Tensor("int64", [0n, 0n, 0n, 0n], [2, 2]); + expect(result).toEqual(target); + }); + }); + + describe("full_like", () => { + it("should create a tensor filled with a number, matching the shape of the original", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [2, 2]); + const result = full_like(t1, 10); + const target = new Tensor("float32", [10, 10, 10, 10], [2, 2]); + expect(result).toEqual(target); + }); + it("should create a boolean tensor with the same shape", () => { + const t2 = new Tensor("bool", [true, false], [2]); + const result = full_like(t2, true); + const target = new Tensor("bool", [true, true], [2]); + expect(result).toEqual(target); + }); + + it("should create a bigint tensor with the same shape", () => { + const t3 = new Tensor("int64", [1n, 2n], [2]); + const result = full_like(t3, 123n); + const target = new Tensor("int64", [123n, 123n], [2]); + expect(result).toEqual(target); + }); + }); + + describe("rand", () => { + it("should create a tensor of random values between 0 and 1 with the given shape", () => { + const shape = [2, 2]; + const random = rand(shape); + expect(random.type).toBe("float32"); + expect(random.dims).toEqual(shape); + random.data.forEach((val) => { + expect(val).toBeGreaterThanOrEqual(0); + expect(val).toBeLessThan(1); + }); + }); + }); + + describe("to", () => { + it("float32 to int32 (number to number)", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + + const target = new Tensor("int32", [1, 2, 3, 4, 5, 6], [2, 3]); + + const t2 = t1.to("int32"); + expect(t2).toEqual(target); + }); + it("float32 to int64 (number to bigint)", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + + const target = new Tensor("int64", [1n, 2n, 3n, 4n, 5n, 6n], [2, 3]); + + const t2 = t1.to("int64"); + expect(t2).toEqual(target); + }); + it("int64 to float32 (bigint to number)", () => { + const t1 = new Tensor("int64", [1n, 2n, 3n, 4n, 5n, 6n], [2, 3]); + + const target = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + + const t2 = t1.to("float32"); + expect(t2).toEqual(target); + }); + it("int32 to uint32", () => { + const t1 = new Tensor("int32", [-1, 2, -3, 4, -5, 6], [2, 3]); + + const target = new Tensor("uint32", [4294967295, 2, 4294967293, 4, 4294967291, 6], [2, 3]); + + const t2 = t1.to("uint32"); + expect(t2).toEqual(target); + }); + it("int16 to int8 (overflow)", () => { + const t1 = new Tensor("int16", [0, 1, 128, 256, 257, 512], [2, 3]); + + const target = new Tensor("int8", [0, 1, -128, 0, 1, 0], [2, 3]); + + const t2 = t1.to("int8"); + expect(t2).toEqual(target); + }); + }); + + describe("repeat", () => { + it("should repeat a 1D tensor", () => { + const t1 = new Tensor("float32", [1, 2, 3], [3]); + const t2 = t1.repeat(2); + const target = new Tensor("float32", [1, 2, 3, 1, 2, 3], [6]); + expect(t2).toEqual(target); + }); + + it("should repeat a 2D tensor along both dimensions", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [2, 2]); + const t2 = t1.repeat(2, 3); + // Shape becomes [4, 6] + // [[1, 2], [3, 4]] repeated 2x along dim0 and 3x along dim1 + const target = new Tensor("float32", [1, 2, 1, 2, 1, 2, 3, 4, 3, 4, 3, 4, 1, 2, 1, 2, 1, 2, 3, 4, 3, 4, 3, 4], [4, 6]); + expect(t2).toEqual(target); + }); + + it("should repeat with more dimensions than tensor has (prepend 1s)", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [2, 2]); + const t2 = t1.repeat(2, 1, 1); + // Tensor is treated as [1, 2, 2], repeated [2, 1, 1] -> [2, 2, 2] + const target = new Tensor("float32", [1, 2, 3, 4, 1, 2, 3, 4], [2, 2, 2]); + expect(t2).toEqual(target); + }); + + it("should repeat a 3D tensor", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [1, 2, 2]); + const t2 = t1.repeat(2, 1, 1); + const target = new Tensor("float32", [1, 2, 3, 4, 1, 2, 3, 4], [2, 2, 2]); + expect(t2).toEqual(target); + }); + + it("should repeat with ones (no actual repetition)", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + const t2 = t1.repeat(1, 1); + const target = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + expect(t2).toEqual(target); + }); + + it("should repeat a scalar tensor", () => { + const t1 = new Tensor("float32", [5], [1]); + const t2 = t1.repeat(4); + const target = new Tensor("float32", [5, 5, 5, 5], [4]); + expect(t2).toEqual(target); + }); + + it("should handle int64 tensor type", () => { + const t1 = new Tensor("int64", [1n, 2n, 3n], [3]); + const t2 = t1.repeat(2); + const target = new Tensor("int64", [1n, 2n, 3n, 1n, 2n, 3n], [6]); + expect(t2).toEqual(target); + }); + + it("should throw error when repeats has fewer dimensions than tensor", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + expect(() => t1.repeat(2)).toThrow(); + }); + + it("should optimize when all repeats are ones (same dims)", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + const t2 = t1.repeat(1, 1); + const target = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + expect(t2).toEqual(target); + // Verify it's a copy, not the same reference + expect(t2.data).not.toBe(t1.data); + }); + + it("should optimize when all repeats are ones (expanded dims)", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [2, 2]); + const t2 = t1.repeat(1, 1, 1); + const target = new Tensor("float32", [1, 2, 3, 4], [1, 2, 2]); + expect(t2).toEqual(target); + }); + }); + + describe("tile", () => { + it("should tile a 1D tensor", () => { + const t1 = new Tensor("float32", [1, 2, 3], [3]); + const t2 = t1.tile(2); + const target = new Tensor("float32", [1, 2, 3, 1, 2, 3], [6]); + expect(t2).toEqual(target); + }); + + it("should tile a 2D tensor along both dimensions", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [2, 2]); + const t2 = t1.tile(2, 3); + const target = new Tensor("float32", [1, 2, 1, 2, 1, 2, 3, 4, 3, 4, 3, 4, 1, 2, 1, 2, 1, 2, 3, 4, 3, 4, 3, 4], [4, 6]); + expect(t2).toEqual(target); + }); + + it("should tile with fewer repeats than tensor dims (prepend 1s to repeats)", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + // Only tile along the last dimension + const t2 = t1.tile(2); + const target = new Tensor("float32", [1, 2, 3, 1, 2, 3, 4, 5, 6, 4, 5, 6], [2, 6]); + expect(t2).toEqual(target); + }); + + it("should tile with more repeats than tensor dims (prepend 1s to dims)", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4], [2, 2]); + const t2 = t1.tile(2, 1, 1); + const target = new Tensor("float32", [1, 2, 3, 4, 1, 2, 3, 4], [2, 2, 2]); + expect(t2).toEqual(target); + }); + + it("should tile with ones (no actual tiling)", () => { + const t1 = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + const t2 = t1.tile(1, 1); + const target = new Tensor("float32", [1, 2, 3, 4, 5, 6], [2, 3]); + expect(t2).toEqual(target); + }); + + it("should handle int64 tensor type", () => { + const t1 = new Tensor("int64", [1n, 2n, 3n], [3]); + const t2 = t1.tile(2); + const target = new Tensor("int64", [1n, 2n, 3n, 1n, 2n, 3n], [6]); + expect(t2).toEqual(target); + }); + }); +}); diff --git a/packages/transformers/tests/utils/tensor_ops.test.js b/packages/transformers/tests/utils/tensor_ops.test.js new file mode 100644 index 0000000..a0ad1f0 --- /dev/null +++ b/packages/transformers/tests/utils/tensor_ops.test.js @@ -0,0 +1,323 @@ +import { Tensor, interpolate_4d, matmul, rfft, slice } from "../../src/transformers.js"; +import { init } from "../init.js"; + +// Initialise the testing environment +init(); + +function expectToBeCloseToArray(actual, expected) { + expect(actual.length).toEqual(expected.length); + actual.forEach((x, i) => expect(x).toBeCloseTo(expected[i]), 6); +} + +function range(start, stop = undefined, step = 1) { + if (stop === undefined) { + stop = start; + start = 0; + } + + const result = []; + for (let i = start; i < stop; i += step) { + result.push(i); + } + return result; +} + +describe("Tensor operations", () => { + describe("interpolate", () => { + describe("downscale", () => { + const input = new Tensor( + "float32", + new Float32Array(2 * 3 * 4 * 5).map((_, i) => i), + [2, 3, 4, 5], + ); + + const size = [2, 3, 3, 2]; + it("bilinear", async () => { + const resized = await interpolate_4d(input, { mode: "bilinear", size }); + const target = new Float32Array( + [ + [ + [ + [1.5833335, 4.0833335], + [8.25, 10.75], + [14.916668, 17.416668], + ], + [ + [21.583332, 24.083334], + [28.25, 30.75], + [34.916668, 37.416668], + ], + [ + [41.583332, 44.083332], + [48.25, 50.75], + [54.916668, 57.416668], + ], + ], + [ + [ + [61.583332, 64.083336], + [68.25, 70.75], + [74.916664, 77.41667], + ], + [ + [81.58333, 84.083336], + [88.25, 90.75], + [94.91667, 97.41667], + ], + [ + [101.583336, 104.08333], + [108.25, 110.75], + [114.916664, 117.416664], + ], + ], + ].flat(Infinity), + ); + + expectToBeCloseToArray(target, resized.data); + }); + + it("bicubic", async () => { + const resized = await interpolate_4d(input, { mode: "bicubic", size }); + + const target = new Float32Array( + [ + [ + [ + [1.2987545, 3.9628172], + [8.167969, 10.832031], + [15.037184, 17.701244], + ], + [ + [21.298756, 23.962818], + [28.167969, 30.832031], + [35.037186, 37.701252], + ], + [ + [41.298756, 43.96282], + [48.16797, 50.83203], + [55.037193, 57.701256], + ], + ], + [ + [ + [61.29875, 63.96282], + [68.16797, 70.83203], + [75.03719, 77.701256], + ], + [ + [81.29875, 83.96282], + [88.16797, 90.83203], + [95.03721, 97.70126], + ], + [ + [101.29875, 103.962814], + [108.16797, 110.83203], + [115.03721, 117.70127], + ], + ], + ].flat(Infinity), + ); + + expectToBeCloseToArray(target, resized.data); + }); + }); + describe("upscale", () => { + const input = new Tensor( + "float32", + new Float32Array(2 * 3 * 3 * 2).map((_, i) => i), + [2, 3, 3, 2], + ); + + const size = [2, 3, 4, 5]; + it("bilinear", async () => { + const resized = await interpolate_4d(input, { mode: "bilinear", size }); + const target = new Float32Array( + [ + [ + [ + [0.0, 0.1, 0.5, 0.9, 1.0], + [1.25, 1.35, 1.75, 2.15, 2.25], + [2.75, 2.85, 3.25, 3.65, 3.75], + [4.0, 4.1, 4.5, 4.9, 5.0], + ], + [ + [6.0, 6.1, 6.5, 6.9, 7.0], + [7.25, 7.35, 7.75, 8.15, 8.25], + [8.75, 8.85, 9.25, 9.65, 9.75], + [10.0, 10.1, 10.5, 10.9, 11.0], + ], + [ + [12.0, 12.1, 12.5, 12.9, 13.0], + [13.25, 13.35, 13.75, 14.15, 14.25], + [14.75, 14.85, 15.25, 15.65, 15.75], + [16.0, 16.1, 16.5, 16.9, 17.0], + ], + ], + [ + [ + [18.0, 18.1, 18.5, 18.9, 19.0], + [19.25, 19.35, 19.75, 20.15, 20.25], + [20.75, 20.85, 21.25, 21.65, 21.75], + [22.0, 22.1, 22.5, 22.9, 23.0], + ], + [ + [24.0, 24.1, 24.5, 24.9, 25.0], + [25.25, 25.35, 25.75, 26.15, 26.25], + [26.75, 26.85, 27.25, 27.65, 27.75], + [28.0, 28.1, 28.5, 28.9, 29.0], + ], + [ + [30.0, 30.1, 30.5, 30.9, 31.0], + [31.25, 31.35, 31.75, 32.15, 32.25], + [32.75, 32.85, 33.25, 33.65, 33.75], + [34.0, 34.1, 34.5, 34.9, 35.0], + ], + ], + ].flat(Infinity), + ); + + expectToBeCloseToArray(target, resized.data); + }); + + it("bicubic", async () => { + const resized = await interpolate_4d(input, { mode: "bicubic", size }); + + const target = new Float32Array( + [ + [ + [ + [-0.253804475069046, -0.06155451014637947, 0.3564453125, 0.7744455337524414, 0.9666945934295654], + [0.9493208527565002, 1.1415706872940063, 1.5595703125, 1.977570652961731, 2.1698191165924072], + [2.8301806449890137, 3.022430181503296, 3.4404296875, 3.8584301471710205, 4.050677299499512], + [4.033306121826172, 4.225555419921875, 4.6435546875, 5.061554908752441, 5.253802299499512], + ], + [ + [5.746196269989014, 5.938446998596191, 6.3564453125, 6.774445533752441, 6.966691493988037], + [6.94932222366333, 7.14157247543335, 7.5595703125, 7.977570056915283, 8.169816970825195], + [8.830181121826172, 9.022432327270508, 9.4404296875, 9.858429908752441, 10.050675392150879], + [10.033307075500488, 10.225557327270508, 10.6435546875, 11.061556816101074, 11.253799438476562], + ], + [ + [11.746198654174805, 11.938446998596191, 12.3564453125, 12.774446487426758, 12.966689109802246], + [12.949322700500488, 13.141572952270508, 13.5595703125, 13.977571487426758, 14.16981315612793], + [14.830183029174805, 15.022432327270508, 15.4404296875, 15.858430862426758, 16.05067253112793], + [16.033309936523438, 16.225557327270508, 16.6435546875, 17.061555862426758, 17.25379753112793], + ], + ], + [ + [ + [17.746200561523438, 17.938447952270508, 18.3564453125, 18.774446487426758, 18.966686248779297], + [18.949325561523438, 19.14157485961914, 19.5595703125, 19.977571487426758, 20.169809341430664], + [20.830184936523438, 21.02243423461914, 21.4404296875, 21.858430862426758, 22.050668716430664], + [22.03331184387207, 22.225557327270508, 22.6435546875, 23.061555862426758, 23.25379180908203], + ], + [ + [23.746200561523438, 23.93844985961914, 24.3564453125, 24.77444839477539, 24.96668243408203], + [24.949325561523438, 25.141576766967773, 25.5595703125, 25.977571487426758, 26.1698055267334], + [26.830184936523438, 27.022436141967773, 27.4404296875, 27.858430862426758, 28.05066680908203], + [28.033313751220703, 28.225557327270508, 28.6435546875, 29.061555862426758, 29.25379180908203], + ], + [ + [29.74620246887207, 29.93844985961914, 30.3564453125, 30.77444839477539, 30.96668243408203], + [30.949325561523438, 31.141578674316406, 31.5595703125, 31.977571487426758, 32.16980743408203], + [32.8301887512207, 33.022438049316406, 33.4404296875, 33.858428955078125, 34.050662994384766], + [34.03330993652344, 34.22556686401367, 34.6435546875, 35.06155014038086, 35.253787994384766], + ], + ], + ].flat(Infinity), + ); + + expectToBeCloseToArray(target, resized.data); + }); + }); + }); + + describe("matmul", () => { + it("(2, 5) @ (5, 4) -> (2, 4)", async () => { + const a = new Tensor("float32", range(10), [2, 5]); + const b = new Tensor("float32", range(20), [5, 4]); + + const result = await matmul(a, b); + + const target = new Float32Array( + [ + [120.0, 130.0, 140.0, 150.0], + [320.0, 355.0, 390.0, 425.0], + ].flat(), + ); + + expectToBeCloseToArray(target, result.data); + }); + }); + + describe("rfft", () => { + it("non-power of 2", async () => { + const rows = 2; + const cols = 3; + const input = new Tensor("float32", range(rows * cols), [rows, cols]); + const dim = new Tensor("int64", [-1n], []); + const result = await rfft(input, dim); + + const target = new Float32Array( + [ + [ + [3, 0], + [-1.5, 0.8660262823104858], + ], + [ + [12, 0], + [-1.5, 0.866027295589447], + ], + ].flat(Infinity), + ); + + expectToBeCloseToArray(target, result.data); + }); + + it("power of 2", async () => { + const rows = 2; + const cols = 4; + const input = new Tensor("float32", range(rows * cols), [rows, cols]); + const dim = new Tensor("int64", [-1n], []); + + const result = await rfft(input, dim); + const target = new Float32Array( + [ + [ + [6, 0], + [-2, 2], + [-2, 0], + ], + [ + [22, 0], + [-2, 2], + [-2, 0], + ], + ].flat(Infinity), + ); + + expectToBeCloseToArray(target, result.data); + }); + }); + + describe("slice", () => { + it("should slice", async () => { + const input = new Tensor("float32", [1, 2, 3, 4, 5, 6, 7, 8, 9], [3, 3]); + + const target = new Float32Array( + [ + [1, 2], + [4, 5], + ].flat(Infinity), + ); + + const starts = [0, 0]; + const ends = [2, 2]; + const axes = [0, 1]; + const steps = [1, 1]; + + const result = await slice(input, starts, ends, axes, steps); + expectToBeCloseToArray(target, result.data); + }); + }); +}); diff --git a/packages/transformers/tests/utils/utils.test.js b/packages/transformers/tests/utils/utils.test.js new file mode 100644 index 0000000..f3113c0 --- /dev/null +++ b/packages/transformers/tests/utils/utils.test.js @@ -0,0 +1,120 @@ +import { AutoProcessor } from "../../src/transformers.js"; +import { hamming, hanning, mel_filter_bank } from "../../src/utils/audio.js"; +import { getFile } from "../../src/utils/hub.js"; +import { RawImage } from "../../src/utils/image.js"; + +import { MAX_TEST_EXECUTION_TIME } from "../init.js"; + +describe("Utilities", () => { + describe("Audio utilities", () => { + it( + "should calculate MEL filters", + async () => { + // NOTE: Uses official HF implementation as reference: + const processor = await AutoProcessor.from_pretrained("openai/whisper-tiny.en"); + const config = processor.feature_extractor.config; + + // True MEL filters + const original_mel_filters = config.mel_filters; + + // Calculated MEL filters + const calculated_mel_filters = mel_filter_bank( + Math.floor(1 + config.n_fft / 2), // num_frequency_bins + config.feature_size, // num_mel_filters + 0.0, // min_frequency + 8000.0, // max_frequency + config.sampling_rate, // sampling_rate + "slaney", // norm + "slaney", // mel_scale + ); + + const original = original_mel_filters.flat(); + const calculated = calculated_mel_filters.flat(); + + // Compute max difference + const maxdiff = original.reduce((maxdiff, _, i) => { + const diff = Math.abs(original[i] - calculated[i]); + return Math.max(maxdiff, diff); + }, -Infinity); + expect(maxdiff).toBeGreaterThanOrEqual(0); + expect(maxdiff).toBeLessThan(1e-6); + }, + MAX_TEST_EXECUTION_TIME, + ); + + it( + "should calculate window", + async () => { + expect(hanning(10)).toBeCloseToNested(new Float64Array([0.0, 0.11697777844051105, 0.41317591116653485, 0.75, 0.9698463103929542, 0.9698463103929542, 0.75, 0.41317591116653485, 0.11697777844051105, 0.0])); + expect(hamming(10)).toBeCloseToNested(new Float64Array([0.08000000000000002, 0.1876195561652702, 0.46012183827321207, 0.7700000000000001, 0.9722586055615179, 0.9722586055615179, 0.7700000000000001, 0.46012183827321207, 0.1876195561652702, 0.08000000000000002])); + }, + MAX_TEST_EXECUTION_TIME, + ); + }); + + describe("Hub utilities", () => { + it("Read data from blob", async () => { + const blob = new Blob(["Hello, world!"], { type: "text/plain" }); + const blobUrl = URL.createObjectURL(blob); + const data = await getFile(blobUrl); + expect(await data.text()).toBe("Hello, world!"); + }); + }); + + describe("Image utilities", () => { + const [width, height, channels] = [2, 2, 3]; + const data = Uint8Array.from({ length: width * height * channels }, (_, i) => i % 5); + const tiny_image = new RawImage(data, width, height, channels); + + let image; + beforeAll(async () => { + image = await RawImage.fromURL("https://picsum.photos/300/200"); + }); + + it("Can split image into separate channels", async () => { + const image_data = tiny_image.split().map((x) => x.data); + + const target = [ + new Uint8Array([0, 3, 1, 4]), // Reds + new Uint8Array([1, 4, 2, 0]), // Greens + new Uint8Array([2, 0, 3, 1]), // Blues + ]; + + expect(image_data).toEqual(target); + }); + + it("Can splits channels for grayscale", async () => { + const image_data = tiny_image + .grayscale() + .split() + .map((x) => x.data); + const target = [new Uint8ClampedArray([1, 3, 2, 1])]; + + expect(image_data).toEqual(target); + }); + + it("Read image from URL", async () => { + expect(image.width).toBe(300); + expect(image.height).toBe(200); + expect(image.channels).toBe(3); + }); + + it("Can resize image", async () => { + const resized = await image.resize(150, 100); + expect(resized.width).toBe(150); + expect(resized.height).toBe(100); + }); + + it("Can resize with aspect ratio", async () => { + const resized = await image.resize(150, null); + expect(resized.width).toBe(150); + expect(resized.height).toBe(100); + }); + + it("Returns original image if width and height are null", async () => { + const resized = await image.resize(null, null); + expect(resized.width).toBe(300); + expect(resized.height).toBe(200); + }); + }); +}); diff --git a/packages/transformers/tsconfig.json b/packages/transformers/tsconfig.json new file mode 100644 index 0000000..07153d7 --- /dev/null +++ b/packages/transformers/tsconfig.json @@ -0,0 +1,24 @@ +{ + // Only include files in the src directory + "include": ["src/**/*"], + "compilerOptions": { + // Tells the compiler to check JS files + "checkJs": true, + "target": "esnext", + "module": "esnext", + "moduleResolution": "bundler", + "outDir": "types", + "rootDir": "src", + "strict": false, + "skipLibCheck": true, + "declaration": true, + "declarationMap": true, + "noEmit": false, + "emitDeclarationOnly": true, + "esModuleInterop": true, + "composite": true + }, + "typeAcquisition": { + "include": ["jest"] + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..bea3479 --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,4195 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + devDependencies: + prettier: + specifier: 3.8.1 + version: 3.8.1 + typescript: + specifier: 5.9.3 + version: 5.9.3 + + packages/transformers: + dependencies: + '@huggingface/jinja': + specifier: ^0.5.6 + version: 0.5.6 + '@huggingface/tokenizers': + specifier: ^0.1.3 + version: 0.1.3 + onnxruntime-node: + specifier: 1.24.3 + version: 1.24.3 + onnxruntime-web: + specifier: 1.26.0-dev.20260416-b7804b056c + version: 1.26.0-dev.20260416-b7804b056c + sharp: + specifier: ^0.34.5 + version: 0.34.5 + devDependencies: + '@types/jest': + specifier: ^30.0.0 + version: 30.0.0 + '@types/node': + specifier: ^24.1.0 + version: 24.10.9 + '@webgpu/types': + specifier: ^0.1.69 + version: 0.1.69 + esbuild: + specifier: ^0.27.2 + version: 0.27.2 + jest: + specifier: ^30.2.0 + version: 30.2.0(@types/node@24.10.9) + jest-environment-node: + specifier: ^30.2.0 + version: 30.2.0 + jsdoc-to-markdown: + specifier: ^9.1.3 + version: 9.1.3 + typescript: + specifier: 5.9.3 + version: 5.9.3 + +packages: + + '@babel/code-frame@7.28.6': + resolution: {integrity: sha512-JYgintcMjRiCvS8mMECzaEn+m3PfoQiyqukOMCCVQtoJGYJw8j/8LBJEiqkHLkfwCcs74E3pbAUFNg7d9VNJ+Q==} + engines: {node: '>=6.9.0'} + + '@babel/compat-data@7.28.6': + resolution: {integrity: sha512-2lfu57JtzctfIrcGMz992hyLlByuzgIk58+hhGCxjKZ3rWI82NnVLjXcaTqkI2NvlcvOskZaiZ5kjUALo3Lpxg==} + engines: {node: '>=6.9.0'} + + '@babel/core@7.28.6': + resolution: {integrity: sha512-H3mcG6ZDLTlYfaSNi0iOKkigqMFvkTKlGUYlD8GW7nNOYRrevuA46iTypPyv+06V3fEmvvazfntkBU34L0azAw==} + engines: {node: '>=6.9.0'} + + '@babel/generator@7.28.6': + resolution: {integrity: sha512-lOoVRwADj8hjf7al89tvQ2a1lf53Z+7tiXMgpZJL3maQPDxh0DgLMN62B2MKUOFcoodBHLMbDM6WAbKgNy5Suw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-compilation-targets@7.28.6': + resolution: {integrity: sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==} + engines: {node: '>=6.9.0'} + + '@babel/helper-globals@7.28.0': + resolution: {integrity: sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-module-imports@7.28.6': + resolution: {integrity: sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-module-transforms@7.28.6': + resolution: {integrity: sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + + '@babel/helper-plugin-utils@7.28.6': + resolution: {integrity: sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==} + engines: {node: '>=6.9.0'} + + '@babel/helper-string-parser@7.27.1': + resolution: {integrity: sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.28.5': + resolution: {integrity: sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-option@7.27.1': + resolution: {integrity: sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==} + engines: {node: '>=6.9.0'} + + '@babel/helpers@7.28.6': + resolution: {integrity: sha512-xOBvwq86HHdB7WUDTfKfT/Vuxh7gElQ+Sfti2Cy6yIWNW05P8iUslOVcZ4/sKbE+/jQaukQAdz/gf3724kYdqw==} + engines: {node: '>=6.9.0'} + + '@babel/parser@7.28.6': + resolution: {integrity: sha512-TeR9zWR18BvbfPmGbLampPMW+uW1NZnJlRuuHso8i87QZNq2JRF9i6RgxRqtEq+wQGsS19NNTWr2duhnE49mfQ==} + engines: {node: '>=6.0.0'} + hasBin: true + + '@babel/plugin-syntax-async-generators@7.8.4': + resolution: {integrity: sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-bigint@7.8.3': + resolution: {integrity: sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-class-properties@7.12.13': + resolution: {integrity: sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-class-static-block@7.14.5': + resolution: {integrity: sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-import-attributes@7.28.6': + resolution: {integrity: sha512-jiLC0ma9XkQT3TKJ9uYvlakm66Pamywo+qwL+oL8HJOvc6TWdZXVfhqJr8CCzbSGUAbDOzlGHJC1U+vRfLQDvw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-import-meta@7.10.4': + resolution: {integrity: sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-json-strings@7.8.3': + resolution: {integrity: sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-jsx@7.28.6': + resolution: {integrity: sha512-wgEmr06G6sIpqr8YDwA2dSRTE3bJ+V0IfpzfSY3Lfgd7YWOaAdlykvJi13ZKBt8cZHfgH1IXN+CL656W3uUa4w==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-logical-assignment-operators@7.10.4': + resolution: {integrity: sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-nullish-coalescing-operator@7.8.3': + resolution: {integrity: sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-numeric-separator@7.10.4': + resolution: {integrity: sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-object-rest-spread@7.8.3': + resolution: {integrity: sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-optional-catch-binding@7.8.3': + resolution: {integrity: sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-optional-chaining@7.8.3': + resolution: {integrity: sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-private-property-in-object@7.14.5': + resolution: {integrity: sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-top-level-await@7.14.5': + resolution: {integrity: sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-syntax-typescript@7.28.6': + resolution: {integrity: sha512-+nDNmQye7nlnuuHDboPbGm00Vqg3oO8niRRL27/4LYHUsHYh0zJ1xWOz0uRwNFmM1Avzk8wZbc6rdiYhomzv/A==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/template@7.28.6': + resolution: {integrity: sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==} + engines: {node: '>=6.9.0'} + + '@babel/traverse@7.28.6': + resolution: {integrity: sha512-fgWX62k02qtjqdSNTAGxmKYY/7FSL9WAS1o2Hu5+I5m9T0yxZzr4cnrfXQ/MX0rIifthCSs6FKTlzYbJcPtMNg==} + engines: {node: '>=6.9.0'} + + '@babel/types@7.28.6': + resolution: {integrity: sha512-0ZrskXVEHSWIqZM/sQZ4EV3jZJXRkio/WCxaqKZP1g//CEWEPSfeZFcms4XeKBCHU0ZKnIkdJeU/kF+eRp5lBg==} + engines: {node: '>=6.9.0'} + + '@bcoe/v8-coverage@0.2.3': + resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} + + '@emnapi/core@1.8.1': + resolution: {integrity: sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==} + + '@emnapi/runtime@1.8.1': + resolution: {integrity: sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==} + + '@emnapi/wasi-threads@1.1.0': + resolution: {integrity: sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==} + + '@esbuild/aix-ppc64@0.27.2': + resolution: {integrity: sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.27.2': + resolution: {integrity: sha512-pvz8ZZ7ot/RBphf8fv60ljmaoydPU12VuXHImtAs0XhLLw+EXBi2BLe3OYSBslR4rryHvweW5gmkKFwTiFy6KA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.27.2': + resolution: {integrity: sha512-DVNI8jlPa7Ujbr1yjU2PfUSRtAUZPG9I1RwW4F4xFB1Imiu2on0ADiI/c3td+KmDtVKNbi+nffGDQMfcIMkwIA==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.27.2': + resolution: {integrity: sha512-z8Ank4Byh4TJJOh4wpz8g2vDy75zFL0TlZlkUkEwYXuPSgX8yzep596n6mT7905kA9uHZsf/o2OJZubl2l3M7A==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.27.2': + resolution: {integrity: sha512-davCD2Zc80nzDVRwXTcQP/28fiJbcOwvdolL0sOiOsbwBa72kegmVU0Wrh1MYrbuCL98Omp5dVhQFWRKR2ZAlg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.27.2': + resolution: {integrity: sha512-ZxtijOmlQCBWGwbVmwOF/UCzuGIbUkqB1faQRf5akQmxRJ1ujusWsb3CVfk/9iZKr2L5SMU5wPBi1UWbvL+VQA==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.27.2': + resolution: {integrity: sha512-lS/9CN+rgqQ9czogxlMcBMGd+l8Q3Nj1MFQwBZJyoEKI50XGxwuzznYdwcav6lpOGv5BqaZXqvBSiB/kJ5op+g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.27.2': + resolution: {integrity: sha512-tAfqtNYb4YgPnJlEFu4c212HYjQWSO/w/h/lQaBK7RbwGIkBOuNKQI9tqWzx7Wtp7bTPaGC6MJvWI608P3wXYA==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.27.2': + resolution: {integrity: sha512-hYxN8pr66NsCCiRFkHUAsxylNOcAQaxSSkHMMjcpx0si13t1LHFphxJZUiGwojB1a/Hd5OiPIqDdXONia6bhTw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.27.2': + resolution: {integrity: sha512-vWfq4GaIMP9AIe4yj1ZUW18RDhx6EPQKjwe7n8BbIecFtCQG4CfHGaHuh7fdfq+y3LIA2vGS/o9ZBGVxIDi9hw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.27.2': + resolution: {integrity: sha512-MJt5BRRSScPDwG2hLelYhAAKh9imjHK5+NE/tvnRLbIqUWa+0E9N4WNMjmp/kXXPHZGqPLxggwVhz7QP8CTR8w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.27.2': + resolution: {integrity: sha512-lugyF1atnAT463aO6KPshVCJK5NgRnU4yb3FUumyVz+cGvZbontBgzeGFO1nF+dPueHD367a2ZXe1NtUkAjOtg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.27.2': + resolution: {integrity: sha512-nlP2I6ArEBewvJ2gjrrkESEZkB5mIoaTswuqNFRv/WYd+ATtUpe9Y09RnJvgvdag7he0OWgEZWhviS1OTOKixw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.27.2': + resolution: {integrity: sha512-C92gnpey7tUQONqg1n6dKVbx3vphKtTHJaNG2Ok9lGwbZil6DrfyecMsp9CrmXGQJmZ7iiVXvvZH6Ml5hL6XdQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.27.2': + resolution: {integrity: sha512-B5BOmojNtUyN8AXlK0QJyvjEZkWwy/FKvakkTDCziX95AowLZKR6aCDhG7LeF7uMCXEJqwa8Bejz5LTPYm8AvA==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.27.2': + resolution: {integrity: sha512-p4bm9+wsPwup5Z8f4EpfN63qNagQ47Ua2znaqGH6bqLlmJ4bx97Y9JdqxgGZ6Y8xVTixUnEkoKSHcpRlDnNr5w==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.27.2': + resolution: {integrity: sha512-uwp2Tip5aPmH+NRUwTcfLb+W32WXjpFejTIOWZFw/v7/KnpCDKG66u4DLcurQpiYTiYwQ9B7KOeMJvLCu/OvbA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.27.2': + resolution: {integrity: sha512-Kj6DiBlwXrPsCRDeRvGAUb/LNrBASrfqAIok+xB0LxK8CHqxZ037viF13ugfsIpePH93mX7xfJp97cyDuTZ3cw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.27.2': + resolution: {integrity: sha512-HwGDZ0VLVBY3Y+Nw0JexZy9o/nUAWq9MlV7cahpaXKW6TOzfVno3y3/M8Ga8u8Yr7GldLOov27xiCnqRZf0tCA==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.27.2': + resolution: {integrity: sha512-DNIHH2BPQ5551A7oSHD0CKbwIA/Ox7+78/AWkbS5QoRzaqlev2uFayfSxq68EkonB+IKjiuxBFoV8ESJy8bOHA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.27.2': + resolution: {integrity: sha512-/it7w9Nb7+0KFIzjalNJVR5bOzA9Vay+yIPLVHfIQYG/j+j9VTH84aNB8ExGKPU4AzfaEvN9/V4HV+F+vo8OEg==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.27.2': + resolution: {integrity: sha512-LRBbCmiU51IXfeXk59csuX/aSaToeG7w48nMwA6049Y4J4+VbWALAuXcs+qcD04rHDuSCSRKdmY63sruDS5qag==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.27.2': + resolution: {integrity: sha512-kMtx1yqJHTmqaqHPAzKCAkDaKsffmXkPHThSfRwZGyuqyIeBvf08KSsYXl+abf5HDAPMJIPnbBfXvP2ZC2TfHg==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.27.2': + resolution: {integrity: sha512-Yaf78O/B3Kkh+nKABUF++bvJv5Ijoy9AN1ww904rOXZFLWVc5OLOfL56W+C8F9xn5JQZa3UX6m+IktJnIb1Jjg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.27.2': + resolution: {integrity: sha512-Iuws0kxo4yusk7sw70Xa2E2imZU5HoixzxfGCdxwBdhiDgt9vX9VUCBhqcwY7/uh//78A1hMkkROMJq9l27oLQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.27.2': + resolution: {integrity: sha512-sRdU18mcKf7F+YgheI/zGf5alZatMUTKj/jNS6l744f9u3WFu4v7twcUI9vu4mknF4Y9aDlblIie0IM+5xxaqQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@huggingface/jinja@0.5.6': + resolution: {integrity: sha512-MyMWyLnjqo+KRJYSH7oWNbsOn5onuIvfXYPcc0WOGxU0eHUV7oAYUoQTl2BMdu7ml+ea/bu11UM+EshbeHwtIA==} + engines: {node: '>=18'} + + '@huggingface/tokenizers@0.1.3': + resolution: {integrity: sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA==} + + '@img/colour@1.0.0': + resolution: {integrity: sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw==} + engines: {node: '>=18'} + + '@img/sharp-darwin-arm64@0.34.5': + resolution: {integrity: sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.34.5': + resolution: {integrity: sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-darwin-arm64@1.2.4': + resolution: {integrity: sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.2.4': + resolution: {integrity: sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.2.4': + resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linux-arm@1.2.4': + resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} + cpu: [arm] + os: [linux] + + '@img/sharp-libvips-linux-ppc64@1.2.4': + resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} + cpu: [ppc64] + os: [linux] + + '@img/sharp-libvips-linux-riscv64@1.2.4': + resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} + cpu: [riscv64] + os: [linux] + + '@img/sharp-libvips-linux-s390x@1.2.4': + resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} + cpu: [s390x] + os: [linux] + + '@img/sharp-libvips-linux-x64@1.2.4': + resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} + cpu: [x64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} + cpu: [x64] + os: [linux] + + '@img/sharp-linux-arm64@0.34.5': + resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linux-arm@0.34.5': + resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm] + os: [linux] + + '@img/sharp-linux-ppc64@0.34.5': + resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ppc64] + os: [linux] + + '@img/sharp-linux-riscv64@0.34.5': + resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [riscv64] + os: [linux] + + '@img/sharp-linux-s390x@0.34.5': + resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [s390x] + os: [linux] + + '@img/sharp-linux-x64@0.34.5': + resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-linuxmusl-arm64@0.34.5': + resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linuxmusl-x64@0.34.5': + resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-wasm32@0.34.5': + resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.34.5': + resolution: {integrity: sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.34.5': + resolution: {integrity: sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.34.5': + resolution: {integrity: sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [win32] + + '@isaacs/cliui@8.0.2': + resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} + engines: {node: '>=12'} + + '@istanbuljs/load-nyc-config@1.1.0': + resolution: {integrity: sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==} + engines: {node: '>=8'} + + '@istanbuljs/schema@0.1.3': + resolution: {integrity: sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==} + engines: {node: '>=8'} + + '@jest/console@30.2.0': + resolution: {integrity: sha512-+O1ifRjkvYIkBqASKWgLxrpEhQAAE7hY77ALLUufSk5717KfOShg6IbqLmdsLMPdUiFvA2kTs0R7YZy+l0IzZQ==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/core@30.2.0': + resolution: {integrity: sha512-03W6IhuhjqTlpzh/ojut/pDB2LPRygyWX8ExpgHtQA8H/3K7+1vKmcINx5UzeOX1se6YEsBsOHQ1CRzf3fOwTQ==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + peerDependencies: + node-notifier: ^8.0.1 || ^9.0.0 || ^10.0.0 + peerDependenciesMeta: + node-notifier: + optional: true + + '@jest/diff-sequences@30.0.1': + resolution: {integrity: sha512-n5H8QLDJ47QqbCNn5SuFjCRDrOLEZ0h8vAHCK5RL9Ls7Xa8AQLa/YxAc9UjFqoEDM48muwtBGjtMY5cr0PLDCw==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/environment@30.2.0': + resolution: {integrity: sha512-/QPTL7OBJQ5ac09UDRa3EQes4gt1FTEG/8jZ/4v5IVzx+Cv7dLxlVIvfvSVRiiX2drWyXeBjkMSR8hvOWSog5g==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/expect-utils@30.2.0': + resolution: {integrity: sha512-1JnRfhqpD8HGpOmQp180Fo9Zt69zNtC+9lR+kT7NVL05tNXIi+QC8Csz7lfidMoVLPD3FnOtcmp0CEFnxExGEA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/expect@30.2.0': + resolution: {integrity: sha512-V9yxQK5erfzx99Sf+7LbhBwNWEZ9eZay8qQ9+JSC0TrMR1pMDHLMY+BnVPacWU6Jamrh252/IKo4F1Xn/zfiqA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/fake-timers@30.2.0': + resolution: {integrity: sha512-HI3tRLjRxAbBy0VO8dqqm7Hb2mIa8d5bg/NJkyQcOk7V118ObQML8RC5luTF/Zsg4474a+gDvhce7eTnP4GhYw==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/get-type@30.1.0': + resolution: {integrity: sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/globals@30.2.0': + resolution: {integrity: sha512-b63wmnKPaK+6ZZfpYhz9K61oybvbI1aMcIs80++JI1O1rR1vaxHUCNqo3ITu6NU0d4V34yZFoHMn/uoKr/Rwfw==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/pattern@30.0.1': + resolution: {integrity: sha512-gWp7NfQW27LaBQz3TITS8L7ZCQ0TLvtmI//4OwlQRx4rnWxcPNIYjxZpDcN4+UlGxgm3jS5QPz8IPTCkb59wZA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/reporters@30.2.0': + resolution: {integrity: sha512-DRyW6baWPqKMa9CzeiBjHwjd8XeAyco2Vt8XbcLFjiwCOEKOvy82GJ8QQnJE9ofsxCMPjH4MfH8fCWIHHDKpAQ==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + peerDependencies: + node-notifier: ^8.0.1 || ^9.0.0 || ^10.0.0 + peerDependenciesMeta: + node-notifier: + optional: true + + '@jest/schemas@30.0.5': + resolution: {integrity: sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/snapshot-utils@30.2.0': + resolution: {integrity: sha512-0aVxM3RH6DaiLcjj/b0KrIBZhSX1373Xci4l3cW5xiUWPctZ59zQ7jj4rqcJQ/Z8JuN/4wX3FpJSa3RssVvCug==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/source-map@30.0.1': + resolution: {integrity: sha512-MIRWMUUR3sdbP36oyNyhbThLHyJ2eEDClPCiHVbrYAe5g3CHRArIVpBw7cdSB5fr+ofSfIb2Tnsw8iEHL0PYQg==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/test-result@30.2.0': + resolution: {integrity: sha512-RF+Z+0CCHkARz5HT9mcQCBulb1wgCP3FBvl9VFokMX27acKphwyQsNuWH3c+ojd1LeWBLoTYoxF0zm6S/66mjg==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/test-sequencer@30.2.0': + resolution: {integrity: sha512-wXKgU/lk8fKXMu/l5Hog1R61bL4q5GCdT6OJvdAFz1P+QrpoFuLU68eoKuVc4RbrTtNnTL5FByhWdLgOPSph+Q==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/transform@30.2.0': + resolution: {integrity: sha512-XsauDV82o5qXbhalKxD7p4TZYYdwcaEXC77PPD2HixEFF+6YGppjrAAQurTl2ECWcEomHBMMNS9AH3kcCFx8jA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jest/types@30.2.0': + resolution: {integrity: sha512-H9xg1/sfVvyfU7o3zMfBEjQ1gcsdeTMgqHoYdN79tuLqfTtuu7WckRA1R5whDwOzxaZAeMKTYWqP+WCAi0CHsg==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + '@jridgewell/gen-mapping@0.3.13': + resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} + + '@jridgewell/remapping@2.3.5': + resolution: {integrity: sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==} + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + + '@jsdoc/salty@0.2.9': + resolution: {integrity: sha512-yYxMVH7Dqw6nO0d5NIV8OQWnitU8k6vXH8NtgqAfIa/IUqRMxRv/NUJJ08VEKbAakwxlgBl5PJdrU0dMPStsnw==} + engines: {node: '>=v12.0.0'} + + '@napi-rs/wasm-runtime@0.2.12': + resolution: {integrity: sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==} + + '@nodelib/fs.scandir@2.1.5': + resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} + engines: {node: '>= 8'} + + '@nodelib/fs.stat@2.0.5': + resolution: {integrity: sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==} + engines: {node: '>= 8'} + + '@nodelib/fs.walk@1.2.8': + resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} + engines: {node: '>= 8'} + + '@pkgjs/parseargs@0.11.0': + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} + engines: {node: '>=14'} + + '@pkgr/core@0.2.9': + resolution: {integrity: sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==} + engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} + + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.4': + resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + + '@protobufjs/eventemitter@1.1.0': + resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + + '@protobufjs/fetch@1.1.0': + resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/inquire@1.1.0': + resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.0': + resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + + '@sinclair/typebox@0.34.48': + resolution: {integrity: sha512-kKJTNuK3AQOrgjjotVxMrCn1sUJwM76wMszfq1kdU4uYVJjvEWuFQ6HgvLt4Xz3fSmZlTOxJ/Ie13KnIcWQXFA==} + + '@sinonjs/commons@3.0.1': + resolution: {integrity: sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==} + + '@sinonjs/fake-timers@13.0.5': + resolution: {integrity: sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw==} + + '@tybys/wasm-util@0.10.1': + resolution: {integrity: sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==} + + '@types/babel__core@7.20.5': + resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==} + + '@types/babel__generator@7.27.0': + resolution: {integrity: sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==} + + '@types/babel__template@7.4.4': + resolution: {integrity: sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==} + + '@types/babel__traverse@7.28.0': + resolution: {integrity: sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==} + + '@types/istanbul-lib-coverage@2.0.6': + resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==} + + '@types/istanbul-lib-report@3.0.3': + resolution: {integrity: sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==} + + '@types/istanbul-reports@3.0.4': + resolution: {integrity: sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==} + + '@types/jest@30.0.0': + resolution: {integrity: sha512-XTYugzhuwqWjws0CVz8QpM36+T+Dz5mTEBKhNs/esGLnCIlGdRy+Dq78NRjd7ls7r8BC8ZRMOrKlkO1hU0JOwA==} + + '@types/linkify-it@5.0.0': + resolution: {integrity: sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==} + + '@types/markdown-it@14.1.2': + resolution: {integrity: sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==} + + '@types/mdurl@2.0.0': + resolution: {integrity: sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==} + + '@types/node@24.10.9': + resolution: {integrity: sha512-ne4A0IpG3+2ETuREInjPNhUGis1SFjv1d5asp8MzEAGtOZeTeHVDOYqOgqfhvseqg/iXty2hjBf1zAOb7RNiNw==} + + '@types/stack-utils@2.0.3': + resolution: {integrity: sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==} + + '@types/yargs-parser@21.0.3': + resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} + + '@types/yargs@17.0.35': + resolution: {integrity: sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==} + + '@ungap/structured-clone@1.3.0': + resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} + + '@unrs/resolver-binding-android-arm-eabi@1.11.1': + resolution: {integrity: sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==} + cpu: [arm] + os: [android] + + '@unrs/resolver-binding-android-arm64@1.11.1': + resolution: {integrity: sha512-lCxkVtb4wp1v+EoN+HjIG9cIIzPkX5OtM03pQYkG+U5O/wL53LC4QbIeazgiKqluGeVEeBlZahHalCaBvU1a2g==} + cpu: [arm64] + os: [android] + + '@unrs/resolver-binding-darwin-arm64@1.11.1': + resolution: {integrity: sha512-gPVA1UjRu1Y/IsB/dQEsp2V1pm44Of6+LWvbLc9SDk1c2KhhDRDBUkQCYVWe6f26uJb3fOK8saWMgtX8IrMk3g==} + cpu: [arm64] + os: [darwin] + + '@unrs/resolver-binding-darwin-x64@1.11.1': + resolution: {integrity: sha512-cFzP7rWKd3lZaCsDze07QX1SC24lO8mPty9vdP+YVa3MGdVgPmFc59317b2ioXtgCMKGiCLxJ4HQs62oz6GfRQ==} + cpu: [x64] + os: [darwin] + + '@unrs/resolver-binding-freebsd-x64@1.11.1': + resolution: {integrity: sha512-fqtGgak3zX4DCB6PFpsH5+Kmt/8CIi4Bry4rb1ho6Av2QHTREM+47y282Uqiu3ZRF5IQioJQ5qWRV6jduA+iGw==} + cpu: [x64] + os: [freebsd] + + '@unrs/resolver-binding-linux-arm-gnueabihf@1.11.1': + resolution: {integrity: sha512-u92mvlcYtp9MRKmP+ZvMmtPN34+/3lMHlyMj7wXJDeXxuM0Vgzz0+PPJNsro1m3IZPYChIkn944wW8TYgGKFHw==} + cpu: [arm] + os: [linux] + + '@unrs/resolver-binding-linux-arm-musleabihf@1.11.1': + resolution: {integrity: sha512-cINaoY2z7LVCrfHkIcmvj7osTOtm6VVT16b5oQdS4beibX2SYBwgYLmqhBjA1t51CarSaBuX5YNsWLjsqfW5Cw==} + cpu: [arm] + os: [linux] + + '@unrs/resolver-binding-linux-arm64-gnu@1.11.1': + resolution: {integrity: sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ==} + cpu: [arm64] + os: [linux] + + '@unrs/resolver-binding-linux-arm64-musl@1.11.1': + resolution: {integrity: sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w==} + cpu: [arm64] + os: [linux] + + '@unrs/resolver-binding-linux-ppc64-gnu@1.11.1': + resolution: {integrity: sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA==} + cpu: [ppc64] + os: [linux] + + '@unrs/resolver-binding-linux-riscv64-gnu@1.11.1': + resolution: {integrity: sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ==} + cpu: [riscv64] + os: [linux] + + '@unrs/resolver-binding-linux-riscv64-musl@1.11.1': + resolution: {integrity: sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew==} + cpu: [riscv64] + os: [linux] + + '@unrs/resolver-binding-linux-s390x-gnu@1.11.1': + resolution: {integrity: sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg==} + cpu: [s390x] + os: [linux] + + '@unrs/resolver-binding-linux-x64-gnu@1.11.1': + resolution: {integrity: sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w==} + cpu: [x64] + os: [linux] + + '@unrs/resolver-binding-linux-x64-musl@1.11.1': + resolution: {integrity: sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA==} + cpu: [x64] + os: [linux] + + '@unrs/resolver-binding-wasm32-wasi@1.11.1': + resolution: {integrity: sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + + '@unrs/resolver-binding-win32-arm64-msvc@1.11.1': + resolution: {integrity: sha512-nRcz5Il4ln0kMhfL8S3hLkxI85BXs3o8EYoattsJNdsX4YUU89iOkVn7g0VHSRxFuVMdM4Q1jEpIId1Ihim/Uw==} + cpu: [arm64] + os: [win32] + + '@unrs/resolver-binding-win32-ia32-msvc@1.11.1': + resolution: {integrity: sha512-DCEI6t5i1NmAZp6pFonpD5m7i6aFrpofcp4LA2i8IIq60Jyo28hamKBxNrZcyOwVOZkgsRp9O2sXWBWP8MnvIQ==} + cpu: [ia32] + os: [win32] + + '@unrs/resolver-binding-win32-x64-msvc@1.11.1': + resolution: {integrity: sha512-lrW200hZdbfRtztbygyaq/6jP6AKE8qQN2KvPcJ+x7wiD038YtnYtZ82IMNJ69GJibV7bwL3y9FgK+5w/pYt6g==} + cpu: [x64] + os: [win32] + + '@webgpu/types@0.1.69': + resolution: {integrity: sha512-RPmm6kgRbI8e98zSD3RVACvnuktIja5+yLgDAkTmxLr90BEwdTXRQWNLF3ETTTyH/8mKhznZuN5AveXYFEsMGQ==} + + adm-zip@0.5.16: + resolution: {integrity: sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==} + engines: {node: '>=12.0'} + + ansi-escapes@4.3.2: + resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==} + engines: {node: '>=8'} + + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + + ansi-regex@6.2.2: + resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==} + engines: {node: '>=12'} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + ansi-styles@5.2.0: + resolution: {integrity: sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==} + engines: {node: '>=10'} + + ansi-styles@6.2.3: + resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} + engines: {node: '>=12'} + + anymatch@3.1.3: + resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} + engines: {node: '>= 8'} + + argparse@1.0.10: + resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} + + argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + + array-back@6.2.2: + resolution: {integrity: sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==} + engines: {node: '>=12.17'} + + babel-jest@30.2.0: + resolution: {integrity: sha512-0YiBEOxWqKkSQWL9nNGGEgndoeL0ZpWrbLMNL5u/Kaxrli3Eaxlt3ZtIDktEvXt4L/R9r3ODr2zKwGM/2BjxVw==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + peerDependencies: + '@babel/core': ^7.11.0 || ^8.0.0-0 + + babel-plugin-istanbul@7.0.1: + resolution: {integrity: sha512-D8Z6Qm8jCvVXtIRkBnqNHX0zJ37rQcFJ9u8WOS6tkYOsRdHBzypCstaxWiu5ZIlqQtviRYbgnRLSoCEvjqcqbA==} + engines: {node: '>=12'} + + babel-plugin-jest-hoist@30.2.0: + resolution: {integrity: sha512-ftzhzSGMUnOzcCXd6WHdBGMyuwy15Wnn0iyyWGKgBDLxf9/s5ABuraCSpBX2uG0jUg4rqJnxsLc5+oYBqoxVaA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + babel-preset-current-node-syntax@1.2.0: + resolution: {integrity: sha512-E/VlAEzRrsLEb2+dv8yp3bo4scof3l9nR4lrld+Iy5NyVqgVYUJnDAmunkhPMisRI32Qc4iRiz425d8vM++2fg==} + peerDependencies: + '@babel/core': ^7.0.0 || ^8.0.0-0 + + babel-preset-jest@30.2.0: + resolution: {integrity: sha512-US4Z3NOieAQumwFnYdUWKvUKh8+YSnS/gB3t6YBiz0bskpu7Pine8pPCheNxlPEW4wnUkma2a94YuW2q3guvCQ==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + peerDependencies: + '@babel/core': ^7.11.0 || ^8.0.0-beta.1 + + balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + + baseline-browser-mapping@2.9.18: + resolution: {integrity: sha512-e23vBV1ZLfjb9apvfPk4rHVu2ry6RIr2Wfs+O324okSidrX7pTAnEJPCh/O5BtRlr7QtZI7ktOP3vsqr7Z5XoA==} + hasBin: true + + bluebird@3.7.2: + resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==} + + boolean@3.2.0: + resolution: {integrity: sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. + + brace-expansion@1.1.12: + resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} + + brace-expansion@2.0.2: + resolution: {integrity: sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==} + + braces@3.0.3: + resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} + engines: {node: '>=8'} + + browserslist@4.28.1: + resolution: {integrity: sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==} + engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} + hasBin: true + + bser@2.1.1: + resolution: {integrity: sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==} + + buffer-from@1.1.2: + resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + + cache-point@3.0.1: + resolution: {integrity: sha512-itTIMLEKbh6Dw5DruXbxAgcyLnh/oPGVLBfTPqBOftASxHe8bAeXy7JkO4F0LvHqht7XqP5O/09h5UcHS2w0FA==} + engines: {node: '>=12.17'} + peerDependencies: + '@75lb/nature': latest + peerDependenciesMeta: + '@75lb/nature': + optional: true + + callsites@3.1.0: + resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} + engines: {node: '>=6'} + + camelcase@5.3.1: + resolution: {integrity: sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==} + engines: {node: '>=6'} + + camelcase@6.3.0: + resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==} + engines: {node: '>=10'} + + caniuse-lite@1.0.30001766: + resolution: {integrity: sha512-4C0lfJ0/YPjJQHagaE9x2Elb69CIqEPZeG0anQt9SIvIoOH4a4uaRl73IavyO+0qZh6MDLH//DrXThEYKHkmYA==} + + catharsis@0.9.0: + resolution: {integrity: sha512-prMTQVpcns/tzFgFVkVp6ak6RykZyWb3gu8ckUpd6YkTlacOd3DXGJjIpD4Q6zJirizvaiAjSSHlOsA+6sNh2A==} + engines: {node: '>= 10'} + + chalk-template@0.4.0: + resolution: {integrity: sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==} + engines: {node: '>=12'} + + chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + + char-regex@1.0.2: + resolution: {integrity: sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==} + engines: {node: '>=10'} + + ci-info@4.3.1: + resolution: {integrity: sha512-Wdy2Igu8OcBpI2pZePZ5oWjPC38tmDVx5WKUXKwlLYkA0ozo85sLsLvkBbBn/sZaSCMFOGZJ14fvW9t5/d7kdA==} + engines: {node: '>=8'} + + cjs-module-lexer@2.2.0: + resolution: {integrity: sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==} + + cliui@8.0.1: + resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} + engines: {node: '>=12'} + + co@4.6.0: + resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} + engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} + + collect-v8-coverage@1.0.3: + resolution: {integrity: sha512-1L5aqIkwPfiodaMgQunkF1zRhNqifHBmtbbbxcr6yVxxBnliw4TDOW6NxpO8DJLgJ16OT+Y4ztZqP6p/FtXnAw==} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + command-line-args@6.0.1: + resolution: {integrity: sha512-Jr3eByUjqyK0qd8W0SGFW1nZwqCaNCtbXjRo2cRJC1OYxWl3MZ5t1US3jq+cO4sPavqgw4l9BMGX0CBe+trepg==} + engines: {node: '>=12.20'} + peerDependencies: + '@75lb/nature': latest + peerDependenciesMeta: + '@75lb/nature': + optional: true + + command-line-usage@7.0.3: + resolution: {integrity: sha512-PqMLy5+YGwhMh1wS04mVG44oqDsgyLRSKJBdOo1bnYhMKBW65gZF1dRp2OZRhiTjgUHljy99qkO7bsctLaw35Q==} + engines: {node: '>=12.20.0'} + + common-sequence@3.0.0: + resolution: {integrity: sha512-g/CgSYk93y+a1IKm50tKl7kaT/OjjTYVQlEbUlt/49ZLV1mcKpUU7iyDiqTAeLdb4QDtQfq3ako8y8v//fzrWQ==} + engines: {node: '>=12.17'} + + concat-map@0.0.1: + resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + + config-master@3.1.0: + resolution: {integrity: sha512-n7LBL1zBzYdTpF1mx5DNcZnZn05CWIdsdvtPL4MosvqbBUK3Rq6VWEtGUuF3Y0s9/CIhMejezqlSkP6TnCJ/9g==} + + convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + + current-module-paths@1.1.3: + resolution: {integrity: sha512-7AH+ZTRKikdK4s1RmY0l6067UD/NZc7p3zZVZxvmnH80G31kr0y0W0E6ibYM4IS01MEm8DiC5FnTcgcgkbFHoA==} + engines: {node: '>=12.17'} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + dedent@1.7.1: + resolution: {integrity: sha512-9JmrhGZpOlEgOLdQgSm0zxFaYoQon408V1v49aqTWuXENVlnCuY9JBZcXZiCsZQWDjTm5Qf/nIvAy77mXDAjEg==} + peerDependencies: + babel-plugin-macros: ^3.1.0 + peerDependenciesMeta: + babel-plugin-macros: + optional: true + + deepmerge@4.3.1: + resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} + engines: {node: '>=0.10.0'} + + define-data-property@1.1.4: + resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} + engines: {node: '>= 0.4'} + + define-properties@1.2.1: + resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} + engines: {node: '>= 0.4'} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + detect-newline@3.1.0: + resolution: {integrity: sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==} + engines: {node: '>=8'} + + detect-node@2.1.0: + resolution: {integrity: sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==} + + dmd@7.1.1: + resolution: {integrity: sha512-Ap2HP6iuOek7eShReDLr9jluNJm9RMZESlt29H/Xs1qrVMkcS9X6m5h1mBC56WMxNiSo0wvjGICmZlYUSFjwZQ==} + engines: {node: '>=12.17'} + peerDependencies: + '@75lb/nature': latest + peerDependenciesMeta: + '@75lb/nature': + optional: true + + eastasianwidth@0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + + electron-to-chromium@1.5.279: + resolution: {integrity: sha512-0bblUU5UNdOt5G7XqGiJtpZMONma6WAfq9vsFmtn9x1+joAObr6x1chfqyxFSDCAFwFhCQDrqeAr6MYdpwJ9Hg==} + + emittery@0.13.1: + resolution: {integrity: sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==} + engines: {node: '>=12'} + + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + + emoji-regex@9.2.2: + resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + + entities@4.5.0: + resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} + engines: {node: '>=0.12'} + + error-ex@1.3.4: + resolution: {integrity: sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es6-error@4.1.1: + resolution: {integrity: sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==} + + esbuild@0.27.2: + resolution: {integrity: sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==} + engines: {node: '>=18'} + hasBin: true + + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} + engines: {node: '>=6'} + + escape-string-regexp@2.0.0: + resolution: {integrity: sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==} + engines: {node: '>=8'} + + escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + + esprima@4.0.1: + resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} + engines: {node: '>=4'} + hasBin: true + + execa@5.1.1: + resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} + engines: {node: '>=10'} + + exit-x@0.2.2: + resolution: {integrity: sha512-+I6B/IkJc1o/2tiURyz/ivu/O0nKNEArIUB5O7zBrlDVJr22SCLH3xTeEry428LvFhRzIA1g8izguxJ/gbNcVQ==} + engines: {node: '>= 0.8.0'} + + expect@30.2.0: + resolution: {integrity: sha512-u/feCi0GPsI+988gU2FLcsHyAHTU0MX1Wg68NhAnN7z/+C5wqG+CY8J53N9ioe8RXgaoz0nBR/TYMf3AycUuPw==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + fast-glob@3.3.3: + resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} + engines: {node: '>=8.6.0'} + + fast-json-stable-stringify@2.1.0: + resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + + fastq@1.20.1: + resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} + + fb-watchman@2.0.2: + resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==} + + file-set@5.3.0: + resolution: {integrity: sha512-FKCxdjLX0J6zqTWdT0RXIxNF/n7MyXXnsSUp0syLEOCKdexvPZ02lNNv2a+gpK9E3hzUYF3+eFZe32ci7goNUg==} + engines: {node: '>=12.17'} + peerDependencies: + '@75lb/nature': latest + peerDependenciesMeta: + '@75lb/nature': + optional: true + + fill-range@7.1.1: + resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} + engines: {node: '>=8'} + + find-replace@5.0.2: + resolution: {integrity: sha512-Y45BAiE3mz2QsrN2fb5QEtO4qb44NcS7en/0y9PEVsg351HsLeVclP8QPMH79Le9sH3rs5RSwJu99W0WPZO43Q==} + engines: {node: '>=14'} + peerDependencies: + '@75lb/nature': latest + peerDependenciesMeta: + '@75lb/nature': + optional: true + + find-up@4.1.0: + resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} + engines: {node: '>=8'} + + flatbuffers@25.9.23: + resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==} + + foreground-child@3.3.1: + resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} + engines: {node: '>=14'} + + fs.realpath@1.0.0: + resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + gensync@1.0.0-beta.2: + resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} + engines: {node: '>=6.9.0'} + + get-caller-file@2.0.5: + resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} + engines: {node: 6.* || 8.* || >= 10.*} + + get-package-type@0.1.0: + resolution: {integrity: sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==} + engines: {node: '>=8.0.0'} + + get-stream@6.0.1: + resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} + engines: {node: '>=10'} + + glob-parent@5.1.2: + resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} + engines: {node: '>= 6'} + + glob@10.5.0: + resolution: {integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==} + hasBin: true + + glob@7.2.3: + resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} + deprecated: Glob versions prior to v9 are no longer supported + + global-agent@3.0.0: + resolution: {integrity: sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==} + engines: {node: '>=10.0'} + + globalthis@1.0.4: + resolution: {integrity: sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==} + engines: {node: '>= 0.4'} + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + + graceful-fs@4.2.11: + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + + guid-typescript@1.0.9: + resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==} + + handlebars@4.7.8: + resolution: {integrity: sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==} + engines: {node: '>=0.4.7'} + hasBin: true + + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + + has-property-descriptors@1.0.2: + resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==} + + html-escaper@2.0.2: + resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} + + human-signals@2.1.0: + resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} + engines: {node: '>=10.17.0'} + + import-local@3.2.0: + resolution: {integrity: sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==} + engines: {node: '>=8'} + hasBin: true + + imurmurhash@0.1.4: + resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} + engines: {node: '>=0.8.19'} + + inflight@1.0.6: + resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} + deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. + + inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + + is-arrayish@0.2.1: + resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} + + is-extglob@2.1.1: + resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} + engines: {node: '>=0.10.0'} + + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + + is-generator-fn@2.1.0: + resolution: {integrity: sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==} + engines: {node: '>=6'} + + is-glob@4.0.3: + resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} + engines: {node: '>=0.10.0'} + + is-number@7.0.0: + resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} + engines: {node: '>=0.12.0'} + + is-stream@2.0.1: + resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} + engines: {node: '>=8'} + + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + + istanbul-lib-coverage@3.2.2: + resolution: {integrity: sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==} + engines: {node: '>=8'} + + istanbul-lib-instrument@6.0.3: + resolution: {integrity: sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==} + engines: {node: '>=10'} + + istanbul-lib-report@3.0.1: + resolution: {integrity: sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==} + engines: {node: '>=10'} + + istanbul-lib-source-maps@5.0.6: + resolution: {integrity: sha512-yg2d+Em4KizZC5niWhQaIomgf5WlL4vOOjZ5xGCmF8SnPE/mDWWXgvRExdcpCgh9lLRRa1/fSYp2ymmbJ1pI+A==} + engines: {node: '>=10'} + + istanbul-reports@3.2.0: + resolution: {integrity: sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==} + engines: {node: '>=8'} + + jackspeak@3.4.3: + resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + + jest-changed-files@30.2.0: + resolution: {integrity: sha512-L8lR1ChrRnSdfeOvTrwZMlnWV8G/LLjQ0nG9MBclwWZidA2N5FviRki0Bvh20WRMOX31/JYvzdqTJrk5oBdydQ==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-circus@30.2.0: + resolution: {integrity: sha512-Fh0096NC3ZkFx05EP2OXCxJAREVxj1BcW/i6EWqqymcgYKWjyyDpral3fMxVcHXg6oZM7iULer9wGRFvfpl+Tg==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-cli@30.2.0: + resolution: {integrity: sha512-Os9ukIvADX/A9sLt6Zse3+nmHtHaE6hqOsjQtNiugFTbKRHYIYtZXNGNK9NChseXy7djFPjndX1tL0sCTlfpAA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + hasBin: true + peerDependencies: + node-notifier: ^8.0.1 || ^9.0.0 || ^10.0.0 + peerDependenciesMeta: + node-notifier: + optional: true + + jest-config@30.2.0: + resolution: {integrity: sha512-g4WkyzFQVWHtu6uqGmQR4CQxz/CH3yDSlhzXMWzNjDx843gYjReZnMRanjRCq5XZFuQrGDxgUaiYWE8BRfVckA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + peerDependencies: + '@types/node': '*' + esbuild-register: '>=3.4.0' + ts-node: '>=9.0.0' + peerDependenciesMeta: + '@types/node': + optional: true + esbuild-register: + optional: true + ts-node: + optional: true + + jest-diff@30.2.0: + resolution: {integrity: sha512-dQHFo3Pt4/NLlG5z4PxZ/3yZTZ1C7s9hveiOj+GCN+uT109NC2QgsoVZsVOAvbJ3RgKkvyLGXZV9+piDpWbm6A==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-docblock@30.2.0: + resolution: {integrity: sha512-tR/FFgZKS1CXluOQzZvNH3+0z9jXr3ldGSD8bhyuxvlVUwbeLOGynkunvlTMxchC5urrKndYiwCFC0DLVjpOCA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-each@30.2.0: + resolution: {integrity: sha512-lpWlJlM7bCUf1mfmuqTA8+j2lNURW9eNafOy99knBM01i5CQeY5UH1vZjgT9071nDJac1M4XsbyI44oNOdhlDQ==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-environment-node@30.2.0: + resolution: {integrity: sha512-ElU8v92QJ9UrYsKrxDIKCxu6PfNj4Hdcktcn0JX12zqNdqWHB0N+hwOnnBBXvjLd2vApZtuLUGs1QSY+MsXoNA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-haste-map@30.2.0: + resolution: {integrity: sha512-sQA/jCb9kNt+neM0anSj6eZhLZUIhQgwDt7cPGjumgLM4rXsfb9kpnlacmvZz3Q5tb80nS+oG/if+NBKrHC+Xw==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-leak-detector@30.2.0: + resolution: {integrity: sha512-M6jKAjyzjHG0SrQgwhgZGy9hFazcudwCNovY/9HPIicmNSBuockPSedAP9vlPK6ONFJ1zfyH/M2/YYJxOz5cdQ==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-matcher-utils@30.2.0: + resolution: {integrity: sha512-dQ94Nq4dbzmUWkQ0ANAWS9tBRfqCrn0bV9AMYdOi/MHW726xn7eQmMeRTpX2ViC00bpNaWXq+7o4lIQ3AX13Hg==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-message-util@30.2.0: + resolution: {integrity: sha512-y4DKFLZ2y6DxTWD4cDe07RglV88ZiNEdlRfGtqahfbIjfsw1nMCPx49Uev4IA/hWn3sDKyAnSPwoYSsAEdcimw==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-mock@30.2.0: + resolution: {integrity: sha512-JNNNl2rj4b5ICpmAcq+WbLH83XswjPbjH4T7yvGzfAGCPh1rw+xVNbtk+FnRslvt9lkCcdn9i1oAoKUuFsOxRw==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-pnp-resolver@1.2.3: + resolution: {integrity: sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==} + engines: {node: '>=6'} + peerDependencies: + jest-resolve: '*' + peerDependenciesMeta: + jest-resolve: + optional: true + + jest-regex-util@30.0.1: + resolution: {integrity: sha512-jHEQgBXAgc+Gh4g0p3bCevgRCVRkB4VB70zhoAE48gxeSr1hfUOsM/C2WoJgVL7Eyg//hudYENbm3Ne+/dRVVA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-resolve-dependencies@30.2.0: + resolution: {integrity: sha512-xTOIGug/0RmIe3mmCqCT95yO0vj6JURrn1TKWlNbhiAefJRWINNPgwVkrVgt/YaerPzY3iItufd80v3lOrFJ2w==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-resolve@30.2.0: + resolution: {integrity: sha512-TCrHSxPlx3tBY3hWNtRQKbtgLhsXa1WmbJEqBlTBrGafd5fiQFByy2GNCEoGR+Tns8d15GaL9cxEzKOO3GEb2A==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-runner@30.2.0: + resolution: {integrity: sha512-PqvZ2B2XEyPEbclp+gV6KO/F1FIFSbIwewRgmROCMBo/aZ6J1w8Qypoj2pEOcg3G2HzLlaP6VUtvwCI8dM3oqQ==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-runtime@30.2.0: + resolution: {integrity: sha512-p1+GVX/PJqTucvsmERPMgCPvQJpFt4hFbM+VN3n8TMo47decMUcJbt+rgzwrEme0MQUA/R+1de2axftTHkKckg==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-snapshot@30.2.0: + resolution: {integrity: sha512-5WEtTy2jXPFypadKNpbNkZ72puZCa6UjSr/7djeecHWOu7iYhSXSnHScT8wBz3Rn8Ena5d5RYRcsyKIeqG1IyA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-util@30.2.0: + resolution: {integrity: sha512-QKNsM0o3Xe6ISQU869e+DhG+4CK/48aHYdJZGlFQVTjnbvgpcKyxpzk29fGiO7i/J8VENZ+d2iGnSsvmuHywlA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-validate@30.2.0: + resolution: {integrity: sha512-FBGWi7dP2hpdi8nBoWxSsLvBFewKAg0+uSQwBaof4Y4DPgBabXgpSYC5/lR7VmnIlSpASmCi/ntRWPbv7089Pw==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-watcher@30.2.0: + resolution: {integrity: sha512-PYxa28dxJ9g777pGm/7PrbnMeA0Jr7osHP9bS7eJy9DuAjMgdGtxgf0uKMyoIsTWAkIbUW5hSDdJ3urmgXBqxg==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest-worker@30.2.0: + resolution: {integrity: sha512-0Q4Uk8WF7BUwqXHuAjc23vmopWJw5WH7w2tqBoUOZpOjW/ZnR44GXXd1r82RvnmI2GZge3ivrYXk/BE2+VtW2g==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + jest@30.2.0: + resolution: {integrity: sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + hasBin: true + peerDependencies: + node-notifier: ^8.0.1 || ^9.0.0 || ^10.0.0 + peerDependenciesMeta: + node-notifier: + optional: true + + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + + js-yaml@3.14.2: + resolution: {integrity: sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==} + hasBin: true + + js2xmlparser@4.0.2: + resolution: {integrity: sha512-6n4D8gLlLf1n5mNLQPRfViYzu9RATblzPEtm1SthMX1Pjao0r9YI9nw7ZIfRxQMERS87mcswrg+r/OYrPRX6jA==} + + jsdoc-api@9.3.5: + resolution: {integrity: sha512-TQwh1jA8xtCkIbVwm/XA3vDRAa5JjydyKx1cC413Sh3WohDFxcMdwKSvn4LOsq2xWyAmOU/VnSChTQf6EF0R8g==} + engines: {node: '>=12.17'} + peerDependencies: + '@75lb/nature': latest + peerDependenciesMeta: + '@75lb/nature': + optional: true + + jsdoc-parse@6.2.5: + resolution: {integrity: sha512-8JaSNjPLr2IuEY4Das1KM6Z4oLHZYUnjRrr27hKSa78Cj0i5Lur3DzNnCkz+DfrKBDoljGMoWOiBVQbtUZJBPw==} + engines: {node: '>=12'} + + jsdoc-to-markdown@9.1.3: + resolution: {integrity: sha512-i9wi+6WHX0WKziv0ar88T8h7OmxA0LWdQaV23nY6uQyKvdUPzVt0o6YAaOceFuKRF5Rvlju5w/KnZBfdpDAlnw==} + engines: {node: '>=12.17'} + hasBin: true + peerDependencies: + '@75lb/nature': latest + peerDependenciesMeta: + '@75lb/nature': + optional: true + + jsdoc@4.0.5: + resolution: {integrity: sha512-P4C6MWP9yIlMiK8nwoZvxN84vb6MsnXcHuy7XzVOvQoCizWX5JFCBsWIIWKXBltpoRZXddUOVQmCTOZt9yDj9g==} + engines: {node: '>=12.0.0'} + hasBin: true + + jsesc@3.1.0: + resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==} + engines: {node: '>=6'} + hasBin: true + + json-parse-even-better-errors@2.3.1: + resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} + + json-stringify-safe@5.0.1: + resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} + + json5@2.2.3: + resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} + engines: {node: '>=6'} + hasBin: true + + klaw@3.0.0: + resolution: {integrity: sha512-0Fo5oir+O9jnXu5EefYbVK+mHMBeEVEy2cmctR1O1NECcCkPRreJKrS6Qt/j3KC2C148Dfo9i3pCmCMsdqGr0g==} + + leven@3.1.0: + resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} + engines: {node: '>=6'} + + lines-and-columns@1.2.4: + resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + + linkify-it@5.0.0: + resolution: {integrity: sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==} + + locate-path@5.0.0: + resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==} + engines: {node: '>=8'} + + lodash.camelcase@4.3.0: + resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + + lodash@4.17.23: + resolution: {integrity: sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==} + + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + + lru-cache@5.1.1: + resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} + + make-dir@4.0.0: + resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} + engines: {node: '>=10'} + + makeerror@1.0.12: + resolution: {integrity: sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==} + + markdown-it-anchor@8.6.7: + resolution: {integrity: sha512-FlCHFwNnutLgVTflOYHPW2pPcl2AACqVzExlkGQNsi4CJgqOHN7YTgDd4LuhgN1BFO3TS0vLAruV1Td6dwWPJA==} + peerDependencies: + '@types/markdown-it': '*' + markdown-it: '*' + + markdown-it@14.1.0: + resolution: {integrity: sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==} + hasBin: true + + marked@4.3.0: + resolution: {integrity: sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==} + engines: {node: '>= 12'} + hasBin: true + + matcher@3.0.0: + resolution: {integrity: sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==} + engines: {node: '>=10'} + + mdurl@2.0.0: + resolution: {integrity: sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==} + + merge-stream@2.0.0: + resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + + merge2@1.4.1: + resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} + engines: {node: '>= 8'} + + micromatch@4.0.8: + resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} + engines: {node: '>=8.6'} + + mimic-fn@2.1.0: + resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} + engines: {node: '>=6'} + + minimatch@3.1.2: + resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} + + minimatch@9.0.5: + resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} + engines: {node: '>=16 || 14 >=14.17'} + + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + + minipass@7.1.2: + resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} + engines: {node: '>=16 || 14 >=14.17'} + + mkdirp@1.0.4: + resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} + engines: {node: '>=10'} + hasBin: true + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + napi-postinstall@0.3.4: + resolution: {integrity: sha512-PHI5f1O0EP5xJ9gQmFGMS6IZcrVvTjpXjz7Na41gTE7eE2hK11lg04CECCYEEjdc17EV4DO+fkGEtt7TpTaTiQ==} + engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} + hasBin: true + + natural-compare@1.4.0: + resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + + neo-async@2.6.2: + resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} + + node-int64@0.4.0: + resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} + + node-releases@2.0.27: + resolution: {integrity: sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==} + + normalize-path@3.0.0: + resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} + engines: {node: '>=0.10.0'} + + npm-run-path@4.0.1: + resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==} + engines: {node: '>=8'} + + object-keys@1.1.1: + resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} + engines: {node: '>= 0.4'} + + object-to-spawn-args@2.0.1: + resolution: {integrity: sha512-6FuKFQ39cOID+BMZ3QaphcC8Y4cw6LXBLyIgPU+OhIYwviJamPAn+4mITapnSBQrejB+NNp+FMskhD8Cq+Ys3w==} + engines: {node: '>=8.0.0'} + + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + + onetime@5.1.2: + resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} + engines: {node: '>=6'} + + onnxruntime-common@1.24.0-dev.20251116-b39e144322: + resolution: {integrity: sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==} + + onnxruntime-common@1.24.3: + resolution: {integrity: sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==} + + onnxruntime-node@1.24.3: + resolution: {integrity: sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==} + os: [win32, darwin, linux] + + onnxruntime-web@1.26.0-dev.20260416-b7804b056c: + resolution: {integrity: sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw==} + + p-limit@2.3.0: + resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} + engines: {node: '>=6'} + + p-limit@3.1.0: + resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} + engines: {node: '>=10'} + + p-locate@4.1.0: + resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} + engines: {node: '>=8'} + + p-try@2.2.0: + resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} + engines: {node: '>=6'} + + package-json-from-dist@1.0.1: + resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} + + parse-json@5.2.0: + resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} + engines: {node: '>=8'} + + path-exists@4.0.0: + resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} + engines: {node: '>=8'} + + path-is-absolute@1.0.1: + resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} + engines: {node: '>=0.10.0'} + + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + + path-scurry@1.11.1: + resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} + engines: {node: '>=16 || 14 >=14.18'} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@2.3.1: + resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} + engines: {node: '>=8.6'} + + picomatch@4.0.3: + resolution: {integrity: sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==} + engines: {node: '>=12'} + + pirates@4.0.7: + resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==} + engines: {node: '>= 6'} + + pkg-dir@4.2.0: + resolution: {integrity: sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==} + engines: {node: '>=8'} + + platform@1.3.6: + resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} + + prettier@3.8.1: + resolution: {integrity: sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==} + engines: {node: '>=14'} + hasBin: true + + pretty-format@30.2.0: + resolution: {integrity: sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + + protobufjs@7.5.4: + resolution: {integrity: sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==} + engines: {node: '>=12.0.0'} + + punycode.js@2.3.1: + resolution: {integrity: sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==} + engines: {node: '>=6'} + + pure-rand@7.0.1: + resolution: {integrity: sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ==} + + queue-microtask@1.2.3: + resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} + + react-is@18.3.1: + resolution: {integrity: sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==} + + require-directory@2.1.1: + resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} + engines: {node: '>=0.10.0'} + + requizzle@0.2.4: + resolution: {integrity: sha512-JRrFk1D4OQ4SqovXOgdav+K8EAhSB/LJZqCz8tbX0KObcdeM15Ss59ozWMBWmmINMagCwmqn4ZNryUGpBsl6Jw==} + + resolve-cwd@3.0.0: + resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==} + engines: {node: '>=8'} + + resolve-from@5.0.0: + resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} + engines: {node: '>=8'} + + reusify@1.1.0: + resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} + engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + + roarr@2.15.4: + resolution: {integrity: sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==} + engines: {node: '>=8.0'} + + run-parallel@1.2.0: + resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} + + semver-compare@1.0.0: + resolution: {integrity: sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==} + + semver@6.3.1: + resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} + hasBin: true + + semver@7.7.3: + resolution: {integrity: sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==} + engines: {node: '>=10'} + hasBin: true + + serialize-error@7.0.1: + resolution: {integrity: sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==} + engines: {node: '>=10'} + + sharp@0.34.5: + resolution: {integrity: sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + + signal-exit@3.0.7: + resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + + slash@3.0.0: + resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} + engines: {node: '>=8'} + + sort-array@5.1.1: + resolution: {integrity: sha512-EltS7AIsNlAFIM9cayrgKrM6XP94ATWwXP4LCL4IQbvbYhELSt2hZTrixg+AaQwnWFs/JGJgqU3rxMcNNWxGAA==} + engines: {node: '>=12.17'} + peerDependencies: + '@75lb/nature': ^0.1.1 + peerDependenciesMeta: + '@75lb/nature': + optional: true + + source-map-support@0.5.13: + resolution: {integrity: sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==} + + source-map@0.6.1: + resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} + engines: {node: '>=0.10.0'} + + sprintf-js@1.0.3: + resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + + sprintf-js@1.1.3: + resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==} + + stack-utils@2.0.6: + resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} + engines: {node: '>=10'} + + string-length@4.0.2: + resolution: {integrity: sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==} + engines: {node: '>=10'} + + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + + string-width@5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + + strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + + strip-ansi@7.1.2: + resolution: {integrity: sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==} + engines: {node: '>=12'} + + strip-bom@4.0.0: + resolution: {integrity: sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==} + engines: {node: '>=8'} + + strip-final-newline@2.0.0: + resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} + engines: {node: '>=6'} + + strip-json-comments@3.1.1: + resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} + engines: {node: '>=8'} + + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + + supports-color@8.1.1: + resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} + engines: {node: '>=10'} + + synckit@0.11.12: + resolution: {integrity: sha512-Bh7QjT8/SuKUIfObSXNHNSK6WHo6J1tHCqJsuaFDP7gP0fkzSfTxI8y85JrppZ0h8l0maIgc2tfuZQ6/t3GtnQ==} + engines: {node: ^14.18.0 || >=16.0.0} + + table-layout@4.1.1: + resolution: {integrity: sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==} + engines: {node: '>=12.17'} + + test-exclude@6.0.0: + resolution: {integrity: sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==} + engines: {node: '>=8'} + + tmpl@1.0.5: + resolution: {integrity: sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==} + + to-regex-range@5.0.1: + resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} + engines: {node: '>=8.0'} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + type-detect@4.0.8: + resolution: {integrity: sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==} + engines: {node: '>=4'} + + type-fest@0.13.1: + resolution: {integrity: sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==} + engines: {node: '>=10'} + + type-fest@0.21.3: + resolution: {integrity: sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==} + engines: {node: '>=10'} + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + typical@7.3.0: + resolution: {integrity: sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==} + engines: {node: '>=12.17'} + + uc.micro@2.1.0: + resolution: {integrity: sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==} + + uglify-js@3.19.3: + resolution: {integrity: sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==} + engines: {node: '>=0.8.0'} + hasBin: true + + underscore@1.13.7: + resolution: {integrity: sha512-GMXzWtsc57XAtguZgaQViUOzs0KTkk8ojr3/xAxXLITqf/3EMwxC0inyETfDFjH/Krbhuep0HNbbjI9i/q3F3g==} + + undici-types@7.16.0: + resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} + + unrs-resolver@1.11.1: + resolution: {integrity: sha512-bSjt9pjaEBnNiGgc9rUiHGKv5l4/TGzDmYw3RhnkJGtLhbnnA/5qJj7x3dNDCRx/PJxu774LlH8lCOlB4hEfKg==} + + update-browserslist-db@1.2.3: + resolution: {integrity: sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==} + hasBin: true + peerDependencies: + browserslist: '>= 4.21.0' + + v8-to-istanbul@9.3.0: + resolution: {integrity: sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==} + engines: {node: '>=10.12.0'} + + walk-back@2.0.1: + resolution: {integrity: sha512-Nb6GvBR8UWX1D+Le+xUq0+Q1kFmRBIWVrfLnQAOmcpEzA9oAxwJ9gIr36t9TWYfzvWRvuMtjHiVsJYEkXWaTAQ==} + engines: {node: '>=0.10.0'} + + walk-back@5.1.1: + resolution: {integrity: sha512-e/FRLDVdZQWFrAzU6Hdvpm7D7m2ina833gIKLptQykRK49mmCYHLHq7UqjPDbxbKLZkTkW1rFqbengdE3sLfdw==} + engines: {node: '>=12.17'} + + walker@1.0.8: + resolution: {integrity: sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==} + + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + + wordwrap@1.0.0: + resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} + + wordwrapjs@5.1.1: + resolution: {integrity: sha512-0yweIbkINJodk27gX9LBGMzyQdBDan3s/dEAiwBOj+Mf0PPyWL6/rikalkv8EeD0E8jm4o5RXEOrFTP3NXbhJg==} + engines: {node: '>=12.17'} + + wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + + wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + + write-file-atomic@5.0.1: + resolution: {integrity: sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==} + engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + + xmlcreate@2.0.4: + resolution: {integrity: sha512-nquOebG4sngPmGPICTS5EnxqhKbCmz5Ox5hsszI2T6U5qdrJizBc+0ilYSEjTSzU0yZcmvppztXe/5Al5fUwdg==} + + y18n@5.0.8: + resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} + engines: {node: '>=10'} + + yallist@3.1.1: + resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + + yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + + yargs@17.7.2: + resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} + engines: {node: '>=12'} + + yocto-queue@0.1.0: + resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} + engines: {node: '>=10'} + +snapshots: + + '@babel/code-frame@7.28.6': + dependencies: + '@babel/helper-validator-identifier': 7.28.5 + js-tokens: 4.0.0 + picocolors: 1.1.1 + + '@babel/compat-data@7.28.6': {} + + '@babel/core@7.28.6': + dependencies: + '@babel/code-frame': 7.28.6 + '@babel/generator': 7.28.6 + '@babel/helper-compilation-targets': 7.28.6 + '@babel/helper-module-transforms': 7.28.6(@babel/core@7.28.6) + '@babel/helpers': 7.28.6 + '@babel/parser': 7.28.6 + '@babel/template': 7.28.6 + '@babel/traverse': 7.28.6 + '@babel/types': 7.28.6 + '@jridgewell/remapping': 2.3.5 + convert-source-map: 2.0.0 + debug: 4.4.3 + gensync: 1.0.0-beta.2 + json5: 2.2.3 + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + + '@babel/generator@7.28.6': + dependencies: + '@babel/parser': 7.28.6 + '@babel/types': 7.28.6 + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + jsesc: 3.1.0 + + '@babel/helper-compilation-targets@7.28.6': + dependencies: + '@babel/compat-data': 7.28.6 + '@babel/helper-validator-option': 7.27.1 + browserslist: 4.28.1 + lru-cache: 5.1.1 + semver: 6.3.1 + + '@babel/helper-globals@7.28.0': {} + + '@babel/helper-module-imports@7.28.6': + dependencies: + '@babel/traverse': 7.28.6 + '@babel/types': 7.28.6 + transitivePeerDependencies: + - supports-color + + '@babel/helper-module-transforms@7.28.6(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-module-imports': 7.28.6 + '@babel/helper-validator-identifier': 7.28.5 + '@babel/traverse': 7.28.6 + transitivePeerDependencies: + - supports-color + + '@babel/helper-plugin-utils@7.28.6': {} + + '@babel/helper-string-parser@7.27.1': {} + + '@babel/helper-validator-identifier@7.28.5': {} + + '@babel/helper-validator-option@7.27.1': {} + + '@babel/helpers@7.28.6': + dependencies: + '@babel/template': 7.28.6 + '@babel/types': 7.28.6 + + '@babel/parser@7.28.6': + dependencies: + '@babel/types': 7.28.6 + + '@babel/plugin-syntax-async-generators@7.8.4(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-bigint@7.8.3(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-class-properties@7.12.13(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-class-static-block@7.14.5(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-import-attributes@7.28.6(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-import-meta@7.10.4(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-json-strings@7.8.3(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-jsx@7.28.6(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-logical-assignment-operators@7.10.4(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-nullish-coalescing-operator@7.8.3(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-numeric-separator@7.10.4(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-object-rest-spread@7.8.3(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-optional-catch-binding@7.8.3(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-optional-chaining@7.8.3(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-private-property-in-object@7.14.5(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-top-level-await@7.14.5(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-syntax-typescript@7.28.6(@babel/core@7.28.6)': + dependencies: + '@babel/core': 7.28.6 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/template@7.28.6': + dependencies: + '@babel/code-frame': 7.28.6 + '@babel/parser': 7.28.6 + '@babel/types': 7.28.6 + + '@babel/traverse@7.28.6': + dependencies: + '@babel/code-frame': 7.28.6 + '@babel/generator': 7.28.6 + '@babel/helper-globals': 7.28.0 + '@babel/parser': 7.28.6 + '@babel/template': 7.28.6 + '@babel/types': 7.28.6 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + '@babel/types@7.28.6': + dependencies: + '@babel/helper-string-parser': 7.27.1 + '@babel/helper-validator-identifier': 7.28.5 + + '@bcoe/v8-coverage@0.2.3': {} + + '@emnapi/core@1.8.1': + dependencies: + '@emnapi/wasi-threads': 1.1.0 + tslib: 2.8.1 + optional: true + + '@emnapi/runtime@1.8.1': + dependencies: + tslib: 2.8.1 + optional: true + + '@emnapi/wasi-threads@1.1.0': + dependencies: + tslib: 2.8.1 + optional: true + + '@esbuild/aix-ppc64@0.27.2': + optional: true + + '@esbuild/android-arm64@0.27.2': + optional: true + + '@esbuild/android-arm@0.27.2': + optional: true + + '@esbuild/android-x64@0.27.2': + optional: true + + '@esbuild/darwin-arm64@0.27.2': + optional: true + + '@esbuild/darwin-x64@0.27.2': + optional: true + + '@esbuild/freebsd-arm64@0.27.2': + optional: true + + '@esbuild/freebsd-x64@0.27.2': + optional: true + + '@esbuild/linux-arm64@0.27.2': + optional: true + + '@esbuild/linux-arm@0.27.2': + optional: true + + '@esbuild/linux-ia32@0.27.2': + optional: true + + '@esbuild/linux-loong64@0.27.2': + optional: true + + '@esbuild/linux-mips64el@0.27.2': + optional: true + + '@esbuild/linux-ppc64@0.27.2': + optional: true + + '@esbuild/linux-riscv64@0.27.2': + optional: true + + '@esbuild/linux-s390x@0.27.2': + optional: true + + '@esbuild/linux-x64@0.27.2': + optional: true + + '@esbuild/netbsd-arm64@0.27.2': + optional: true + + '@esbuild/netbsd-x64@0.27.2': + optional: true + + '@esbuild/openbsd-arm64@0.27.2': + optional: true + + '@esbuild/openbsd-x64@0.27.2': + optional: true + + '@esbuild/openharmony-arm64@0.27.2': + optional: true + + '@esbuild/sunos-x64@0.27.2': + optional: true + + '@esbuild/win32-arm64@0.27.2': + optional: true + + '@esbuild/win32-ia32@0.27.2': + optional: true + + '@esbuild/win32-x64@0.27.2': + optional: true + + '@huggingface/jinja@0.5.6': {} + + '@huggingface/tokenizers@0.1.3': {} + + '@img/colour@1.0.0': {} + + '@img/sharp-darwin-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.2.4 + optional: true + + '@img/sharp-darwin-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.2.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-darwin-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm@1.2.4': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-riscv64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-s390x@1.2.4': + optional: true + + '@img/sharp-libvips-linux-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + optional: true + + '@img/sharp-linux-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.2.4 + optional: true + + '@img/sharp-linux-arm@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.2.4 + optional: true + + '@img/sharp-linux-ppc64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-ppc64': 1.2.4 + optional: true + + '@img/sharp-linux-riscv64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-riscv64': 1.2.4 + optional: true + + '@img/sharp-linux-s390x@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.2.4 + optional: true + + '@img/sharp-linux-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + optional: true + + '@img/sharp-wasm32@0.34.5': + dependencies: + '@emnapi/runtime': 1.8.1 + optional: true + + '@img/sharp-win32-arm64@0.34.5': + optional: true + + '@img/sharp-win32-ia32@0.34.5': + optional: true + + '@img/sharp-win32-x64@0.34.5': + optional: true + + '@isaacs/cliui@8.0.2': + dependencies: + string-width: 5.1.2 + string-width-cjs: string-width@4.2.3 + strip-ansi: 7.1.2 + strip-ansi-cjs: strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: wrap-ansi@7.0.0 + + '@istanbuljs/load-nyc-config@1.1.0': + dependencies: + camelcase: 5.3.1 + find-up: 4.1.0 + get-package-type: 0.1.0 + js-yaml: 3.14.2 + resolve-from: 5.0.0 + + '@istanbuljs/schema@0.1.3': {} + + '@jest/console@30.2.0': + dependencies: + '@jest/types': 30.2.0 + '@types/node': 24.10.9 + chalk: 4.1.2 + jest-message-util: 30.2.0 + jest-util: 30.2.0 + slash: 3.0.0 + + '@jest/core@30.2.0': + dependencies: + '@jest/console': 30.2.0 + '@jest/pattern': 30.0.1 + '@jest/reporters': 30.2.0 + '@jest/test-result': 30.2.0 + '@jest/transform': 30.2.0 + '@jest/types': 30.2.0 + '@types/node': 24.10.9 + ansi-escapes: 4.3.2 + chalk: 4.1.2 + ci-info: 4.3.1 + exit-x: 0.2.2 + graceful-fs: 4.2.11 + jest-changed-files: 30.2.0 + jest-config: 30.2.0(@types/node@24.10.9) + jest-haste-map: 30.2.0 + jest-message-util: 30.2.0 + jest-regex-util: 30.0.1 + jest-resolve: 30.2.0 + jest-resolve-dependencies: 30.2.0 + jest-runner: 30.2.0 + jest-runtime: 30.2.0 + jest-snapshot: 30.2.0 + jest-util: 30.2.0 + jest-validate: 30.2.0 + jest-watcher: 30.2.0 + micromatch: 4.0.8 + pretty-format: 30.2.0 + slash: 3.0.0 + transitivePeerDependencies: + - babel-plugin-macros + - esbuild-register + - supports-color + - ts-node + + '@jest/diff-sequences@30.0.1': {} + + '@jest/environment@30.2.0': + dependencies: + '@jest/fake-timers': 30.2.0 + '@jest/types': 30.2.0 + '@types/node': 24.10.9 + jest-mock: 30.2.0 + + '@jest/expect-utils@30.2.0': + dependencies: + '@jest/get-type': 30.1.0 + + '@jest/expect@30.2.0': + dependencies: + expect: 30.2.0 + jest-snapshot: 30.2.0 + transitivePeerDependencies: + - supports-color + + '@jest/fake-timers@30.2.0': + dependencies: + '@jest/types': 30.2.0 + '@sinonjs/fake-timers': 13.0.5 + '@types/node': 24.10.9 + jest-message-util: 30.2.0 + jest-mock: 30.2.0 + jest-util: 30.2.0 + + '@jest/get-type@30.1.0': {} + + '@jest/globals@30.2.0': + dependencies: + '@jest/environment': 30.2.0 + '@jest/expect': 30.2.0 + '@jest/types': 30.2.0 + jest-mock: 30.2.0 + transitivePeerDependencies: + - supports-color + + '@jest/pattern@30.0.1': + dependencies: + '@types/node': 24.10.9 + jest-regex-util: 30.0.1 + + '@jest/reporters@30.2.0': + dependencies: + '@bcoe/v8-coverage': 0.2.3 + '@jest/console': 30.2.0 + '@jest/test-result': 30.2.0 + '@jest/transform': 30.2.0 + '@jest/types': 30.2.0 + '@jridgewell/trace-mapping': 0.3.31 + '@types/node': 24.10.9 + chalk: 4.1.2 + collect-v8-coverage: 1.0.3 + exit-x: 0.2.2 + glob: 10.5.0 + graceful-fs: 4.2.11 + istanbul-lib-coverage: 3.2.2 + istanbul-lib-instrument: 6.0.3 + istanbul-lib-report: 3.0.1 + istanbul-lib-source-maps: 5.0.6 + istanbul-reports: 3.2.0 + jest-message-util: 30.2.0 + jest-util: 30.2.0 + jest-worker: 30.2.0 + slash: 3.0.0 + string-length: 4.0.2 + v8-to-istanbul: 9.3.0 + transitivePeerDependencies: + - supports-color + + '@jest/schemas@30.0.5': + dependencies: + '@sinclair/typebox': 0.34.48 + + '@jest/snapshot-utils@30.2.0': + dependencies: + '@jest/types': 30.2.0 + chalk: 4.1.2 + graceful-fs: 4.2.11 + natural-compare: 1.4.0 + + '@jest/source-map@30.0.1': + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + callsites: 3.1.0 + graceful-fs: 4.2.11 + + '@jest/test-result@30.2.0': + dependencies: + '@jest/console': 30.2.0 + '@jest/types': 30.2.0 + '@types/istanbul-lib-coverage': 2.0.6 + collect-v8-coverage: 1.0.3 + + '@jest/test-sequencer@30.2.0': + dependencies: + '@jest/test-result': 30.2.0 + graceful-fs: 4.2.11 + jest-haste-map: 30.2.0 + slash: 3.0.0 + + '@jest/transform@30.2.0': + dependencies: + '@babel/core': 7.28.6 + '@jest/types': 30.2.0 + '@jridgewell/trace-mapping': 0.3.31 + babel-plugin-istanbul: 7.0.1 + chalk: 4.1.2 + convert-source-map: 2.0.0 + fast-json-stable-stringify: 2.1.0 + graceful-fs: 4.2.11 + jest-haste-map: 30.2.0 + jest-regex-util: 30.0.1 + jest-util: 30.2.0 + micromatch: 4.0.8 + pirates: 4.0.7 + slash: 3.0.0 + write-file-atomic: 5.0.1 + transitivePeerDependencies: + - supports-color + + '@jest/types@30.2.0': + dependencies: + '@jest/pattern': 30.0.1 + '@jest/schemas': 30.0.5 + '@types/istanbul-lib-coverage': 2.0.6 + '@types/istanbul-reports': 3.0.4 + '@types/node': 24.10.9 + '@types/yargs': 17.0.35 + chalk: 4.1.2 + + '@jridgewell/gen-mapping@0.3.13': + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/remapping@2.3.5': + dependencies: + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@jridgewell/trace-mapping@0.3.31': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + + '@jsdoc/salty@0.2.9': + dependencies: + lodash: 4.17.23 + + '@napi-rs/wasm-runtime@0.2.12': + dependencies: + '@emnapi/core': 1.8.1 + '@emnapi/runtime': 1.8.1 + '@tybys/wasm-util': 0.10.1 + optional: true + + '@nodelib/fs.scandir@2.1.5': + dependencies: + '@nodelib/fs.stat': 2.0.5 + run-parallel: 1.2.0 + + '@nodelib/fs.stat@2.0.5': {} + + '@nodelib/fs.walk@1.2.8': + dependencies: + '@nodelib/fs.scandir': 2.1.5 + fastq: 1.20.1 + + '@pkgjs/parseargs@0.11.0': + optional: true + + '@pkgr/core@0.2.9': {} + + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.4': {} + + '@protobufjs/eventemitter@1.1.0': {} + + '@protobufjs/fetch@1.1.0': + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/inquire': 1.1.0 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/inquire@1.1.0': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.0': {} + + '@sinclair/typebox@0.34.48': {} + + '@sinonjs/commons@3.0.1': + dependencies: + type-detect: 4.0.8 + + '@sinonjs/fake-timers@13.0.5': + dependencies: + '@sinonjs/commons': 3.0.1 + + '@tybys/wasm-util@0.10.1': + dependencies: + tslib: 2.8.1 + optional: true + + '@types/babel__core@7.20.5': + dependencies: + '@babel/parser': 7.28.6 + '@babel/types': 7.28.6 + '@types/babel__generator': 7.27.0 + '@types/babel__template': 7.4.4 + '@types/babel__traverse': 7.28.0 + + '@types/babel__generator@7.27.0': + dependencies: + '@babel/types': 7.28.6 + + '@types/babel__template@7.4.4': + dependencies: + '@babel/parser': 7.28.6 + '@babel/types': 7.28.6 + + '@types/babel__traverse@7.28.0': + dependencies: + '@babel/types': 7.28.6 + + '@types/istanbul-lib-coverage@2.0.6': {} + + '@types/istanbul-lib-report@3.0.3': + dependencies: + '@types/istanbul-lib-coverage': 2.0.6 + + '@types/istanbul-reports@3.0.4': + dependencies: + '@types/istanbul-lib-report': 3.0.3 + + '@types/jest@30.0.0': + dependencies: + expect: 30.2.0 + pretty-format: 30.2.0 + + '@types/linkify-it@5.0.0': {} + + '@types/markdown-it@14.1.2': + dependencies: + '@types/linkify-it': 5.0.0 + '@types/mdurl': 2.0.0 + + '@types/mdurl@2.0.0': {} + + '@types/node@24.10.9': + dependencies: + undici-types: 7.16.0 + + '@types/stack-utils@2.0.3': {} + + '@types/yargs-parser@21.0.3': {} + + '@types/yargs@17.0.35': + dependencies: + '@types/yargs-parser': 21.0.3 + + '@ungap/structured-clone@1.3.0': {} + + '@unrs/resolver-binding-android-arm-eabi@1.11.1': + optional: true + + '@unrs/resolver-binding-android-arm64@1.11.1': + optional: true + + '@unrs/resolver-binding-darwin-arm64@1.11.1': + optional: true + + '@unrs/resolver-binding-darwin-x64@1.11.1': + optional: true + + '@unrs/resolver-binding-freebsd-x64@1.11.1': + optional: true + + '@unrs/resolver-binding-linux-arm-gnueabihf@1.11.1': + optional: true + + '@unrs/resolver-binding-linux-arm-musleabihf@1.11.1': + optional: true + + '@unrs/resolver-binding-linux-arm64-gnu@1.11.1': + optional: true + + '@unrs/resolver-binding-linux-arm64-musl@1.11.1': + optional: true + + '@unrs/resolver-binding-linux-ppc64-gnu@1.11.1': + optional: true + + '@unrs/resolver-binding-linux-riscv64-gnu@1.11.1': + optional: true + + '@unrs/resolver-binding-linux-riscv64-musl@1.11.1': + optional: true + + '@unrs/resolver-binding-linux-s390x-gnu@1.11.1': + optional: true + + '@unrs/resolver-binding-linux-x64-gnu@1.11.1': + optional: true + + '@unrs/resolver-binding-linux-x64-musl@1.11.1': + optional: true + + '@unrs/resolver-binding-wasm32-wasi@1.11.1': + dependencies: + '@napi-rs/wasm-runtime': 0.2.12 + optional: true + + '@unrs/resolver-binding-win32-arm64-msvc@1.11.1': + optional: true + + '@unrs/resolver-binding-win32-ia32-msvc@1.11.1': + optional: true + + '@unrs/resolver-binding-win32-x64-msvc@1.11.1': + optional: true + + '@webgpu/types@0.1.69': {} + + adm-zip@0.5.16: {} + + ansi-escapes@4.3.2: + dependencies: + type-fest: 0.21.3 + + ansi-regex@5.0.1: {} + + ansi-regex@6.2.2: {} + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + ansi-styles@5.2.0: {} + + ansi-styles@6.2.3: {} + + anymatch@3.1.3: + dependencies: + normalize-path: 3.0.0 + picomatch: 2.3.1 + + argparse@1.0.10: + dependencies: + sprintf-js: 1.0.3 + + argparse@2.0.1: {} + + array-back@6.2.2: {} + + babel-jest@30.2.0(@babel/core@7.28.6): + dependencies: + '@babel/core': 7.28.6 + '@jest/transform': 30.2.0 + '@types/babel__core': 7.20.5 + babel-plugin-istanbul: 7.0.1 + babel-preset-jest: 30.2.0(@babel/core@7.28.6) + chalk: 4.1.2 + graceful-fs: 4.2.11 + slash: 3.0.0 + transitivePeerDependencies: + - supports-color + + babel-plugin-istanbul@7.0.1: + dependencies: + '@babel/helper-plugin-utils': 7.28.6 + '@istanbuljs/load-nyc-config': 1.1.0 + '@istanbuljs/schema': 0.1.3 + istanbul-lib-instrument: 6.0.3 + test-exclude: 6.0.0 + transitivePeerDependencies: + - supports-color + + babel-plugin-jest-hoist@30.2.0: + dependencies: + '@types/babel__core': 7.20.5 + + babel-preset-current-node-syntax@1.2.0(@babel/core@7.28.6): + dependencies: + '@babel/core': 7.28.6 + '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.28.6) + '@babel/plugin-syntax-bigint': 7.8.3(@babel/core@7.28.6) + '@babel/plugin-syntax-class-properties': 7.12.13(@babel/core@7.28.6) + '@babel/plugin-syntax-class-static-block': 7.14.5(@babel/core@7.28.6) + '@babel/plugin-syntax-import-attributes': 7.28.6(@babel/core@7.28.6) + '@babel/plugin-syntax-import-meta': 7.10.4(@babel/core@7.28.6) + '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.28.6) + '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.28.6) + '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.28.6) + '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.28.6) + '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.28.6) + '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.28.6) + '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.28.6) + '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.28.6) + '@babel/plugin-syntax-top-level-await': 7.14.5(@babel/core@7.28.6) + + babel-preset-jest@30.2.0(@babel/core@7.28.6): + dependencies: + '@babel/core': 7.28.6 + babel-plugin-jest-hoist: 30.2.0 + babel-preset-current-node-syntax: 1.2.0(@babel/core@7.28.6) + + balanced-match@1.0.2: {} + + baseline-browser-mapping@2.9.18: {} + + bluebird@3.7.2: {} + + boolean@3.2.0: {} + + brace-expansion@1.1.12: + dependencies: + balanced-match: 1.0.2 + concat-map: 0.0.1 + + brace-expansion@2.0.2: + dependencies: + balanced-match: 1.0.2 + + braces@3.0.3: + dependencies: + fill-range: 7.1.1 + + browserslist@4.28.1: + dependencies: + baseline-browser-mapping: 2.9.18 + caniuse-lite: 1.0.30001766 + electron-to-chromium: 1.5.279 + node-releases: 2.0.27 + update-browserslist-db: 1.2.3(browserslist@4.28.1) + + bser@2.1.1: + dependencies: + node-int64: 0.4.0 + + buffer-from@1.1.2: {} + + cache-point@3.0.1: + dependencies: + array-back: 6.2.2 + + callsites@3.1.0: {} + + camelcase@5.3.1: {} + + camelcase@6.3.0: {} + + caniuse-lite@1.0.30001766: {} + + catharsis@0.9.0: + dependencies: + lodash: 4.17.23 + + chalk-template@0.4.0: + dependencies: + chalk: 4.1.2 + + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + char-regex@1.0.2: {} + + ci-info@4.3.1: {} + + cjs-module-lexer@2.2.0: {} + + cliui@8.0.1: + dependencies: + string-width: 4.2.3 + strip-ansi: 6.0.1 + wrap-ansi: 7.0.0 + + co@4.6.0: {} + + collect-v8-coverage@1.0.3: {} + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + command-line-args@6.0.1: + dependencies: + array-back: 6.2.2 + find-replace: 5.0.2 + lodash.camelcase: 4.3.0 + typical: 7.3.0 + + command-line-usage@7.0.3: + dependencies: + array-back: 6.2.2 + chalk-template: 0.4.0 + table-layout: 4.1.1 + typical: 7.3.0 + + common-sequence@3.0.0: {} + + concat-map@0.0.1: {} + + config-master@3.1.0: + dependencies: + walk-back: 2.0.1 + + convert-source-map@2.0.0: {} + + cross-spawn@7.0.6: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + + current-module-paths@1.1.3: {} + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + dedent@1.7.1: {} + + deepmerge@4.3.1: {} + + define-data-property@1.1.4: + dependencies: + es-define-property: 1.0.1 + es-errors: 1.3.0 + gopd: 1.2.0 + + define-properties@1.2.1: + dependencies: + define-data-property: 1.1.4 + has-property-descriptors: 1.0.2 + object-keys: 1.1.1 + + detect-libc@2.1.2: {} + + detect-newline@3.1.0: {} + + detect-node@2.1.0: {} + + dmd@7.1.1: + dependencies: + array-back: 6.2.2 + cache-point: 3.0.1 + common-sequence: 3.0.0 + file-set: 5.3.0 + handlebars: 4.7.8 + marked: 4.3.0 + walk-back: 5.1.1 + + eastasianwidth@0.2.0: {} + + electron-to-chromium@1.5.279: {} + + emittery@0.13.1: {} + + emoji-regex@8.0.0: {} + + emoji-regex@9.2.2: {} + + entities@4.5.0: {} + + error-ex@1.3.4: + dependencies: + is-arrayish: 0.2.1 + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es6-error@4.1.1: {} + + esbuild@0.27.2: + optionalDependencies: + '@esbuild/aix-ppc64': 0.27.2 + '@esbuild/android-arm': 0.27.2 + '@esbuild/android-arm64': 0.27.2 + '@esbuild/android-x64': 0.27.2 + '@esbuild/darwin-arm64': 0.27.2 + '@esbuild/darwin-x64': 0.27.2 + '@esbuild/freebsd-arm64': 0.27.2 + '@esbuild/freebsd-x64': 0.27.2 + '@esbuild/linux-arm': 0.27.2 + '@esbuild/linux-arm64': 0.27.2 + '@esbuild/linux-ia32': 0.27.2 + '@esbuild/linux-loong64': 0.27.2 + '@esbuild/linux-mips64el': 0.27.2 + '@esbuild/linux-ppc64': 0.27.2 + '@esbuild/linux-riscv64': 0.27.2 + '@esbuild/linux-s390x': 0.27.2 + '@esbuild/linux-x64': 0.27.2 + '@esbuild/netbsd-arm64': 0.27.2 + '@esbuild/netbsd-x64': 0.27.2 + '@esbuild/openbsd-arm64': 0.27.2 + '@esbuild/openbsd-x64': 0.27.2 + '@esbuild/openharmony-arm64': 0.27.2 + '@esbuild/sunos-x64': 0.27.2 + '@esbuild/win32-arm64': 0.27.2 + '@esbuild/win32-ia32': 0.27.2 + '@esbuild/win32-x64': 0.27.2 + + escalade@3.2.0: {} + + escape-string-regexp@2.0.0: {} + + escape-string-regexp@4.0.0: {} + + esprima@4.0.1: {} + + execa@5.1.1: + dependencies: + cross-spawn: 7.0.6 + get-stream: 6.0.1 + human-signals: 2.1.0 + is-stream: 2.0.1 + merge-stream: 2.0.0 + npm-run-path: 4.0.1 + onetime: 5.1.2 + signal-exit: 3.0.7 + strip-final-newline: 2.0.0 + + exit-x@0.2.2: {} + + expect@30.2.0: + dependencies: + '@jest/expect-utils': 30.2.0 + '@jest/get-type': 30.1.0 + jest-matcher-utils: 30.2.0 + jest-message-util: 30.2.0 + jest-mock: 30.2.0 + jest-util: 30.2.0 + + fast-glob@3.3.3: + dependencies: + '@nodelib/fs.stat': 2.0.5 + '@nodelib/fs.walk': 1.2.8 + glob-parent: 5.1.2 + merge2: 1.4.1 + micromatch: 4.0.8 + + fast-json-stable-stringify@2.1.0: {} + + fastq@1.20.1: + dependencies: + reusify: 1.1.0 + + fb-watchman@2.0.2: + dependencies: + bser: 2.1.1 + + file-set@5.3.0: + dependencies: + array-back: 6.2.2 + fast-glob: 3.3.3 + + fill-range@7.1.1: + dependencies: + to-regex-range: 5.0.1 + + find-replace@5.0.2: {} + + find-up@4.1.0: + dependencies: + locate-path: 5.0.0 + path-exists: 4.0.0 + + flatbuffers@25.9.23: {} + + foreground-child@3.3.1: + dependencies: + cross-spawn: 7.0.6 + signal-exit: 4.1.0 + + fs.realpath@1.0.0: {} + + fsevents@2.3.3: + optional: true + + gensync@1.0.0-beta.2: {} + + get-caller-file@2.0.5: {} + + get-package-type@0.1.0: {} + + get-stream@6.0.1: {} + + glob-parent@5.1.2: + dependencies: + is-glob: 4.0.3 + + glob@10.5.0: + dependencies: + foreground-child: 3.3.1 + jackspeak: 3.4.3 + minimatch: 9.0.5 + minipass: 7.1.2 + package-json-from-dist: 1.0.1 + path-scurry: 1.11.1 + + glob@7.2.3: + dependencies: + fs.realpath: 1.0.0 + inflight: 1.0.6 + inherits: 2.0.4 + minimatch: 3.1.2 + once: 1.4.0 + path-is-absolute: 1.0.1 + + global-agent@3.0.0: + dependencies: + boolean: 3.2.0 + es6-error: 4.1.1 + matcher: 3.0.0 + roarr: 2.15.4 + semver: 7.7.3 + serialize-error: 7.0.1 + + globalthis@1.0.4: + dependencies: + define-properties: 1.2.1 + gopd: 1.2.0 + + gopd@1.2.0: {} + + graceful-fs@4.2.11: {} + + guid-typescript@1.0.9: {} + + handlebars@4.7.8: + dependencies: + minimist: 1.2.8 + neo-async: 2.6.2 + source-map: 0.6.1 + wordwrap: 1.0.0 + optionalDependencies: + uglify-js: 3.19.3 + + has-flag@4.0.0: {} + + has-property-descriptors@1.0.2: + dependencies: + es-define-property: 1.0.1 + + html-escaper@2.0.2: {} + + human-signals@2.1.0: {} + + import-local@3.2.0: + dependencies: + pkg-dir: 4.2.0 + resolve-cwd: 3.0.0 + + imurmurhash@0.1.4: {} + + inflight@1.0.6: + dependencies: + once: 1.4.0 + wrappy: 1.0.2 + + inherits@2.0.4: {} + + is-arrayish@0.2.1: {} + + is-extglob@2.1.1: {} + + is-fullwidth-code-point@3.0.0: {} + + is-generator-fn@2.1.0: {} + + is-glob@4.0.3: + dependencies: + is-extglob: 2.1.1 + + is-number@7.0.0: {} + + is-stream@2.0.1: {} + + isexe@2.0.0: {} + + istanbul-lib-coverage@3.2.2: {} + + istanbul-lib-instrument@6.0.3: + dependencies: + '@babel/core': 7.28.6 + '@babel/parser': 7.28.6 + '@istanbuljs/schema': 0.1.3 + istanbul-lib-coverage: 3.2.2 + semver: 7.7.3 + transitivePeerDependencies: + - supports-color + + istanbul-lib-report@3.0.1: + dependencies: + istanbul-lib-coverage: 3.2.2 + make-dir: 4.0.0 + supports-color: 7.2.0 + + istanbul-lib-source-maps@5.0.6: + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + debug: 4.4.3 + istanbul-lib-coverage: 3.2.2 + transitivePeerDependencies: + - supports-color + + istanbul-reports@3.2.0: + dependencies: + html-escaper: 2.0.2 + istanbul-lib-report: 3.0.1 + + jackspeak@3.4.3: + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + + jest-changed-files@30.2.0: + dependencies: + execa: 5.1.1 + jest-util: 30.2.0 + p-limit: 3.1.0 + + jest-circus@30.2.0: + dependencies: + '@jest/environment': 30.2.0 + '@jest/expect': 30.2.0 + '@jest/test-result': 30.2.0 + '@jest/types': 30.2.0 + '@types/node': 24.10.9 + chalk: 4.1.2 + co: 4.6.0 + dedent: 1.7.1 + is-generator-fn: 2.1.0 + jest-each: 30.2.0 + jest-matcher-utils: 30.2.0 + jest-message-util: 30.2.0 + jest-runtime: 30.2.0 + jest-snapshot: 30.2.0 + jest-util: 30.2.0 + p-limit: 3.1.0 + pretty-format: 30.2.0 + pure-rand: 7.0.1 + slash: 3.0.0 + stack-utils: 2.0.6 + transitivePeerDependencies: + - babel-plugin-macros + - supports-color + + jest-cli@30.2.0(@types/node@24.10.9): + dependencies: + '@jest/core': 30.2.0 + '@jest/test-result': 30.2.0 + '@jest/types': 30.2.0 + chalk: 4.1.2 + exit-x: 0.2.2 + import-local: 3.2.0 + jest-config: 30.2.0(@types/node@24.10.9) + jest-util: 30.2.0 + jest-validate: 30.2.0 + yargs: 17.7.2 + transitivePeerDependencies: + - '@types/node' + - babel-plugin-macros + - esbuild-register + - supports-color + - ts-node + + jest-config@30.2.0(@types/node@24.10.9): + dependencies: + '@babel/core': 7.28.6 + '@jest/get-type': 30.1.0 + '@jest/pattern': 30.0.1 + '@jest/test-sequencer': 30.2.0 + '@jest/types': 30.2.0 + babel-jest: 30.2.0(@babel/core@7.28.6) + chalk: 4.1.2 + ci-info: 4.3.1 + deepmerge: 4.3.1 + glob: 10.5.0 + graceful-fs: 4.2.11 + jest-circus: 30.2.0 + jest-docblock: 30.2.0 + jest-environment-node: 30.2.0 + jest-regex-util: 30.0.1 + jest-resolve: 30.2.0 + jest-runner: 30.2.0 + jest-util: 30.2.0 + jest-validate: 30.2.0 + micromatch: 4.0.8 + parse-json: 5.2.0 + pretty-format: 30.2.0 + slash: 3.0.0 + strip-json-comments: 3.1.1 + optionalDependencies: + '@types/node': 24.10.9 + transitivePeerDependencies: + - babel-plugin-macros + - supports-color + + jest-diff@30.2.0: + dependencies: + '@jest/diff-sequences': 30.0.1 + '@jest/get-type': 30.1.0 + chalk: 4.1.2 + pretty-format: 30.2.0 + + jest-docblock@30.2.0: + dependencies: + detect-newline: 3.1.0 + + jest-each@30.2.0: + dependencies: + '@jest/get-type': 30.1.0 + '@jest/types': 30.2.0 + chalk: 4.1.2 + jest-util: 30.2.0 + pretty-format: 30.2.0 + + jest-environment-node@30.2.0: + dependencies: + '@jest/environment': 30.2.0 + '@jest/fake-timers': 30.2.0 + '@jest/types': 30.2.0 + '@types/node': 24.10.9 + jest-mock: 30.2.0 + jest-util: 30.2.0 + jest-validate: 30.2.0 + + jest-haste-map@30.2.0: + dependencies: + '@jest/types': 30.2.0 + '@types/node': 24.10.9 + anymatch: 3.1.3 + fb-watchman: 2.0.2 + graceful-fs: 4.2.11 + jest-regex-util: 30.0.1 + jest-util: 30.2.0 + jest-worker: 30.2.0 + micromatch: 4.0.8 + walker: 1.0.8 + optionalDependencies: + fsevents: 2.3.3 + + jest-leak-detector@30.2.0: + dependencies: + '@jest/get-type': 30.1.0 + pretty-format: 30.2.0 + + jest-matcher-utils@30.2.0: + dependencies: + '@jest/get-type': 30.1.0 + chalk: 4.1.2 + jest-diff: 30.2.0 + pretty-format: 30.2.0 + + jest-message-util@30.2.0: + dependencies: + '@babel/code-frame': 7.28.6 + '@jest/types': 30.2.0 + '@types/stack-utils': 2.0.3 + chalk: 4.1.2 + graceful-fs: 4.2.11 + micromatch: 4.0.8 + pretty-format: 30.2.0 + slash: 3.0.0 + stack-utils: 2.0.6 + + jest-mock@30.2.0: + dependencies: + '@jest/types': 30.2.0 + '@types/node': 24.10.9 + jest-util: 30.2.0 + + jest-pnp-resolver@1.2.3(jest-resolve@30.2.0): + optionalDependencies: + jest-resolve: 30.2.0 + + jest-regex-util@30.0.1: {} + + jest-resolve-dependencies@30.2.0: + dependencies: + jest-regex-util: 30.0.1 + jest-snapshot: 30.2.0 + transitivePeerDependencies: + - supports-color + + jest-resolve@30.2.0: + dependencies: + chalk: 4.1.2 + graceful-fs: 4.2.11 + jest-haste-map: 30.2.0 + jest-pnp-resolver: 1.2.3(jest-resolve@30.2.0) + jest-util: 30.2.0 + jest-validate: 30.2.0 + slash: 3.0.0 + unrs-resolver: 1.11.1 + + jest-runner@30.2.0: + dependencies: + '@jest/console': 30.2.0 + '@jest/environment': 30.2.0 + '@jest/test-result': 30.2.0 + '@jest/transform': 30.2.0 + '@jest/types': 30.2.0 + '@types/node': 24.10.9 + chalk: 4.1.2 + emittery: 0.13.1 + exit-x: 0.2.2 + graceful-fs: 4.2.11 + jest-docblock: 30.2.0 + jest-environment-node: 30.2.0 + jest-haste-map: 30.2.0 + jest-leak-detector: 30.2.0 + jest-message-util: 30.2.0 + jest-resolve: 30.2.0 + jest-runtime: 30.2.0 + jest-util: 30.2.0 + jest-watcher: 30.2.0 + jest-worker: 30.2.0 + p-limit: 3.1.0 + source-map-support: 0.5.13 + transitivePeerDependencies: + - supports-color + + jest-runtime@30.2.0: + dependencies: + '@jest/environment': 30.2.0 + '@jest/fake-timers': 30.2.0 + '@jest/globals': 30.2.0 + '@jest/source-map': 30.0.1 + '@jest/test-result': 30.2.0 + '@jest/transform': 30.2.0 + '@jest/types': 30.2.0 + '@types/node': 24.10.9 + chalk: 4.1.2 + cjs-module-lexer: 2.2.0 + collect-v8-coverage: 1.0.3 + glob: 10.5.0 + graceful-fs: 4.2.11 + jest-haste-map: 30.2.0 + jest-message-util: 30.2.0 + jest-mock: 30.2.0 + jest-regex-util: 30.0.1 + jest-resolve: 30.2.0 + jest-snapshot: 30.2.0 + jest-util: 30.2.0 + slash: 3.0.0 + strip-bom: 4.0.0 + transitivePeerDependencies: + - supports-color + + jest-snapshot@30.2.0: + dependencies: + '@babel/core': 7.28.6 + '@babel/generator': 7.28.6 + '@babel/plugin-syntax-jsx': 7.28.6(@babel/core@7.28.6) + '@babel/plugin-syntax-typescript': 7.28.6(@babel/core@7.28.6) + '@babel/types': 7.28.6 + '@jest/expect-utils': 30.2.0 + '@jest/get-type': 30.1.0 + '@jest/snapshot-utils': 30.2.0 + '@jest/transform': 30.2.0 + '@jest/types': 30.2.0 + babel-preset-current-node-syntax: 1.2.0(@babel/core@7.28.6) + chalk: 4.1.2 + expect: 30.2.0 + graceful-fs: 4.2.11 + jest-diff: 30.2.0 + jest-matcher-utils: 30.2.0 + jest-message-util: 30.2.0 + jest-util: 30.2.0 + pretty-format: 30.2.0 + semver: 7.7.3 + synckit: 0.11.12 + transitivePeerDependencies: + - supports-color + + jest-util@30.2.0: + dependencies: + '@jest/types': 30.2.0 + '@types/node': 24.10.9 + chalk: 4.1.2 + ci-info: 4.3.1 + graceful-fs: 4.2.11 + picomatch: 4.0.3 + + jest-validate@30.2.0: + dependencies: + '@jest/get-type': 30.1.0 + '@jest/types': 30.2.0 + camelcase: 6.3.0 + chalk: 4.1.2 + leven: 3.1.0 + pretty-format: 30.2.0 + + jest-watcher@30.2.0: + dependencies: + '@jest/test-result': 30.2.0 + '@jest/types': 30.2.0 + '@types/node': 24.10.9 + ansi-escapes: 4.3.2 + chalk: 4.1.2 + emittery: 0.13.1 + jest-util: 30.2.0 + string-length: 4.0.2 + + jest-worker@30.2.0: + dependencies: + '@types/node': 24.10.9 + '@ungap/structured-clone': 1.3.0 + jest-util: 30.2.0 + merge-stream: 2.0.0 + supports-color: 8.1.1 + + jest@30.2.0(@types/node@24.10.9): + dependencies: + '@jest/core': 30.2.0 + '@jest/types': 30.2.0 + import-local: 3.2.0 + jest-cli: 30.2.0(@types/node@24.10.9) + transitivePeerDependencies: + - '@types/node' + - babel-plugin-macros + - esbuild-register + - supports-color + - ts-node + + js-tokens@4.0.0: {} + + js-yaml@3.14.2: + dependencies: + argparse: 1.0.10 + esprima: 4.0.1 + + js2xmlparser@4.0.2: + dependencies: + xmlcreate: 2.0.4 + + jsdoc-api@9.3.5: + dependencies: + array-back: 6.2.2 + cache-point: 3.0.1 + current-module-paths: 1.1.3 + file-set: 5.3.0 + jsdoc: 4.0.5 + object-to-spawn-args: 2.0.1 + walk-back: 5.1.1 + + jsdoc-parse@6.2.5: + dependencies: + array-back: 6.2.2 + find-replace: 5.0.2 + sort-array: 5.1.1 + transitivePeerDependencies: + - '@75lb/nature' + + jsdoc-to-markdown@9.1.3: + dependencies: + array-back: 6.2.2 + command-line-args: 6.0.1 + command-line-usage: 7.0.3 + config-master: 3.1.0 + dmd: 7.1.1 + jsdoc-api: 9.3.5 + jsdoc-parse: 6.2.5 + walk-back: 5.1.1 + + jsdoc@4.0.5: + dependencies: + '@babel/parser': 7.28.6 + '@jsdoc/salty': 0.2.9 + '@types/markdown-it': 14.1.2 + bluebird: 3.7.2 + catharsis: 0.9.0 + escape-string-regexp: 2.0.0 + js2xmlparser: 4.0.2 + klaw: 3.0.0 + markdown-it: 14.1.0 + markdown-it-anchor: 8.6.7(@types/markdown-it@14.1.2)(markdown-it@14.1.0) + marked: 4.3.0 + mkdirp: 1.0.4 + requizzle: 0.2.4 + strip-json-comments: 3.1.1 + underscore: 1.13.7 + + jsesc@3.1.0: {} + + json-parse-even-better-errors@2.3.1: {} + + json-stringify-safe@5.0.1: {} + + json5@2.2.3: {} + + klaw@3.0.0: + dependencies: + graceful-fs: 4.2.11 + + leven@3.1.0: {} + + lines-and-columns@1.2.4: {} + + linkify-it@5.0.0: + dependencies: + uc.micro: 2.1.0 + + locate-path@5.0.0: + dependencies: + p-locate: 4.1.0 + + lodash.camelcase@4.3.0: {} + + lodash@4.17.23: {} + + long@5.3.2: {} + + lru-cache@10.4.3: {} + + lru-cache@5.1.1: + dependencies: + yallist: 3.1.1 + + make-dir@4.0.0: + dependencies: + semver: 7.7.3 + + makeerror@1.0.12: + dependencies: + tmpl: 1.0.5 + + markdown-it-anchor@8.6.7(@types/markdown-it@14.1.2)(markdown-it@14.1.0): + dependencies: + '@types/markdown-it': 14.1.2 + markdown-it: 14.1.0 + + markdown-it@14.1.0: + dependencies: + argparse: 2.0.1 + entities: 4.5.0 + linkify-it: 5.0.0 + mdurl: 2.0.0 + punycode.js: 2.3.1 + uc.micro: 2.1.0 + + marked@4.3.0: {} + + matcher@3.0.0: + dependencies: + escape-string-regexp: 4.0.0 + + mdurl@2.0.0: {} + + merge-stream@2.0.0: {} + + merge2@1.4.1: {} + + micromatch@4.0.8: + dependencies: + braces: 3.0.3 + picomatch: 2.3.1 + + mimic-fn@2.1.0: {} + + minimatch@3.1.2: + dependencies: + brace-expansion: 1.1.12 + + minimatch@9.0.5: + dependencies: + brace-expansion: 2.0.2 + + minimist@1.2.8: {} + + minipass@7.1.2: {} + + mkdirp@1.0.4: {} + + ms@2.1.3: {} + + napi-postinstall@0.3.4: {} + + natural-compare@1.4.0: {} + + neo-async@2.6.2: {} + + node-int64@0.4.0: {} + + node-releases@2.0.27: {} + + normalize-path@3.0.0: {} + + npm-run-path@4.0.1: + dependencies: + path-key: 3.1.1 + + object-keys@1.1.1: {} + + object-to-spawn-args@2.0.1: {} + + once@1.4.0: + dependencies: + wrappy: 1.0.2 + + onetime@5.1.2: + dependencies: + mimic-fn: 2.1.0 + + onnxruntime-common@1.24.0-dev.20251116-b39e144322: {} + + onnxruntime-common@1.24.3: {} + + onnxruntime-node@1.24.3: + dependencies: + adm-zip: 0.5.16 + global-agent: 3.0.0 + onnxruntime-common: 1.24.3 + + onnxruntime-web@1.26.0-dev.20260416-b7804b056c: + dependencies: + flatbuffers: 25.9.23 + guid-typescript: 1.0.9 + long: 5.3.2 + onnxruntime-common: 1.24.0-dev.20251116-b39e144322 + platform: 1.3.6 + protobufjs: 7.5.4 + + p-limit@2.3.0: + dependencies: + p-try: 2.2.0 + + p-limit@3.1.0: + dependencies: + yocto-queue: 0.1.0 + + p-locate@4.1.0: + dependencies: + p-limit: 2.3.0 + + p-try@2.2.0: {} + + package-json-from-dist@1.0.1: {} + + parse-json@5.2.0: + dependencies: + '@babel/code-frame': 7.28.6 + error-ex: 1.3.4 + json-parse-even-better-errors: 2.3.1 + lines-and-columns: 1.2.4 + + path-exists@4.0.0: {} + + path-is-absolute@1.0.1: {} + + path-key@3.1.1: {} + + path-scurry@1.11.1: + dependencies: + lru-cache: 10.4.3 + minipass: 7.1.2 + + picocolors@1.1.1: {} + + picomatch@2.3.1: {} + + picomatch@4.0.3: {} + + pirates@4.0.7: {} + + pkg-dir@4.2.0: + dependencies: + find-up: 4.1.0 + + platform@1.3.6: {} + + prettier@3.8.1: {} + + pretty-format@30.2.0: + dependencies: + '@jest/schemas': 30.0.5 + ansi-styles: 5.2.0 + react-is: 18.3.1 + + protobufjs@7.5.4: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.4 + '@protobufjs/eventemitter': 1.1.0 + '@protobufjs/fetch': 1.1.0 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.0 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.0 + '@types/node': 24.10.9 + long: 5.3.2 + + punycode.js@2.3.1: {} + + pure-rand@7.0.1: {} + + queue-microtask@1.2.3: {} + + react-is@18.3.1: {} + + require-directory@2.1.1: {} + + requizzle@0.2.4: + dependencies: + lodash: 4.17.23 + + resolve-cwd@3.0.0: + dependencies: + resolve-from: 5.0.0 + + resolve-from@5.0.0: {} + + reusify@1.1.0: {} + + roarr@2.15.4: + dependencies: + boolean: 3.2.0 + detect-node: 2.1.0 + globalthis: 1.0.4 + json-stringify-safe: 5.0.1 + semver-compare: 1.0.0 + sprintf-js: 1.1.3 + + run-parallel@1.2.0: + dependencies: + queue-microtask: 1.2.3 + + semver-compare@1.0.0: {} + + semver@6.3.1: {} + + semver@7.7.3: {} + + serialize-error@7.0.1: + dependencies: + type-fest: 0.13.1 + + sharp@0.34.5: + dependencies: + '@img/colour': 1.0.0 + detect-libc: 2.1.2 + semver: 7.7.3 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.34.5 + '@img/sharp-darwin-x64': 0.34.5 + '@img/sharp-libvips-darwin-arm64': 1.2.4 + '@img/sharp-libvips-darwin-x64': 1.2.4 + '@img/sharp-libvips-linux-arm': 1.2.4 + '@img/sharp-libvips-linux-arm64': 1.2.4 + '@img/sharp-libvips-linux-ppc64': 1.2.4 + '@img/sharp-libvips-linux-riscv64': 1.2.4 + '@img/sharp-libvips-linux-s390x': 1.2.4 + '@img/sharp-libvips-linux-x64': 1.2.4 + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + '@img/sharp-linux-arm': 0.34.5 + '@img/sharp-linux-arm64': 0.34.5 + '@img/sharp-linux-ppc64': 0.34.5 + '@img/sharp-linux-riscv64': 0.34.5 + '@img/sharp-linux-s390x': 0.34.5 + '@img/sharp-linux-x64': 0.34.5 + '@img/sharp-linuxmusl-arm64': 0.34.5 + '@img/sharp-linuxmusl-x64': 0.34.5 + '@img/sharp-wasm32': 0.34.5 + '@img/sharp-win32-arm64': 0.34.5 + '@img/sharp-win32-ia32': 0.34.5 + '@img/sharp-win32-x64': 0.34.5 + + shebang-command@2.0.0: + dependencies: + shebang-regex: 3.0.0 + + shebang-regex@3.0.0: {} + + signal-exit@3.0.7: {} + + signal-exit@4.1.0: {} + + slash@3.0.0: {} + + sort-array@5.1.1: + dependencies: + array-back: 6.2.2 + typical: 7.3.0 + + source-map-support@0.5.13: + dependencies: + buffer-from: 1.1.2 + source-map: 0.6.1 + + source-map@0.6.1: {} + + sprintf-js@1.0.3: {} + + sprintf-js@1.1.3: {} + + stack-utils@2.0.6: + dependencies: + escape-string-regexp: 2.0.0 + + string-length@4.0.2: + dependencies: + char-regex: 1.0.2 + strip-ansi: 6.0.1 + + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + + string-width@5.1.2: + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.1.2 + + strip-ansi@6.0.1: + dependencies: + ansi-regex: 5.0.1 + + strip-ansi@7.1.2: + dependencies: + ansi-regex: 6.2.2 + + strip-bom@4.0.0: {} + + strip-final-newline@2.0.0: {} + + strip-json-comments@3.1.1: {} + + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + + supports-color@8.1.1: + dependencies: + has-flag: 4.0.0 + + synckit@0.11.12: + dependencies: + '@pkgr/core': 0.2.9 + + table-layout@4.1.1: + dependencies: + array-back: 6.2.2 + wordwrapjs: 5.1.1 + + test-exclude@6.0.0: + dependencies: + '@istanbuljs/schema': 0.1.3 + glob: 7.2.3 + minimatch: 3.1.2 + + tmpl@1.0.5: {} + + to-regex-range@5.0.1: + dependencies: + is-number: 7.0.0 + + tslib@2.8.1: + optional: true + + type-detect@4.0.8: {} + + type-fest@0.13.1: {} + + type-fest@0.21.3: {} + + typescript@5.9.3: {} + + typical@7.3.0: {} + + uc.micro@2.1.0: {} + + uglify-js@3.19.3: + optional: true + + underscore@1.13.7: {} + + undici-types@7.16.0: {} + + unrs-resolver@1.11.1: + dependencies: + napi-postinstall: 0.3.4 + optionalDependencies: + '@unrs/resolver-binding-android-arm-eabi': 1.11.1 + '@unrs/resolver-binding-android-arm64': 1.11.1 + '@unrs/resolver-binding-darwin-arm64': 1.11.1 + '@unrs/resolver-binding-darwin-x64': 1.11.1 + '@unrs/resolver-binding-freebsd-x64': 1.11.1 + '@unrs/resolver-binding-linux-arm-gnueabihf': 1.11.1 + '@unrs/resolver-binding-linux-arm-musleabihf': 1.11.1 + '@unrs/resolver-binding-linux-arm64-gnu': 1.11.1 + '@unrs/resolver-binding-linux-arm64-musl': 1.11.1 + '@unrs/resolver-binding-linux-ppc64-gnu': 1.11.1 + '@unrs/resolver-binding-linux-riscv64-gnu': 1.11.1 + '@unrs/resolver-binding-linux-riscv64-musl': 1.11.1 + '@unrs/resolver-binding-linux-s390x-gnu': 1.11.1 + '@unrs/resolver-binding-linux-x64-gnu': 1.11.1 + '@unrs/resolver-binding-linux-x64-musl': 1.11.1 + '@unrs/resolver-binding-wasm32-wasi': 1.11.1 + '@unrs/resolver-binding-win32-arm64-msvc': 1.11.1 + '@unrs/resolver-binding-win32-ia32-msvc': 1.11.1 + '@unrs/resolver-binding-win32-x64-msvc': 1.11.1 + + update-browserslist-db@1.2.3(browserslist@4.28.1): + dependencies: + browserslist: 4.28.1 + escalade: 3.2.0 + picocolors: 1.1.1 + + v8-to-istanbul@9.3.0: + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + '@types/istanbul-lib-coverage': 2.0.6 + convert-source-map: 2.0.0 + + walk-back@2.0.1: {} + + walk-back@5.1.1: {} + + walker@1.0.8: + dependencies: + makeerror: 1.0.12 + + which@2.0.2: + dependencies: + isexe: 2.0.0 + + wordwrap@1.0.0: {} + + wordwrapjs@5.1.1: {} + + wrap-ansi@7.0.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + wrap-ansi@8.1.0: + dependencies: + ansi-styles: 6.2.3 + string-width: 5.1.2 + strip-ansi: 7.1.2 + + wrappy@1.0.2: {} + + write-file-atomic@5.0.1: + dependencies: + imurmurhash: 0.1.4 + signal-exit: 4.1.0 + + xmlcreate@2.0.4: {} + + y18n@5.0.8: {} + + yallist@3.1.1: {} + + yargs-parser@21.1.1: {} + + yargs@17.7.2: + dependencies: + cliui: 8.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + require-directory: 2.1.1 + string-width: 4.2.3 + y18n: 5.0.8 + yargs-parser: 21.1.1 + + yocto-queue@0.1.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..89595a5 --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,12 @@ +packages: + - packages/* + +onlyBuiltDependencies: + - esbuild + - onnxruntime-node + - protobufjs + - sharp + - unrs-resolver + +publicHoistPattern: + - onnxruntime-common diff --git a/scripts/dev.mjs b/scripts/dev.mjs new file mode 100644 index 0000000..5a86ed0 --- /dev/null +++ b/scripts/dev.mjs @@ -0,0 +1,102 @@ +#!/usr/bin/env node + +/** + * This scripts looks at all packages and starts their dev command in parallel, if they have one. + */ + +import { spawn } from "node:child_process"; +import { readdirSync, readFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { colors, log } from "./logger.mjs"; + +const processes = []; + +// Cleanup function +const cleanup = () => { + console.log(`\n\n${colors.yellow}[stop]${colors.reset} Stopping all dev servers...`); + processes.forEach((proc) => { + try { + proc.kill("SIGINT"); + } catch (error) { + // Ignore errors during cleanup + } + }); + process.exit(0); +}; + +// Handle various termination signals +process.on("SIGINT", cleanup); +process.on("SIGTERM", cleanup); +process.on("exit", cleanup); + +// Find all packages with dev scripts +const findPackagesWithDevScript = () => { + const packagesDir = "packages"; + const packagesWithDev = []; + + if (!existsSync(packagesDir)) { + log.error(`Packages directory not found: ${packagesDir}`); + return packagesWithDev; + } + + const packages = readdirSync(packagesDir, { withFileTypes: true }) + .filter((dirent) => dirent.isDirectory()) + .map((dirent) => dirent.name); + + for (const pkg of packages) { + const packageJsonPath = join(packagesDir, pkg, "package.json"); + + if (existsSync(packageJsonPath)) { + try { + const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8")); + if (packageJson.scripts && packageJson.scripts.dev) { + packagesWithDev.push({ + name: pkg, + displayName: packageJson.name || pkg, + }); + } + } catch (error) { + log.error(`Failed to read package.json for ${pkg}: ${error.message}`); + } + } + } + + return packagesWithDev; +}; + +log.section("DEV SERVERS"); +log.info("Discovering packages with dev scripts...\n"); + +const packagesWithDev = findPackagesWithDevScript(); + +if (packagesWithDev.length === 0) { + log.error("No packages found with dev scripts!"); + process.exit(1); +} + +log.info(`Found ${packagesWithDev.length} package(s) with dev scripts:`); +packagesWithDev.forEach((pkg) => { + log.info(` - ${pkg.displayName} (${pkg.name})`); +}); +log.info("\nStarting development servers...\n"); + +// Start dev server for each package +packagesWithDev.forEach((pkg) => { + const proc = spawn("pnpm", ["--filter", pkg.displayName, "dev"], { + stdio: "inherit", + shell: true, + }); + + processes.push(proc); + + // Handle process exits + proc.on("exit", (code) => { + if (code !== 0 && code !== null) { + log.error(`${pkg.displayName} dev server exited with code ${code}`); + cleanup(); + } + }); +}); + +// Keep process alive +process.stdin.resume(); diff --git a/scripts/logger.mjs b/scripts/logger.mjs new file mode 100644 index 0000000..34954d0 --- /dev/null +++ b/scripts/logger.mjs @@ -0,0 +1,42 @@ +/** + * Shared logging utility with colored output + */ + +// ANSI color codes +export const colors = { + reset: "\x1b[0m", + bright: "\x1b[1m", + dim: "\x1b[2m", + cyan: "\x1b[36m", + green: "\x1b[32m", + yellow: "\x1b[33m", + blue: "\x1b[34m", + magenta: "\x1b[35m", + red: "\x1b[31m", + gray: "\x1b[90m", +}; + +/** + * Creates a logger with optional package prefix + * @param {string} [prefix] - Optional prefix to prepend to all logs (e.g., "transformers", "react") + * @returns {Object} Logger object with various log methods + */ +export function createLogger(prefix = "") { + const formatPrefix = prefix ? `${colors.magenta}[${prefix}]${colors.reset} ` : ""; + + return { + section: (text) => console.log(`\n${formatPrefix}${colors.bright}${colors.cyan}=== ${text} ===${colors.reset}`), + info: (text) => console.log(`${formatPrefix}${colors.blue}[info]${colors.reset} ${text}`), + success: (text) => console.log(`${formatPrefix}${colors.green}✓${colors.reset} ${text}`), + warning: (text) => console.log(`${formatPrefix}${colors.yellow}[warn]${colors.reset} ${text}`), + error: (text) => console.log(`${formatPrefix}${colors.red}[error]${colors.reset} ${text}`), + dim: (text) => console.log(`${formatPrefix}${colors.dim}${text}${colors.reset}`), + url: (text) => console.log(`${formatPrefix} ${colors.cyan}→${colors.reset} ${colors.bright}${text}${colors.reset}`), + file: (text) => console.log(`${formatPrefix} ${colors.gray}-${colors.reset} ${text}`), + build: (text) => console.log(`${formatPrefix}${colors.cyan}[build]${colors.reset} ${text}`), + done: (text) => console.log(`${formatPrefix}${colors.green}[done]${colors.reset} ${text}`), + }; +} + +// Default logger without prefix for backward compatibility +export const log = createLogger(); diff --git a/scripts/prepareOutDir.mjs b/scripts/prepareOutDir.mjs new file mode 100644 index 0000000..5f9302d --- /dev/null +++ b/scripts/prepareOutDir.mjs @@ -0,0 +1,9 @@ +import { existsSync, mkdirSync, rmSync } from "node:fs"; + +export default function prepareOutDir(dir) { + if (existsSync(dir)) { + rmSync(dir, { recursive: true, force: true }); + } + + mkdirSync(dir, { recursive: true }); +} diff --git a/scripts/rebuildPlugin.mjs b/scripts/rebuildPlugin.mjs new file mode 100644 index 0000000..a271400 --- /dev/null +++ b/scripts/rebuildPlugin.mjs @@ -0,0 +1,42 @@ +import { colors } from "./logger.mjs"; + +/** + * Plugin to log rebuild information in watch mode + * @param {string} targetName - Name of the build target + * @param {Object} log - Logger instance + * @param {Object} options - Plugin options + * @param {boolean} options.logFirstBuild - Whether to log the first build completion (default: true) + */ +export function rebuildPlugin(targetName, log, options = {}) { + const { logFirstBuild = true } = options; + + return { + name: "rebuild-logger", + setup(build) { + let startTime = 0; + let isFirstBuild = true; + + build.onStart(() => { + startTime = performance.now(); + if (!isFirstBuild) { + log.build(`${colors.gray}Rebuilding ${targetName}...${colors.reset}`); + } + }); + + build.onEnd((result) => { + const endTime = performance.now(); + const duration = (endTime - startTime).toFixed(2); + + if (result.errors.length > 0) { + log.error(`${colors.bright}${targetName}${colors.reset} - Build failed with ${result.errors.length} error(s) in ${duration}ms`); + } else if (!isFirstBuild) { + log.done(`${colors.bright}${targetName}${colors.reset} - Rebuilt in ${colors.gray}${duration}ms${colors.reset}`); + } else if (logFirstBuild) { + log.done(`${colors.bright}${targetName}${colors.reset} - Built in ${colors.gray}${duration}ms${colors.reset}`); + } + + isFirstBuild = false; + }); + }, + }; +} diff --git a/scripts/reportSize.mjs b/scripts/reportSize.mjs new file mode 100644 index 0000000..4aad985 --- /dev/null +++ b/scripts/reportSize.mjs @@ -0,0 +1,21 @@ +import { readFileSync } from "node:fs"; +import { gzipSync } from "node:zlib"; +import path from "node:path"; +import { colors } from "./logger.mjs"; + +export const formatSize = (bytes) => { + if (bytes < 1024) return `${bytes}b`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}kb`; + return `${(bytes / (1024 * 1024)).toFixed(2)}mb`; +}; + +export const reportSize = (outfile, log) => { + const content = readFileSync(outfile); + const size = content.length; + const gzipSize = gzipSync(content).length; + const filename = path.basename(outfile); + + log.done( + `${colors.bright}${filename}${colors.reset} ${colors.gray}${formatSize(size)}${colors.reset} ${colors.dim}(gzip: ${formatSize(gzipSize)})${colors.reset}`, + ); +}; diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..e007748 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,26 @@ +{ + // Only include files in the src directory + "include": ["./packages/**/src/**/*"], + "compilerOptions": { + // Tells the compiler to check JS files + "checkJs": true, + "target": "esnext", + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["ES2020", "DOM"], + "outDir": "types", + "strict": false, + "skipLibCheck": true, + "declaration": true, + "declarationMap": true, + "noEmit": false, + "emitDeclarationOnly": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "allowSyntheticDefaultImports": true, + "forceConsistentCasingInFileNames": true + }, + "typeAcquisition": { + "include": ["jest"] + } +}