chore: import upstream snapshot with attribution

2026-07-13 12:42:51 +08:00
commit f73e710e38
276 changed files with 36598 additions and 0 deletions
@@ -0,0 +1,38 @@
 name: Build site and push to gh-pages
 on:
  push:
    branches:
      - main
 jobs:
  build:
    name: Build site
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - name: Configuring build Environment
      run: |
        sudo apt-get update
        python -m pip install -U pip
    - name: Setup Ruby
      uses: ruby/setup-ruby@v1
      with:
        ruby-version: '3.0'
    - name: Installing dependencies
      run: |
        python -m pip install -r docs/requirements.txt
        gem install jekyll jekyll-remote-theme jekyll-sass-converter
    - name: Build and deploy site
      if: github.ref == 'refs/heads/main'
      run: |
        git remote set-url origin https://x-access-token:${{ secrets.MLC_GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
        git config --global user.email "mlc-gh-actions-bot@nomail"
        git config --global user.name "mlc-gh-actions-bot"
        ./scripts/gh_deploy_site.sh
@@ -0,0 +1,38 @@
 name: Build
 on:
  pull_request:
    branches:
      - main
  push:
    branches:
      - main
  workflow_dispatch:
 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
 jobs:
  build:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version-file: ".nvmrc"
          cache: npm
      - name: Install dependencies
        run: npm ci
      - name: Build package
        run: npm run build
      - name: Validate package contents
        run: npm pack --dry-run
@@ -0,0 +1,27 @@
 name: Linter
 on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
 jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
      - name: Set up Node.js
        uses: actions/setup-node@v3
        with:
          node-version-file: ".nvmrc"
      - name: Install dependencies
        run: npm install
      - name: Run lint
        run: npm run lint
@@ -0,0 +1,73 @@
 name: Security
 on:
  pull_request:
    branches:
      - main
  push:
    branches:
      - main
  schedule:
    - cron: "25 5 * * 1"
  workflow_dispatch:
 permissions:
  contents: read
 jobs:
  dependency-review:
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: read
    steps:
      - name: Dependency review
        uses: actions/dependency-review-action@v4
        with:
          fail-on-severity: high
  npm-audit:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version-file: ".nvmrc"
          cache: npm
      - name: Install dependencies
        run: npm ci
      - name: Run npm audit (production dependencies)
        run: npm audit --omit=dev --audit-level=high
  codeql:
    if: github.event_name != 'pull_request'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      actions: read
      contents: read
      security-events: write
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Initialize CodeQL
        uses: github/codeql-action/init@v3
        with:
          languages: javascript-typescript
      - name: Autobuild
        uses: github/codeql-action/autobuild@v3
      - name: Analyze
        uses: github/codeql-action/analyze@v3
@@ -0,0 +1,45 @@
 name: Tests
 on:
  pull_request:
    branches:
      - main
  push:
    branches:
      - main
  workflow_dispatch:
 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
 jobs:
  test:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version-file: ".nvmrc"
          cache: npm
      - name: Install dependencies
        run: npm ci
      - name: Run test suite
        env:
          CI: "true"
        run: npm run test -- --ci
      - name: Upload coverage artifact
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: coverage-${{ github.run_id }}
          path: coverage
          if-no-files-found: ignore
@@ -0,0 +1,328 @@
 scratch/
 dist/
 params/
 *.bak
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
 *$py.class
 *.S
 # C extensions
 *.so
 *.ll
 .npm
 # Distribution / packaging
 .Python
 env/
 build/
 build-*/
 develop-eggs/
 dist/
 downloads/
 eggs/
 .eggs/
 lib/
 lib64/
 parts/
 sdist/
 var/
 wheels/
 pip-wheel-metadata/
 share/python-wheels/
 *.egg-info/
 .installed.cfg
 *.egg
 MANIFEST
 .conda/
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
 # Generated by python/gen_requirements.py
 python/requirements/*.txt
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt
 # Unit test / coverage reports
 htmlcov/
 .tox/
 .nox/
 .coverage
 .coverage.*
 .cache
 nosetests.xml
 coverage.xml
 *.cover
 *.py,cover
 .hypothesis/
 .pytest_cache/
 # Translations
 *.mo
 *.pot
 # Django stuff:
 *.log
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
 # Flask stuff:
 instance/
 .webassets-cache
 # Scrapy stuff:
 .scrapy
 # Sphinx documentation
 docs/_build/
 docs/_staging/
 # PyBuilder
 target/
 /target/
 # Jupyter Notebook
 .ipynb_checkpoints
 # IPython
 profile_default/
 ipython_config.py
 # pyenv
 .python-version
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 #   However, in case of collaboration, if having platform-specific dependencies or dependencies
 #   having no cross-platform support, pipenv may install dependencies that don't work, or not
 #   install all needed dependencies.
 #Pipfile.lock
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow
 __pypackages__/
 # Celery stuff
 celerybeat-schedule
 celerybeat.pid
 # SageMath parsed files
 *.sage.py
 # Environments
 .env
 .venv
 env/
 venv/
 ENV/
 env.bak/
 venv.bak/
 # Spyder project settings
 .spyderproject
 .spyproject
 # Rope project settings
 .ropeproject
 *~
 *.pyc
 *~
 config.mk
 config.cmake
 Win32
 *.dir
 perf
 *.wasm
 .emscripten
 ## IOS
 DerivedData/
 ## Java
 *.class
 jvm/*/target/
 jvm/*/*/target/
 jvm/native/*/generated
 jvm/native/src/main/native/org_apache_tvm_native_c_api.h
 *.worksheet
 *.idea
 *.iml
 *.classpath
 *.project
 *.settings
 */node_modules/
 ## Various settings
 *.pbxuser
 !default.pbxuser
 *.mode1v3
 !default.mode1v3
 *.mode2v3
 !default.mode2v3
 *.perspectivev3
 !default.perspectivev3
 xcuserdata/
 .pkl_memoize_*
 .emscripten*
 .m2
 # Compiled Dynamic libraries
 *.so
 *.dylib
 *.dll
 # Compiled Object files
 *.slo
 *.lo
 *.o
 *.obj
 # Precompiled Headers
 *.gch
 *.pch
 # Compiled Static libraries
 *.lai
 *.la
 *.a
 *.lib
 # Executables
 *.exe
 *.out
 *.app
 ## Other
 *.moved-aside
 *.xccheckout
 *.xcscmblueprint
 .DS_Store
 tags
 cscope*
 *.lock
 # vim temporary files
 *.swp
 *.swo
 # TVM generated code
 perf
 .bash_history
 # *.json
 *.params
 *.ro
 *.onnx
 *.h5
 synset.txt
 cat.jpg
 cat.png
 docs.tgz
 cat.png
 *.mlmodel
 tvm_u.*
 tvm_t.*
 # Mac OS X
 .DS_Store
 # Jetbrain
 .idea
 .ipython
 .jupyter
 .nv
 .pylint.d
 .python_history
 .pytest_cache
 .local
 cmake-build-debug
 # Visual Studio
 .vs
 # Visual Studio Code
 .vscode
 # tmp file
 .nfs*
 # keys
 *.pem
 *.p12
 *.pfx
 *.cer
 *.crt
 *.der
 # patch sentinel
 patched.txt
 # Python type checking
 .mypy_cache/
 .pyre/
 # pipenv files
 Pipfile
 Pipfile.lock
 # conda package artifacts
 conda/Dockerfile.cuda*
 conda/pkg
 .node_repl_history
 # nix files
 .envrc
 *.nix
 # Docker files
 .sudo_as_admin_successful
 # Downloaded models/datasets
 .tvm_test_data
 .dgl
 .caffe2
 # Local docs build
 _docs/
 jvm/target
 .config/configstore/
 .ci-py-scripts/
 # Generated Hexagon files
 src/runtime/hexagon/rpc/hexagon_rpc.h
 src/runtime/hexagon/rpc/hexagon_rpc_skel.c
 src/runtime/hexagon/rpc/hexagon_rpc_stub.c
 # Local tvm-site checkout
 tvm-site/
 # Generated docs files
 gallery/how_to/work_with_microtvm/micro_tvmc.py
 # Test sample data files
 !tests/python/ci/sample_prs/*.json
 # Used in CI to communicate between Python and Jenkins
 .docker-image-names/
 # Printed TIR code on disk
 *.tir
 # GDB history file
 .gdb_history
 3rdparty
 dist
 tvm_home
 node_modules
 lib
 .parcel-cache
 **/.next
 coverage
@@ -0,0 +1 @@
 npx lint-staged
@@ -0,0 +1,3 @@
 {
  "./**/*.{js,ts,jsx,tsx,json}": ["eslint --fix", "prettier --write"]
 }
@@ -0,0 +1 @@
 v24.11.1
@@ -0,0 +1,8 @@
 dist
 debug
 lib
 build
 node_modules
 3rdparty
 .eslintrc.cjs
 **/.next
@@ -0,0 +1,3 @@
 {
  "trailingComma": "all"
 }
@@ -0,0 +1,143 @@
 # Contributing to WebLLM
 Thank you for your interest in contributing to WebLLM. This guide helps contributors get set up quickly and make high-impact changes that are easy to review and merge.
 ## Ways To Contribute
 We welcome contributions across the project, including:
 - Bug reports with clear reproduction steps
 - Bug fixes and reliability improvements
 - New features and API improvements
 - Performance and memory optimizations
 - Tests and test coverage improvements
 - Documentation updates and tutorials
 - New or improved examples in `examples/`
 - Model integration and configuration improvements
 - Code review and issue triage support
 If you are unsure where to start, look for open issues in the repository and propose a plan in the issue thread before implementation.
 ## Community Principles
 WebLLM is part of a broader open-source ecosystem and follows collaborative, public-first development norms.
 - Keep technical discussion in public, archivable channels (issues and pull requests)
 - Use clear technical reasoning and seek consensus on non-trivial changes
 - For major design changes, start with an issue or RFC-style proposal before coding
 - Review other contributors' PRs when possible
 Additional reference: Apache TVM community guidelines
 - https://tvm.apache.org/docs/contribute/community.html
 ## Development Setup
 ### Prerequisites
 - Node.js (see `.nvmrc` for the required version)
 - npm
 - Git
 Optional:
 - Python 3 (for docs build)
 - Emscripten/toolchain setup
 ### Local Setup
 ```bash
 git clone https://github.com/mlc-ai/web-llm.git
 cd web-llm
 npm install
 ```
 ### Build, Lint, and Test
 ```bash
 npm run build
 npm run lint
 npm test
 ```
 Notes:
 - `npm test` runs Jest with coverage thresholds.
 - For quick iteration on a single test file, you can run:
 ```bash
 npx jest --coverage=false tests/<file>.test.ts
 ```
 ### Auto-formatting
 If lint or style checks fail, run:
 ```bash
 npm run format
 ```
 Pre-commit hooks (Husky + lint-staged) are configured in this repo.
 ## Testing Changes In Examples
 To test local package changes inside an example app:
 1. Edit `examples/<example>/package.json` and set `"@mlc-ai/web-llm"` to `"../.."` (or `"file:../.."` if needed).
 2. Install and run the example.
 ```bash
 cd examples/<example>
 npm install
 npm run start
 ```
 ## Documentation Contributions
 Docs are in `docs/` and built with Sphinx.
 ```bash
 cd docs
 pip3 install -r requirements.txt
 make html
 ```
 Open the built docs from `docs/_build/html`.
 ## Pull Request Guidelines
 Before opening a PR:
 1. Keep the change scoped to one problem or feature.
 2. Add or update tests for behavior changes.
 3. Update docs/examples for user-facing changes.
 4. Run `npm run lint` and `npm test` locally.
 5. Include a clear PR description with:
   - Problem statement
   - Proposed solution
   - Validation steps and results
   - Backward-compatibility considerations
 During review:
 - Respond to comments with concrete follow-ups
 - Prefer additional tests over assumptions
 - Keep commit history understandable (small, logical commits)
 ## Reporting Bugs and Requesting Features
 - Use GitHub Issues for bug reports and feature requests.
 - Include environment details, expected vs. actual behavior, and minimal reproduction steps.
 - For substantial feature additions, open an issue first to align on design and scope.
 ## Security Reporting
 Please do not report security vulnerabilities in public issues. Report vulnerabilities via email to `mlc-llm-private@googlegroups.com`.
 Reference:
 - https://github.com/mlc-ai/web-llm/blob/main/SECURITY.md
 ## License
 By contributing, you agree that your contributions are provided under the repository's Apache-2.0 license.
@@ -0,0 +1,211 @@
                                 Apache License
                           Version 2.0, January 2004
                        http://www.apache.org/licenses/
   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
   1. Definitions.
      "License" shall mean the terms and conditions for use, reproduction,
      and distribution as defined by Sections 1 through 9 of this document.
      "Licensor" shall mean the copyright owner or entity authorized by
      the copyright owner that is granting the License.
      "Legal Entity" shall mean the union of the acting entity and all
      other entities that control, are controlled by, or are under common
      control with that entity. For the purposes of this definition,
      "control" means (i) the power, direct or indirect, to cause the
      direction or management of such entity, whether by contract or
      otherwise, or (ii) ownership of fifty percent (50%) or more of the
      outstanding shares, or (iii) beneficial ownership of such entity.
      "You" (or "Your") shall mean an individual or Legal Entity
      exercising permissions granted by this License.
      "Source" form shall mean the preferred form for making modifications,
      including but not limited to software source code, documentation
      source, and configuration files.
      "Object" form shall mean any form resulting from mechanical
      transformation or translation of a Source form, including but
      not limited to compiled object code, generated documentation,
      and conversions to other media types.
      "Work" shall mean the work of authorship, whether in Source or
      Object form, made available under the License, as indicated by a
      copyright notice that is included in or attached to the work
      (an example is provided in the Appendix below).
      "Derivative Works" shall mean any work, whether in Source or Object
      form, that is based on (or derived from) the Work and for which the
      editorial revisions, annotations, elaborations, or other modifications
      represent, as a whole, an original work of authorship. For the purposes
      of this License, Derivative Works shall not include works that remain
      separable from, or merely link (or bind by name) to the interfaces of,
      the Work and Derivative Works thereof.
      "Contribution" shall mean any work of authorship, including
      the original version of the Work and any modifications or additions
      to that Work or Derivative Works thereof, that is intentionally
      submitted to Licensor for inclusion in the Work by the copyright owner
      or by an individual or Legal Entity authorized to submit on behalf of
      the copyright owner. For the purposes of this definition, "submitted"
      means any form of electronic, verbal, or written communication sent
      to the Licensor or its representatives, including but not limited to
      communication on electronic mailing lists, source code control systems,
      and issue tracking systems that are managed by, or on behalf of, the
      Licensor for the purpose of discussing and improving the Work, but
      excluding communication that is conspicuously marked or otherwise
      designated in writing by the copyright owner as "Not a Contribution."
      "Contributor" shall mean Licensor and any individual or Legal Entity
      on behalf of whom a Contribution has been received by Licensor and
      subsequently incorporated within the Work.
   2. Grant of Copyright License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      copyright license to reproduce, prepare Derivative Works of,
      publicly display, publicly perform, sublicense, and distribute the
      Work and such Derivative Works in Source or Object form.
   3. Grant of Patent License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      (except as stated in this section) patent license to make, have made,
      use, offer to sell, sell, import, and otherwise transfer the Work,
      where such license applies only to those patent claims licensable
      by such Contributor that are necessarily infringed by their
      Contribution(s) alone or by combination of their Contribution(s)
      with the Work to which such Contribution(s) was submitted. If You
      institute patent litigation against any entity (including a
      cross-claim or counterclaim in a lawsuit) alleging that the Work
      or a Contribution incorporated within the Work constitutes direct
      or contributory patent infringement, then any patent licenses
      granted to You under this License for that Work shall terminate
      as of the date such litigation is filed.
   4. Redistribution. You may reproduce and distribute copies of the
      Work or Derivative Works thereof in any medium, with or without
      modifications, and in Source or Object form, provided that You
      meet the following conditions:
      (a) You must give any other recipients of the Work or
          Derivative Works a copy of this License; and
      (b) You must cause any modified files to carry prominent notices
          stating that You changed the files; and
      (c) You must retain, in the Source form of any Derivative Works
          that You distribute, all copyright, patent, trademark, and
          attribution notices from the Source form of the Work,
          excluding those notices that do not pertain to any part of
          the Derivative Works; and
      (d) If the Work includes a "NOTICE" text file as part of its
          distribution, then any Derivative Works that You distribute must
          include a readable copy of the attribution notices contained
          within such NOTICE file, excluding those notices that do not
          pertain to any part of the Derivative Works, in at least one
          of the following places: within a NOTICE text file distributed
          as part of the Derivative Works; within the Source form or
          documentation, if provided along with the Derivative Works; or,
          within a display generated by the Derivative Works, if and
          wherever such third-party notices normally appear. The contents
          of the NOTICE file are for informational purposes only and
          do not modify the License. You may add Your own attribution
          notices within Derivative Works that You distribute, alongside
          or as an addendum to the NOTICE text from the Work, provided
          that such additional attribution notices cannot be construed
          as modifying the License.
      You may add Your own copyright statement to Your modifications and
      may provide additional or different license terms and conditions
      for use, reproduction, or distribution of Your modifications, or
      for any such Derivative Works as a whole, provided Your use,
      reproduction, and distribution of the Work otherwise complies with
      the conditions stated in this License.
   5. Submission of Contributions. Unless You explicitly state otherwise,
      any Contribution intentionally submitted for inclusion in the Work
      by You to the Licensor shall be under the terms and conditions of
      this License, without any additional terms or conditions.
      Notwithstanding the above, nothing herein shall supersede or modify
      the terms of any separate license agreement you may have executed
      with Licensor regarding such Contributions.
   6. Trademarks. This License does not grant permission to use the trade
      names, trademarks, service marks, or product names of the Licensor,
      except as required for reasonable and customary use in describing the
      origin of the Work and reproducing the content of the NOTICE file.
   7. Disclaimer of Warranty. Unless required by applicable law or
      agreed to in writing, Licensor provides the Work (and each
      Contributor provides its Contributions) on an "AS IS" BASIS,
      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
      implied, including, without limitation, any warranties or conditions
      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
      PARTICULAR PURPOSE. You are solely responsible for determining the
      appropriateness of using or redistributing the Work and assume any
      risks associated with Your exercise of permissions under this License.
   8. Limitation of Liability. In no event and under no legal theory,
      whether in tort (including negligence), contract, or otherwise,
      unless required by applicable law (such as deliberate and grossly
      negligent acts) or agreed to in writing, shall any Contributor be
      liable to You for damages, including any direct, indirect, special,
      incidental, or consequential damages of any character arising as a
      result of this License or out of the use or inability to use the
      Work (including but not limited to damages for loss of goodwill,
      work stoppage, computer failure or malfunction, or any and all
      other commercial damages or losses), even if such Contributor
      has been advised of the possibility of such damages.
   9. Accepting Warranty or Additional Liability. While redistributing
      the Work or Derivative Works thereof, You may choose to offer,
      and charge a fee for, acceptance of support, warranty, indemnity,
      or other liability obligations and/or rights consistent with this
      License. However, in accepting such obligations, You may act only
      on Your own behalf and on Your sole responsibility, not on behalf
      of any other Contributor, and only if You agree to indemnify,
      defend, and hold each Contributor harmless for any liability
      incurred by, or claims asserted against, such Contributor by reason
      of your accepting any such warranty or additional liability.
   END OF TERMS AND CONDITIONS
   APPENDIX: How to apply the Apache License to your work.
      To apply the Apache License to your work, attach the following
      boilerplate notice, with the fields enclosed by brackets "[]"
      replaced with your own identifying information. (Don't include
      the brackets!)  The text should be enclosed in the appropriate
      comment syntax for the file format. We also recommend that a
      file or class name and description of purpose be included on the
      same "printed page" as the copyright notice for easier
      identification within third-party archives.
   Copyright [yyyy] [name of copyright owner]
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
       http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 ------------------------------------------------------------------------------------
 This product bundles various third-party components under other open source licenses.
 This section summarizes those components and their licenses. See licenses/
 for text of these licenses.
 Apache Software Foundation License 2.0
 --------------------------------------
 src/openai_api_protocols
@@ -0,0 +1,567 @@
 <div align="center" id="top">
 # WebLLM
 [![NPM Package](https://img.shields.io/badge/NPM_Package-Published-cc3534)](https://www.npmjs.com/package/@mlc-ai/web-llm)
 [!["WebLLM Chat Deployed"](https://img.shields.io/badge/WebLLM_Chat-Deployed-%2332a852)](https://chat.webllm.ai/)
 [![Join Discord](https://img.shields.io/badge/Join-Discord-7289DA?logo=discord&logoColor=white)](https://discord.gg/9Xpy2HGBuD)
 [![Related Repository: WebLLM Chat](https://img.shields.io/badge/Related_Repo-WebLLM_Chat-fafbfc?logo=github)](https://github.com/mlc-ai/web-llm-chat/)
 [![Related Repository: MLC LLM](https://img.shields.io/badge/Related_Repo-MLC_LLM-fafbfc?logo=github)](https://github.com/mlc-ai/mlc-llm/)
 **High-Performance In-Browser LLM Inference Engine.**
 [Documentation](https://webllm.mlc.ai/docs/) | [Blogpost](https://blog.mlc.ai/2024/06/13/webllm-a-high-performance-in-browser-llm-inference-engine) | [Paper](https://arxiv.org/abs/2412.15803) | [Examples](examples)
 </div>
 ## Overview
 WebLLM is a high-performance in-browser LLM inference engine that brings language model inference directly onto web browsers with hardware acceleration.
 Everything runs inside the browser with no server support and is accelerated with WebGPU.
 WebLLM is **fully compatible with [OpenAI API](https://platform.openai.com/docs/api-reference/chat).**
 That is, you can use the same OpenAI API on **any open source models** locally, with functionalities
 including streaming, JSON-mode, function-calling (WIP), etc.
 We can bring a lot of fun opportunities to build AI assistants for everyone and enable privacy while enjoying GPU acceleration.
 You can use WebLLM as a base [npm package](https://www.npmjs.com/package/@mlc-ai/web-llm) and build your own web application on top of it by following the examples below. This project is a companion project of [MLC LLM](https://github.com/mlc-ai/mlc-llm), which enables universal deployment of LLM across hardware environments.
 <div align="center">
 **[Check out WebLLM Chat to try it out!](https://chat.webllm.ai/)**
 </div>
 ## Key Features
 - **In-Browser Inference**: WebLLM is a high-performance, in-browser language model inference engine that leverages WebGPU for hardware acceleration, enabling powerful LLM operations directly within web browsers without server-side processing.
 - [**Full OpenAI API Compatibility**](#full-openai-compatibility): Seamlessly integrate your app with WebLLM using OpenAI API with functionalities such as streaming, JSON-mode, logit-level control, seeding, and more.
 - **Structured JSON Generation**: WebLLM supports state-of-the-art JSON mode structured generation, implemented in the WebAssembly portion of the model library for optimal performance. Check [WebLLM JSON Playground](https://huggingface.co/spaces/mlc-ai/WebLLM-JSON-Playground) on HuggingFace to try generating JSON output with custom JSON schema.
 - [**Extensive Model Support**](#built-in-models): WebLLM natively supports a range of models including Llama 3, Phi 3, Gemma, Mistral, Qwen(通义千问), and many others, making it versatile for various AI tasks. For the complete supported model list, check [MLC Models](https://mlc.ai/models).
 - [**Custom Model Integration**](#custom-models): Easily integrate and deploy custom models in MLC format, allowing you to adapt WebLLM to specific needs and scenarios, enhancing flexibility in model deployment.
 - **Plug-and-Play Integration**: Easily integrate WebLLM into your projects using package managers like NPM and Yarn, or directly via CDN, complete with comprehensive [examples](./examples/) and a modular design for connecting with UI components.
 - **Streaming & Real-Time Interactions**: Supports streaming chat completions, allowing real-time output generation which enhances interactive applications like chatbots and virtual assistants.
 - **Web Worker & Service Worker Support**: Optimize UI performance and manage the lifecycle of models efficiently by offloading computations to separate worker threads or service workers.
 - **Chrome Extension Support**: Extend the functionality of web browsers through custom Chrome extensions using WebLLM, with examples available for building both basic and advanced extensions.
 ## Built-in Models
 Check the complete list of available models on [MLC Models](https://mlc.ai/models). WebLLM supports a subset of these available models and the list can be accessed at [`prebuiltAppConfig.model_list`](https://github.com/mlc-ai/web-llm/blob/main/src/config.ts#L293).
 Here are the primary families of models currently supported:
 - **Llama**: Llama 3, Llama 2, Hermes-2-Pro-Llama-3
 - **Phi**: Phi 3, Phi 2, Phi 1.5
 - **Gemma**: Gemma-2B
 - **Mistral**: Mistral-7B-v0.3, Hermes-2-Pro-Mistral-7B, NeuralHermes-2.5-Mistral-7B, OpenHermes-2.5-Mistral-7B
 - **Qwen (通义千问)**: Qwen2 0.5B, 1.5B, 7B
 If you need more models, [request a new model via opening an issue](https://github.com/mlc-ai/web-llm/issues/new/choose) or check [Custom Models](#custom-models) for how to compile and use your own models with WebLLM.
 ## Jumpstart with Examples
 Learn how to use WebLLM to integrate large language models into your application and generate chat completions through this simple Chatbot example:
 [![Example Chatbot on JSFiddle](https://img.shields.io/badge/Example-JSFiddle-blue?logo=jsfiddle&logoColor=white)](https://jsfiddle.net/neetnestor/4nmgvsa2/)
 [![Example Chatbot on Codepen](https://img.shields.io/badge/Example-Codepen-gainsboro?logo=codepen)](https://codepen.io/neetnestor/pen/vYwgZaG)
 For an advanced example of a larger, more complicated project, check [WebLLM Chat](https://github.com/mlc-ai/web-llm-chat/blob/main/app/client/webllm.ts).
 More examples for different use cases are available in the [examples](./examples/) folder.
 ## Get Started
 WebLLM offers a minimalist and modular interface to access the chatbot in the browser.
 The package is designed in a modular way to hook to any of the UI components.
 ### Installation
 #### Package Manager
 ```sh
 # npm
 npm install @mlc-ai/web-llm
 # yarn
 yarn add @mlc-ai/web-llm
 # or pnpm
 pnpm install @mlc-ai/web-llm
 ```
 Then import the module in your code.
 ```typescript
 // Import everything
 import * as webllm from "@mlc-ai/web-llm";
 // Or only import what you need
 import { CreateMLCEngine } from "@mlc-ai/web-llm";
 ```
 #### CDN Delivery
 Thanks to [jsdelivr.com](https://www.jsdelivr.com/package/npm/@mlc-ai/web-llm), WebLLM can be imported directly through URL and work out-of-the-box on cloud development platforms like [jsfiddle.net](https://jsfiddle.net/), [Codepen.io](https://codepen.io/), and [Scribbler](https://scribbler.live):
 ```javascript
 import * as webllm from "https://esm.run/@mlc-ai/web-llm";
 ```
 It can also be dynamically imported as:
 ```javascript
 const webllm = await import("https://esm.run/@mlc-ai/web-llm");
 ```
 ### Create MLCEngine
 Most operations in WebLLM are invoked through the `MLCEngine` interface. You can create an `MLCEngine` instance and loading the model by calling the `CreateMLCEngine()` factory function.
 (Note that loading models requires downloading and it can take a significant amount of time for the very first run without caching previously. You should properly handle this asynchronous call.)
 ```typescript
 import { CreateMLCEngine } from "@mlc-ai/web-llm";
 // Callback function to update model loading progress
 const initProgressCallback = (initProgress) => {
  console.log(initProgress);
 };
 const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
 const engine = await CreateMLCEngine(
  selectedModel,
  { initProgressCallback: initProgressCallback }, // engineConfig
 );
 ```
 Under the hood, this factory function does the following steps for first creating an engine instance (synchronous) and then loading the model (asynchronous). You can also do them separately in your application.
 ```typescript
 import { MLCEngine } from "@mlc-ai/web-llm";
 // This is a synchronous call that returns immediately
 const engine = new MLCEngine({
  initProgressCallback: initProgressCallback,
 });
 // This is an asynchronous call and can take a long time to finish
 await engine.reload(selectedModel);
 ```
 ### Cache Backend Policy
 WebLLM supports four cache backends through `AppConfig.cacheBackend`:
 - `"cache"`: browser [Cache API](https://developer.mozilla.org/en-US/docs/Web/API/Cache) (default).
 - `"indexeddb"`: browser [IndexedDB](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API).
 - `"opfs"`: browser [Origin Private File System (OPFS)](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system).
 - `"cross-origin"`: experimental Chrome [Cross-Origin Storage API](https://github.com/WICG/cross-origin-storage) extension backend. Install the [Cross-Origin Storage extension](https://chromewebstore.google.com/detail/cross-origin-storage/denpnpcgjgikjpoglpjefakmdcbmlgih) to use it. (If the extension isn't installed, WebLLM falls back to the default cache automatically.)
 Example:
 ```typescript
 import { CreateMLCEngine, prebuiltAppConfig } from "@mlc-ai/web-llm";
 const appConfig = { ...prebuiltAppConfig, cacheBackend: "cross-origin" };
 const engine = await CreateMLCEngine("Llama-3.1-8B-Instruct-q4f32_1-MLC", {
  appConfig,
 });
 ```
 Notes:
 - If `"opfs"` is selected in an environment without OPFS support, cache operations fail with an OPFS availability error.
 - When using `"opfs"`, `appConfig.opfsAccessMode` can be set to `"auto"` to use OPFS sync access handles where supported, or `"sync"` to require sync access handles. The default is `"async"`.
 - The `"cross-origin"` backend requires installing and enabling a compatible browser extension.
 - Cross-origin backend currently does not support programmatic tensor-cache deletion; clearing is extension-managed.
 ### Chat Completion
 After successfully initializing the engine, you can now invoke chat completions using OpenAI style chat APIs through the `engine.chat.completions` interface. For the full list of parameters and their descriptions, check [section below](#full-openai-compatibility) and [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create).
 (Note: The `model` parameter is not supported and will be ignored here. Instead, call `CreateMLCEngine(model)` or `engine.reload(model)` instead as shown in the [Create MLCEngine](#create-mlcengine) above.)
 ```typescript
 const messages = [
  { role: "system", content: "You are a helpful AI assistant." },
  { role: "user", content: "Hello!" },
 ];
 const reply = await engine.chat.completions.create({
  messages,
 });
 console.log(reply.choices[0].message);
 console.log(reply.usage);
 ```
 ### Streaming
 WebLLM also supports streaming chat completion generating. To use it, simply pass `stream: true` to the `engine.chat.completions.create` call.
 ```typescript
 const messages = [
  { role: "system", content: "You are a helpful AI assistant." },
  { role: "user", content: "Hello!" },
 ];
 // Chunks is an AsyncGenerator object
 const chunks = await engine.chat.completions.create({
  messages,
  temperature: 1,
  stream: true, // <-- Enable streaming
  stream_options: { include_usage: true },
 });
 let reply = "";
 for await (const chunk of chunks) {
  reply += chunk.choices[0]?.delta.content || "";
  console.log(reply);
  if (chunk.usage) {
    console.log(chunk.usage); // only last chunk has usage
  }
 }
 const fullReply = await engine.getMessage();
 console.log(fullReply);
 ```
 ## Advanced Usage
 ### Using Workers
 You can put the heavy computation in a worker script to optimize your application performance. To do so, you need to:
 1. Create a handler in the worker thread that communicates with the frontend while handling the requests.
 2. Create a Worker Engine in your main application, which under the hood sends messages to the handler in the worker thread.
 For detailed implementations of different kinds of Workers, check the following sections.
 #### Dedicated Web Worker
 WebLLM comes with API support for WebWorker so you can hook
 the generation process into a separate worker thread so that
 the computing in the worker thread won't disrupt the UI.
 We create a handler in the worker thread that communicates with the frontend while handling the requests.
 ```typescript
 // worker.ts
 import { WebWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
 // A handler that resides in the worker thread
 const handler = new WebWorkerMLCEngineHandler();
 self.onmessage = (msg: MessageEvent) => {
  handler.onmessage(msg);
 };
 ```
 In the main logic, we create a `WebWorkerMLCEngine` that
 implements the same `MLCEngineInterface`. The rest of the logic remains the same.
 ```typescript
 // main.ts
 import { CreateWebWorkerMLCEngine } from "@mlc-ai/web-llm";
 async function main() {
  // Use a WebWorkerMLCEngine instead of MLCEngine here
  const engine = await CreateWebWorkerMLCEngine(
    new Worker(new URL("./worker.ts", import.meta.url), {
      type: "module",
    }),
    selectedModel,
    { initProgressCallback }, // engineConfig
  );
  // everything else remains the same
 }
 ```
 ### Use Service Worker
 WebLLM comes with API support for ServiceWorker so you can hook the generation process
 into a service worker to avoid reloading the model in every page visit and optimize
 your application's offline experience.
 (Note, Service Worker's life cycle is managed by the browser and can be killed any time without notifying the webapp. `ServiceWorkerMLCEngine` will try to keep the service worker thread alive by periodically sending heartbeat events, but your application should also include proper error handling. Check `keepAliveMs` and `missedHeatbeat` in [`ServiceWorkerMLCEngine`](https://github.com/mlc-ai/web-llm/blob/main/src/service_worker.ts#L234) for more details.)
 We create a handler in the worker thread that communicates with the frontend while handling the requests.
 ```typescript
 // sw.ts
 import { ServiceWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
 let handler: ServiceWorkerMLCEngineHandler;
 self.addEventListener("activate", function (event) {
  handler = new ServiceWorkerMLCEngineHandler();
  console.log("Service Worker is ready");
 });
 ```
 Then in the main logic, we register the service worker and create the engine using
 `CreateServiceWorkerMLCEngine` function. The rest of the logic remains the same.
 ```typescript
 // main.ts
 import {
  MLCEngineInterface,
  CreateServiceWorkerMLCEngine,
 } from "@mlc-ai/web-llm";
 if ("serviceWorker" in navigator) {
  navigator.serviceWorker.register(
    new URL("sw.ts", import.meta.url), // worker script
    { type: "module" },
  );
 }
 const engine: MLCEngineInterface = await CreateServiceWorkerMLCEngine(
  selectedModel,
  { initProgressCallback }, // engineConfig
 );
 ```
 You can find a complete example on how to run WebLLM in service worker in [examples/service-worker](examples/service-worker/).
 ### Chrome Extension
 You can also find examples of building Chrome extension with WebLLM in [examples/chrome-extension](examples/chrome-extension/) and [examples/chrome-extension-webgpu-service-worker](examples/chrome-extension-webgpu-service-worker/). The latter one leverages service worker, so the extension is persistent in the background. Additionally, you can explore another full project of a Chrome extension, WebLLM Assistant, which leverages WebLLM [here](https://github.com/mlc-ai/web-llm-assistant).
 ## Full OpenAI Compatibility
 WebLLM is designed to be fully compatible with [OpenAI API](https://platform.openai.com/docs/api-reference/chat). Thus, besides building a simple chatbot, you can also have the following functionalities with WebLLM:
 - [streaming](examples/streaming): return output as chunks in real-time in the form of an AsyncGenerator
 - [json-mode](examples/json-mode): efficiently ensure output is in JSON format, see [OpenAI Reference](https://platform.openai.com/docs/guides/text-generation/chat-completions-api) for more.
 - [seed-to-reproduce](examples/seed-to-reproduce): use seeding to ensure a reproducible output with fields `seed`.
 - [function-calling](examples/function-calling) (WIP): function calling with fields `tools` and `tool_choice` (with preliminary support); or manual function calling without `tools` or `tool_choice` (keeps the most flexibility).
 ## Integrity Verification
 WebLLM supports optional integrity verification for model artifacts using
 [SRI (Subresource Integrity)](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) hashes.
 When the `integrity` field is set on a `ModelRecord`, WebLLM will verify the downloaded config,
 WASM, and tokenizer files against the provided hashes before loading.
 ```typescript
 import { CreateMLCEngine } from "@mlc-ai/web-llm";
 const appConfig = {
  model_list: [
    {
      model: "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_1-MLC",
      model_id: "Llama-3.2-1B-Instruct-q4f16_1-MLC",
      model_lib:
        "https://raw.githubusercontent.com/user/model-libs/main/model.wasm",
      integrity: {
        config: "sha256-<base64-hash-of-mlc-chat-config.json>",
        model_lib: "sha256-<base64-hash-of-wasm-file>",
        tokenizer: {
          "tokenizer.json": "sha256-<base64-hash-of-tokenizer.json>",
        },
        onFailure: "error", // "error" (default) throws IntegrityError, "warn" logs and continues
      },
    },
  ],
 };
 const engine = await CreateMLCEngine("Llama-3.2-1B-Instruct-q4f16_1-MLC", {
  appConfig,
 });
 ```
 You can generate SRI hashes for model files with:
 ```bash
 # SHA-256
 openssl dgst -sha256 -binary <file> | openssl base64 -A | sed 's/^/sha256-/'
 # SHA-384
 openssl dgst -sha384 -binary <file> | openssl base64 -A | sed 's/^/sha384-/'
 # SHA-512
 openssl dgst -sha512 -binary <file> | openssl base64 -A | sed 's/^/sha512-/'
 ```
 > The `openssl` commands require a Unix-like shell (macOS/Linux). On Windows, run `openssl` via [Git Bash](https://gitforwindows.org/) or [WSL](https://learn.microsoft.com/en-us/windows/wsl/).
 If a hash does not match, an `IntegrityError` is thrown (or a warning is logged when `onFailure: "warn"`).
 All fields in `integrity` are optional — only specified artifacts will be verified.
 When the `integrity` field is omitted entirely, WebLLM behaves exactly as before (no verification).
 See the [integrity-verification example](examples/integrity-verification/) for a complete working demo.
 ## Custom Models
 WebLLM works as a companion project of [MLC LLM](https://github.com/mlc-ai/mlc-llm) and it supports custom models in MLC format.
 It reuses the model artifact and builds the flow of MLC LLM. To compile and use your own models with WebLLM, please check out
 [MLC LLM document](https://llm.mlc.ai/docs/deploy/webllm.html)
 on how to compile and deploy new model weights and libraries to WebLLM.
 Here, we go over the high-level idea. There are two elements of the WebLLM package that enable new models and weight variants.
 - `model`: Contains a URL to model artifacts, such as weights and meta-data.
 - `model_lib`: A URL to the web assembly library (i.e. wasm file) that contains the executables to accelerate the model computations.
 Both are customizable in the WebLLM.
 ```typescript
 import { CreateMLCEngine } from "@mlc-ai/web-llm";
 async main() {
  const appConfig = {
    "model_list": [
      {
        "model": "/url/to/my/llama",
        "model_id": "MyLlama-3b-v1-q4f32_0",
        "model_lib": "/url/to/myllama3b.wasm",
      }
    ],
  };
  // override default
  const chatOpts = {
    "repetition_penalty": 1.01
  };
  // load a prebuilt model
  // with a chat option override and app config
  // under the hood, it will load the model from myLlamaUrl
  // and cache it in the browser cache
  // The chat will also load the model library from "/url/to/myllama3b.wasm",
  // assuming that it is compatible to the model in myLlamaUrl.
  const engine = await CreateMLCEngine(
    "MyLlama-3b-v1-q4f32_0",
    { appConfig }, // engineConfig
    chatOpts,
  );
 }
 ```
 In many cases, we only want to supply the model weight variant, but
 not necessarily a new model (e.g. `NeuralHermes-Mistral` can reuse `Mistral`'s
 model library). For examples of how a model library can be shared by different model variants,
 see `webllm.prebuiltAppConfig`.
 ## Build WebLLM Package From Source
 NOTE: you don't need to build from source unless you would like to modify the WebLLM package.
 To use the npm, simply follow [Get Started](#get-started) or any of the [examples](examples) instead.
 To build from source, simply run:
 ```bash
 npm install
 npm run build
 ```
 Then, to test the effects of your code change in an example, inside `examples/get-started/package.json`, change from `"@mlc-ai/web-llm": "^0.2.84"` to `"@mlc-ai/web-llm": ../..`.
 Then run:
 ```bash
 cd examples/get-started
 npm install
 npm start
 ```
 Note that sometimes you would need to switch between `file:../..` and `../..` to trigger npm to recognize new changes. In the worst case, you can run:
 ```bash
 cd examples/get-started
 rm -rf node_modules dist package-lock.json .parcel-cache
 npm install
 npm start
 ```
 ### In case you need to build TVMjs from source
 WebLLM's runtime largely depends on TVMjs: https://github.com/apache/tvm/tree/main/web
 While it is also available as an npm package: https://www.npmjs.com/package/@mlc-ai/web-runtime, you can build it from source if needed by following the steps below.
 1. Install [emscripten](https://emscripten.org). It is an LLVM-based compiler that compiles C/C++ source code to WebAssembly.
   - Follow the [installation instruction](https://emscripten.org/docs/getting_started/downloads.html#installation-instructions-using-the-emsdk-recommended) to install the latest emsdk.
   - Source `emsdk_env.sh` by `source path/to/emsdk_env.sh`, so that `emcc` is reachable from PATH and the command `emcc` works.
   We can verify the successful installation by trying out `emcc` terminal.
   Note: We recently found that using the latest `emcc` version may run into issues during runtime. Use `./emsdk install 3.1.56` instead of `./emsdk install latest` for now as a workaround. The error may look like
   ```
   Init error, LinkError: WebAssembly.instantiate(): Import #6 module="wasi_snapshot_preview1"
   function="proc_exit": function import requires a callable
   ```
 2. In `./package.json`, change from `"@mlc-ai/web-runtime": "0.18.0-dev2",` to `"@mlc-ai/web-runtime": "file:./tvm_home/web",`.
 3. Setup necessary environment
   Prepare all the necessary dependencies for web build:
   ```shell
   ./scripts/prep_deps.sh
   ```
   In this step, if `$TVM_SOURCE_DIR` is not defined in the environment, we will execute the following line to build `tvmjs` dependency:
   ```shell
   git clone https://github.com/mlc-ai/relax 3rdparty/tvm-unity --recursive
   ```
   This clones the current HEAD of `mlc-ai/relax`. However, it may not always be the correct branch or commit to clone. To build a specific npm version from source, refer to the version bump PR, which states which branch (i.e. `mlc-ai/relax` or `apache/tvm`) and which commit the current WebLLM version depends on. For instance, version 0.2.52, according to its version bump PR https://github.com/mlc-ai/web-llm/pull/521, is built by checking out the following commit https://github.com/apache/tvm/commit/e6476847753c80e054719ac47bc2091c888418b6 in `apache/tvm`, rather than the HEAD of `mlc-ai/relax`.
   Besides, `--recursive` is necessary and important. Otherwise, you may encounter errors like `fatal error: 'dlpack/dlpack.h' file not found`.
 4. Build WebLLM Package
   ```shell
   npm run build
   ```
 5. Validate some of the sub-packages
   You can then go to the subfolders in [examples](examples) to validate some of the sub-packages.
   We use Parcelv2 for bundling. Although Parcel is not very good at tracking parent directory
   changes sometimes. When you make a change in the WebLLM package, try to edit the `package.json`
   of the subfolder and save it, which will trigger Parcel to rebuild.
 ## Links
 - [Demo App: WebLLM Chat](https://chat.webllm.ai/)
 - If you want to run LLM on native runtime, check out [MLC-LLM](https://github.com/mlc-ai/mlc-llm)
 - You might also be interested in [Web Stable Diffusion](https://github.com/mlc-ai/web-stable-diffusion/).
 ## Acknowledgement
 This project is initiated by members from CMU Catalyst, UW SAMPL, SJTU, OctoML, and the MLC community. We would love to continue developing and supporting the open-source ML community.
 This project is only possible thanks to the shoulders open-source ecosystems that we stand on. We want to thank the Apache TVM community and developers of the TVM Unity effort. The open-source ML community members made these models publicly available. PyTorch and Hugging Face communities make these models accessible. We would like to thank the teams behind Vicuna, SentencePiece, LLaMA, and Alpaca. We also would like to thank the WebAssembly, Emscripten, and WebGPU communities. Finally, thanks to Dawn and WebGPU developers.
 ## Citation
 If you find this project to be useful, please cite:
 ```
@misc{ruan2026webllmhighperformanceinbrowserllm,
      title={WebLLM: A High-Performance In-Browser LLM Inference Engine},
      author={Charlie F. Ruan and Yucheng Qin and Akaash R. Parthasarathy and Xun Zhou and Ruihang Lai and Hongyi Jin and Yixin Dong and Bohan Hou and Meng-Shiun Yu and Yiyan Zhai and Sudeep Agarwal and Hangrui Cao and Siyuan Feng and Tianqi Chen},
      year={2026},
      eprint={2412.15803},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2412.15803},
 }
 ```
 ## Contributors
 <a href="https://github.com/mlc-ai/web-llm/graphs/contributors">
  <img alt="contributors" src="https://contrib.rocks/image?repo=mlc-ai/web-llm"/>
 </a>
 <p align="right">
  <a href="#top">⬆ Back to Top ⬆</a>
 </p>
@@ -0,0 +1,7 @@
 # WeHub 来源说明
 - 原始项目：`mlc-ai/web-llm`
 - 原始仓库：https://github.com/mlc-ai/web-llm
 - 导入方式：上游默认分支的最新快照
 - 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
 - 本文件仅用于记录来源，不代表 WeHub 是原项目作者
@@ -0,0 +1,5 @@
 # Security Policy
 ## Reporting a Vulnerability
 For security concerns or vulnerability reports, please send email to `mlc-llm-private@googlegroups.com`.
@@ -0,0 +1,39 @@
 # Remove instances of string "const{createRequire:createRequire}=await import('module');"
 # This is required to allow background workers packaged with Parcel for the chrome extension
 # to run the `ChatModule`.
 sed -e s/"const{createRequire:createRequire}=await import('module');"//g -i.backup lib/index.js
 sed -e s/"const{createRequire:createRequire}=await import('module');"//g -i.backup lib/index.js.map
 # Replace scriptDirectory init that Parcel cannot resolve ("new URL('./', import.meta.url)") with a plain relative string
 sed -e s~"require(\\\"url\\\").fileURLToPath(new URL(\\\"\\.\\/\\\",import.meta.url))"~"\\\"./\\\""~g -i.backup lib/index.js
 sed -e s~"require(\\\"url\\\").fileURLToPath(new URL(\\\"\\.\\/\\\",import.meta.url))"~'\\\".\\\"'~g -i.backup lib/index.js.map
 # Replace string "new (require('u' + 'rl').URL)('file:' + __filename).href" with "MLC_DUMMY_PATH"
 # This is required for building nextJS projects -- its compile time would complain about `require()`
 # See https://github.com/mlc-ai/web-llm/issues/383 and the fixing PR's description for more.
 sed -e s/"new (require('u' + 'rl').URL)('file:' + __filename).href"/"\"MLC_DUMMY_PATH\""/g -i.backup lib/index.js
 # Replace rollup's newer output "require('u' + 'rl').pathToFileURL(__filename).href" with "MLC_DUMMY_PATH"
 # This avoids `require is not defined` in ESM SSR contexts (e.g. SvelteKit/Astro with bun/node).
 sed -e s/"require('u' + 'rl')\\.pathToFileURL(__filename)\\.href"/"\"MLC_DUMMY_PATH\""/g -i.backup lib/index.js
 # Replace with \"MLC_DUMMY_PATH\"
 sed -e s/"new (require('u' + 'rl').URL)('file:' + __filename).href"/'\\\"MLC_DUMMY_PATH\\\"'/g -i.backup lib/index.js.map
 sed -e s/"require('u' + 'rl')\\.pathToFileURL(__filename)\\.href"/'\\\"MLC_DUMMY_PATH\\\"'/g -i.backup lib/index.js.map
 # Replace "import require$$N from 'perf_hooks';" with "const require$$N = "MLC_DUMMY_REQUIRE_VAR""
 # This is to prevent `perf_hooks` not found error
 # For more see https://github.com/mlc-ai/web-llm/issues/258 and https://github.com/mlc-ai/web-llm/issues/127
 # Use a regex to match any variable number (require$$0, require$$1, etc.)
 sed -E -i.backup "s/import (require\\\$\\\$[0-9]+) from 'perf_hooks';/const \1 = \"MLC_DUMMY_REQUIRE_VAR\"/g" lib/index.js
 # Similarly replace `const performanceNode = require(\"perf_hooks\")` with `const performanceNode = \"MLC_DUMMY_REQUIRE_VAR\"`
 sed -e s/'require(\\\"perf_hooks\\\")'/'\\\"MLC_DUMMY_REQUIRE_VAR\\\"'/g -i.backup lib/index.js.map
 # Below is added when we include dependency @mlc-ai/web-runtime, rather than using local tvm_home
 # Replace "import require$$N from 'ws'" with "const require$$N = "MLC_DUMMY_REQUIRE_VAR""
 # This is to prevent error `Cannot find module 'ws'`
 sed -E -i.backup "s/import (require\\\$\\\$[0-9]+) from 'ws';/const \1 = \"MLC_DUMMY_REQUIRE_VAR\"/g" lib/index.js
 # Similarly replace `const WebSocket = require(\"ws\")` with `const WebSocket = \"MLC_DUMMY_REQUIRE_VAR\"`
 sed -e s/'require(\\\"ws\\\")'/'\\\"MLC_DUMMY_REQUIRE_VAR\\\"'/g -i.backup lib/index.js.map
 # Cleanup backup files
 rm -f lib/index.js.backup
 rm -f lib/index.js.map.backup
@@ -0,0 +1,20 @@
 # Minimal makefile for Sphinx documentation
 #
 # You can set these variables from the command line, and also
 # from the environment for the first two.
 SPHINXOPTS    ?=
 SPHINXBUILD   ?= python -m sphinx
 SOURCEDIR     = .
 BUILDDIR      = _build
 # Put it first so that "make" without argument is like "make help".
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 .PHONY: help Makefile
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
@@ -0,0 +1,30 @@
 # WebLLM Documentation
 The documentation was built upon [Sphinx](https://www.sphinx-doc.org/en/master/).
 ## Dependencies
 Run the following command in this directory to install dependencies first:
 ```bash
 pip3 install -r requirements.txt
 ```
 ## Build the Documentation
 Then you can build the documentation by running:
 ```bash
 make html
 ```
 ## View the Documentation
 Run the following command to start a simple HTTP server:
 ```bash
 cd _build/html
 python3 -m http.server
 ```
 Then you can view the documentation in your browser at `http://localhost:8000` (the port can be customized by appending ` -p PORT_NUMBER` in the python command above).
@@ -0,0 +1,87 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <svg
   xmlns:i="&amp;#38;ns_ai;"
   xmlns:dc="http://purl.org/dc/elements/1.1/"
   xmlns:cc="http://creativecommons.org/ns#"
   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
   xmlns:svg="http://www.w3.org/2000/svg"
   xmlns="http://www.w3.org/2000/svg"
   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
   version="1.1"
   id="Layer_1"
   x="0px"
   y="0px"
   viewBox="0 0 523.56958 171.35398"
   xml:space="preserve"
   sodipodi:docname="mlc-logo-with-text-landscape.svg"
   width="523.56958"
   height="171.35399"
   inkscape:version="1.0.1 (1.0.1+r75)"><metadata
   id="metadata23"><rdf:RDF><cc:Work
       rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
         rdf:resource="http://purl.org/dc/dcmitype/StillImage" /></cc:Work></rdf:RDF></metadata><defs
   id="defs21" /><sodipodi:namedview
   pagecolor="#ffffff"
   bordercolor="#666666"
   borderopacity="1"
   objecttolerance="10"
   gridtolerance="10"
   guidetolerance="10"
   inkscape:pageopacity="0"
   inkscape:pageshadow="2"
   inkscape:window-width="1853"
   inkscape:window-height="1025"
   id="namedview19"
   showgrid="false"
   inkscape:zoom="1.42"
   inkscape:cx="201.79468"
   inkscape:cy="119.53208"
   inkscape:window-x="67"
   inkscape:window-y="27"
   inkscape:window-maximized="1"
   inkscape:current-layer="Layer_1"
   inkscape:document-rotation="0" />
 <style
   type="text/css"
   id="style2">
 	.st0{fill-rule:evenodd;clip-rule:evenodd;fill:#062578;}
 	.st1{fill:#062578;}
 </style>
 <switch
   id="switch16"
   transform="translate(19.160879,-130.46791)">
 	<foreignObject
   requiredExtensions="http://ns.adobe.com/AdobeIllustrator/10.0/"
   x="0"
   y="0"
   width="1"
   height="1">
 	</foreignObject>
 	<g
   i:extraneous="self"
   id="g14">
 		<g
   id="g12">
 			<path
   class="st0"
   d="M 124.8,208.2 H 82.7 c -1.4,0 -2.6,1.2 -2.6,2.6 v 7.5 7.5 3.4 c 1.7,-0.2 3.3,0.1 4.7,0.9 v -11.8 -5.4 h 37.9 v 49.9 H 84.8 v -4.6 -11.1 c -1.1,0.8 -2.3,1.4 -3.4,2 -0.4,0.2 -0.8,0.4 -1.2,0.5 v 9.5 3.6 2.1 c 0,1.4 1.2,2.6 2.6,2.6 h 42.1 c 1.4,0 2.6,-1.1 2.6,-2.6 v -54.2 c -0.1,-1.2 -1.3,-2.4 -2.7,-2.4 z m -44.6,38.5 c -3,1.4 -8.3,2.7 -11.5,3.4 -3.6,0.8 -14.2,3.2 -15.1,-2.5 -0.7,-4.7 11.6,-9.9 14.8,-11.3 2.9,-1.3 5.9,-2.4 8.8,-3.6 3.2,-1.2 6.6,-1.7 8.2,2.3 0.7,1.7 0.9,3.2 0.9,5 v 0.2 0.2 c -0.4,3.1 -3.4,5.1 -6.1,6.3 z m 10.6,-51.3 v 0 c 1.4,-0.1 2.7,0.9 2.8,2.4 l 0.4,4.9 c 0.1,1.4 -0.9,2.7 -2.4,2.8 v 0 c -1.4,0.1 -2.7,-0.9 -2.8,-2.4 l -0.4,-4.9 c -0.1,-1.4 1,-2.6 2.4,-2.8 z m 18.8,10.9 c 0,-2.9 -0.2,-5.7 -0.4,-8.4 v 0 c -0.1,-0.9 -0.2,-1.8 -0.3,-2.7 v -0.1 -0.2 c -0.5,-3.3 -1.1,-6.6 -1.9,-10 -0.8,-3.3 -3.7,-5.4 -7,-5.5 -6.1,-0.1 -12.3,-0.2 -18.4,0 -1,-1.9 -4.2,-3.1 -7.9,-2.9 L 69.5,167 c 0.8,-0.7 1.2,-1.8 1.1,-3 -0.2,-2 -1.9,-3.5 -3.9,-3.3 -2,0.2 -3.5,1.9 -3.3,3.9 0.2,1.9 1.7,3.3 3.6,3.3 l 4,9 c -2.6,0.7 -4.6,2.2 -5.1,3.9 -5.8,0.8 -11.5,2 -17.2,3.1 -0.7,0.1 -1.3,0.4 -1.9,0.6 -6.4,1.6 -13,5.1 -13,5.1 -0.1,0.8 -0.2,1.7 -0.2,2.5 0.3,-0.1 0.6,-0.1 1,-0.2 5,-0.4 9.6,4.7 10.2,11.5 0.6,6.8 -3,12.7 -8,13.1 -0.4,0 -0.8,0 -1.1,0 0.2,0.9 0.4,1.8 0.7,2.6 4.2,2.3 9.9,3.8 13.4,4.6 v 0 c 0.8,0.3 1.7,0.5 2.6,0.5 8.6,0.2 17.3,0.3 25.9,-0.5 h 0.3 v -6.3 c -5.8,0.5 -11.8,0.5 -20.8,0.3 -1.7,0 -2.9,-1.1 -3.4,-2.5 -2,-7.2 -2.5,-14.8 -2,-22.9 0.1,-1.5 1.2,-2.8 2.9,-3.1 1.6,-0.3 3.2,-0.6 4.6,-0.9 0.7,-0.1 2.5,-0.5 3.8,-0.7 v 0 c 1,-0.2 2.1,-0.4 3.1,-0.6 3.3,0.6 5.6,5.4 10.5,4.6 5,-0.1 6.4,-5.2 9.5,-6.3 2.3,0 4.6,0 6.9,0.1 1.1,0 2.1,0 3.9,0.1 1.7,0 3,1 3.4,2.5 0.4,1.6 0.7,3.2 1,4.8 0.6,5.4 0.9,9.8 0.8,13.2 h 6.8 z m -75.1,-7.8 c 2,-0.2 3.8,2.5 4.1,6 0.3,3.5 -1,6.4 -3,6.6 -1,0.1 -1.9,-0.5 -2.6,-1.5 0.4,0.3 0.9,0.5 1.3,0.4 1.5,-0.1 2.6,-2.5 2.3,-5.3 -0.2,-2.8 -1.7,-5 -3.2,-4.8 -0.5,0 -0.9,0.3 -1.2,0.7 0.5,-1.3 1.3,-2.1 2.3,-2.1 z M 77,182.4 c 1.6,0 2.9,1.3 2.9,2.9 0,1.6 -1.3,2.9 -2.9,2.9 -1.6,0 -2.9,-1.3 -2.9,-2.9 -0.1,-1.6 1.3,-2.9 2.9,-2.9 z m -10.3,15.2 v 0 c 1.4,-0.1 2.7,0.9 2.8,2.4 l 0.4,4.9 c 0.1,1.4 -0.9,2.7 -2.4,2.8 v 0 c -1.4,0.1 -2.7,-0.9 -2.8,-2.4 l -0.4,-4.9 c -0.1,-1.4 1,-2.7 2.4,-2.8 z m -32.2,-3.1 c 3.7,-0.3 7.1,3.9 7.6,9.4 0.5,5.5 -2.1,10.2 -5.9,10.6 -3.7,0.3 -7.1,-3.9 -7.6,-9.4 -0.5,-5.6 2.1,-10.3 5.9,-10.6 z m 43.7,66.1 c -3.9,-1 -7.3,-3.6 -8.8,-7.9 2.7,-0.6 5.9,-1.3 8.8,-2.3 z m -10.9,-26.8 -0.7,-7.3 11.6,-0.6 v 3.6 c -0.6,0.2 -1.3,0.4 -1.9,0.6 -3,1.1 -6,2.3 -8.9,3.6 z m -13.1,-0.2 c 3.5,-2.6 7.9,-2.5 9.8,0.1 0.3,0.4 0.5,0.8 0.6,1.3 -4.3,1.9 -10.3,5 -12.6,8.8 -0.3,-0.2 -0.6,-0.5 -0.8,-0.8 -1.9,-2.6 -0.6,-6.8 3,-9.4 z m 48.6,10.7 c 0.7,-0.2 1.4,-0.1 2.1,0 l 0.8,-1.4 0.3,0.1 c 0.7,0.2 1.3,0.6 1.9,1.1 l 0.3,0.2 -0.8,1.4 c 0.2,0.3 0.4,0.5 0.6,0.8 0.2,0.3 0.3,0.6 0.4,0.9 h 1.6 l 0.1,0.4 c 0.1,0.7 0.1,1.5 0,2.2 l -0.1,0.4 h -1.6 c -0.2,0.7 -0.6,1.3 -1,1.8 l 0.8,1.4 -0.3,0.2 c -0.3,0.2 -0.6,0.5 -0.9,0.6 -0.3,0.2 -0.6,0.3 -1,0.5 l -0.3,0.1 -0.8,-1.4 c -0.7,0.1 -1.4,0.1 -2.1,0 l -0.8,1.4 -0.3,-0.1 c -0.7,-0.3 -1.3,-0.6 -1.9,-1.1 l -0.3,-0.2 0.8,-1.4 c -0.2,-0.3 -0.4,-0.5 -0.6,-0.8 -0.2,-0.3 -0.3,-0.6 -0.4,-0.9 h -1.6 l -0.1,-0.4 c -0.1,-0.7 -0.1,-1.5 0,-2.2 l 0.1,-0.4 h 1.6 c 0.2,-0.7 0.6,-1.3 1,-1.8 l -0.8,-1.4 0.3,-0.2 c 0.3,-0.2 0.6,-0.5 0.9,-0.6 0.3,-0.2 0.6,-0.3 1,-0.5 l 0.3,-0.1 z M 88.4,217.9 h 29.9 v 3.2 H 88.4 Z m 0,7.2 h 12.1 v 3.2 H 88.4 Z m 0,7.3 h 9.7 v 3.2 h -9.7 l -0.1,-0.3 v -2.9 z m 25.7,-4.8 c 0.9,0.3 1.7,0.8 2.5,1.4 l 1.9,-1.1 0.3,0.4 c 0.7,0.8 1.2,1.7 1.5,2.6 l 0.2,0.5 -1.9,1.1 c 0.1,0.5 0.2,0.9 0.2,1.4 0,0.5 -0.1,1 -0.2,1.4 l 1.9,1.1 -0.2,0.5 c -0.3,0.9 -0.9,1.8 -1.5,2.6 l -0.3,0.4 -1.9,-1.1 c -0.7,0.6 -1.5,1.1 -2.5,1.4 v 2.2 l -0.5,0.1 c -0.5,0.1 -1,0.1 -1.5,0.1 -0.5,0 -1,0 -1.5,-0.1 l -0.5,-0.1 v -2.2 c -0.9,-0.3 -1.7,-0.8 -2.5,-1.4 l -1.9,1.1 -0.3,-0.4 c -0.6,-0.8 -1.2,-1.7 -1.5,-2.6 l -0.2,-0.5 1.9,-1.1 c -0.1,-0.5 -0.2,-0.9 -0.2,-1.4 0,-0.5 0.1,-0.9 0.2,-1.4 l -1.9,-1.1 0.2,-0.5 c 0.3,-0.9 0.9,-1.8 1.5,-2.6 l 0.3,-0.4 1.9,1.1 c 0.7,-0.6 1.5,-1.1 2.5,-1.4 v -2.2 l 0.5,-0.1 c 0.5,-0.1 1,-0.1 1.5,-0.1 0.5,0 1,0 1.5,0.1 l 0.5,0.1 z m -2,3 c -1.8,0 -3.3,1.5 -3.3,3.3 0,1.8 1.5,3.3 3.3,3.3 1.8,0 3.3,-1.5 3.3,-3.3 0,-1.8 -1.5,-3.3 -3.3,-3.3 z m -9.5,16.3 c -1.2,0.7 -1.6,2.2 -0.9,3.3 0.7,1.2 2.2,1.6 3.3,0.9 1.2,-0.7 1.6,-2.2 0.9,-3.3 -0.7,-1.2 -2.1,-1.6 -3.3,-0.9 z"
   id="path4" />
 			<path
   class="st1"
   d="m 172,209.4 v 22.9 h -4.7 v -13.2 c 0,-0.3 0,-0.6 0,-1 0,-0.4 0,-0.7 0.1,-1.1 l -6.1,11.8 c -0.2,0.4 -0.4,0.6 -0.8,0.8 -0.3,0.2 -0.7,0.3 -1.1,0.3 h -0.7 c -0.4,0 -0.8,-0.1 -1.1,-0.3 -0.3,-0.2 -0.6,-0.5 -0.8,-0.8 L 150.7,217 c 0,0.4 0.1,0.7 0.1,1.1 0,0.4 0,0.7 0,1 v 13.2 h -4.7 v -22.9 h 4.1 c 0.2,0 0.4,0 0.6,0 0.2,0 0.3,0 0.5,0.1 0.1,0.1 0.3,0.1 0.4,0.2 0.1,0.1 0.2,0.3 0.3,0.5 l 5.9,11.6 c 0.2,0.4 0.4,0.8 0.6,1.2 0.2,0.4 0.4,0.9 0.6,1.3 0.2,-0.5 0.4,-0.9 0.6,-1.4 0.2,-0.4 0.4,-0.9 0.6,-1.3 l 5.9,-11.6 c 0.1,-0.2 0.2,-0.4 0.3,-0.5 0.1,-0.1 0.2,-0.2 0.4,-0.2 0.1,-0.1 0.3,-0.1 0.5,-0.1 0.2,0 0.4,0 0.6,0 h 4 z m 16.6,14.1 -2.1,-6.3 c -0.2,-0.4 -0.3,-0.9 -0.5,-1.4 -0.2,-0.5 -0.4,-1.1 -0.5,-1.8 -0.2,0.6 -0.3,1.2 -0.5,1.8 -0.2,0.5 -0.3,1 -0.5,1.4 l -2.1,6.2 h 6.2 z m 8.4,8.8 h -4.1 c -0.5,0 -0.8,-0.1 -1.1,-0.3 -0.3,-0.2 -0.5,-0.5 -0.6,-0.8 l -1.4,-4 h -8.7 l -1.4,4 c -0.1,0.3 -0.3,0.6 -0.6,0.8 -0.3,0.2 -0.7,0.4 -1.1,0.4 h -4.2 l 8.9,-22.9 h 5.4 z m 16.5,-6 c 0.1,0 0.3,0 0.4,0.1 0.1,0 0.2,0.1 0.4,0.2 l 2.1,2.2 c -0.9,1.2 -2.1,2.1 -3.5,2.7 -1.4,0.6 -3,0.9 -4.9,0.9 -1.7,0 -3.3,-0.3 -4.7,-0.9 -1.4,-0.6 -2.5,-1.4 -3.5,-2.5 -1,-1 -1.7,-2.3 -2.2,-3.7 -0.5,-1.4 -0.8,-3 -0.8,-4.7 0,-1.7 0.3,-3.3 0.8,-4.7 0.6,-1.4 1.3,-2.7 2.3,-3.7 1,-1 2.2,-1.8 3.6,-2.4 1.4,-0.6 3,-0.9 4.6,-0.9 0.9,0 1.7,0.1 2.4,0.2 0.8,0.2 1.5,0.4 2.1,0.6 0.7,0.3 1.3,0.6 1.8,1 0.6,0.4 1,0.8 1.5,1.2 l -1.8,2.4 c -0.1,0.1 -0.3,0.3 -0.4,0.4 -0.2,0.1 -0.4,0.2 -0.7,0.2 -0.2,0 -0.4,0 -0.5,-0.1 -0.2,-0.1 -0.3,-0.2 -0.5,-0.3 -0.2,-0.1 -0.4,-0.3 -0.6,-0.4 -0.2,-0.1 -0.5,-0.3 -0.8,-0.4 -0.3,-0.1 -0.7,-0.2 -1.1,-0.3 -0.4,-0.1 -0.9,-0.1 -1.5,-0.1 -0.9,0 -1.7,0.2 -2.4,0.5 -0.7,0.3 -1.4,0.8 -1.9,1.4 -0.5,0.6 -0.9,1.4 -1.2,2.3 -0.3,0.9 -0.4,1.9 -0.4,3.1 0,1.2 0.2,2.2 0.5,3.1 0.3,0.9 0.8,1.7 1.3,2.3 0.6,0.6 1.2,1.1 1.9,1.4 0.7,0.3 1.5,0.5 2.4,0.5 0.5,0 0.9,0 1.3,-0.1 0.4,0 0.8,-0.1 1.1,-0.2 0.3,-0.1 0.7,-0.3 1,-0.4 0.3,-0.2 0.6,-0.4 0.9,-0.7 0.1,-0.1 0.3,-0.2 0.4,-0.3 0.3,0.2 0.5,0.1 0.6,0.1 z m 25.2,-16.9 v 22.9 h -5.3 v -9.7 h -9.3 v 9.7 h -5.4 v -22.9 h 5.4 v 9.6 h 9.3 v -9.6 z m 9.8,22.9 h -5.4 v -22.9 h 5.4 z M 273,209.4 v 22.9 h -2.8 c -0.4,0 -0.8,-0.1 -1,-0.2 -0.3,-0.1 -0.6,-0.4 -0.8,-0.7 l -10.8,-13.7 c 0,0.4 0.1,0.8 0.1,1.2 0,0.4 0,0.7 0,1.1 v 12.3 H 253 v -22.9 h 2.8 c 0.2,0 0.4,0 0.6,0 0.2,0 0.3,0.1 0.4,0.1 0.1,0.1 0.2,0.1 0.4,0.2 0.1,0.1 0.2,0.2 0.4,0.4 l 10.9,13.8 c -0.1,-0.4 -0.1,-0.9 -0.1,-1.3 0,-0.4 0,-0.8 0,-1.2 v -12.1 h 4.6 z m 9.8,4 v 5.4 h 7.2 v 3.9 h -7.2 v 5.5 h 9.4 v 4.1 h -14.8 v -22.9 h 14.8 v 4.1 h -9.4 z"
   id="path6" />
 			<path
   class="st1"
   d="m 316.5,228 v 4.2 H 302.7 V 209.3 H 308 V 228 Z m 8.1,-14.6 v 5.4 h 7.2 v 3.9 h -7.2 v 5.5 h 9.4 v 4.1 H 319.2 V 209.4 H 334 v 4.1 h -9.4 z m 24.8,10.1 -2.1,-6.3 c -0.2,-0.4 -0.3,-0.9 -0.5,-1.4 -0.2,-0.5 -0.4,-1.1 -0.5,-1.8 -0.2,0.6 -0.3,1.2 -0.5,1.8 -0.2,0.5 -0.3,1 -0.5,1.4 l -2.1,6.2 h 6.2 z m 8.4,8.8 h -4.1 c -0.5,0 -0.8,-0.1 -1.1,-0.3 -0.3,-0.2 -0.5,-0.5 -0.6,-0.8 l -1.4,-4 h -8.7 l -1.4,4 c -0.1,0.3 -0.3,0.6 -0.6,0.8 -0.3,0.2 -0.7,0.4 -1.1,0.4 h -4.2 l 8.9,-22.9 h 5.4 z m 9.3,-12.2 c 0.7,0 1.3,-0.1 1.9,-0.3 0.5,-0.2 0.9,-0.4 1.2,-0.8 0.3,-0.3 0.6,-0.7 0.7,-1.1 0.1,-0.4 0.2,-0.9 0.2,-1.4 0,-1 -0.3,-1.8 -1,-2.4 -0.7,-0.6 -1.7,-0.8 -3,-0.8 H 365 v 6.8 z m 11.3,12.2 h -4.8 c -0.9,0 -1.5,-0.3 -1.9,-1 l -3.8,-6.7 c -0.2,-0.3 -0.4,-0.5 -0.6,-0.6 -0.2,-0.1 -0.5,-0.2 -0.9,-0.2 H 365 v 8.5 h -5.3 v -22.9 h 7.5 c 1.7,0 3.1,0.2 4.2,0.5 1.2,0.3 2.1,0.8 2.9,1.4 0.7,0.6 1.3,1.3 1.6,2.2 0.3,0.8 0.5,1.7 0.5,2.7 0,0.7 -0.1,1.4 -0.3,2.1 -0.2,0.7 -0.5,1.3 -0.9,1.8 -0.4,0.6 -0.9,1.1 -1.4,1.5 -0.6,0.4 -1.2,0.8 -2,1.1 0.3,0.2 0.7,0.4 1,0.7 0.3,0.3 0.6,0.6 0.8,1 z m 22,-22.9 v 22.9 h -2.8 c -0.4,0 -0.8,-0.1 -1,-0.2 -0.3,-0.1 -0.6,-0.4 -0.8,-0.7 L 385,217.7 c 0,0.4 0.1,0.8 0.1,1.2 0,0.4 0,0.7 0,1.1 v 12.3 h -4.7 v -22.9 h 2.8 c 0.2,0 0.4,0 0.6,0 0.2,0 0.3,0.1 0.4,0.1 0.1,0.1 0.2,0.1 0.4,0.2 0.1,0.1 0.2,0.2 0.4,0.4 l 10.9,13.8 c -0.1,-0.4 -0.1,-0.9 -0.1,-1.3 0,-0.4 0,-0.8 0,-1.2 v -12.1 h 4.6 z m 9.7,22.9 h -5.4 v -22.9 h 5.4 z m 24.5,-22.9 v 22.9 h -2.8 c -0.4,0 -0.7,-0.1 -1,-0.2 -0.3,-0.1 -0.6,-0.4 -0.8,-0.7 l -10.8,-13.7 c 0,0.4 0.1,0.8 0.1,1.2 0,0.4 0,0.7 0,1.1 v 12.3 h -4.7 v -22.9 h 2.8 c 0.2,0 0.4,0 0.6,0 0.2,0 0.3,0.1 0.4,0.1 0.1,0.1 0.2,0.1 0.4,0.2 0.1,0.1 0.2,0.2 0.4,0.4 l 10.9,13.8 c -0.1,-0.4 -0.1,-0.9 -0.1,-1.3 0,-0.4 0,-0.8 0,-1.2 v -12.1 h 4.6 z m 15.5,11 h 8.2 v 9.7 c -1.2,0.9 -2.4,1.5 -3.8,1.9 -1.3,0.4 -2.7,0.6 -4.2,0.6 -1.9,0 -3.6,-0.3 -5.2,-0.9 -1.5,-0.6 -2.9,-1.4 -4,-2.5 -1.1,-1 -2,-2.3 -2.5,-3.7 -0.6,-1.4 -0.9,-3 -0.9,-4.7 0,-1.7 0.3,-3.3 0.9,-4.7 0.6,-1.4 1.4,-2.7 2.4,-3.7 1.1,-1 2.3,-1.8 3.8,-2.4 1.5,-0.6 3.2,-0.9 5,-0.9 1,0 1.9,0.1 2.7,0.2 0.8,0.2 1.6,0.4 2.3,0.6 0.7,0.3 1.4,0.6 1.9,1 0.6,0.4 1.1,0.8 1.6,1.2 l -1.5,2.3 c -0.2,0.4 -0.6,0.6 -0.9,0.7 -0.4,0.1 -0.8,0 -1.2,-0.3 -0.4,-0.3 -0.8,-0.5 -1.2,-0.7 -0.4,-0.2 -0.8,-0.3 -1.1,-0.4 -0.4,-0.1 -0.8,-0.2 -1.2,-0.3 -0.4,-0.1 -0.9,-0.1 -1.4,-0.1 -1,0 -1.9,0.2 -2.7,0.5 -0.8,0.4 -1.5,0.9 -2,1.5 -0.6,0.6 -1,1.4 -1.3,2.3 -0.3,0.9 -0.5,1.9 -0.5,3 0,1.2 0.2,2.3 0.5,3.2 0.3,0.9 0.8,1.7 1.4,2.4 0.6,0.7 1.3,1.1 2.2,1.5 0.9,0.3 1.8,0.5 2.8,0.5 0.6,0 1.2,-0.1 1.7,-0.2 0.5,-0.1 1,-0.3 1.5,-0.5 V 224 h -2.3 c -0.3,0 -0.6,-0.1 -0.8,-0.3 -0.2,-0.2 -0.3,-0.4 -0.3,-0.7 v -2.6 z"
   id="path8" />
 			<path
   class="st1"
   d="m 155.6,253.7 c 0.1,0 0.2,0 0.3,0.1 0.1,0 0.2,0.1 0.3,0.2 l 1.5,1.6 c -0.7,0.9 -1.5,1.5 -2.5,2 -1,0.4 -2.2,0.7 -3.6,0.7 -1.3,0 -2.4,-0.2 -3.4,-0.7 -1,-0.4 -1.9,-1 -2.6,-1.8 -0.7,-0.8 -1.2,-1.7 -1.6,-2.7 -0.4,-1 -0.6,-2.2 -0.6,-3.4 0,-1.2 0.2,-2.4 0.6,-3.4 0.4,-1 1,-1.9 1.7,-2.7 0.7,-0.8 1.6,-1.3 2.6,-1.8 1,-0.4 2.2,-0.6 3.4,-0.6 0.6,0 1.2,0.1 1.8,0.2 0.6,0.1 1.1,0.3 1.6,0.5 0.5,0.2 0.9,0.4 1.3,0.7 0.4,0.3 0.8,0.6 1.1,0.9 l -1.3,1.8 c -0.1,0.1 -0.2,0.2 -0.3,0.3 -0.1,0.1 -0.3,0.1 -0.5,0.1 -0.1,0 -0.3,0 -0.4,-0.1 -0.1,-0.1 -0.3,-0.1 -0.4,-0.2 -0.1,-0.1 -0.3,-0.2 -0.5,-0.3 -0.2,-0.1 -0.4,-0.2 -0.6,-0.3 -0.2,-0.1 -0.5,-0.2 -0.8,-0.2 -0.3,-0.1 -0.7,-0.1 -1.1,-0.1 -0.6,0 -1.2,0.1 -1.8,0.4 -0.5,0.2 -1,0.6 -1.4,1 -0.4,0.5 -0.7,1 -0.9,1.7 -0.2,0.7 -0.3,1.4 -0.3,2.3 0,0.9 0.1,1.6 0.4,2.3 0.2,0.7 0.6,1.2 1,1.7 0.4,0.5 0.9,0.8 1.4,1 0.5,0.2 1.1,0.4 1.7,0.4 0.4,0 0.7,0 1,-0.1 0.3,0 0.6,-0.1 0.8,-0.2 0.3,-0.1 0.5,-0.2 0.7,-0.3 0.2,-0.1 0.4,-0.3 0.7,-0.5 0.1,-0.1 0.2,-0.1 0.3,-0.2 0.2,-0.3 0.3,-0.3 0.4,-0.3 z m 36.1,-4.1 c 0,1.2 -0.2,2.3 -0.6,3.4 -0.4,1 -1,1.9 -1.8,2.7 -0.8,0.8 -1.7,1.4 -2.7,1.8 -1.1,0.4 -2.3,0.7 -3.6,0.7 -1.3,0 -2.5,-0.2 -3.6,-0.7 -1.1,-0.4 -2,-1 -2.7,-1.8 -0.8,-0.8 -1.4,-1.7 -1.8,-2.7 -0.4,-1 -0.6,-2.2 -0.6,-3.4 0,-1.2 0.2,-2.3 0.6,-3.4 0.4,-1 1,-1.9 1.8,-2.7 0.8,-0.8 1.7,-1.4 2.7,-1.8 1.1,-0.4 2.3,-0.7 3.6,-0.7 1.3,0 2.5,0.2 3.6,0.7 1.1,0.4 2,1 2.7,1.8 0.8,0.8 1.3,1.7 1.8,2.7 0.4,1.1 0.6,2.2 0.6,3.4 z m -3.9,0 c 0,-0.8 -0.1,-1.6 -0.3,-2.2 -0.2,-0.7 -0.5,-1.2 -0.9,-1.7 -0.4,-0.5 -0.9,-0.8 -1.5,-1.1 -0.6,-0.2 -1.2,-0.4 -2,-0.4 -0.8,0 -1.4,0.1 -2,0.4 -0.6,0.2 -1.1,0.6 -1.5,1.1 -0.4,0.5 -0.7,1 -0.9,1.7 -0.2,0.7 -0.3,1.4 -0.3,2.2 0,0.8 0.1,1.6 0.3,2.2 0.2,0.7 0.5,1.2 0.9,1.7 0.4,0.5 0.9,0.8 1.5,1 0.6,0.2 1.3,0.4 2,0.4 0.7,0 1.4,-0.1 2,-0.4 0.6,-0.2 1.1,-0.6 1.5,-1 0.4,-0.5 0.7,-1 0.9,-1.7 0.2,-0.6 0.3,-1.3 0.3,-2.2 z m 41,-8.3 V 258 h -3.4 v -9.6 c 0,-0.2 0,-0.5 0,-0.7 0,-0.3 0,-0.5 0.1,-0.8 l -4.4,8.6 c -0.1,0.3 -0.3,0.5 -0.6,0.6 -0.2,0.1 -0.5,0.2 -0.8,0.2 h -0.5 c -0.3,0 -0.6,-0.1 -0.8,-0.2 -0.2,-0.1 -0.4,-0.3 -0.6,-0.6 l -4.4,-8.6 c 0,0.3 0,0.5 0.1,0.8 0,0.3 0,0.5 0,0.7 v 9.6 h -3.4 v -16.7 h 3 c 0.2,0 0.3,0 0.4,0 0.1,0 0.2,0 0.3,0.1 0.1,0 0.2,0.1 0.3,0.2 0.1,0.1 0.2,0.2 0.2,0.3 l 4.3,8.5 c 0.2,0.3 0.3,0.6 0.4,0.9 0.1,0.3 0.3,0.6 0.4,1 0.1,-0.3 0.3,-0.7 0.4,-1 0.1,-0.3 0.3,-0.6 0.5,-0.9 l 4.3,-8.5 c 0.1,-0.1 0.2,-0.3 0.2,-0.3 0.1,-0.1 0.2,-0.1 0.3,-0.2 0.1,0 0.2,-0.1 0.3,-0.1 0.1,0 0.3,0 0.4,0 z m 25,8.2 c 1,0 1.7,-0.2 2.2,-0.7 0.4,-0.5 0.7,-1.2 0.7,-2 0,-0.4 -0.1,-0.7 -0.2,-1 -0.1,-0.3 -0.3,-0.6 -0.5,-0.8 -0.2,-0.2 -0.5,-0.4 -0.9,-0.5 -0.4,-0.1 -0.8,-0.2 -1.3,-0.2 h -2 v 5.3 h 2 z m 0,-8.2 c 1.2,0 2.2,0.1 3,0.4 0.8,0.3 1.5,0.7 2.1,1.2 0.5,0.5 0.9,1.1 1.2,1.7 0.3,0.7 0.4,1.4 0.4,2.2 0,0.8 -0.1,1.6 -0.4,2.3 -0.3,0.7 -0.7,1.3 -1.2,1.8 -0.6,0.5 -1.2,0.9 -2.1,1.2 -0.8,0.3 -1.8,0.4 -3,0.4 h -2 v 5.6 h -3.9 v -16.7 h 5.9 z m 27.9,16.7 h -3.9 v -16.7 h 3.9 z m 29.3,-3.1 v 3.1 h -10.1 v -16.7 h 3.9 v 13.6 z m 25.2,-3.3 -1.6,-4.6 c -0.1,-0.3 -0.2,-0.6 -0.4,-1 -0.1,-0.4 -0.3,-0.8 -0.4,-1.3 -0.1,0.5 -0.2,0.9 -0.4,1.3 -0.1,0.4 -0.3,0.7 -0.4,1 l -1.5,4.6 z m 6.2,6.4 h -3 c -0.3,0 -0.6,-0.1 -0.8,-0.2 -0.2,-0.2 -0.4,-0.4 -0.5,-0.6 l -1,-2.9 h -6.4 l -1,2.9 c -0.1,0.2 -0.2,0.4 -0.5,0.6 -0.2,0.2 -0.5,0.3 -0.8,0.3 h -3 l 6.5,-16.7 h 4 z m 27.8,-13.6 h -4.7 V 258 h -3.9 v -13.7 h -4.7 v -3.1 h 13.3 z m 21.1,13.6 h -3.9 v -16.7 h 3.9 z m 35.5,-8.4 c 0,1.2 -0.2,2.3 -0.6,3.4 -0.4,1 -1,1.9 -1.8,2.7 -0.8,0.8 -1.7,1.4 -2.7,1.8 -1.1,0.4 -2.3,0.7 -3.6,0.7 -1.3,0 -2.5,-0.2 -3.6,-0.7 -1.1,-0.4 -2,-1 -2.7,-1.8 -0.8,-0.8 -1.4,-1.7 -1.8,-2.7 -0.4,-1 -0.6,-2.2 -0.6,-3.4 0,-1.2 0.2,-2.3 0.6,-3.4 0.4,-1 1,-1.9 1.8,-2.7 0.8,-0.8 1.7,-1.4 2.7,-1.8 1.1,-0.4 2.3,-0.7 3.6,-0.7 1.3,0 2.5,0.2 3.6,0.7 1.1,0.4 2,1 2.7,1.8 0.8,0.8 1.3,1.7 1.8,2.7 0.4,1.1 0.6,2.2 0.6,3.4 z m -3.9,0 c 0,-0.8 -0.1,-1.6 -0.3,-2.2 -0.2,-0.7 -0.5,-1.2 -0.9,-1.7 -0.4,-0.5 -0.9,-0.8 -1.5,-1.1 -0.6,-0.2 -1.2,-0.4 -2,-0.4 -0.8,0 -1.4,0.1 -2,0.4 -0.6,0.2 -1.1,0.6 -1.5,1.1 -0.4,0.5 -0.7,1 -0.9,1.7 -0.2,0.7 -0.3,1.4 -0.3,2.2 0,0.8 0.1,1.6 0.3,2.2 0.2,0.7 0.5,1.2 0.9,1.7 0.4,0.5 0.9,0.8 1.5,1 0.6,0.2 1.3,0.4 2,0.4 0.7,0 1.4,-0.1 2,-0.4 0.6,-0.2 1.1,-0.6 1.5,-1 0.4,-0.5 0.7,-1 0.9,-1.7 0.2,-0.6 0.3,-1.3 0.3,-2.2 z m 36.8,-8.3 V 258 h -2 c -0.3,0 -0.5,-0.1 -0.8,-0.1 -0.2,-0.1 -0.4,-0.3 -0.6,-0.5 l -7.9,-10 c 0,0.3 0.1,0.6 0.1,0.9 0,0.3 0,0.5 0,0.8 v 9 h -3.4 v -16.7 h 2 c 0.2,0 0.3,0 0.4,0 0.1,0 0.2,0 0.3,0.1 0.1,0 0.2,0.1 0.3,0.2 0.1,0.1 0.2,0.2 0.3,0.3 l 8,10.1 c 0,-0.3 -0.1,-0.6 -0.1,-0.9 0,-0.3 0,-0.6 0,-0.9 v -8.9 h 3.4 z"
   id="path10" />
 		</g>
 	</g>
 </switch>
 </svg>
@@ -0,0 +1,102 @@
 # -*- coding: utf-8 -*-
 import os
 import sys
 import tlcpack_sphinx_addon
 # -- General configuration ------------------------------------------------
 sys.path.insert(0, os.path.abspath("../python"))
 sys.path.insert(0, os.path.abspath("../"))
 autodoc_mock_imports = ["torch"]
 # General information about the project.
 project = "web-llm"
 author = "WebLLM Contributors"
 copyright = "2023, %s" % author
 # Version information.
 version = "0.2.84"
 release = "0.2.84"
 extensions = [
    "sphinx_tabs.tabs",
    "sphinx_toolbox.collapse",
    "sphinxcontrib.httpdomain",
    "sphinx.ext.autodoc",
    "sphinx.ext.napoleon",
    "sphinx_reredirects",
 ]
 redirects = {"get_started/try_out": "../index.html#getting-started"}
 source_suffix = [".rst"]
 language = "en"
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = "sphinx"
 # A list of ignored prefixes for module index sorting.
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = False
 # -- Options for HTML output ----------------------------------------------
 # The theme is set by the make target
 import sphinx_rtd_theme
 html_theme = "sphinx_rtd_theme"
 html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 templates_path = []
 html_static_path = []
 footer_copyright = "© 2023 MLC LLM"
 footer_note = " "
 html_logo = "_static/img/mlc-logo-with-text-landscape.svg"
 html_theme_options = {
    "logo_only": True,
 }
 header_links = [
    ("Home", "https://webllm.mlc.ai/"),
    ("GitHub", "https://github.com/mlc-ai/web-llm"),
    ("Discord", "https://discord.gg/9Xpy2HGBuD"),
 ]
 header_dropdown = {
    "name": "Other Resources",
    "items": [
        ("WebLLM Chat", "https://chat.webllm.ai/"),
        ("MLC Course", "https://mlc.ai/"),
        ("MLC Blog", "https://blog.mlc.ai/"),
        ("MLC LLM", "https://llm.mlc.ai/"),
    ],
 }
 html_context = {
    "footer_copyright": footer_copyright,
    "footer_note": footer_note,
    "header_links": header_links,
    "header_dropdown": header_dropdown,
    "display_github": True,
    "github_user": "mlc-ai",
    "github_repo": "web-llm",
    "github_version": "main/docs/",
    "theme_vcs_pageview_mode": "edit",
    # "header_logo": "/path/to/logo",
    # "header_logo_link": "",
    # "version_selecter": "",
 }
 # add additional overrides
 templates_path += [tlcpack_sphinx_addon.get_templates_path()]
 html_static_path += [tlcpack_sphinx_addon.get_static_path()]
@@ -0,0 +1,6 @@
 Adding Models
 =============
 WebLLM allows you to compile custom language models using `MLC-LLM <https://llm.mlc.ai/>`_ and then serve the compiled model through WebLLM.
 For instructions on how to compile and add custom models to WebLLM, please refer to the `MLC-LLM documentation <https://llm.mlc.ai/docs/deploy/webllm.html>`_. 
@@ -0,0 +1,35 @@
 Building From Source
 ====================
 Clone the Repository
 ---------------------
 .. code-block:: bash
   git clone https://github.com/mlc-ai/web-llm.git
   cd web-llm
 Install Dependencies
 ---------------------
 .. code-block:: bash
   npm install
 Build the Project
 -----------------
 .. code-block:: bash
   npm run build
 Test Changes
 ------------
 To test your changes, you can reuse an existing example or create a new example that specifically tests the new functionality you wish to provide.
 To test the effects of your code change in an example, inside ``examples/<example>/package.json``, change ``"@mlc-ai/web-llm": "^0.2.xx"`` to ``"@mlc-ai/web-llm": "../.."`` to let it reference your local code. Note that sometimes you may need to switch between ``"file:../.."`` and ``"../.."`` to trigger npm to recognize new changes.
 .. code-block:: bash
   cd examples/<example>
   # Modify package.json as described
   npm install
   npm start
@@ -0,0 +1,35 @@
 👋 Welcome to WebLLM
 ====================
 `GitHub <https://github.com/mlc-ai/web-llm>`_ | `WebLLM Chat <https://chat.webllm.ai/>`_ | `NPM <https://www.npmjs.com/package/@mlc-ai/web-llm>`_ | `Discord <https://discord.gg/9Xpy2HGBuD>`_
 WebLLM is a high-performance in-browser language model inference engine that brings large language models (LLMs) to web browsers with hardware acceleration. With WebGPU support, it allows developers to build AI-powered applications directly within the browser environment, removing the need for server-side processing and ensuring privacy.
 It provides a specialized runtime for the web backend of MLCEngine, leverages
 `WebGPU <https://www.w3.org/TR/webgpu/>`_ for local acceleration, offers OpenAI-compatible API,
 and provides built-in support for web workers to separate heavy computation from the UI flow.
 Key Features
 ------------
 - 🌐 In-Browser Inference: Run LLMs directly in the browser
 - 🚀 WebGPU Acceleration: Leverage hardware acceleration for optimal performance
 - 🔄 OpenAI API Compatibility: Seamless integration with standard AI workflows
 - 📦 Multiple Model Support: Works with Llama, Phi, Gemma, Mistral, and more
 Start exploring WebLLM by `chatting with WebLLM Chat <https://chat.webllm.ai/>`_, and start building webapps with high-performance local LLM inference with the following guides and tutorials.
 .. toctree::
   :maxdepth: 2
   :caption: User Guide
   user/get_started.rst
   user/basic_usage.rst
   user/advanced_usage.rst
   user/api_reference.rst
 .. toctree::
   :maxdepth: 2
   :caption: Developer Guide
   developer/building_from_source.rst
   developer/add_models.rst
@@ -0,0 +1,35 @@
@ECHO OFF
 pushd %~dp0
 REM Command file for Sphinx documentation
 if "%SPHINXBUILD%" == "" (
 	set SPHINXBUILD=sphinx-build
 )
 set SOURCEDIR=.
 set BUILDDIR=_build
 %SPHINXBUILD% >NUL 2>NUL
 if errorlevel 9009 (
 	echo.
 	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
 	echo.installed, then set the SPHINXBUILD environment variable to point
 	echo.to the full path of the 'sphinx-build' executable. Alternatively you
 	echo.may add the Sphinx directory to PATH.
 	echo.
 	echo.If you don't have Sphinx installed, grab it from
 	echo.https://www.sphinx-doc.org/
 	exit /b 1
 )
 if "%1" == "" goto help
 %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
 goto end
 :help
 %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
 :end
 popd
@@ -0,0 +1,8 @@
 sphinx-tabs == 3.4.1
 sphinx-rtd-theme
 sphinx == 5.2.3
 sphinx-toolbox == 3.4.0
 tlcpack-sphinx-addon==0.2.2
 sphinxcontrib_httpdomain==1.8.1
 sphinxcontrib-napoleon==0.7
 sphinx-reredirects==0.1.2
@@ -0,0 +1,153 @@
 Advanced Use Cases
 ==================
 Using Workers
 -------------
 You can put the heavy computation in a worker script to optimize your application performance. To do so, you need to:
 Create a handler in the worker thread that communicates with the frontend while handling the requests.
 Create a worker engine in your main application that sends messages to the handler in the worker thread under the hood.
 For detailed implementations of different kinds of workers, look at the following sections.
 Using Web Workers
 ^^^^^^^^^^^^^^^^^
 WebLLM comes with API support for `Web Workers <https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers>`_ so you can offload the computation-heavy generation work into a separate worker thread. WebLLM has implemented cross-thread communication through messages under the hood, so manual implementation is not required.
 In the worker script, import and instantiate a ``WebWorkerMLCEngineHandler``, which handles communication with other scripts and processes incoming requests.
 .. code-block:: typescript
   // worker.ts
   import { WebWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
   const handler = new WebWorkerMLCEngineHandler();
   self.onmessage = (msg: MessageEvent) => {
       handler.onmessage(msg);
   };
 In the main script, import and instantiate a ``WebWorkerMLCEngine`` that implements the same ``MLCEngineInterface`` and exposes the same APIs. Then, simply use it as you would a normal ``MLCEngine``.
 .. code-block:: typescript
   import { CreateWebWorkerMLCEngine } from "@mlc-ai/web-llm";
   async function runWorker() {
       const engine = await CreateWebWorkerMLCEngine(
           new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }),
           "Llama-3.1-8B-Instruct"
       );
       const messages = [{ role: "user", content: "How does WebLLM use workers?" }];
       const reply = await engine.chat.completions.create({ messages });
       console.log(reply.choices[0].message.content);
   }
   runWorker();
 Under the hood, ``WebWorkerMLCEngine`` does **not** perform any computation. It translates all calls into messages and sends them to the ``WebWorkerMLCEngineHandler`` for processing. The worker thread receives these messages and processes the actual computation using a hidden engine, and returns the result to the main thread using messages.
 Service Workers
 ^^^^^^^^^^^^^^^
 WebLLM also supports offloading computation using `Service Workers <https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API>`_. This allows you to avoid reloading the model between page refreshes and optimize your application's offline experience.
 (Note, the lifecycle of a Service Worker is managed by the browser and can be killed any time without notifying the web application. WebLLM's ``ServiceWorkerMLCEngine`` attempts to keep the service worker thread alive by periodically sending heartbeat events. However, the script could still be killed at any time by Chrome, and your application should include proper error handling. Check `keepAliveMs` and `missedHeartbeat` in `ServiceWorkerMLCEngine <https://github.com/mlc-ai/web-llm/blob/main/src/service_worker.ts#L218>`_ for more details.)
 In the worker script, import and instantiate ``ServiceWorkerMLCEngineHandler``, which handles communication with page scripts and processes incoming requests.
 .. code-block:: typescript
   // sw.ts
   import { ServiceWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
   self.addEventListener("activate", () => {
       const handler = new ServiceWorkerMLCEngineHandler();
       console.log("Service Worker activated!");
   });
 Then, in the main page script, register the service worker and instantiate the engine using the ``CreateServiceWorkerMLCEngine`` factory function that implements the same ``MLCEngineInterface`` and exposes the same APIs. Then, simply use it as you would a normal ``MLCEngine``.
 .. code-block:: typescript
    // main.ts
    import { MLCEngineInterface, CreateServiceWorkerMLCEngine } from "@mlc-ai/web-llm";
    if ("serviceWorker" in navigator) {
    navigator.serviceWorker.register(
        new URL("sw.ts", import.meta.url),  // worker script
        { type: "module" },
    );
    }
    const engine: MLCEngineInterface =
    await CreateServiceWorkerMLCEngine(
        selectedModel,
        { initProgressCallback }, // engineConfig
    );
 Similar to the ``WebWorkerMLCEngine`` above, the ``ServiceWorkerMLCEngine`` is also a proxy and does not perform any actual computation. Instead, it forwards all calls to the service worker thread and receives the result through messages.
 Chrome Extension
 ----------------
 WebLLM can be used in Chrome extensions to empower local LLM inference. You can find examples of building Chrome extension using WebLLM in `examples/chrome-extension <https://github.com/mlc-ai/web-llm/blob/main/examples/chrome-extension>`_ and `examples/chrome-extension-webgpu-service-worker <https://github.com/mlc-ai/web-llm/blob/main/examples/chrome-extension-webgpu-service-worker>`_. The latter leverages Service Worker, so the extension is persistent in the background.
 Additionally, we have a full Chrome extension project, `WebLLM Assistant <https://github.com/mlc-ai/web-llm-assistant>`_, which leverages WebLLM to provide a personal web browsing copilot assistant experience. Feel free to check it out and contribute if you are interested.
 Additional Customization
 ------------------------
 Using IndexedDB Cache
 ^^^^^^^^^^^^^^^^^^^^^
 By default, WebLLM caches model artifacts using the `Cache API <https://developer.mozilla.org/en-US/docs/Web/API/Cache>`_ for faster subsequent model loads. You can alternatively use `IndexedDB caching <https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API>`_ by setting ``appConfig.cacheBackend = "indexeddb"``. When changing only the cache backend, preserve the prebuilt model list by spreading ``prebuiltAppConfig``.
 .. code-block:: typescript
   import { AppConfig, CreateMLCEngine, prebuiltAppConfig } from "@mlc-ai/web-llm";
   const appConfig: AppConfig = {
       ...prebuiltAppConfig,
       cacheBackend: "indexeddb",
   };
   const engine = await CreateMLCEngine("Llama-3.1-8B-Instruct-q4f32_1-MLC", {
       appConfig,
   });
 Using Cross-Origin Storage Cache
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 WebLLM also supports caching model artifacts across different origins using the experimental Cross-Origin Storage API. You can enable this cache backend by setting ``appConfig.cacheBackend = "cross-origin"``. For users with the `Cross-Origin Storage browser extension <https://chromewebstore.google.com/detail/cross-origin-storage/denpnpcgjgikjpoglpjefakmdcbmlgih>`_ installed, resources will then be cached and shared across origins. This means two independent apps opted into this cache backend using the same AI model will download and cache the required resources only once. See the `cache usage example <https://github.com/mlc-ai/web-llm/tree/main/examples/cache-usage>`_ for more details. If Cross-Origin Storage isn't available, WebLLM will automatically fall back to using the default cache.
 .. code-block:: typescript
   import { AppConfig, CreateMLCEngine, prebuiltAppConfig } from "@mlc-ai/web-llm";
   const appConfig: AppConfig = {
       ...prebuiltAppConfig,
       cacheBackend: "cross-origin",
   };
   const engine = await CreateMLCEngine("Llama-3.1-8B-Instruct-q4f32_1-MLC", {
       appConfig,
   });
 Customizing Token Behavior
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 You can modify `logit_bias` in `GenerationConfig` to control token likelihood. Setting a token's bias to a positive value increases its likelihood of being generated, while a negative value decreases it. A large negative value (e.g., -100) can effectively prevent the token from being generated.
 .. code-block:: typescript
   const messages = [
       { role: "user", content: "Describe WebLLM in detail." },
   ];
   const response = await engine.chatCompletion({
       messages,
       logit_bias: { "50256": -100 }, // Example: Prevent specific token generation
   });
@@ -0,0 +1,239 @@
 .. _api-reference:
 WebLLM API Reference
 ====================
 The ``MLCEngine`` class is the core interface of WebLLM. It enables model loading, chat completions, embeddings, and other operations. Below, we document its methods, along with the associated configuration interfaces.
 Interfaces
 ----------
 The following interfaces are used as parameters or configurations within ``MLCEngine`` methods. They are linked to their respective methods for reference.
 MLCEngineConfig
 ^^^^^^^^^^^^^^^
 Optional configurations for ``CreateMLCEngine()`` and ``CreateWebWorkerMLCEngine()``.
 - **Fields**:
    - ``appConfig``: Configure the app, including the list of models and whether to use IndexedDB cache.
    - ``initProgressCallback``: A callback for showing model loading progress.
    - ``logitProcessorRegistry``: A registry for stateful logit processors (see ``webllm.LogitProcessor``).
 - **Usage**:
    - ``appConfig``: Contains application-specific settings, including:
        - Model configurations.
        - IndexedDB caching preferences.
    - ``initProgressCallback``: Allows developers to visualize model loading progress by implementing a callback.
    - ``logitProcessorRegistry``: A ``Map`` object for registering custom logit processors. Only applies to ``MLCEngine``.
 .. note:: All fields are optional, and ``logitProcessorRegistry`` is only used in ``MLCEngine``.
 Example:
 .. code-block:: typescript
   const engine = await CreateMLCEngine("Llama-3.1-8B-Instruct", {
       appConfig: { /* app-specific config */ },
       initProgressCallback: (progress) => console.log(progress),
   });
 GenerationConfig
 ^^^^^^^^^^^^^^^^
 Configurations for a single generation task, primarily used in chat completions.
 - **Fields**:
    - ``repetition_penalty``, ``ignore_eos``: Parameters specific to MLC models.
    - ``top_p``, ``temperature``, ``max_tokens``, ``stop``: Common parameters shared with OpenAI APIs.
    - ``frequency_penalty``, ``presence_penalty``: Tune repetition behavior following OpenAI semantics.
    - ``logit_bias``, ``n``, ``logprobs``, ``top_logprobs``: Advanced sampling controls.
    - ``response_format``, ``enable_thinking``, ``enable_latency_breakdown``: Additional OpenAI-style request features.
 - **Usage**:
    - Fields like ``repetition_penalty`` and ``ignore_eos`` give explicit control over repetition handling and whether the model stops at the EOS token, respectively.
    - Common parameters shared with OpenAI APIs (e.g., ``temperature``, ``top_p``) ensure compatibility while still falling back to the values configured during ``MLCEngine.reload()`` when omitted.
    - ``frequency_penalty`` and ``presence_penalty`` mirror OpenAI's bounds ``[-2, 2]``; providing only one will default the other to ``0``.
    - ``response_format`` (for JSON or other schema outputs), ``enable_thinking``, and ``enable_latency_breakdown`` pass through directly to the engine and surface enhanced telemetry or structured responses when the underlying model supports them.
 Example:
 .. code-block:: typescript
   const messages = [
       { role: "system", content: "You are a helpful assistant." },
       { role: "user", content: "Explain WebLLM." },
   ];
   const response = await engine.chatCompletion({
       messages,
       top_p: 0.9,
       temperature: 0.8,
       max_tokens: 150,
   });
 ChatConfig
 ^^^^^^^^^^
 Model's baseline configuration loaded from ``mlc-chat-config.json`` when ``MLCEngine.reload()`` runs. ``ChatOptions`` (and therefore the ``chatOpts`` argument to ``reload``) can override any subset of these fields.
 - **Fields** (subset):
    - ``tokenizer_files``, ``tokenizer_info``: Files and parameters required to initialize the tokenizer.
    - ``conv_template``, ``conv_config``: Conversation templates that define prompts, separators, and role formatting.
    - ``context_window_size``, ``sliding_window_size``, ``attention_sink_size``: KV-cache and memory settings.
    - Default generation knobs such as ``repetition_penalty``, ``frequency_penalty``, ``presence_penalty``, ``top_p``, and ``temperature``.
 - **Usage**:
    - Loaded automatically for each model; provides defaults that ``GenerationConfig`` falls back to when fields are omitted.
    - Override selected values per model load by supplying ``chatOpts`` (``Partial<ChatConfig>``) to ``MLCEngine.reload()``.
 Example:
 .. code-block:: typescript
   await engine.reload("Llama-3.1-8B-Instruct", {
       temperature: 0.7,
       repetition_penalty: 1.1,
       context_window_size: 4096,
   });
 ChatCompletionRequest
 ^^^^^^^^^^^^^^^^^^^^^
 Defines the structure for chat completion requests.
 - **Base Interface**: ``ChatCompletionRequestBase``
    - Contains parameters such as ``messages``, ``stream``, ``frequency_penalty``, and ``presence_penalty``.
 - **Sub-interfaces**:
    - ``ChatCompletionRequestNonStreaming``: For non-streaming completions.
    - ``ChatCompletionRequestStreaming``: For streaming completions.
 - **Usage**:
    - Combines settings from ``GenerationConfig`` and ``ChatCompletionRequestBase`` to provide complete control over chat behavior.
    - The ``stream`` parameter enables streaming responses, improving interactivity in conversational agents.
    - The ``logit_bias`` feature allows controlling token generation probabilities, providing a mechanism to restrict or encourage specific outputs.
 Example:
 .. code-block:: typescript
   const response = await engine.chatCompletion({
       messages: [
           { role: "user", content: "Tell me about WebLLM." },
       ],
       stream: true,
   });
 Model Loading
 -------------
 ``MLCEngine.reload(modelId: string | string[], chatOpts?: ChatOptions | ChatOptions[]): Promise<void>``
 Loads the specified model(s) into the engine. Uses ``MLCEngineConfig`` during initialization.
 - Parameters:
    - ``modelId``: Identifier(s) for the model(s) to load.
    - ``chatOpts``: Configuration for generation (see ``ChatConfig``).
 Example:
 .. code-block:: typescript
   await engine.reload(["Llama-3.1-8B", "Gemma-2B"], [
       { temperature: 0.7 },
       { top_p: 0.9 },
   ]);
 ``MLCEngine.unload(): Promise<void>``
 Unloads all loaded models and clears their associated configurations.
 Example:
 .. code-block:: typescript
   await engine.unload();
 ---
 Chat Completions
 ----------------
 ``MLCEngine.chat.completions.create(request: ChatCompletionRequest): Promise<ChatCompletion | AsyncIterable<ChatCompletionChunk>>``
 Generates chat-based completions using a specified request configuration.
 - Parameters:
  - ``request``: A ``ChatCompletionRequest`` instance.
 Example:
 .. code-block:: typescript
   const response = await engine.chat.completions.create({
       messages: [
           { role: "system", content: "You are a helpful AI assistant." },
           { role: "user", content: "What is WebLLM?" },
       ],
       temperature: 0.8,
       stream: false,
   });
 ---
 Utility Methods
 ^^^^^^^^^^^^^^^
 ``MLCEngine.getMessage(modelId?: string): Promise<string>``
 Retrieves the current output message from the specified model.
 - Parameters:
    - ``modelId``: (Optional) Identifier of model to query. Omitting modelId only works when the engine currently has a single model loaded.
 ``MLCEngine.resetChat(keepStats?: boolean, modelId?: string): Promise<void>``
 Resets the chat history and optionally retains usage statistics.
 - Parameters:
    - ``keepStats``: (Optional) If true, retains usage statistics.
    - ``modelId``: (Optional) Identifier of the model to reset. Omitting modelId only works when the engine currently has a single model loaded.
 GPU Information
 ----------------
 The following methods provide detailed information about the GPU used for WebLLM computations.
 ``MLCEngine.getGPUVendor(): Promise<string>``
 Retrieves the vendor name of the GPU used for computations. This is useful for understanding hardware capabilities during inference.
 - **Returns**: A string indicating the GPU vendor (e.g., "Intel", "NVIDIA").
 Example:
 .. code-block:: typescript
   const gpuVendor = await engine.getGPUVendor();
   console.log(``GPU Vendor: ${gpuVendor}``);
 ``MLCEngine.getMaxStorageBufferBindingSize(): Promise<number>``
 Returns the maximum storage buffer size supported by the GPU. This is important when working with larger models that require significant memory for processing.
 - **Returns**: A number representing the maximum size in bytes.
 Example:
 .. code-block:: typescript
   const maxBufferSize = await engine.getMaxStorageBufferBindingSize();
   console.log(``Max Storage Buffer Binding Size: ${maxBufferSize}``);
@@ -0,0 +1,120 @@
 Basic Usage
 ================
 Model Records in WebLLM
 -----------------------
 Each of the model available WebLLM is registered as an instance of
 ``ModelRecord`` and can be accessed at
 `webllm.prebuiltAppConfig.model_list <https://github.com/mlc-ai/web-llm/blob/main/src/config.ts#L313>`__.
 Creating an MLCEngine
 ---------------------
 WebLLM APIs are exposed through the ``MLCEngine`` interface. You can create an ``MLCEngine`` instance and load the model by calling the CreateMLCEngine() factory function.
 (Note that loading models requires downloading and it can take a significant amount of time for the very first run without previous caching. You should properly handle this asynchronous call.)
 ``MLCEngine`` can be instantiated in two ways:
 1. Using the factory function ``CreateMLCEngine``.
 2. Instantiating the ``MLCEngine`` class directly and using ``reload()`` to load models.
 .. code-block:: typescript
   import { CreateMLCEngine, MLCEngine } from "@mlc-ai/web-llm";
    // Initialize with a progress callback
    const initProgressCallback = (progress) => {
        console.log("Model loading progress:", progress);
    };
   // Using CreateMLCEngine
   const engine = await CreateMLCEngine("Llama-3.1-8B-Instruct", { initProgressCallback });
   // Direct instantiation
   const engineInstance = new MLCEngine({ initProgressCallback });
   await engineInstance.reload("Llama-3.1-8B-Instruct");
 Under the hood, this factory function ``CreateMLCEngine`` does the following steps for first creating an engine instance (synchronous) and then loading the model (asynchronous). You can also do them separately in your application.
 .. code-block:: typescript
    import { MLCEngine } from "@mlc-ai/web-llm";
    // This is a synchronous call that returns immediately
    const engine = new MLCEngine({
        initProgressCallback: initProgressCallback
    });
    // This is an asynchronous call and can take a long time to finish
    await engine.reload(selectedModel);
 Chat Completion
 ---------------
 Chat completions can be invoked using OpenAI style chat APIs through the ``engine.chat.completions`` interface of an initialized ``MLCEngine``. For the full list of parameters and their descriptions, check :ref:`api-reference` for full list of parameters.
 (Note: Since the model is determined during ``MLCEngine`` instantiation, the ``model`` parameter is not supported and will be **ignored**. Instead, call ``CreateMLCEngine(model)`` or ``engine.reload(model)`` to reinitialize the engine to use a specific model.)
 .. code-block:: typescript
    const messages = [
        { role: "system", content: "You are a helpful AI assistant." },
        { role: "user", content: "Hello!" }
    ];
    const reply = await engine.chat.completions.create({
        messages,
    });
    console.log(reply.choices[0].message);
    console.log(reply.usage);
 Streaming Chat Completion
 -------------------------
 Streaming chat completion could be enabled by passsing ``stream: true`` parameter to the `engine.chat.completions.create` call configuration. Check :ref:`api-reference` for full list of parameters.
 .. code-block:: typescript
    const messages = [
        { role: "system", content: "You are a helpful AI assistant." },
        { role: "user", content: "Hello!" },
    ]
    // chunks is an AsyncGenerator object
    const chunks = await engine.chat.completions.create({
        messages,
        temperature: 1,
        stream: true, // <-- Enable streaming
        stream_options: { include_usage: true },
    });
    let reply = "";
    for await (const chunk of chunks) {
        reply += chunk.choices[0]?.delta.content || "";
        console.log(reply);
        if (chunk.usage) {
            console.log(chunk.usage); // only last chunk has usage
        }
    }
    const fullReply = await engine.getMessage();
    console.log(fullReply);
 Chatbot Examples
 ----------------
 Learn how to use WebLLM to integrate large language models into your applications and generate chat completions through this simple Chatbot example:
 - `Example in JSFiddle <https://jsfiddle.net/neetnestor/4nmgvsa2/>`_
 - `Example in CodePen <https://codepen.io/neetnestor/pen/vYwgZaG>`_
 For an advanced example of a larger, more complicated project, look at `WebLLM Chat <https://github.com/mlc-ai/web-llm-chat/blob/main/app/client/webllm.ts>`_.
 More examples for different use cases are available in the `WebLLM examples folder <https://github.com/mlc-ai/web-llm/tree/main/examples>`_.
@@ -0,0 +1,75 @@
 Getting Started with WebLLM
 ===========================
 This guide will help you set up WebLLM in your project, install necessary dependencies, and verify your setup.
 WebLLM Chat
 -----------
 If you want to experience AI Chat supported by local LLM inference and understand how WebLLM works, try out `WebLLM Chat <https://chat.webllm.ai/>`__, which provides a great example
 of integrating WebLLM into a full web application.
 A WebGPU-compatible browser is needed to run WebLLM-powered web applications.
 You can download the latest Google Chrome and use `WebGPU Report <https://webgpureport.org/>`__
 to verify the functionality of WebGPU on your browser.
 Installation
 ------------
 WebLLM offers a minimalist and modular interface to access the chatbot in the browser. The package is designed in a modular way to hook to any of the UI components.
 WebLLM is available as an `npm package <https://www.npmjs.com/package/@mlc-ai/web-llm>`_ and is also CDN-delivered. Therefore, you can install WebLLM using Node.js package managers like npm, yarn, or pnpm, or directly import the pacakge via CDN.
 Using Package Managers
 ^^^^^^^^^^^^^^^^^^^^^^
 Install WebLLM via your preferred package manager:
 .. code-block:: bash
   # npm
   npm install @mlc-ai/web-llm
   # yarn
   yarn add @mlc-ai/web-llm
   # pnpm
   pnpm install @mlc-ai/web-llm
 Import WebLLM into your project:
 .. code-block:: javascript
   // Import everything
   import * as webllm from "@mlc-ai/web-llm";
   // Or only import what you need
   import { CreateMLCEngine } from "@mlc-ai/web-llm";
 Using CDN
 ^^^^^^^^^
 Thanks to `jsdelivr.com <https://www.jsdelivr.com/package/npm/@mlc-ai/web-llm>`_, WebLLM can be imported directly through URL and work out-of-the-box on cloud development platforms like `jsfiddle.net <https://jsfiddle.net/>`_, `Codepen.io <https://codepen.io/>`_, and `Scribbler <https://scribbler.live/>`_:
 .. code-block:: javascript
   import * as webllm from "https://esm.run/@mlc-ai/web-llm";
 This method is especially useful for online environments like CodePen, JSFiddle, or local experiments.
 Verifying Installation
 ^^^^^^^^^^^^^^^^^^^^^^
 Run the following script to verify the installation:
 .. code-block:: javascript
   import { CreateMLCEngine } from "@mlc-ai/web-llm";
   console.log("WebLLM loaded successfully!");
 Online IDE Sandbox
 ------------------
 Instead of setting WebLLM locally, you can also try it on online Javascript IDE sandboxes like:
 - `Example in JSFiddle <https://jsfiddle.net/neetnestor/4nmgvsa2/>`_
 - `Example in CodePen <https://codepen.io/neetnestor/pen/vYwgZaG>`_
@@ -0,0 +1,56 @@
 const {
    defineConfig,
    globalIgnores,
 } = require("eslint/config");
 const tsParser = require("@typescript-eslint/parser");
 const typescriptEslint = require("@typescript-eslint/eslint-plugin");
 const js = require("@eslint/js");
 const {
    FlatCompat,
 } = require("@eslint/eslintrc");
 const compat = new FlatCompat({
    baseDirectory: __dirname,
    recommendedConfig: js.configs.recommended,
    allConfig: js.configs.all
 });
 module.exports = defineConfig([{
    extends: compat.extends(
        "eslint:recommended",
        "plugin:@typescript-eslint/recommended",
        "plugin:prettier/recommended",
    ),
    languageOptions: {
        parser: tsParser,
    },
    plugins: {
        "@typescript-eslint": typescriptEslint,
    },
    rules: {
        "@typescript-eslint/no-explicit-any": "off",
        "@typescript-eslint/no-empty-function": "off",
        "@typescript-eslint/no-non-null-assertion": "off",
    },
 }, {
    files: ["examples/**/*.js", "examples/**/*.ts"],
    "rules": {
        "no-undef": "off",
        "@typescript-eslint/no-unused-vars": "off",
    },
 }, globalIgnores([
    "**/dist",
    "**/debug",
    "**/lib",
    "**/build",
    "**/node_modules",
    "**/3rdparty",
    "**/.eslintrc.cjs",
    "**/.next",
 ])]);
@@ -0,0 +1 @@
 package-lock.json
@@ -0,0 +1,57 @@
 # Awesome WebLLM
 This page contains a curated list of examples, tutorials, blogs about WebLLM usecases.
 Please send a pull request if you find things that belong here.
 ## Example Projects
 Note that all examples below run in-browser and use WebGPU as a backend.
 #### Project List
 - [get-started](get-started): minimum get started example with chat completion.
  [![Open in JSFiddle](https://img.shields.io/badge/open-JSFiddle-blue?logo=jsfiddle&logoColor=white)](https://jsfiddle.net/neetnestor/yac9gbwf/)
  [![Open in Codepen](https://img.shields.io/badge/open-codepen-gainsboro?logo=codepen)](https://codepen.io/neetnestor/pen/NWVdgey)
 - [simple-chat-js](simple-chat-js): a mininum and complete chat bot app in vanilla JavaScript.
  [![Open in JSFiddle](https://img.shields.io/badge/open-JSFiddle-blue?logo=jsfiddle&logoColor=white)](https://jsfiddle.net/neetnestor/4nmgvsa2/)
  [![Open in Codepen](https://img.shields.io/badge/open-codepen-gainsboro?logo=codepen)](https://codepen.io/neetnestor/pen/vYwgZaG)
 - [simple-chat-ts](simple-chat-ts): a mininum and complete chat bot app in TypeScript.
 - [get-started-web-worker](get-started-web-worker): same as get-started, but using web worker.
 - [next-simple-chat](next-simple-chat): a mininum and complete chat bot app with [Next.js](https://nextjs.org/).
 - [subgroups-usage](subgroups-usage): capability-based routing between baseline and subgroup WebGPU WASM builds.
 - [multi-round-chat](multi-round-chat): while APIs are functional, we internally optimize so that multi round chat usage can reuse KV cache
 - [text-completion](text-completion): demonstrates API `engine.completions.create()`, which is pure text completion with no conversation, as opposed to `engine.chat.completions.create()`
 - [embeddings](embeddings): demonstrates API `engine.embeddings.create()`, integration with `EmbeddingsInterface` and `MemoryVectorStore` of [Langchain.js](https://js.langchain.com), and RAG with Langchain.js using WebLLM for both LLM and Embedding in a single engine
 - [multi-models](multi-models): demonstrates loading multiple models in a single engine concurrently
 #### Advanced OpenAI API Capabilities
 These examples demonstrate various capabilities via WebLLM's OpenAI-like API.
 - [streaming](streaming): return output as chunks in real-time in the form of an AsyncGenerator
 - [json-mode](json-mode): efficiently ensure output is in json format, see [OpenAI Reference](https://platform.openai.com/docs/guides/text-generation/chat-completions-api) for more.
 - [json-schema](json-schema): besides guaranteeing output to be in JSON, ensure output to adhere to a specific JSON schema specified the user
 - [seed-to-reproduce](seed-to-reproduce): use seeding to ensure reproducible output with fields `seed`.
 - [function-calling](function-calling) (WIP): function calling with fields `tools` and `tool_choice` (with preliminary support).
 - [vision-model](vision-model): process request with image input using Vision Language Model (e.g. Phi3.5-vision)
 #### Chrome Extension
 - [chrome-extension](chrome-extension): chrome extension that does not have a persistent background
 - [chrome-extension-webgpu-service-worker](chrome-extension-webgpu-service-worker): chrome extension using service worker, hence having a persistent background
 #### Others
 - [logit-processor](logit-processor): while `logit_bias` is supported, we additionally support stateful logit processing where users can specify their own rules. We also expose low-level API `forwardTokensAndSample()`.
 - [cache-usage](cache-usage): demonstrates how WebLLM supports multiple cache backends. Choose between the [Cache API](https://developer.mozilla.org/en-US/docs/Web/API/Cache), [IndexedDB cache](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API), [OPFS](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system), or the experimental Chrome [Cross-Origin Storage](https://github.com/WICG/cross-origin-storage) extension via `appConfig.cacheBackend`. Also demonstrates various cache utils such as checking
  whether a model is cached, deleting a model's weights from cache, deleting a model library wasm from cache, etc. Note: cross-origin backend currently does not support programmatic tensor-cache deletion.
 - [simple-chat-upload](simple-chat-upload): demonstrates how to upload local models to WebLLM instead of downloading via a URL link
 ## Demo Spaces
 - [web-llm-embed](https://huggingface.co/spaces/matthoffner/web-llm-embed): document chat prototype using react-llm with transformers.js embeddings
 - [DeVinci](https://x6occ-biaaa-aaaai-acqzq-cai.icp0.io/): AI chat app based on WebLLM and hosted on decentralized cloud platform
@@ -0,0 +1,13 @@
 # WebLLM Get Started App
 This folder provides a demo for cancelling model fetching after calling `engine.reload()`.
 ```bash
 npm install
 npm start
 ```
 Note if you would like to hack WebLLM core package.
 You can change web-llm dependencies as `"file:../.."`, and follow the build from source
 instruction in the project to build webllm locally. This option is only recommended
 if you would like to hack WebLLM core package.
@@ -0,0 +1,20 @@
 {
  "name": "get-started",
  "version": "0.1.0",
  "private": true,
  "scripts": {
    "start": "parcel src/get_started.html  --port 8887",
    "build": "parcel build src/get_started.html --dist-dir lib"
  },
  "devDependencies": {
    "buffer": "^5.7.1",
    "parcel": "^2.8.3",
    "process": "^0.11.10",
    "tslib": "^2.3.1",
    "typescript": "^4.9.5",
    "url": "^0.11.3"
  },
  "dependencies": {
    "@mlc-ai/web-llm": "^0.2.84"
  }
 }
@@ -0,0 +1,23 @@
 <!doctype html>
 <html>
  <script>
    webLLMGlobal = {};
  </script>
  <body>
    <h2>WebLLM Test Page</h2>
    Open console to see output
    <br />
    <br />
    <label id="init-label"> </label>
    <h3>Prompt</h3>
    <label id="prompt-label"> </label>
    <h3>Response</h3>
    <label id="generate-label"> </label>
    <br />
    <label id="stats-label"> </label>
    <script type="module" src="./get_started.js"></script>
  </body>
 </html>
@@ -0,0 +1,32 @@
 import * as webllm from "@mlc-ai/web-llm";
 import { error } from "loglevel";
 let engine;
 function setLabel(id, text) {
  const label = document.getElementById(id);
  if (label == null) {
    throw Error("Cannot find label " + id);
  }
  label.innerText = text;
 }
 async function main() {
  const initProgressCallback = (report) => {
    console.log(report.text);
    setLabel("init-label", report.text);
  };
  // Option 1: If we do not specify appConfig, we use `prebuiltAppConfig` defined in `config.ts`
  const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
  engine = new webllm.MLCEngine({
    initProgressCallback,
  });
  engine.reload(selectedModel);
 }
 main();
 setTimeout(() => {
  console.log("calling unload");
  engine.unload().catch((err) => {
    console.log(err);
  });
 }, 5000);
@@ -0,0 +1,30 @@
 # WebLLM Cache Usage
 WebLLM supports multiple persistent cache backends. You can pick the classic Cache API, IndexedDB, [Origin Private File System (OPFS)](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system), or the experimental Chrome [Cross-Origin Storage](https://github.com/WICG/cross-origin-storage) extension by
 setting `AppConfig.cacheBackend` to `"cache"`, `"indexeddb"`, `"opfs"`, or `"cross-origin"`.
 This folder provides an example on how different caches are used in WebLLM. We also
 demonstrate the utility cache functions such as deleting models, checking if models are in cache, etc.
 > **Note:** The cross-origin backend requires installation of the [Cross-Origin Storage browser extension](https://chromewebstore.google.com/detail/cross-origin-storage/denpnpcgjgikjpoglpjefakmdcbmlgih) ([source code](https://github.com/web-ai-community/cross-origin-storage-extension)). This does not currently support programmatic tensor-cache deletion; deletion is extension-managed.
 > **Note:** If `"opfs"` is selected in an environment without OPFS support, cache operations fail with an OPFS availability error.
 > Use `appConfig.opfsAccessMode = "auto"` to use OPFS sync access handles where supported, or `"sync"` to require sync access handles. The default is `"async"`.
 For more information about Cache API and IndexedDB, see:
 https://developer.mozilla.org/en-US/docs/Web/API/Storage_API/Storage_quotas_and_eviction_criteria#what_technologies_store_data_in_the_browser.
 To inspect the downloaded artifacts in your browser, open up developer console, go to application,
 and you will find the artifacts under `IndexedDB`, `Cache storage`, or OPFS (under the browser's origin-private file system). When `"cross-origin"` is selected,
 the extension displays origins and resource hashes.
 To run the example, you can do the following steps under this folder
 ```bash
 npm install
 npm start
 ```
 Note if you would like to hack WebLLM core package.
 You can change web-llm dependencies as `"file:../.."`, and follow the build from source
 instruction in the project to build webllm locally. This option is only recommended
 if you would like to hack WebLLM core package.
@@ -0,0 +1,20 @@
 {
  "name": "cache-usage",
  "version": "0.1.0",
  "private": true,
  "scripts": {
    "start": "parcel src/cache_usage.html  --port 8889",
    "build": "parcel build src/cache_usage.html --dist-dir lib"
  },
  "devDependencies": {
    "buffer": "^5.7.1",
    "parcel": "2.8.3",
    "process": "^0.11.10",
    "tslib": "^2.3.1",
    "typescript": "^4.9.5",
    "url": "^0.11.3"
  },
  "dependencies": {
    "@mlc-ai/web-llm": "^0.2.84"
  }
 }
@@ -0,0 +1,24 @@
 <!doctype html>
 <html>
  <script>
    webLLMGlobal = {};
  </script>
  <body>
    <h2>WebLLM Test Page</h2>
    Open console to see output
    <br />
    <br />
    <label id="init-label"> </label>
    <h3>Prompt</h3>
    <label id="prompt-label"> </label>
    <h3>Response</h3>
    <label id="generate-label"> </label>
    <br />
    <label id="stats-label"> </label>
    <script type="module" src="./cache_usage.ts"></script>
  </body>
 </html>
@@ -0,0 +1,90 @@
 import * as webllm from "@mlc-ai/web-llm";
 function setLabel(id: string, text: string) {
  const label = document.getElementById(id);
  if (label == null) {
    throw Error("Cannot find label " + id);
  }
  label.innerText = text;
 }
 const initProgressCallback = (report: webllm.InitProgressReport) => {
  setLabel("init-label", report.text);
 };
 async function main() {
  const appConfig = webllm.prebuiltAppConfig;
  // CHANGE THIS TO SEE THE EFFECTS OF EACH, CODE BELOW DOES NOT NEED TO CHANGE
  appConfig.cacheBackend = "cache"; // "indexeddb" or "cache" or "cross-origin" or "opfs"
  const cacheBackend = appConfig.cacheBackend as string;
  if (cacheBackend === "indexeddb") {
    console.log("Using IndexedDB Cache");
  } else if (cacheBackend === "cache") {
    console.log("Using Cache API");
  } else if (cacheBackend === "cross-origin") {
    console.log("Using Cross-Origin Storage");
  } else if (cacheBackend === "opfs") {
    console.log("Using Origin Private File System");
  }
  // 1. This triggers downloading and caching the model with either Cache or IndexedDB Cache
  const selectedModel = "Llama-3.2-1B-Instruct-q4f16_1-MLC";
  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
    selectedModel,
    { initProgressCallback: initProgressCallback, appConfig: appConfig },
  );
  const request: webllm.ChatCompletionRequest = {
    stream: false,
    messages: [
      {
        role: "user",
        content: "Write an analogy between mathematics and a lighthouse.",
      },
    ],
    n: 1,
    temperature: 0,
  };
  let reply = await engine.chat.completions.create(request);
  console.log(reply);
  // 2. Check whether model weights are cached
  let modelCached = await webllm.hasModelInCache(selectedModel, appConfig);
  console.log("hasModelInCache: ", modelCached);
  if (!modelCached) {
    throw Error("Expect hasModelInCache() to be true, but got: " + modelCached);
  }
  // 3. We reload, and we should see this time it is much faster because the weights are cached.
  console.log("Reload model start");
  await engine.reload(selectedModel);
  console.log("Reload model end");
  reply = await engine.chat.completions.create(request);
  console.log(reply);
  // Cross-Origin Storage does not support deletion
  if (cacheBackend === "cross-origin") {
    return;
  }
  // 4. Delete everything about this model from cache
  // You can also delete only the model library wasm, only the model weights, or only the config file
  await webllm.deleteModelAllInfoInCache(selectedModel, appConfig);
  modelCached = await webllm.hasModelInCache(selectedModel, appConfig);
  console.log("After deletion, hasModelInCache: ", modelCached);
  if (modelCached) {
    throw Error(
      "Expect hasModelInCache() to be false, but got: " + modelCached,
    );
  }
  // 5. If we reload, we should expect the model to start downloading again
  console.log("Reload model start");
  await engine.reload(selectedModel);
  console.log("Reload model end");
  reply = await engine.chat.completions.create(request);
  console.log(reply);
 }
 main();
@@ -0,0 +1,29 @@
 # WebLLM Chrome Extension using WebGPU Running on Service Worker
 ![Chrome Extension](https://github.com/mlc-ai/mlc-llm/assets/11940172/0d94cc73-eff1-4128-a6e4-70dc879f04e0)
 > [!WARNING]  
 > Service worker support in WebGPU is enabled by default in [Chrome 124](https://chromiumdash.appspot.com/commit/8d78510e4aca5ac3cd8ee4a33e96b404eaa43246).
 > If you are using Chrome 123, go to `chrome://flags/#enable-experimental-web-platform-features`, enable the `#enable-experimental-web-platform-features` flag, and **relaunch the browser**.
 This example shows how we can create a Chrome extension using WebGPU and service worker.
 - The project structure is as follows:
  - `manifest.json`: A required file that lists important information about the structure and behavior of that extension. Here we are using manifest V3.
  - `popup.ts`: Script of the extension pop-up window.
  - `background.ts`: Script of the service worker. An extension service worker is loaded when it is needed, and unloaded when it goes dormant.
  - `content.js`: Content script that interacts with DOM.
 - Run
  ```bash
  npm install
  npm run build
  ```
  This will create a new directory at `./dist/`. To load the extension into Chrome, go to Extensions > Manage Extensions and select Load Unpacked. Add the `./dist/` directory. You can now pin the extension to your toolbar and use it to chat with your favorite model!
 **Note**: This example disables chatting using the contents of the active tab by default.
 To enable it, set `useContext` in `popup.ts` to `true`. More info about this feature can be found
 [here](https://github.com/mlc-ai/web-llm/pull/190).
 However, if the web content is too large, it might run into issues. We recommend using `example.html` to
 test this feature.
@@ -0,0 +1,23 @@
 {
  "name": "chrome-extension",
  "version": "1.0.0",
  "description": "",
  "private": true,
  "scripts": {
    "build": "parcel build src/manifest.json --config @parcel/config-webextension"
  },
  "author": "",
  "license": "ISC",
  "devDependencies": {
    "@parcel/config-webextension": "^2.9.3",
    "@types/chrome": "^0.0.242",
    "buffer": "^6.0.3",
    "parcel": "^2.9.3",
    "process": "^0.11.10",
    "url": "^0.11.1"
  },
  "dependencies": {
    "@mlc-ai/web-llm": "^0.2.84",
    "progressbar.js": "^1.1.0"
  }
 }
@@ -0,0 +1,14 @@
 import { ExtensionServiceWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
 // Hookup an engine to a service worker handler
 let handler;
 chrome.runtime.onConnect.addListener(function (port) {
  console.assert(port.name === "web_llm_service_worker");
  if (handler === undefined) {
    handler = new ExtensionServiceWorkerMLCEngineHandler(port);
  } else {
    handler.setPort(port);
  }
  port.onMessage.addListener(handler.onmessage.bind(handler));
 });
@@ -0,0 +1,6 @@
 // Only the content script is able to access the DOM
 chrome.runtime.onConnect.addListener(function (port) {
  port.onMessage.addListener(function (msg) {
    port.postMessage({ contents: document.body.innerHTML });
  });
 });
@@ -0,0 +1,20 @@
 In the year 2154, humanity had colonized several planets in the distant reaches
 of the galaxy. The planet of Xylophia-IV was one of the most remote and
 inhospitable, with temperatures often dropping to -200 degrees Celsius. Despite
 these harsh conditions, a team of scientists had established a research station
 on the planet to study the unique geological formations and exotic flora and
 fauna. One day, while conducting a routine survey of the planet's surface, the
 team discovered an strange object buried deep in the ice. As they examined it
 closer, they realized it was a small, metallic capsule with a glowing blue
 symbol etched onto its surface. The team's leader, a brilliant scientist named
 Dr. Maria Rodriguez, was immediately intrigued by the capsule's mysterious
 origins. She ordered her team to bring it back to the research station for
 further analysis. After weeks of studying the capsule, the team finally cracked
 the code to the symbol etched onto its surface. It was a message from an alien
 race, warning Earth of an impending attack from an unknown threat. The team was
 shocked and dismayed by the news, but they knew they had to act quickly to warn
 the rest of humanity. They transmitted the message to the nearest space station,
 which relayed it to Earth's government. As the threat of attack loomed near, the
 team remained on high alert, ready to face whatever dangers lay ahead. They had
 uncovered a secrets of the universe, and now they were determined to protect
 their planet and its inhabitants at all costs.
@@ -0,0 +1,30 @@
 {
  "manifest_version": 3,
  "name": "MLCBot",
  "version": "0.1.0",
  "description": "Chat with your browser",
  "icons": {
    "16": "icons/icon-16.png",
    "32": "icons/icon-32.png",
    "64": "icons/icon-64.png",
    "128": "icons/icon-128.png"
  },
  "content_security_policy": {
    "extension_pages": "style-src-elem 'self' https://cdnjs.cloudflare.com; font-src 'self' https://cdnjs.cloudflare.com; script-src 'self' 'wasm-unsafe-eval'; default-src 'self' data:; connect-src 'self' data: http://localhost:8000 https://huggingface.co https://cdn-lfs.huggingface.co https://cdn-lfs-us-1.huggingface.co https://raw.githubusercontent.com https://cdn-lfs-us-1.hf.co https://cas-bridge.xethub.hf.co"
  },
  "action": {
    "default_title": "MLCBot",
    "default_popup": "popup.html"
  },
  "content_scripts": [
    {
      "matches": ["<all_urls>"],
      "js": ["content.js"]
    }
  ],
  "background": {
    "service_worker": "background.ts",
    "type": "module"
  },
  "permissions": ["storage", "tabs", "webNavigation"]
 }
@@ -0,0 +1,240 @@
 *,
 *::before,
 *::after {
  margin: 0;
  padding: 0;
  box-sizing: border-box;
 }
 html {
  font-family:
    -apple-system,
    BlinkMacSystemFont,
    Segoe UI,
    Helvetica,
    Arial,
    sans-serif;
  color: #222;
 }
 body {
  margin: 0;
  padding: 0.5rem;
  background-color: #778da9;
  width: 320px;
  font-size: small;
 }
 p {
  margin: 0;
 }
 /* LOADING BAR */
 #loadingContainer {
  margin-bottom: 15px;
  width: 300px;
  height: 8px;
 }
 /* INPUT AREA */
 #query-input {
  border: 1px solid #ccc;
  border-radius: 4px;
 }
 .input-container {
  display: flex;
  flex-direction: row;
  align-items: center;
 }
 .input-container input {
  width: 100%;
  outline: none;
  padding: 0.5rem;
  margin-right: 0.5rem;
 }
 /* SUBMIT BUTTON */
 .btn {
  background-color: #1b263b;
  color: white;
  font-size: small;
  cursor: pointer;
  border-radius: 4px;
  border: none;
  padding: 0.5rem;
 }
 .btn:hover {
  background-color: #d0d0d0;
 }
 .btn:disabled {
  background-color: #a7a7a7;
  color: rgb(255, 255, 255);
  cursor: default;
 }
 .btn img {
  width: 1rem;
  height: 1rem;
 }
 /* LOADING */
 .stage {
  display: flex;
  justify-content: center;
  align-items: center;
  position: relative;
  margin: 0 -5%;
  overflow: hidden;
 }
 #loading-indicator {
  display: none;
  color: white;
  margin-top: 0.5rem;
 }
 .dot-flashing {
  position: relative;
  width: 10px;
  height: 10px;
  border-radius: 5px;
  background-color: #1b263b;
  color: #1b263b;
  animation: dot-flashing 0.4s infinite linear alternate;
  animation-delay: 0.2s;
 }
 .dot-flashing::before,
 .dot-flashing::after {
  content: "";
  display: inline-block;
  position: absolute;
  top: 0;
 }
 .dot-flashing::before {
  left: -15px;
  width: 10px;
  height: 10px;
  border-radius: 5px;
  background-color: #1b263b;
  color: #1b263b;
  animation: dot-flashing 0.4s infinite alternate;
  animation-delay: 0s;
 }
 .dot-flashing::after {
  left: 15px;
  width: 10px;
  height: 10px;
  border-radius: 5px;
  background-color: #1b263b;
  color: #1b263b;
  animation: dot-flashing 0.4s infinite alternate;
  animation-delay: 0.4s;
 }
@keyframes dot-flashing {
  0% {
    background-color: #1b263b;
  }
  50%,
  100% {
    background-color: #415a77;
  }
 }
 /* ANSWERS */
 #queriesAnswersContainer {
  display: block;
  color: white;
  margin-top: 0.5rem;
 }
 #answer {
  color: #333333;
 }
 #answerWrapper {
  display: none;
  background-color: #ffd166;
  border-radius: 8px;
  padding: 0.5rem;
  margin-top: 0.5rem;
 }
 .queriesAnswers {
  border-radius: 8px;
  background-color: #ffd166;
  padding: 0.5rem;
  color: #333333;
 }
 #lastQuery {
  color: rgb(188, 188, 188);
 }
 #lastAnswer {
  color: white;
  margin-top: 0.5rem;
 }
 #lastRequest {
  padding: 0.5rem;
  margin-top: 0.5rem;
  background-color: #333333;
  border-radius: 4px;
 }
 /* ANSWER OPTIONS */
 .timeStamp {
  color: #9a8c98;
 }
 .copyRow {
  display: flex;
  flex-direction: row;
  align-items: end;
  justify-content: space-between;
  color: #a7a7a7;
  margin-top: 0.5rem;
 }
 .copyText {
  display: none;
  color: #a7a7a7;
  margin-right: 0.5rem;
 }
 .copyButton {
  color: #415a77;
  background-color: transparent;
  border: none;
  cursor: pointer;
  padding: 0;
  margin-left: 0.5rem;
 }
 .copyButton:hover {
  color: #5e80a7;
  background-color: transparent;
 }
 .removeButton {
  color: #415a77;
  background-color: transparent;
  border: none;
  cursor: pointer;
  padding: 0;
 }
 .removeButton:hover {
  color: #5e80a7;
  background-color: transparent;
 }
@@ -0,0 +1,46 @@
 <!doctype html>
 <html lang="en">
  <head>
    <meta charset="UTF-8" />
    <title>Chatbot</title>
    <link rel="stylesheet" href="popup.css" />
    <link
      rel="stylesheet"
      href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css"
    />
  </head>
  <body>
    <div id="loadingContainer"></div>
    <div class="input-container form-group">
      <input
        type="search"
        id="query-input"
        placeholder="What's on your mind?"
      />
      <button id="submit-button" class="btn">
        <i class="fa fa-comments"></i>
      </button>
    </div>
    <div class="stage">
      <div id="loading-indicator" class="dot-flashing"></div>
    </div>
    <div id="answerWrapper">
      <div id="answer"></div>
      <div class="copyRow">
        <span id="timestamp"></span>
        <button
          id="copyAnswer"
          class="btn copyButton"
          title="Copy the Answer to the Clipboard"
        >
          <i class="fa-solid fa-copy fa-lg"></i>
        </button>
      </div>
    </div>
    <script type="module" src="./popup.ts"></script>
  </body>
 </html>
@@ -0,0 +1,167 @@
 "use strict";
 // This code is partially adapted from the openai-chatgpt-chrome-extension repo:
 // https://github.com/jessedi0n/openai-chatgpt-chrome-extension
 import "./popup.css";
 import {
  ChatCompletionMessageParam,
  CreateExtensionServiceWorkerMLCEngine,
  MLCEngineInterface,
  InitProgressReport,
 } from "@mlc-ai/web-llm";
 import { ProgressBar, Line } from "progressbar.js";
 /***************** UI elements *****************/
 // Whether or not to use the content from the active tab as the context
 const useContext = false;
 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
 const queryInput = document.getElementById("query-input")!;
 const submitButton = document.getElementById("submit-button")!;
 let isLoadingParams = false;
 (<HTMLButtonElement>submitButton).disabled = true;
 const progressBar: ProgressBar = new Line("#loadingContainer", {
  strokeWidth: 4,
  easing: "easeInOut",
  duration: 1400,
  color: "#ffd166",
  trailColor: "#eee",
  trailWidth: 1,
  svgStyle: { width: "100%", height: "100%" },
 });
 /***************** Web-LLM MLCEngine Configuration *****************/
 const initProgressCallback = (report: InitProgressReport) => {
  progressBar.animate(report.progress, {
    duration: 50,
  });
  if (report.progress == 1.0) {
    enableInputs();
  }
 };
 const engine: MLCEngineInterface = await CreateExtensionServiceWorkerMLCEngine(
  "Qwen2-0.5B-Instruct-q4f16_1-MLC",
  { initProgressCallback: initProgressCallback },
 );
 const chatHistory: ChatCompletionMessageParam[] = [];
 isLoadingParams = true;
 function enableInputs() {
  if (isLoadingParams) {
    sleep(500);
    (<HTMLButtonElement>submitButton).disabled = false;
    const loadingBarContainer = document.getElementById("loadingContainer")!;
    loadingBarContainer.remove();
    queryInput.focus();
    isLoadingParams = false;
  }
 }
 /***************** Event Listeners *****************/
 // Disable submit button if input field is empty
 queryInput.addEventListener("keyup", () => {
  if ((<HTMLInputElement>queryInput).value === "") {
    (<HTMLButtonElement>submitButton).disabled = true;
  } else {
    (<HTMLButtonElement>submitButton).disabled = false;
  }
 });
 // If user presses enter, click submit button
 queryInput.addEventListener("keyup", (event) => {
  if (event.code === "Enter") {
    event.preventDefault();
    submitButton.click();
  }
 });
 // Listen for clicks on submit button
 async function handleClick() {
  // Get the message from the input field
  const message = (<HTMLInputElement>queryInput).value;
  console.log("message", message);
  chatHistory.push({ role: "user", content: message });
  // Clear the answer
  document.getElementById("answer")!.innerHTML = "";
  // Hide the answer
  document.getElementById("answerWrapper")!.style.display = "none";
  // Show the loading indicator
  document.getElementById("loading-indicator")!.style.display = "block";
  // Send the chat completion message to the engine
  let curMessage = "";
  const completion = await engine.chat.completions.create({
    stream: true,
    messages: chatHistory,
  });
  // Update the answer as the model generates more text
  for await (const chunk of completion) {
    const curDelta = chunk.choices[0].delta.content;
    if (curDelta) {
      curMessage += curDelta;
    }
    updateAnswer(curMessage);
  }
  chatHistory.push({ role: "assistant", content: await engine.getMessage() });
 }
 submitButton.addEventListener("click", handleClick);
 function updateAnswer(answer: string) {
  // Show answer
  document.getElementById("answerWrapper")!.style.display = "block";
  const answerWithBreaks = answer.replace(/\n/g, "<br>");
  document.getElementById("answer")!.innerHTML = answerWithBreaks;
  // Add event listener to copy button
  document.getElementById("copyAnswer")!.addEventListener("click", () => {
    // Get the answer text
    const answerText = answer;
    // Copy the answer text to the clipboard
    navigator.clipboard
      .writeText(answerText)
      .then(() => console.log("Answer text copied to clipboard"))
      .catch((err) => console.error("Could not copy text: ", err));
  });
  const options: Intl.DateTimeFormatOptions = {
    month: "short",
    day: "2-digit",
    hour: "2-digit",
    minute: "2-digit",
    second: "2-digit",
  };
  const time = new Date().toLocaleString("en-US", options);
  // Update timestamp
  document.getElementById("timestamp")!.innerText = time;
  // Hide loading indicator
  document.getElementById("loading-indicator")!.style.display = "none";
 }
 function fetchPageContents() {
  chrome.tabs.query({ currentWindow: true, active: true }, function (tabs) {
    if (tabs[0]?.id) {
      const port = chrome.tabs.connect(tabs[0].id, { name: "channelName" });
      port.postMessage({});
      port.onMessage.addListener(function (msg) {
        console.log("Page contents:", msg.contents);
        chrome.runtime.sendMessage({ context: msg.contents });
      });
    }
  });
 }
 // Grab the page contents when the popup is opened
 window.onload = function () {
  if (useContext) {
    fetchPageContents();
  }
 };
@@ -0,0 +1,12 @@
 # WebLLM Chrome Extension
 ![Chrome Extension](https://github.com/mlc-ai/mlc-llm/assets/11940172/0d94cc73-eff1-4128-a6e4-70dc879f04e0)
 To run the extension, do the following steps under this folder
 ```bash
 npm install
 npm run build
 ```
 This will create a new directory at `chrome-extension/dist/`. To load the extension into Chrome, go to Extensions > Manage Extensions and select Load Unpacked. Add the `chrome-extension/dist/` directory. You can now pin the extension to your toolbar and use the drop-down menu to chat with your favorite model!
@@ -0,0 +1,23 @@
 {
  "name": "chrome-extension",
  "version": "1.0.1",
  "description": "",
  "private": true,
  "scripts": {
    "build": "parcel build src/manifest.json --config @parcel/config-webextension"
  },
  "author": "",
  "license": "ISC",
  "devDependencies": {
    "@parcel/config-webextension": "^2.9.3",
    "@types/chrome": "^0.0.242",
    "buffer": "^6.0.3",
    "parcel": "^2.9.3",
    "process": "^0.11.10",
    "url": "^0.11.1"
  },
  "dependencies": {
    "@mlc-ai/web-llm": "^0.2.84",
    "progressbar.js": "^1.1.0"
  }
 }
@@ -0,0 +1,6 @@
 // Only the content script is able to access the DOM
 chrome.runtime.onConnect.addListener(function (port) {
  port.onMessage.addListener(function (msg) {
    port.postMessage({ contents: document.body.innerText });
  });
 });
@@ -0,0 +1,20 @@
 In the year 2154, humanity had colonized several planets in the distant reaches
 of the galaxy. The planet of Xylophia-IV was one of the most remote and
 inhospitable, with temperatures often dropping to -200 degrees Celsius. Despite
 these harsh conditions, a team of scientists had established a research station
 on the planet to study the unique geological formations and exotic flora and
 fauna. One day, while conducting a routine survey of the planet's surface, the
 team discovered an strange object buried deep in the ice. As they examined it
 closer, they realized it was a small, metallic capsule with a glowing blue
 symbol etched onto its surface. The team's leader, a brilliant scientist named
 Dr. Maria Rodriguez, was immediately intrigued by the capsule's mysterious
 origins. She ordered her team to bring it back to the research station for
 further analysis. After weeks of studying the capsule, the team finally cracked
 the code to the symbol etched onto its surface. It was a message from an alien
 race, warning Earth of an impending attack from an unknown threat. The team was
 shocked and dismayed by the news, but they knew they had to act quickly to warn
 the rest of humanity. They transmitted the message to the nearest space station,
 which relayed it to Earth's government. As the threat of attack loomed near, the
 team remained on high alert, ready to face whatever dangers lay ahead. They had
 uncovered a secrets of the universe, and now they were determined to protect
 their planet and its inhabitants at all costs.
@@ -0,0 +1,27 @@
 {
  "manifest_version": 3,
  "name": "MLCBot",
  "version": "0.1.1",
  "description": "Chat with your browser",
  "icons": {
    "16": "icons/icon-16.png",
    "32": "icons/icon-32.png",
    "64": "icons/icon-64.png",
    "128": "icons/icon-128.png"
  },
  "content_security_policy": {
    "extension_pages": "style-src-elem 'self' https://cdnjs.cloudflare.com; font-src 'self' https://cdnjs.cloudflare.com; script-src 'self' 'wasm-unsafe-eval'; default-src 'self' data:; connect-src 'self' data: http://localhost:8000 https://huggingface.co https://cdn-lfs.huggingface.co https://cdn-lfs-us-1.huggingface.co https://raw.githubusercontent.com https://cdn-lfs-us-1.hf.co https://cas-bridge.xethub.hf.co"
  },
  "action": {
    "default_title": "MLCBot",
    "default_popup": "popup.html"
  },
  "content_scripts": [
    {
      "matches": ["<all_urls>"],
      "js": ["content.js"]
    }
  ],
  "permissions": ["storage", "tabs", "webNavigation", "activeTab", "scripting"],
  "host_permissions": ["http://*/", "https://*/"]
 }
@@ -0,0 +1,23 @@
 {
  "manifest_version": 2,
  "name": "MLCBot",
  "version": "0.1.0",
  "description": "Chat with your browser",
  "icons": {
    "16": "icons/icon-16.png",
    "32": "icons/icon-32.png",
    "64": "icons/icon-64.png",
    "128": "icons/icon-128.png"
  },
  "content_security_policy": "style-src-elem 'self' https://cdnjs.cloudflare.com; font-src 'self' https://cdnjs.cloudflare.com; script-src 'self' 'unsafe-eval' 'wasm-unsafe-eval'; default-src 'self' data:; connect-src 'self' data: http://localhost:8000 https://huggingface.co https://cdn-lfs.huggingface.co https://raw.githubusercontent.com https://cdn-lfs-us-1.hf.co https://cas-bridge.xethub.hf.co",
  "browser_action": {
    "default_popup": "popup.html"
  },
  "content_scripts": [
    {
      "matches": ["<all_urls>"],
      "js": ["content.js"]
    }
  ],
  "permissions": ["storage", "tabs", "webNavigation", "activeTab"]
 }
@@ -0,0 +1,240 @@
 *,
 *::before,
 *::after {
  margin: 0;
  padding: 0;
  box-sizing: border-box;
 }
 html {
  font-family:
    -apple-system,
    BlinkMacSystemFont,
    Segoe UI,
    Helvetica,
    Arial,
    sans-serif;
  color: #222;
 }
 body {
  margin: 0;
  padding: 0.5rem;
  background-color: #778da9;
  width: 335px;
  font-size: small;
 }
 p {
  margin: 0;
 }
 /* LOADING BAR */
 #loadingContainer {
  margin-bottom: 15px;
  width: 315px;
  height: 8px;
 }
 /* INPUT AREA */
 #query-input {
  border: 1px solid #ccc;
  border-radius: 4px;
 }
 .input-container {
  display: flex;
  flex-direction: row;
  align-items: center;
 }
 .input-container input {
  width: 100%;
  outline: none;
  padding: 0.5rem;
  margin-right: 0.5rem;
 }
 /* BUTTON */
 .btn {
  background-color: #1b263b;
  color: white;
  font-size: small;
  cursor: pointer;
  border-radius: 4px;
  border: none;
  padding: 0.5rem;
 }
 .btn:hover {
  background-color: #d0d0d0;
 }
 .btn:disabled {
  background-color: #a7a7a7;
  color: rgb(255, 255, 255);
  cursor: default;
 }
 .btn img {
  width: 1rem;
  height: 1rem;
 }
 /* LOADING */
 .stage {
  display: flex;
  justify-content: center;
  align-items: center;
  position: relative;
  margin: 0 -5%;
  overflow: hidden;
 }
 #loading-indicator {
  display: none;
  color: white;
  margin-top: 0.5rem;
 }
 .dot-flashing {
  position: relative;
  width: 10px;
  height: 10px;
  border-radius: 5px;
  background-color: #1b263b;
  color: #1b263b;
  animation: dot-flashing 0.4s infinite linear alternate;
  animation-delay: 0.2s;
 }
 .dot-flashing::before,
 .dot-flashing::after {
  content: "";
  display: inline-block;
  position: absolute;
  top: 0;
 }
 .dot-flashing::before {
  left: -15px;
  width: 10px;
  height: 10px;
  border-radius: 5px;
  background-color: #1b263b;
  color: #1b263b;
  animation: dot-flashing 0.4s infinite alternate;
  animation-delay: 0s;
 }
 .dot-flashing::after {
  left: 15px;
  width: 10px;
  height: 10px;
  border-radius: 5px;
  background-color: #1b263b;
  color: #1b263b;
  animation: dot-flashing 0.4s infinite alternate;
  animation-delay: 0.4s;
 }
@keyframes dot-flashing {
  0% {
    background-color: #1b263b;
  }
  50%,
  100% {
    background-color: #415a77;
  }
 }
 /* ANSWERS */
 #queriesAnswersContainer {
  display: block;
  color: white;
  margin-top: 0.5rem;
 }
 #answer {
  color: #333333;
 }
 #answerWrapper {
  display: none;
  background-color: #ffd166;
  border-radius: 8px;
  padding: 0.5rem;
  margin-top: 0.5rem;
 }
 .queriesAnswers {
  border-radius: 8px;
  background-color: #ffd166;
  padding: 0.5rem;
  color: #333333;
 }
 #lastQuery {
  color: rgb(188, 188, 188);
 }
 #lastAnswer {
  color: white;
  margin-top: 0.5rem;
 }
 #lastRequest {
  padding: 0.5rem;
  margin-top: 0.5rem;
  background-color: #333333;
  border-radius: 4px;
 }
 /* ANSWER OPTIONS */
 .timeStamp {
  color: #9a8c98;
 }
 .copyRow {
  display: flex;
  flex-direction: row;
  align-items: end;
  justify-content: space-between;
  color: #a7a7a7;
  margin-top: 0.5rem;
 }
 .copyText {
  display: none;
  color: #a7a7a7;
  margin-right: 0.5rem;
 }
 .copyButton {
  color: #415a77;
  background-color: transparent;
  border: none;
  cursor: pointer;
  padding: 0;
  margin-left: 0.5rem;
 }
 .copyButton:hover {
  color: #5e80a7;
  background-color: transparent;
 }
 .removeButton {
  color: #415a77;
  background-color: transparent;
  border: none;
  cursor: pointer;
  padding: 0;
 }
 .removeButton:hover {
  color: #5e80a7;
  background-color: transparent;
 }
@@ -0,0 +1,50 @@
 <!doctype html>
 <html lang="en">
  <head>
    <meta charset="UTF-8" />
    <title>Chatbot</title>
    <link rel="stylesheet" href="popup.css" />
    <link
      rel="stylesheet"
      href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css"
    />
  </head>
  <body>
    <select id="model-selection"></select>
    <div id="loadingBox">
      <p id="init-label">Initializing model...</p>
      <div id="loadingContainer"></div>
    </div>
    <p id="model-name"></p>
    <div class="input-container form-group">
      <input
        type="search"
        id="query-input"
        placeholder="What's on your mind?"
      />
      <button id="submit-button" class="btn">
        <i class="fa fa-comments"></i>
      </button>
    </div>
    <div class="stage">
      <div id="loading-indicator" class="dot-flashing"></div>
    </div>
    <div id="answerWrapper">
      <div id="answer"></div>
      <div class="copyRow">
        <span id="timestamp"></span>
        <button
          id="copyAnswer"
          class="btn copyButton"
          title="Copy the Answer to the Clipboard"
        >
          <i class="fa-solid fa-copy fa-lg"></i>
        </button>
      </div>
    </div>
    <script type="module" src="./popup.ts"></script>
  </body>
 </html>
@@ -0,0 +1,298 @@
 "use strict";
 // This code is partially adapted from the openai-chatgpt-chrome-extension repo:
 // https://github.com/jessedi0n/openai-chatgpt-chrome-extension
 import "./popup.css";
 import {
  MLCEngineInterface,
  InitProgressReport,
  CreateMLCEngine,
  ChatCompletionMessageParam,
  prebuiltAppConfig,
 } from "@mlc-ai/web-llm";
 import { ProgressBar, Line } from "progressbar.js";
 // modified setLabel to not throw error
 function setLabel(id: string, text: string) {
  const label = document.getElementById(id);
  if (label != null) {
    label.innerText = text;
  }
 }
 function getElementAndCheck(id: string): HTMLElement {
  const element = document.getElementById(id);
  if (element == null) {
    throw Error("Cannot find element " + id);
  }
  return element;
 }
 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
 const queryInput = getElementAndCheck("query-input")!;
 const submitButton = getElementAndCheck("submit-button")!;
 const modelName = getElementAndCheck("model-name");
 let context = "";
 let modelDisplayName = "";
 // throws runtime.lastError if you refresh extension AND try to access a webpage that is already open
 fetchPageContents();
 (<HTMLButtonElement>submitButton).disabled = true;
 let progressBar: ProgressBar = new Line("#loadingContainer", {
  strokeWidth: 4,
  easing: "easeInOut",
  duration: 1400,
  color: "#ffd166",
  trailColor: "#eee",
  trailWidth: 1,
  svgStyle: { width: "100%", height: "100%" },
 });
 let isLoadingParams = true;
 let initProgressCallback = (report: InitProgressReport) => {
  setLabel("init-label", report.text);
  progressBar.animate(report.progress, {
    duration: 50,
  });
  if (report.progress == 1.0) {
    enableInputs();
  }
 };
 // initially selected model
 let selectedModel = "Qwen2-0.5B-Instruct-q4f16_1-MLC";
 // populate model-selection
 const modelSelector = getElementAndCheck(
  "model-selection",
 ) as HTMLSelectElement;
 for (let i = 0; i < prebuiltAppConfig.model_list.length; ++i) {
  const model = prebuiltAppConfig.model_list[i];
  const opt = document.createElement("option");
  opt.value = model.model_id;
  opt.innerHTML = model.model_id;
  opt.selected = false;
  // set initial selection as the initially selected model
  if (model.model_id == selectedModel) {
    opt.selected = true;
  }
  modelSelector.appendChild(opt);
 }
 modelName.innerText = "Loading initial model...";
 const engine: MLCEngineInterface = await CreateMLCEngine(selectedModel, {
  initProgressCallback: initProgressCallback,
 });
 modelName.innerText = "Now chatting with " + modelDisplayName;
 let chatHistory: ChatCompletionMessageParam[] = [];
 function enableInputs() {
  if (isLoadingParams) {
    sleep(500);
    isLoadingParams = false;
  }
  // remove loading bar and loading bar descriptors, if exists
  const initLabel = document.getElementById("init-label");
  initLabel?.remove();
  const loadingBarContainer = document.getElementById("loadingContainer")!;
  loadingBarContainer?.remove();
  queryInput.focus();
  const modelNameArray = selectedModel.split("-");
  modelDisplayName = modelNameArray[0];
  let j = 1;
  while (j < modelNameArray.length && modelNameArray[j][0] != "q") {
    modelDisplayName = modelDisplayName + "-" + modelNameArray[j];
    j++;
  }
 }
 let requestInProgress = false;
 // Disable submit button if input field is empty
 queryInput.addEventListener("keyup", () => {
  if (
    (<HTMLInputElement>queryInput).value === "" ||
    requestInProgress ||
    isLoadingParams
  ) {
    (<HTMLButtonElement>submitButton).disabled = true;
  } else {
    (<HTMLButtonElement>submitButton).disabled = false;
  }
 });
 // If user presses enter, click submit button
 queryInput.addEventListener("keyup", (event) => {
  if (event.code === "Enter") {
    event.preventDefault();
    submitButton.click();
  }
 });
 // Listen for clicks on submit button
 async function handleClick() {
  requestInProgress = true;
  (<HTMLButtonElement>submitButton).disabled = true;
  // Get the message from the input field
  const message = (<HTMLInputElement>queryInput).value;
  console.log("message", message);
  // Clear the answer
  document.getElementById("answer")!.innerHTML = "";
  // Hide the answer
  document.getElementById("answerWrapper")!.style.display = "none";
  // Show the loading indicator
  document.getElementById("loading-indicator")!.style.display = "block";
  // Generate response
  let inp = message;
  if (context.length > 0) {
    inp =
      "Use only the following context when answering the question at the end. Don't use any other knowledge.\n" +
      context +
      "\n\nQuestion: " +
      message +
      "\n\nHelpful Answer: ";
  }
  console.log("Input:", inp);
  chatHistory.push({ role: "user", content: inp });
  let curMessage = "";
  const completion = await engine.chat.completions.create({
    stream: true,
    messages: chatHistory,
  });
  for await (const chunk of completion) {
    const curDelta = chunk.choices[0].delta.content;
    if (curDelta) {
      curMessage += curDelta;
    }
    updateAnswer(curMessage);
  }
  const response = await engine.getMessage();
  chatHistory.push({ role: "assistant", content: await engine.getMessage() });
  console.log("response", response);
  requestInProgress = false;
  (<HTMLButtonElement>submitButton).disabled = false;
 }
 submitButton.addEventListener("click", handleClick);
 // listen for changes in modelSelector
 async function handleSelectChange() {
  if (isLoadingParams) {
    return;
  }
  modelName.innerText = "";
  const initLabel = document.createElement("p");
  initLabel.id = "init-label";
  initLabel.innerText = "Initializing model...";
  const loadingContainer = document.createElement("div");
  loadingContainer.id = "loadingContainer";
  const loadingBox = getElementAndCheck("loadingBox");
  loadingBox.appendChild(initLabel);
  loadingBox.appendChild(loadingContainer);
  isLoadingParams = true;
  (<HTMLButtonElement>submitButton).disabled = true;
  if (requestInProgress) {
    engine.interruptGenerate();
  }
  engine.resetChat();
  chatHistory = [];
  await engine.unload();
  selectedModel = modelSelector.value;
  progressBar = new Line("#loadingContainer", {
    strokeWidth: 4,
    easing: "easeInOut",
    duration: 1400,
    color: "#ffd166",
    trailColor: "#eee",
    trailWidth: 1,
    svgStyle: { width: "100%", height: "100%" },
  });
  initProgressCallback = (report: InitProgressReport) => {
    setLabel("init-label", report.text);
    progressBar.animate(report.progress, {
      duration: 50,
    });
    if (report.progress == 1.0) {
      enableInputs();
    }
  };
  engine.setInitProgressCallback(initProgressCallback);
  requestInProgress = true;
  modelName.innerText = "Reloading with new model...";
  await engine.reload(selectedModel);
  requestInProgress = false;
  modelName.innerText = "Now chatting with " + modelDisplayName;
 }
 modelSelector.addEventListener("change", handleSelectChange);
 // Listen for messages from the background script
 chrome.runtime.onMessage.addListener(({ answer, error }) => {
  if (answer) {
    updateAnswer(answer);
  }
 });
 function updateAnswer(answer: string) {
  // Show answer
  document.getElementById("answerWrapper")!.style.display = "block";
  const answerWithBreaks = answer.replace(/\n/g, "<br>");
  document.getElementById("answer")!.innerHTML = answerWithBreaks;
  // Add event listener to copy button
  document.getElementById("copyAnswer")!.addEventListener("click", () => {
    // Get the answer text
    const answerText = answer;
    // Copy the answer text to the clipboard
    navigator.clipboard
      .writeText(answerText)
      .then(() => console.log("Answer text copied to clipboard"))
      .catch((err) => console.error("Could not copy text: ", err));
  });
  const options: Intl.DateTimeFormatOptions = {
    month: "short",
    day: "2-digit",
    hour: "2-digit",
    minute: "2-digit",
    second: "2-digit",
  };
  const time = new Date().toLocaleString("en-US", options);
  // Update timestamp
  document.getElementById("timestamp")!.innerText = time;
  // Hide loading indicator
  document.getElementById("loading-indicator")!.style.display = "none";
 }
 function fetchPageContents() {
  chrome.tabs.query({ currentWindow: true, active: true }, function (tabs) {
    const port = chrome.tabs.connect(tabs[0].id, { name: "channelName" });
    port.postMessage({});
    port.onMessage.addListener(function (msg) {
      console.log("Page contents:", msg.contents);
      context = msg.contents;
    });
  });
 }
@@ -0,0 +1,14 @@
 # WebLLM Get Started App
 This folder provides a minimum demo to show WebLLM API in a webapp setting.
 To try it out, you can do the following steps under this folder
 ```bash
 npm install
 npm start
 ```
 Note if you would like to hack WebLLM core package.
 You can change web-llm dependencies as `"file:../.."`, and follow the build from source
 instruction in the project to build webllm locally. This option is only recommended
 if you would like to hack WebLLM core package.
@@ -0,0 +1,21 @@
 {
  "name": "embeddings-example",
  "version": "0.1.0",
  "private": true,
  "scripts": {
    "start": "parcel src/embeddings.html  --port 8885",
    "build": "parcel build src/embeddings.html --dist-dir lib"
  },
  "devDependencies": {
    "buffer": "^5.7.1",
    "parcel": "^2.8.3",
    "process": "^0.11.10",
    "tslib": "^2.3.1",
    "typescript": "^4.9.5",
    "url": "^0.11.3"
  },
  "dependencies": {
    "@mlc-ai/web-llm": "^0.2.84",
    "langchain": "^0.3.37"
  }
 }
@@ -0,0 +1,23 @@
 <!doctype html>
 <html>
  <script>
    webLLMGlobal = {};
  </script>
  <body>
    <h2>WebLLM Test Page</h2>
    Open console to see output
    <br />
    <br />
    <label id="init-label"> </label>
    <h3>Prompt</h3>
    <label id="prompt-label"> </label>
    <h3>Response</h3>
    <label id="generate-label"> </label>
    <br />
    <label id="stats-label"> </label>
    <script type="module" src="./embeddings.ts"></script>
  </body>
 </html>
@@ -0,0 +1,211 @@
 import * as webllm from "@mlc-ai/web-llm";
 import { MemoryVectorStore } from "langchain/vectorstores/memory";
 import type { EmbeddingsInterface } from "@langchain/core/embeddings";
 import type { Document } from "@langchain/core/documents";
 import { formatDocumentsAsString } from "langchain/util/document";
 import { PromptTemplate } from "@langchain/core/prompts";
 import {
  RunnableSequence,
  RunnablePassthrough,
 } from "@langchain/core/runnables";
 function setLabel(id: string, text: string) {
  const label = document.getElementById(id);
  if (label == null) {
    throw Error("Cannot find label " + id);
  }
  label.innerText = text;
 }
 const initProgressCallback = (report: webllm.InitProgressReport) => {
  setLabel("init-label", report.text);
 };
 // For integration with Langchain
 class WebLLMEmbeddings implements EmbeddingsInterface {
  engine: webllm.MLCEngineInterface;
  modelId: string;
  constructor(engine: webllm.MLCEngineInterface, modelId: string) {
    this.engine = engine;
    this.modelId = modelId;
  }
  async _embed(texts: string[]): Promise<number[][]> {
    const reply = await this.engine.embeddings.create({
      input: texts,
      model: this.modelId,
    });
    const result: number[][] = [];
    for (let i = 0; i < texts.length; i++) {
      result.push(reply.data[i].embedding);
    }
    return result;
  }
  async embedQuery(document: string): Promise<number[]> {
    return this._embed([document]).then((embeddings) => embeddings[0]);
  }
  async embedDocuments(documents: string[]): Promise<number[][]> {
    return this._embed(documents);
  }
 }
 // Prepare inputs
 const documents_og = ["The Data Cloud!", "Mexico City of Course!"];
 const queries_og = ["what is snowflake?", "Where can I get the best tacos?"];
 const documents: string[] = [];
 const queries: string[] = [];
 const query_prefix =
  "Represent this sentence for searching relevant passages: ";
 // Process according to Snowflake model
 documents_og.forEach(function (item, index) {
  documents[index] = `[CLS] ${item} [SEP]`;
 });
 queries_og.forEach(function (item, index) {
  queries[index] = `[CLS] ${query_prefix}${item} [SEP]`;
 });
 console.log("Formatted documents: ", documents);
 console.log("Formatted queries: ", queries);
 // Using webllm's API
 async function webllmAPI() {
  // b4 means the max batch size is compiled as 4. That is, the model can process 4 inputs in a
  // batch. If given more than 4, the model will forward multiple times. The larger the max batch
  // size, the more memory it consumes.
  // const selectedModel = "snowflake-arctic-embed-m-q0f32-MLC-b32";
  const selectedModel = "snowflake-arctic-embed-m-q0f32-MLC-b4";
  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
    selectedModel,
    {
      initProgressCallback: initProgressCallback,
      logLevel: "INFO", // specify the log level
    },
  );
  const docReply = await engine.embeddings.create({ input: documents });
  console.log(docReply);
  console.log(docReply.usage);
  const queryReply = await engine.embeddings.create({ input: queries });
  console.log(queryReply);
  console.log(queryReply.usage);
  // Calculate similarity (we use langchain here, but any method works)
  const vectorStore = await MemoryVectorStore.fromExistingIndex(
    new WebLLMEmbeddings(engine, selectedModel),
  );
  // See score
  for (let i = 0; i < queries_og.length; i++) {
    console.log(`Similarity with: ${queries_og[i]}`);
    for (let j = 0; j < documents_og.length; j++) {
      const similarity = vectorStore.similarity(
        queryReply.data[i].embedding,
        docReply.data[j].embedding,
      );
      console.log(`${documents_og[j]}: ${similarity}`);
    }
  }
 }
 // Alternatively, integrating with Langchain's API
 async function langchainAPI() {
  // b4 means the max batch size is compiled as 4. That is, the model can process 4 inputs in a
  // batch. If given more than 4, the model will forward multiple times. The larger the max batch
  // size, the more memory it consumes.
  // const selectedModel = "snowflake-arctic-embed-m-q0f32-MLC-b32";
  const selectedModel = "snowflake-arctic-embed-m-q0f32-MLC-b4";
  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
    selectedModel,
    {
      initProgressCallback: initProgressCallback,
      logLevel: "INFO", // specify the log level
    },
  );
  const vectorStore = await MemoryVectorStore.fromExistingIndex(
    new WebLLMEmbeddings(engine, selectedModel),
  );
  const document0: Document = {
    pageContent: documents[0],
    metadata: {},
  };
  const document1: Document = {
    pageContent: documents[1],
    metadata: {},
  };
  await vectorStore.addDocuments([document0, document1]);
  const similaritySearchResults0 = await vectorStore.similaritySearch(
    queries[0],
    1,
  );
  for (const doc of similaritySearchResults0) {
    console.log(`* ${doc.pageContent}`);
  }
  const similaritySearchResults1 = await vectorStore.similaritySearch(
    queries[1],
    1,
  );
  for (const doc of similaritySearchResults1) {
    console.log(`* ${doc.pageContent}`);
  }
 }
 // RAG with Langchain.js using WebLLM for both LLM and Embedding in a single engine
 // Followed https://js.langchain.com/v0.1/docs/expression_language/cookbook/retrieval/
 // There are many possible ways to achieve RAG (e.g. degree of integration with Langchain,
 // using WebWorker, etc.). We provide a minimal example here.
 async function simpleRAG() {
  // 0. Load both embedding model and LLM to a single WebLLM Engine
  const embeddingModelId = "snowflake-arctic-embed-m-q0f32-MLC-b4";
  const llmModelId = "gemma-2-2b-it-q4f32_1-MLC-1k";
  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
    [embeddingModelId, llmModelId],
    {
      initProgressCallback: initProgressCallback,
      logLevel: "INFO", // specify the log level
    },
  );
  const vectorStore = await MemoryVectorStore.fromTexts(
    ["mitochondria is the powerhouse of the cell"],
    [{ id: 1 }],
    new WebLLMEmbeddings(engine, embeddingModelId),
  );
  const retriever = vectorStore.asRetriever();
  const prompt =
    PromptTemplate.fromTemplate(`Answer the question based only on the following context:
  {context}
  Question: {question}`);
  const chain = RunnableSequence.from([
    {
      context: retriever.pipe(formatDocumentsAsString),
      question: new RunnablePassthrough(),
    },
    prompt,
  ]);
  const formattedPrompt = (
    await chain.invoke("What is the powerhouse of the cell?")
  ).toString();
  const reply = await engine.chat.completions.create({
    messages: [{ role: "user", content: formattedPrompt }],
    model: llmModelId,
  });
  console.log(reply.choices[0].message.content);
  /*
    "The powerhouse of the cell is the mitochondria."
  */
 }
 // Select one to run
 // webllmAPI();
 // langchainAPI();
 simpleRAG();
@@ -0,0 +1,11 @@
 ### OpenAI API Demos - Function calling
 This folder contains two main ways of using function calling with WebLLM.
 `function-calling-manual` demonstrates how you can use function calling with Llama3.1 and Hermes2
 without using the `tools`, `tool_choice`, and `tool_call` fields. This is the most flexible way and you can follow
 the instruction given by the model releaser and iterate yourself on top of that. However, you need to do parsing on your own, which differs for each model. For instance, Hermes2 models use `<tool_call>` and `</tool_call>` to wrap around a tool call, which may be very different from other models' format.
 `function-calling-openai` conforms to the OpenAI function calling usage, leveraging `tools`, `tool_choice`, and `tool_call`
 fields. This is more usable, but sacrifices the flexibility since we have pre-defined system prompt
 for this.
@@ -0,0 +1,8 @@
 ### Demos - Function calling
 Run `npm install` first, followed by `npm start`.
 Note if you would like to hack WebLLM core package,
 you can change web-llm dependencies as `"file:../../.."`, and follow the build from source
 instruction in the project to build webllm locally. This option is only recommended
 if you would like to hack WebLLM core package.
@@ -0,0 +1,20 @@
 {
  "name": "openai-api",
  "version": "0.1.0",
  "private": true,
  "scripts": {
    "start": "parcel src/function_calling_manual.html  --port 8888",
    "build": "parcel build src/function_calling_manual.html --dist-dir lib"
  },
  "devDependencies": {
    "buffer": "^5.7.1",
    "parcel": "^2.8.3",
    "process": "^0.11.10",
    "tslib": "^2.3.1",
    "typescript": "^4.9.5",
    "url": "^0.11.3"
  },
  "dependencies": {
    "@mlc-ai/web-llm": "^0.2.84"
  }
 }
@@ -0,0 +1,17 @@
 <!doctype html>
 <html>
  <script>
    webLLMGlobal = {};
  </script>
  <body>
    <h2>WebLLM Test Page</h2>
    Open console to see output
    <br />
    <br />
    <label id="init-label"> </label>
    <label id="generate-label"> </label>
    <script type="module" src="./function_calling_manual.ts"></script>
  </body>
 </html>
@@ -0,0 +1,235 @@
 /* eslint-disable no-useless-escape */
 import * as webllm from "@mlc-ai/web-llm";
 // Common helper methods
 function setLabel(id: string, text: string) {
  const label = document.getElementById(id);
  if (label == null) {
    throw Error("Cannot find label " + id);
  }
  label.innerText = text;
 }
 const initProgressCallback = (report: webllm.InitProgressReport) => {
  setLabel("init-label", report.text);
 };
 // Same example as https://huggingface.co/NousResearch/Hermes-2-Theta-Llama-3-8B#prompt-format-for-function-calling
 async function hermes2_example() {
  // 0. Setups
  // Most manual function calling models specify the tools inside the system prompt
  const system_prompt = `You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: <tools> {"type": "function", "function": {"name": "get_stock_fundamentals", "description": "get_stock_fundamentals(symbol: str) -> dict - Get fundamental data for a given stock symbol using yfinance API.\\n\\n    Args:\\n        symbol (str): The stock symbol.\\n\\n    Returns:\\n        dict: A dictionary containing fundamental data.\\n            Keys:\\n                - \'symbol\': The stock symbol.\\n                - \'company_name\': The long name of the company.\\n                - \'sector\': The sector to which the company belongs.\\n                - \'industry\': The industry to which the company belongs.\\n                - \'market_cap\': The market capitalization of the company.\\n                - \'pe_ratio\': The forward price-to-earnings ratio.\\n                - \'pb_ratio\': The price-to-book ratio.\\n                - \'dividend_yield\': The dividend yield.\\n                - \'eps\': The trailing earnings per share.\\n                - \'beta\': The beta value of the stock.\\n                - \'52_week_high\': The 52-week high price of the stock.\\n                - \'52_week_low\': The 52-week low price of the stock.", "parameters": {"type": "object", "properties": {"symbol": {"type": "string"}}, "required": ["symbol"]}}}  </tools> Use the following pydantic model json schema for each tool call you will make: {"properties": {"arguments": {"title": "Arguments", "type": "object"}, "name": {"title": "Name", "type": "string"}}, "required": ["arguments", "name"], "title": "FunctionCall", "type": "object"} For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:\n<tool_call>\n{"arguments": <args-dict>, "name": <function-name>}\n</tool_call>`;
  // Same formatting for Hermes-2-Pro-Llama-3, Hermes-2-Theta-Llama-3
  // const selectedModel = "Hermes-2-Theta-Llama-3-8B-q4f16_1-MLC";
  const selectedModel = "Hermes-2-Pro-Llama-3-8B-q4f16_1-MLC";
  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
    selectedModel,
    { initProgressCallback: initProgressCallback, logLevel: "INFO" },
  );
  const seed = 0;
  // 1. First request, expect to generate tool call
  const messages: webllm.ChatCompletionMessageParam[] = [
    { role: "system", content: system_prompt },
    {
      role: "user",
      content: "Fetch the stock fundamentals data for Tesla (TSLA)",
    },
  ];
  const request1: webllm.ChatCompletionRequest = {
    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
    messages: messages,
    seed: seed,
  };
  const reply1 = await engine.chat.completions.create(request1);
  const response1 = reply1.choices[0].message.content;
  console.log(reply1.usage);
  console.log("Response 1: " + response1);
  messages.push({ role: "assistant", content: response1 });
  // <tool_call>\n{"arguments": {"symbol": "TSLA"}, "name": "get_stock_fundamentals"}\n</tool_call>
  // 2. Call function on your own to get tool response
  const tool_response = `<tool_response>\n{"name": "get_stock_fundamentals", "content": {'symbol': 'TSLA', 'company_name': 'Tesla, Inc.', 'sector': 'Consumer Cyclical', 'industry': 'Auto Manufacturers', 'market_cap': 611384164352, 'pe_ratio': 49.604652, 'pb_ratio': 9.762013, 'dividend_yield': None, 'eps': 4.3, 'beta': 2.427, '52_week_high': 299.29, '52_week_low': 152.37}}\n</tool_response>`;
  messages.push({ role: "tool", content: tool_response, tool_call_id: "0" });
  // 3. Get natural language response
  const request2: webllm.ChatCompletionRequest = {
    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
    messages: messages,
    seed: seed,
  };
  const reply2 = await engine.chat.completions.create(request2);
  const response2 = reply2.choices[0].message.content;
  messages.push({ role: "assistant", content: response2 });
  console.log(reply2.usage);
  console.log("Response 2: " + response2);
  // 4. Another function call
  messages.push({
    role: "user",
    content: "Now do another one with NVIDIA, symbol being NVDA.",
  });
  const request3: webllm.ChatCompletionRequest = {
    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
    messages: messages,
    seed: seed,
  };
  const reply3 = await engine.chat.completions.create(request3);
  const response3 = reply3.choices[0].message.content;
  messages.push({ role: "assistant", content: response3 });
  console.log(reply3.usage);
  console.log("Response 3: " + response3);
  // <tool_call>\n{"arguments": {"symbol": "NVDA"}, "name": "get_stock_fundamentals"}\n</tool_call>
 }
 // Similar example to https://llama.meta.com/docs/model-cards-and-prompt-formats/llama3_1/#user-defined-custom-tool-calling
 async function llama3_1_example() {
  // Follows example, but tweaks the formatting with <function>
  const system_prompt = `Cutting Knowledge Date: December 2023
 Today Date: 23 Jul 2024
 # Tool Instructions
 - When looking for real time information use relevant functions if available
 You have access to the following functions:
 {
    "type": "function",
    "function": {
        "name": "get_current_temperature",
        "description": "Get the current temperature at a location.",
        "parameters": {
            "type": "object",
            "properties": {
                "location": {
                    "type": "string",
                    "description": "The location to get the temperature for, in the format \"City, Country\""
                }
            },
            "required": [
                "location"
            ]
        },
        "return": {
            "type": "number",
            "description": "The current temperature at the specified location in the specified units, as a float."
        }
    }
 }
 {
    "type": "function",
    "function": {
        "name": "send_message",
        "description": "Send a message to a recipient.",
        "parameters": {
            "type": "object",
            "properties": {
                "recipient": {
                    "type": "string",
                    "description": "Name of the recipient of the message"
                }
                "content": {
                    "type": "string",
                    "description": "Content of the message"
                }
            },
            "required": [
                "recipient",
                "content"
            ]
        },
        "return": {
            "type": "None"
        }
    }
 }
 If a you choose to call a function ONLY reply in the following format:
    <function>{"name": function name, "parameters": dictionary of argument name and its value}</function>
 Here is an example,
    <function>{"name": "example_function_name", "parameters": {"example_name": "example_value"}}</function>
 Reminder:
 - Function calls MUST follow the specified format and use BOTH <function> and </function>
 - Required parameters MUST be specified
 - Only call one function at a time
 - When calling a function, do NOT add any other words, ONLY the function calling
 - Put the entire function call reply on one line
 - Always add your sources when using search results to answer the user query
 You are a helpful Assistant.`;
  const selectedModel = "Llama-3.1-8B-Instruct-q4f16_1-MLC";
  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
    selectedModel,
    { initProgressCallback: initProgressCallback, logLevel: "INFO" },
  );
  const seed = 0;
  // 1. First request, expect to generate tool call to get temperature of Paris
  const messages: webllm.ChatCompletionMessageParam[] = [
    { role: "system", content: system_prompt },
    {
      role: "user",
      content: "Hey, what's the temperature in Paris right now?",
    },
  ];
  const request1: webllm.ChatCompletionRequest = {
    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
    messages: messages,
    seed: seed,
  };
  const reply1 = await engine.chat.completions.create(request1);
  const response1 = reply1.choices[0].message.content;
  console.log(reply1.usage);
  console.log("Response 1: " + response1);
  messages.push({ role: "assistant", content: response1 });
  // <function>{"name": "get_current_temperature", "parameters": {"location": "Paris, France"}}</function>
  // 2. Call function on your own to get tool response
  const tool_response = `{"output": 22.5}`;
  messages.push({ role: "tool", content: tool_response, tool_call_id: "0" });
  // 3. Get natural language response
  const request2: webllm.ChatCompletionRequest = {
    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
    messages: messages,
    seed: seed,
  };
  const reply2 = await engine.chat.completions.create(request2);
  const response2 = reply2.choices[0].message.content;
  messages.push({ role: "assistant", content: response2 });
  console.log(reply2.usage);
  console.log("Response 2: " + response2);
  // The current temperature in Paris is 22.5°C.
  // 4. Make another request, expect model to call `send_message`
  messages.push({
    role: "user",
    content: "Send a message to Tom to tell him this information.",
  });
  const request3: webllm.ChatCompletionRequest = {
    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
    messages: messages,
    seed: seed,
  };
  const reply3 = await engine.chat.completions.create(request3);
  const response3 = reply3.choices[0].message.content;
  messages.push({ role: "assistant", content: response3 });
  console.log(reply3.usage);
  console.log("Response 3: " + response3);
  // <function>{"name": "send_message", "parameters": {"recipient": "Tom", "content": "The current temperature in Paris is 22.5°C."}}</function>
  // 5. Call API, which has no return value, so simply prompt model again
  const tool_response2 = `{"output": None}`;
  messages.push({ role: "tool", content: tool_response2, tool_call_id: "1" });
  const request4: webllm.ChatCompletionRequest = {
    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
    messages: messages,
    seed: seed,
  };
  const reply4 = await engine.chat.completions.create(request4);
  const response4 = reply4.choices[0].message.content;
  console.log(reply4.usage);
  console.log("Response 4: " + response4);
  // The message has been sent to Tom.
 }
 // Pick one to run
 // hermes2_example();
 llama3_1_example();
@@ -0,0 +1,8 @@
 ### Demos - Function calling
 Run `npm install` first, followed by `npm start`.
 Note if you would like to hack WebLLM core package,
 you can change web-llm dependencies as `"file:../../.."`, and follow the build from source
 instruction in the project to build webllm locally. This option is only recommended
 if you would like to hack WebLLM core package.
@@ -0,0 +1,20 @@
 {
  "name": "openai-api",
  "version": "0.1.0",
  "private": true,
  "scripts": {
    "start": "parcel src/function_calling_openai.html  --port 8888",
    "build": "parcel build src/function_calling_openai.html --dist-dir lib"
  },
  "devDependencies": {
    "buffer": "^5.7.1",
    "parcel": "^2.8.3",
    "process": "^0.11.10",
    "tslib": "^2.3.1",
    "typescript": "^4.9.5",
    "url": "^0.11.3"
  },
  "dependencies": {
    "@mlc-ai/web-llm": "^0.2.84"
  }
 }
@@ -0,0 +1,17 @@
 <!doctype html>
 <html>
  <script>
    webLLMGlobal = {};
  </script>
  <body>
    <h2>WebLLM Test Page</h2>
    Open console to see output
    <br />
    <br />
    <label id="init-label"> </label>
    <label id="generate-label"> </label>
    <script type="module" src="./function_calling_openai.ts"></script>
  </body>
 </html>
@@ -0,0 +1,80 @@
 import * as webllm from "@mlc-ai/web-llm";
 function setLabel(id: string, text: string) {
  const label = document.getElementById(id);
  if (label == null) {
    throw Error("Cannot find label " + id);
  }
  label.innerText = text;
 }
 async function main() {
  const initProgressCallback = (report: webllm.InitProgressReport) => {
    setLabel("init-label", report.text);
  };
  const selectedModel = "Hermes-2-Pro-Llama-3-8B-q4f16_1-MLC";
  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
    selectedModel,
    { initProgressCallback: initProgressCallback },
  );
  const tools: Array<webllm.ChatCompletionTool> = [
    {
      type: "function",
      function: {
        name: "get_current_weather",
        description: "Get the current weather in a given location",
        parameters: {
          type: "object",
          properties: {
            location: {
              type: "string",
              description: "The city and state, e.g. San Francisco, CA",
            },
            unit: { type: "string", enum: ["celsius", "fahrenheit"] },
          },
          required: ["location"],
        },
      },
    },
  ];
  const request: webllm.ChatCompletionRequest = {
    stream: true, // works with stream as well, where the last chunk returns tool_calls
    stream_options: { include_usage: true },
    messages: [
      {
        role: "user",
        content:
          "What is the current weather in celsius in Pittsburgh and Tokyo?",
      },
    ],
    tool_choice: "auto",
    tools: tools,
  };
  if (!request.stream) {
    const reply0 = await engine.chat.completions.create(request);
    console.log(reply0.choices[0]);
    console.log(reply0.usage);
  } else {
    // If streaming, the last chunk returns tool calls
    const asyncChunkGenerator = await engine.chat.completions.create(request);
    let message = "";
    let lastChunk: webllm.ChatCompletionChunk | undefined;
    let usageChunk: webllm.ChatCompletionChunk | undefined;
    for await (const chunk of asyncChunkGenerator) {
      console.log(chunk);
      message += chunk.choices[0]?.delta?.content || "";
      setLabel("generate-label", message);
      if (!chunk.usage) {
        lastChunk = chunk;
      }
      usageChunk = chunk;
    }
    console.log(lastChunk!.choices[0].delta);
    console.log(usageChunk!.usage);
  }
 }
 main();
@@ -0,0 +1,15 @@
 # WebLLM Get Started App
 This folder provides a minimum demo to show WebLLM API in a webapp setting with
 collection of latency statistics for individual token sampling steps.
 To try it out, you can do the following steps under this folder
 ```bash
 npm install
 npm start
 ```
 Note if you would like to hack WebLLM core package.
 You can change web-llm dependencies as `"file:../.."`, and follow the build from source
 instruction in the project to build webllm locally. This option is only recommended
 if you would like to hack WebLLM core package.
@@ -0,0 +1,20 @@
 {
  "name": "get-started-latency-breakdown",
  "version": "0.1.0",
  "private": true,
  "scripts": {
    "start": "parcel src/get_started_latency_breakdown.html  --port 8888",
    "build": "parcel build src/get_started_latency_breakdown.html --dist-dir lib"
  },
  "devDependencies": {
    "buffer": "^5.7.1",
    "parcel": "^2.8.3",
    "process": "^0.11.10",
    "tslib": "^2.3.1",
    "typescript": "^4.9.5",
    "url": "^0.11.3"
  },
  "dependencies": {
    "@mlc-ai/web-llm": "^0.2.84"
  }
 }
@@ -0,0 +1,23 @@
 <!doctype html>
 <html>
  <script>
    webLLMGlobal = {};
  </script>
  <body>
    <h2>WebLLM Test Page</h2>
    Open console to see output
    <br />
    <br />
    <label id="init-label"> </label>
    <h3>Prompt</h3>
    <label id="prompt-label"> </label>
    <h3>Response</h3>
    <label id="generate-label"> </label>
    <br />
    <label id="stats-label"> </label>
    <script type="module" src="./get_started_latency_breakdown.ts"></script>
  </body>
 </html>
@@ -0,0 +1,135 @@
 import * as webllm from "@mlc-ai/web-llm";
 function setLabel(id: string, text: string) {
  const label = document.getElementById(id);
  if (label == null) {
    throw Error("Cannot find label " + id);
  }
  label.innerText = text;
 }
 type LatencyBreakdown = {
  logitProcessorTime: number[];
  logitBiasTime: number[];
  penaltyTime: number[];
  sampleTime: number[];
  totalTime: number[];
  grammarBitmaskTime: number[];
 };
 function computeStats(
  latency_breakdown: LatencyBreakdown,
 ): Record<string, any> {
  function _computeStats(arr: number[]) {
    if (!arr.length) return undefined;
    const sorted = [...arr].sort((a, b) => a - b);
    const sum = arr.reduce((a, b) => a + b, 0);
    const avg = sum / arr.length;
    const min = sorted[0];
    const max = sorted[sorted.length - 1];
    const p99 = sorted[Math.floor(0.99 * (sorted.length - 1))];
    return { avg, min, max, p99 };
  }
  const latencyStats: Record<string, any> = {};
  for (const key of Object.keys(latency_breakdown)) {
    const arr = (latency_breakdown as any)[key];
    if (Array.isArray(arr) && arr.length > 0) {
      latencyStats[key] = _computeStats(arr);
    }
  }
  return latencyStats;
 }
 async function main() {
  const initProgressCallback = (report: webllm.InitProgressReport) => {
    setLabel("init-label", report.text);
  };
  // Option 1: If we do not specify appConfig, we use `prebuiltAppConfig` defined in `config.ts`
  const selectedModel = "Qwen3-0.6B-q0f32-MLC";
  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
    selectedModel,
    {
      initProgressCallback: initProgressCallback,
      logLevel: "INFO", // specify the log level
    },
    // customize kv cache, use either context_window_size or sliding_window_size (with attention sink)
    {
      context_window_size: 2048,
      // sliding_window_size: 1024,
      // attention_sink_size: 4,
    },
  );
  const latencyBreakdown: LatencyBreakdown = {
    logitProcessorTime: [],
    logitBiasTime: [],
    penaltyTime: [],
    sampleTime: [],
    totalTime: [],
    grammarBitmaskTime: [],
  };
  const decodeTokensPerS: number[] = [];
  const completionTokens: number[] = [];
  const e2eLatencyS: number[] = [];
  const timePerOutputTokenS: number[] = [];
  const numTrials = 20;
  for (let i = 0; i < numTrials; i++) {
    console.log(`Trial ${i + 1} / ${numTrials}`);
    const reply0 = await engine.chat.completions.create({
      messages: [{ role: "user", content: "List twenty US states." }],
      // below configurations are all optional
      n: 1,
      temperature: 0,
      max_tokens: 2048,
      // 46510 and 7188 are "California", and 8421 and 51325 are "Texas" in Llama-3.1-8B-Instruct
      // So we would have a higher chance of seeing the latter two, but never the first in the answer
      // logit_bias: {
      //   "46510": -100,
      //   "7188": -100,
      //   "8421": 5,
      //   "41325": 5,
      // },
      top_p: 0.8,
      logprobs: true,
      top_logprobs: 2,
      frequency_penalty: 1.2,
      presence_penalty: 1.0,
      repetition_penalty: 1.1,
    });
    const logitProcessorTime =
      reply0.usage?.extra.latencyBreakdown?.logitProcessorTime;
    const logitBiasTime = reply0.usage?.extra.latencyBreakdown?.logitBiasTime;
    const penaltyTime = reply0.usage?.extra.latencyBreakdown?.penaltyTime;
    const sampleTime = reply0.usage?.extra.latencyBreakdown?.sampleTime;
    const totalTime = reply0.usage?.extra.latencyBreakdown?.totalTime;
    const grammarBitmaskTime =
      reply0.usage?.extra.latencyBreakdown?.grammarBitmaskTime;
    latencyBreakdown.logitProcessorTime.push(...(logitProcessorTime || []));
    latencyBreakdown.logitBiasTime.push(...(logitBiasTime || []));
    latencyBreakdown.penaltyTime.push(...(penaltyTime || []));
    latencyBreakdown.sampleTime.push(...(sampleTime || []));
    latencyBreakdown.totalTime.push(...(totalTime || []));
    latencyBreakdown.grammarBitmaskTime.push(...(grammarBitmaskTime || []));
    decodeTokensPerS.push(reply0.usage?.extra.decode_tokens_per_s || 0);
    e2eLatencyS.push(reply0.usage?.extra.e2e_latency_s || 0);
    timePerOutputTokenS.push(reply0.usage?.extra.time_per_output_token_s || 0);
    completionTokens.push(reply0.usage?.completion_tokens || 0);
  }
  const latencyStats: { [key: string]: number } =
    computeStats(latencyBreakdown);
  console.log("Latency stats: ", latencyStats);
  console.log("Decode tokens per second: ", decodeTokensPerS);
  console.log("Completion tokens: ", completionTokens);
  console.log("E2E latency (s): ", e2eLatencyS);
  console.log("Time per output token (s): ", timePerOutputTokenS);
  // To change model, either create a new engine via `CreateMLCEngine()`, or call `engine.reload(modelId)`
 }
 main();
@@ -0,0 +1,18 @@
 # WebLLM Get Started with WebWorker
 This folder provides a minimum demo to show WebLLM API using
 [WebWorker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers).
 The main benefit of web worker is that all ML workloads runs on a separate thread as a result
 will less likely block the UI.
 To try it out, you can do the following steps under this folder
 ```bash
 npm install
 npm start
 ```
 Note if you would like to hack WebLLM core package.
 You can change web-llm dependencies as `"file:../.."`, and follow the build from source
 instruction in the project to build webllm locally. This option is only recommended
 if you would like to hack WebLLM core package.
@@ -0,0 +1,20 @@
 {
  "name": "get-started-web-worker",
  "version": "0.1.0",
  "private": true,
  "scripts": {
    "start": "parcel src/get_started.html  --port 8885",
    "build": "parcel build src/get_started.html --dist-dir lib"
  },
  "devDependencies": {
    "buffer": "^6.0.3",
    "parcel": "^2.8.3",
    "process": "^0.11.10",
    "tslib": "^2.3.1",
    "typescript": "^4.9.5",
    "url": "^0.11.3"
  },
  "dependencies": {
    "@mlc-ai/web-llm": "^0.2.84"
  }
 }
@@ -0,0 +1,23 @@
 <!doctype html>
 <html>
  <script>
    webLLMGlobal = {};
  </script>
  <body>
    <h2>WebLLM Test Page</h2>
    Open console to see output
    <br />
    <br />
    <label id="init-label"> </label>
    <h3>Prompt</h3>
    <label id="prompt-label"> </label>
    <h3>Response</h3>
    <label id="generate-label"> </label>
    <br />
    <label id="stats-label"> </label>
    <script type="module" src="./main.ts"></script>
  </body>
 </html>
@@ -0,0 +1,102 @@
 import * as webllm from "@mlc-ai/web-llm";
 function setLabel(id: string, text: string) {
  const label = document.getElementById(id);
  if (label == null) {
    throw Error("Cannot find label " + id);
  }
  label.innerText = text;
 }
 // There are two demonstrations, pick one to run
 /**
 * Chat completion (OpenAI style) without streaming, where we get the entire response at once.
 */
 async function mainNonStreaming() {
  const initProgressCallback = (report: webllm.InitProgressReport) => {
    setLabel("init-label", report.text);
  };
  const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
  const engine: webllm.MLCEngineInterface =
    await webllm.CreateWebWorkerMLCEngine(
      new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }),
      selectedModel,
      { initProgressCallback: initProgressCallback },
    );
  const request: webllm.ChatCompletionRequest = {
    messages: [
      {
        role: "system",
        content:
          "You are a helpful, respectful and honest assistant. " +
          "Be as happy as you can when speaking please. ",
      },
      { role: "user", content: "Provide me three US states." },
      { role: "assistant", content: "California, New York, Pennsylvania." },
      { role: "user", content: "Two more please!" },
    ],
    n: 3,
    temperature: 1.5,
    max_tokens: 256,
  };
  const reply0 = await engine.chat.completions.create(request);
  console.log(reply0);
  console.log(reply0.usage);
 }
 /**
 * Chat completion (OpenAI style) with streaming, where delta is sent while generating response.
 */
 async function mainStreaming() {
  const initProgressCallback = (report: webllm.InitProgressReport) => {
    setLabel("init-label", report.text);
  };
  const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
  const engine: webllm.MLCEngineInterface =
    await webllm.CreateWebWorkerMLCEngine(
      new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }),
      selectedModel,
      { initProgressCallback: initProgressCallback },
    );
  const request: webllm.ChatCompletionRequest = {
    stream: true,
    stream_options: { include_usage: true },
    messages: [
      {
        role: "system",
        content:
          "You are a helpful, respectful and honest assistant. " +
          "Be as happy as you can when speaking please. ",
      },
      { role: "user", content: "Provide me three US states." },
      { role: "assistant", content: "California, New York, Pennsylvania." },
      { role: "user", content: "Two more please!" },
    ],
    temperature: 1.5,
    max_tokens: 256,
  };
  const asyncChunkGenerator = await engine.chat.completions.create(request);
  let message = "";
  for await (const chunk of asyncChunkGenerator) {
    console.log(chunk);
    message += chunk.choices[0]?.delta?.content || "";
    setLabel("generate-label", message);
    if (chunk.usage) {
      console.log(chunk.usage); // only last chunk has usage
    }
    // engine.interruptGenerate();  // works with interrupt as well
  }
  console.log("Final message:\n", await engine.getMessage()); // the concatenated message
 }
 // Run one of the function below
 // mainNonStreaming();
 mainStreaming();
@@ -0,0 +1,7 @@
 import { WebWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
 // Hookup an engine to a worker handler
 const handler = new WebWorkerMLCEngineHandler();
 self.onmessage = (msg: MessageEvent) => {
  handler.onmessage(msg);
 };
@@ -0,0 +1,14 @@
 # WebLLM Get Started App
 This folder provides a minimum demo to show WebLLM API in a webapp setting.
 To try it out, you can do the following steps under this folder
 ```bash
 npm install
 npm start
 ```
 Note if you would like to hack WebLLM core package.
 You can change web-llm dependencies as `"file:../.."`, and follow the build from source
 instruction in the project to build webllm locally. This option is only recommended
 if you would like to hack WebLLM core package.
@@ -0,0 +1,20 @@
 {
  "name": "get-started",
  "version": "0.1.0",
  "private": true,
  "scripts": {
    "start": "parcel src/get_started.html  --port 8888",
    "build": "parcel build src/get_started.html --dist-dir lib"
  },
  "devDependencies": {
    "buffer": "^5.7.1",
    "parcel": "^2.8.3",
    "process": "^0.11.10",
    "tslib": "^2.3.1",
    "typescript": "^4.9.5",
    "url": "^0.11.3"
  },
  "dependencies": {
    "@mlc-ai/web-llm": "^0.2.84"
  }
 }
@@ -0,0 +1,23 @@
 <!doctype html>
 <html>
  <script>
    webLLMGlobal = {};
  </script>
  <body>
    <h2>WebLLM Test Page</h2>
    Open console to see output
    <br />
    <br />
    <label id="init-label"> </label>
    <h3>Prompt</h3>
    <label id="prompt-label"> </label>
    <h3>Response</h3>
    <label id="generate-label"> </label>
    <br />
    <label id="stats-label"> </label>
    <script type="module" src="./get_started.ts"></script>
  </body>
 </html>
@@ -0,0 +1,82 @@
 import * as webllm from "@mlc-ai/web-llm";
 function setLabel(id: string, text: string) {
  const label = document.getElementById(id);
  if (label == null) {
    throw Error("Cannot find label " + id);
  }
  label.innerText = text;
 }
 async function main() {
  const initProgressCallback = (report: webllm.InitProgressReport) => {
    setLabel("init-label", report.text);
  };
  // Option 1: If we do not specify appConfig, we use `prebuiltAppConfig` defined in `config.ts`
  const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
    selectedModel,
    {
      initProgressCallback: initProgressCallback,
      logLevel: "INFO", // specify the log level
    },
    // customize kv cache, use either context_window_size or sliding_window_size (with attention sink)
    {
      context_window_size: 2048,
      // sliding_window_size: 1024,
      // attention_sink_size: 4,
    },
  );
  // Option 2: Specify your own model other than the prebuilt ones
  // const appConfig: webllm.AppConfig = {
  //   model_list: [
  //     {
  //       model: "https://huggingface.co/mlc-ai/Llama-3.1-8B-Instruct-q4f32_1-MLC",
  //       model_id: "Llama-3.1-8B-Instruct-q4f32_1-MLC",
  //       model_lib:
  //         webllm.modelLibURLPrefix +
  //         webllm.modelVersion +
  //         "/Llama-3_1-8B-Instruct-q4f32_1-ctx4k_cs1k-webgpu.wasm",
  //       overrides: {
  //         context_window_size: 2048,
  //       },
  //     },
  //   ],
  // };
  // const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
  //   selectedModel,
  //   { appConfig: appConfig, initProgressCallback: initProgressCallback },
  // );
  // Option 3: Instantiate MLCEngine() and call reload() separately
  // const engine: webllm.MLCEngineInterface = new webllm.MLCEngine({
  //   appConfig: appConfig, // if do not specify, we use webllm.prebuiltAppConfig
  //   initProgressCallback: initProgressCallback,
  // });
  // await engine.reload(selectedModel);
  const reply0 = await engine.chat.completions.create({
    messages: [{ role: "user", content: "List three US states." }],
    // below configurations are all optional
    n: 3,
    temperature: 1.5,
    max_tokens: 256,
    // 46510 and 7188 are "California", and 8421 and 51325 are "Texas" in Llama-3.1-8B-Instruct
    // So we would have a higher chance of seeing the latter two, but never the first in the answer
    logit_bias: {
      "46510": -100,
      "7188": -100,
      "8421": 5,
      "51325": 5,
    },
    logprobs: true,
    top_logprobs: 2,
  });
  console.log(reply0);
  console.log(reply0.usage);
  // To change model, either create a new engine via `CreateMLCEngine()`, or call `engine.reload(modelId)`
 }
 main();
@@ -0,0 +1,20 @@
 {
  "name": "integrity-verification",
  "version": "0.1.0",
  "private": true,
  "scripts": {
    "start": "parcel src/integrity_verification.html --port 8888",
    "build": "parcel build src/integrity_verification.html --dist-dir lib"
  },
  "devDependencies": {
    "buffer": "^5.7.1",
    "parcel": "^2.8.3",
    "process": "^0.11.10",
    "tslib": "^2.3.1",
    "typescript": "^4.9.5",
    "url": "^0.11.3"
  },
  "dependencies": {
    "@mlc-ai/web-llm": "^0.2.84"
  }
 }
@@ -0,0 +1,13 @@
 <!doctype html>
 <html>
  <body>
    <h2>WebLLM Integrity Verification Example</h2>
    <p>
      This example demonstrates how to use the optional
      <code>integrity</code> field on <code>ModelRecord</code> to verify
      downloaded artifacts with SRI hashes.
    </p>
    <div id="status">Loading...</div>
    <script type="module" src="./integrity_verification.ts"></script>
  </body>
 </html>
@@ -0,0 +1,89 @@
 import * as webllm from "@mlc-ai/web-llm";
 const statusEl = document.getElementById("status")!;
 function setStatus(msg: string) {
  console.log(msg);
  statusEl.innerText = msg;
 }
 /**
 * Example: Basic integrity verification using the native `integrity` field.
 *
 * When `integrity` is set on a ModelRecord, WebLLM will verify the downloaded
 * config, WASM, and tokenizer files against the provided SRI hashes before
 * loading the model. If a hash does not match, an IntegrityError is thrown
 * (or a warning is logged if `onFailure: "warn"` is set).
 *
 * Hash-generation commands are documented in README.md under "Integrity Verification".
 * This example uses the same commands:
 * - `openssl dgst -sha256 -binary <file> | openssl base64 -A | sed 's/^/sha256-/'`
 * - `openssl dgst -sha384 -binary <file> | openssl base64 -A | sed 's/^/sha384-/'`
 * - `openssl dgst -sha512 -binary <file> | openssl base64 -A | sed 's/^/sha512-/'`
 */
 async function main() {
  setStatus("Initializing...");
  // Example model configuration with integrity hashes.
  // Replace the placeholder hashes below with real SRI hashes for your model.
  const appConfig: webllm.AppConfig = {
    model_list: [
      {
        model:
          "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_1-MLC",
        model_id: "Llama-3.2-1B-Instruct-q4f16_1-MLC",
        model_lib:
          webllm.modelLibURLPrefix +
          webllm.modelVersion +
          "/Llama-3.2-1B-Instruct-q4f16_1-ctx4k_cs1k-webgpu.wasm",
        low_resource_required: true,
        vram_required_MB: 879.04,
        overrides: {
          context_window_size: 4096,
        },
        // Uncomment and fill in real SRI hashes to enable verification:
        // integrity: {
        //   config: "sha256-<base64-hash-of-mlc-chat-config.json>",
        //   model_lib: "sha256-<base64-hash-of-wasm-file>",
        //   tokenizer: {
        //     "tokenizer.json": "sha256-<base64-hash-of-tokenizer.json>",
        //   },
        //   onFailure: "error",  // or "warn" to log and continue
        // },
      },
    ],
  };
  try {
    const engine = await webllm.CreateMLCEngine(
      "Llama-3.2-1B-Instruct-q4f16_1-MLC",
      {
        appConfig,
        initProgressCallback: (report: webllm.InitProgressReport) => {
          setStatus(report.text);
        },
      },
    );
    setStatus("Model loaded! Generating response...");
    const reply = await engine.chat.completions.create({
      messages: [{ role: "user", content: "Hello! What can you do?" }],
    });
    setStatus("Response: " + reply.choices[0].message.content);
  } catch (error) {
    if (error instanceof webllm.IntegrityError) {
      setStatus(
        `Integrity verification failed!\n` +
          `URL: ${error.url}\n` +
          `Expected: ${error.expected}\n` +
          `Got: ${error.actual}`,
      );
    } else {
      setStatus("Error: " + error);
    }
  }
 }
 main();
@@ -0,0 +1,8 @@
 ### OpenAI API Demos
 Run `npm install` first, followed by `npm start`.
 Note if you would like to hack WebLLM core package,
 you can change web-llm dependencies as `"file:../.."`, and follow the build from source
 instruction in the project to build webllm locally. This option is only recommended
 if you would like to hack WebLLM core package.
@@ -0,0 +1,20 @@
 {
  "name": "openai-api",
  "version": "0.1.0",
  "private": true,
  "scripts": {
    "start": "parcel src/json_mode.html  --port 8888",
    "build": "parcel build src/json_mode.html --dist-dir lib"
  },
  "devDependencies": {
    "buffer": "^5.7.1",
    "parcel": "^2.8.3",
    "process": "^0.11.10",
    "tslib": "^2.3.1",
    "typescript": "^4.9.5",
    "url": "^0.11.3"
  },
  "dependencies": {
    "@mlc-ai/web-llm": "^0.2.84"
  }
 }
@@ -0,0 +1,16 @@
 <!doctype html>
 <html>
  <script>
    webLLMGlobal = {};
  </script>
  <body>
    <h2>WebLLM Test Page</h2>
    Open console to see output.
    <br />
    <br />
    <label id="init-label"> </label>
    <script type="module" src="./json_mode.ts"></script>
  </body>
 </html>
--- a/Show More
+++ b/Show More