chore: import upstream snapshot with attribution
Build site and push to gh-pages / Build site (push) Waiting to run
Build / build (push) Waiting to run
Linter / lint (push) Waiting to run
Security / dependency-review (push) Waiting to run
Security / npm-audit (push) Waiting to run
Security / codeql (push) Waiting to run
Tests / test (push) Waiting to run
Build site and push to gh-pages / Build site (push) Waiting to run
Build / build (push) Waiting to run
Linter / lint (push) Waiting to run
Security / dependency-review (push) Waiting to run
Security / npm-audit (push) Waiting to run
Security / codeql (push) Waiting to run
Tests / test (push) Waiting to run
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,38 @@
|
||||
name: Build site and push to gh-pages
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build site
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Configuring build Environment
|
||||
run: |
|
||||
sudo apt-get update
|
||||
python -m pip install -U pip
|
||||
|
||||
- name: Setup Ruby
|
||||
uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: '3.0'
|
||||
|
||||
- name: Installing dependencies
|
||||
run: |
|
||||
python -m pip install -r docs/requirements.txt
|
||||
gem install jekyll jekyll-remote-theme jekyll-sass-converter
|
||||
|
||||
- name: Build and deploy site
|
||||
if: github.ref == 'refs/heads/main'
|
||||
run: |
|
||||
git remote set-url origin https://x-access-token:${{ secrets.MLC_GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
|
||||
git config --global user.email "mlc-gh-actions-bot@nomail"
|
||||
git config --global user.name "mlc-gh-actions-bot"
|
||||
|
||||
./scripts/gh_deploy_site.sh
|
||||
@@ -0,0 +1,38 @@
|
||||
name: Build
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: ".nvmrc"
|
||||
cache: npm
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Build package
|
||||
run: npm run build
|
||||
|
||||
- name: Validate package contents
|
||||
run: npm pack --dry-run
|
||||
@@ -0,0 +1,27 @@
|
||||
name: Linter
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version-file: ".nvmrc"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm install
|
||||
|
||||
- name: Run lint
|
||||
run: npm run lint
|
||||
@@ -0,0 +1,73 @@
|
||||
name: Security
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
schedule:
|
||||
- cron: "25 5 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
dependency-review:
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
steps:
|
||||
- name: Dependency review
|
||||
uses: actions/dependency-review-action@v4
|
||||
with:
|
||||
fail-on-severity: high
|
||||
|
||||
npm-audit:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: ".nvmrc"
|
||||
cache: npm
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Run npm audit (production dependencies)
|
||||
run: npm audit --omit=dev --audit-level=high
|
||||
|
||||
codeql:
|
||||
if: github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: javascript-typescript
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v3
|
||||
|
||||
- name: Analyze
|
||||
uses: github/codeql-action/analyze@v3
|
||||
@@ -0,0 +1,45 @@
|
||||
name: Tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: ".nvmrc"
|
||||
cache: npm
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Run test suite
|
||||
env:
|
||||
CI: "true"
|
||||
run: npm run test -- --ci
|
||||
|
||||
- name: Upload coverage artifact
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-${{ github.run_id }}
|
||||
path: coverage
|
||||
if-no-files-found: ignore
|
||||
+328
@@ -0,0 +1,328 @@
|
||||
scratch/
|
||||
dist/
|
||||
params/
|
||||
*.bak
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
*.S
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
|
||||
*.ll
|
||||
.npm
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
env/
|
||||
build/
|
||||
build-*/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
.conda/
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Generated by python/gen_requirements.py
|
||||
python/requirements/*.txt
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
docs/_staging/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
/target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
*~
|
||||
*.pyc
|
||||
*~
|
||||
config.mk
|
||||
config.cmake
|
||||
Win32
|
||||
*.dir
|
||||
perf
|
||||
*.wasm
|
||||
.emscripten
|
||||
|
||||
## IOS
|
||||
DerivedData/
|
||||
|
||||
## Java
|
||||
*.class
|
||||
jvm/*/target/
|
||||
jvm/*/*/target/
|
||||
jvm/native/*/generated
|
||||
jvm/native/src/main/native/org_apache_tvm_native_c_api.h
|
||||
*.worksheet
|
||||
*.idea
|
||||
*.iml
|
||||
*.classpath
|
||||
*.project
|
||||
*.settings
|
||||
*/node_modules/
|
||||
|
||||
## Various settings
|
||||
*.pbxuser
|
||||
!default.pbxuser
|
||||
*.mode1v3
|
||||
!default.mode1v3
|
||||
*.mode2v3
|
||||
!default.mode2v3
|
||||
*.perspectivev3
|
||||
!default.perspectivev3
|
||||
xcuserdata/
|
||||
.pkl_memoize_*
|
||||
|
||||
.emscripten*
|
||||
.m2
|
||||
|
||||
# Compiled Dynamic libraries
|
||||
*.so
|
||||
*.dylib
|
||||
*.dll
|
||||
|
||||
# Compiled Object files
|
||||
*.slo
|
||||
*.lo
|
||||
*.o
|
||||
*.obj
|
||||
|
||||
# Precompiled Headers
|
||||
*.gch
|
||||
*.pch
|
||||
|
||||
# Compiled Static libraries
|
||||
*.lai
|
||||
*.la
|
||||
*.a
|
||||
*.lib
|
||||
|
||||
# Executables
|
||||
*.exe
|
||||
*.out
|
||||
*.app
|
||||
|
||||
## Other
|
||||
*.moved-aside
|
||||
*.xccheckout
|
||||
*.xcscmblueprint
|
||||
.DS_Store
|
||||
tags
|
||||
cscope*
|
||||
*.lock
|
||||
|
||||
# vim temporary files
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# TVM generated code
|
||||
perf
|
||||
.bash_history
|
||||
# *.json
|
||||
*.params
|
||||
*.ro
|
||||
*.onnx
|
||||
*.h5
|
||||
synset.txt
|
||||
cat.jpg
|
||||
cat.png
|
||||
docs.tgz
|
||||
cat.png
|
||||
*.mlmodel
|
||||
tvm_u.*
|
||||
tvm_t.*
|
||||
# Mac OS X
|
||||
.DS_Store
|
||||
|
||||
# Jetbrain
|
||||
.idea
|
||||
.ipython
|
||||
.jupyter
|
||||
.nv
|
||||
.pylint.d
|
||||
.python_history
|
||||
.pytest_cache
|
||||
.local
|
||||
cmake-build-debug
|
||||
|
||||
# Visual Studio
|
||||
.vs
|
||||
|
||||
# Visual Studio Code
|
||||
.vscode
|
||||
|
||||
# tmp file
|
||||
.nfs*
|
||||
|
||||
# keys
|
||||
*.pem
|
||||
*.p12
|
||||
*.pfx
|
||||
*.cer
|
||||
*.crt
|
||||
*.der
|
||||
|
||||
# patch sentinel
|
||||
patched.txt
|
||||
|
||||
# Python type checking
|
||||
.mypy_cache/
|
||||
.pyre/
|
||||
|
||||
# pipenv files
|
||||
Pipfile
|
||||
Pipfile.lock
|
||||
|
||||
# conda package artifacts
|
||||
conda/Dockerfile.cuda*
|
||||
conda/pkg
|
||||
.node_repl_history
|
||||
# nix files
|
||||
.envrc
|
||||
*.nix
|
||||
|
||||
# Docker files
|
||||
.sudo_as_admin_successful
|
||||
|
||||
# Downloaded models/datasets
|
||||
.tvm_test_data
|
||||
.dgl
|
||||
.caffe2
|
||||
|
||||
# Local docs build
|
||||
_docs/
|
||||
jvm/target
|
||||
.config/configstore/
|
||||
.ci-py-scripts/
|
||||
|
||||
# Generated Hexagon files
|
||||
src/runtime/hexagon/rpc/hexagon_rpc.h
|
||||
src/runtime/hexagon/rpc/hexagon_rpc_skel.c
|
||||
src/runtime/hexagon/rpc/hexagon_rpc_stub.c
|
||||
|
||||
# Local tvm-site checkout
|
||||
tvm-site/
|
||||
|
||||
# Generated docs files
|
||||
gallery/how_to/work_with_microtvm/micro_tvmc.py
|
||||
|
||||
# Test sample data files
|
||||
!tests/python/ci/sample_prs/*.json
|
||||
|
||||
# Used in CI to communicate between Python and Jenkins
|
||||
.docker-image-names/
|
||||
|
||||
# Printed TIR code on disk
|
||||
*.tir
|
||||
|
||||
# GDB history file
|
||||
.gdb_history
|
||||
|
||||
3rdparty
|
||||
dist
|
||||
tvm_home
|
||||
node_modules
|
||||
lib
|
||||
.parcel-cache
|
||||
|
||||
**/.next
|
||||
coverage
|
||||
@@ -0,0 +1 @@
|
||||
npx lint-staged
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"./**/*.{js,ts,jsx,tsx,json}": ["eslint --fix", "prettier --write"]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
dist
|
||||
debug
|
||||
lib
|
||||
build
|
||||
node_modules
|
||||
3rdparty
|
||||
.eslintrc.cjs
|
||||
**/.next
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"trailingComma": "all"
|
||||
}
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
# Contributing to WebLLM
|
||||
|
||||
Thank you for your interest in contributing to WebLLM. This guide helps contributors get set up quickly and make high-impact changes that are easy to review and merge.
|
||||
|
||||
## Ways To Contribute
|
||||
|
||||
We welcome contributions across the project, including:
|
||||
|
||||
- Bug reports with clear reproduction steps
|
||||
- Bug fixes and reliability improvements
|
||||
- New features and API improvements
|
||||
- Performance and memory optimizations
|
||||
- Tests and test coverage improvements
|
||||
- Documentation updates and tutorials
|
||||
- New or improved examples in `examples/`
|
||||
- Model integration and configuration improvements
|
||||
- Code review and issue triage support
|
||||
|
||||
If you are unsure where to start, look for open issues in the repository and propose a plan in the issue thread before implementation.
|
||||
|
||||
## Community Principles
|
||||
|
||||
WebLLM is part of a broader open-source ecosystem and follows collaborative, public-first development norms.
|
||||
|
||||
- Keep technical discussion in public, archivable channels (issues and pull requests)
|
||||
- Use clear technical reasoning and seek consensus on non-trivial changes
|
||||
- For major design changes, start with an issue or RFC-style proposal before coding
|
||||
- Review other contributors' PRs when possible
|
||||
|
||||
Additional reference: Apache TVM community guidelines
|
||||
|
||||
- https://tvm.apache.org/docs/contribute/community.html
|
||||
|
||||
## Development Setup
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Node.js (see `.nvmrc` for the required version)
|
||||
- npm
|
||||
- Git
|
||||
|
||||
Optional:
|
||||
|
||||
- Python 3 (for docs build)
|
||||
- Emscripten/toolchain setup
|
||||
|
||||
### Local Setup
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mlc-ai/web-llm.git
|
||||
cd web-llm
|
||||
npm install
|
||||
```
|
||||
|
||||
### Build, Lint, and Test
|
||||
|
||||
```bash
|
||||
npm run build
|
||||
npm run lint
|
||||
npm test
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- `npm test` runs Jest with coverage thresholds.
|
||||
- For quick iteration on a single test file, you can run:
|
||||
|
||||
```bash
|
||||
npx jest --coverage=false tests/<file>.test.ts
|
||||
```
|
||||
|
||||
### Auto-formatting
|
||||
|
||||
If lint or style checks fail, run:
|
||||
|
||||
```bash
|
||||
npm run format
|
||||
```
|
||||
|
||||
Pre-commit hooks (Husky + lint-staged) are configured in this repo.
|
||||
|
||||
## Testing Changes In Examples
|
||||
|
||||
To test local package changes inside an example app:
|
||||
|
||||
1. Edit `examples/<example>/package.json` and set `"@mlc-ai/web-llm"` to `"../.."` (or `"file:../.."` if needed).
|
||||
2. Install and run the example.
|
||||
|
||||
```bash
|
||||
cd examples/<example>
|
||||
npm install
|
||||
npm run start
|
||||
```
|
||||
|
||||
## Documentation Contributions
|
||||
|
||||
Docs are in `docs/` and built with Sphinx.
|
||||
|
||||
```bash
|
||||
cd docs
|
||||
pip3 install -r requirements.txt
|
||||
make html
|
||||
```
|
||||
|
||||
Open the built docs from `docs/_build/html`.
|
||||
|
||||
## Pull Request Guidelines
|
||||
|
||||
Before opening a PR:
|
||||
|
||||
1. Keep the change scoped to one problem or feature.
|
||||
2. Add or update tests for behavior changes.
|
||||
3. Update docs/examples for user-facing changes.
|
||||
4. Run `npm run lint` and `npm test` locally.
|
||||
5. Include a clear PR description with:
|
||||
- Problem statement
|
||||
- Proposed solution
|
||||
- Validation steps and results
|
||||
- Backward-compatibility considerations
|
||||
|
||||
During review:
|
||||
|
||||
- Respond to comments with concrete follow-ups
|
||||
- Prefer additional tests over assumptions
|
||||
- Keep commit history understandable (small, logical commits)
|
||||
|
||||
## Reporting Bugs and Requesting Features
|
||||
|
||||
- Use GitHub Issues for bug reports and feature requests.
|
||||
- Include environment details, expected vs. actual behavior, and minimal reproduction steps.
|
||||
- For substantial feature additions, open an issue first to align on design and scope.
|
||||
|
||||
## Security Reporting
|
||||
|
||||
Please do not report security vulnerabilities in public issues. Report vulnerabilities via email to `mlc-llm-private@googlegroups.com`.
|
||||
|
||||
Reference:
|
||||
|
||||
- https://github.com/mlc-ai/web-llm/blob/main/SECURITY.md
|
||||
|
||||
## License
|
||||
|
||||
By contributing, you agree that your contributions are provided under the repository's Apache-2.0 license.
|
||||
@@ -0,0 +1,211 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
------------------------------------------------------------------------------------
|
||||
This product bundles various third-party components under other open source licenses.
|
||||
This section summarizes those components and their licenses. See licenses/
|
||||
for text of these licenses.
|
||||
|
||||
Apache Software Foundation License 2.0
|
||||
--------------------------------------
|
||||
|
||||
src/openai_api_protocols
|
||||
@@ -0,0 +1,567 @@
|
||||
<div align="center" id="top">
|
||||
|
||||
# WebLLM
|
||||
|
||||
[](https://www.npmjs.com/package/@mlc-ai/web-llm)
|
||||
[](https://chat.webllm.ai/)
|
||||
[](https://discord.gg/9Xpy2HGBuD)
|
||||
[](https://github.com/mlc-ai/web-llm-chat/)
|
||||
[](https://github.com/mlc-ai/mlc-llm/)
|
||||
|
||||
**High-Performance In-Browser LLM Inference Engine.**
|
||||
|
||||
[Documentation](https://webllm.mlc.ai/docs/) | [Blogpost](https://blog.mlc.ai/2024/06/13/webllm-a-high-performance-in-browser-llm-inference-engine) | [Paper](https://arxiv.org/abs/2412.15803) | [Examples](examples)
|
||||
|
||||
</div>
|
||||
|
||||
## Overview
|
||||
|
||||
WebLLM is a high-performance in-browser LLM inference engine that brings language model inference directly onto web browsers with hardware acceleration.
|
||||
Everything runs inside the browser with no server support and is accelerated with WebGPU.
|
||||
|
||||
WebLLM is **fully compatible with [OpenAI API](https://platform.openai.com/docs/api-reference/chat).**
|
||||
That is, you can use the same OpenAI API on **any open source models** locally, with functionalities
|
||||
including streaming, JSON-mode, function-calling (WIP), etc.
|
||||
|
||||
We can bring a lot of fun opportunities to build AI assistants for everyone and enable privacy while enjoying GPU acceleration.
|
||||
|
||||
You can use WebLLM as a base [npm package](https://www.npmjs.com/package/@mlc-ai/web-llm) and build your own web application on top of it by following the examples below. This project is a companion project of [MLC LLM](https://github.com/mlc-ai/mlc-llm), which enables universal deployment of LLM across hardware environments.
|
||||
|
||||
<div align="center">
|
||||
|
||||
**[Check out WebLLM Chat to try it out!](https://chat.webllm.ai/)**
|
||||
|
||||
</div>
|
||||
|
||||
## Key Features
|
||||
|
||||
- **In-Browser Inference**: WebLLM is a high-performance, in-browser language model inference engine that leverages WebGPU for hardware acceleration, enabling powerful LLM operations directly within web browsers without server-side processing.
|
||||
|
||||
- [**Full OpenAI API Compatibility**](#full-openai-compatibility): Seamlessly integrate your app with WebLLM using OpenAI API with functionalities such as streaming, JSON-mode, logit-level control, seeding, and more.
|
||||
|
||||
- **Structured JSON Generation**: WebLLM supports state-of-the-art JSON mode structured generation, implemented in the WebAssembly portion of the model library for optimal performance. Check [WebLLM JSON Playground](https://huggingface.co/spaces/mlc-ai/WebLLM-JSON-Playground) on HuggingFace to try generating JSON output with custom JSON schema.
|
||||
|
||||
- [**Extensive Model Support**](#built-in-models): WebLLM natively supports a range of models including Llama 3, Phi 3, Gemma, Mistral, Qwen(通义千问), and many others, making it versatile for various AI tasks. For the complete supported model list, check [MLC Models](https://mlc.ai/models).
|
||||
|
||||
- [**Custom Model Integration**](#custom-models): Easily integrate and deploy custom models in MLC format, allowing you to adapt WebLLM to specific needs and scenarios, enhancing flexibility in model deployment.
|
||||
|
||||
- **Plug-and-Play Integration**: Easily integrate WebLLM into your projects using package managers like NPM and Yarn, or directly via CDN, complete with comprehensive [examples](./examples/) and a modular design for connecting with UI components.
|
||||
|
||||
- **Streaming & Real-Time Interactions**: Supports streaming chat completions, allowing real-time output generation which enhances interactive applications like chatbots and virtual assistants.
|
||||
|
||||
- **Web Worker & Service Worker Support**: Optimize UI performance and manage the lifecycle of models efficiently by offloading computations to separate worker threads or service workers.
|
||||
|
||||
- **Chrome Extension Support**: Extend the functionality of web browsers through custom Chrome extensions using WebLLM, with examples available for building both basic and advanced extensions.
|
||||
|
||||
## Built-in Models
|
||||
|
||||
Check the complete list of available models on [MLC Models](https://mlc.ai/models). WebLLM supports a subset of these available models and the list can be accessed at [`prebuiltAppConfig.model_list`](https://github.com/mlc-ai/web-llm/blob/main/src/config.ts#L293).
|
||||
|
||||
Here are the primary families of models currently supported:
|
||||
|
||||
- **Llama**: Llama 3, Llama 2, Hermes-2-Pro-Llama-3
|
||||
- **Phi**: Phi 3, Phi 2, Phi 1.5
|
||||
- **Gemma**: Gemma-2B
|
||||
- **Mistral**: Mistral-7B-v0.3, Hermes-2-Pro-Mistral-7B, NeuralHermes-2.5-Mistral-7B, OpenHermes-2.5-Mistral-7B
|
||||
- **Qwen (通义千问)**: Qwen2 0.5B, 1.5B, 7B
|
||||
|
||||
If you need more models, [request a new model via opening an issue](https://github.com/mlc-ai/web-llm/issues/new/choose) or check [Custom Models](#custom-models) for how to compile and use your own models with WebLLM.
|
||||
|
||||
## Jumpstart with Examples
|
||||
|
||||
Learn how to use WebLLM to integrate large language models into your application and generate chat completions through this simple Chatbot example:
|
||||
|
||||
[](https://jsfiddle.net/neetnestor/4nmgvsa2/)
|
||||
[](https://codepen.io/neetnestor/pen/vYwgZaG)
|
||||
|
||||
For an advanced example of a larger, more complicated project, check [WebLLM Chat](https://github.com/mlc-ai/web-llm-chat/blob/main/app/client/webllm.ts).
|
||||
|
||||
More examples for different use cases are available in the [examples](./examples/) folder.
|
||||
|
||||
## Get Started
|
||||
|
||||
WebLLM offers a minimalist and modular interface to access the chatbot in the browser.
|
||||
The package is designed in a modular way to hook to any of the UI components.
|
||||
|
||||
### Installation
|
||||
|
||||
#### Package Manager
|
||||
|
||||
```sh
|
||||
# npm
|
||||
npm install @mlc-ai/web-llm
|
||||
# yarn
|
||||
yarn add @mlc-ai/web-llm
|
||||
# or pnpm
|
||||
pnpm install @mlc-ai/web-llm
|
||||
```
|
||||
|
||||
Then import the module in your code.
|
||||
|
||||
```typescript
|
||||
// Import everything
|
||||
import * as webllm from "@mlc-ai/web-llm";
|
||||
// Or only import what you need
|
||||
import { CreateMLCEngine } from "@mlc-ai/web-llm";
|
||||
```
|
||||
|
||||
#### CDN Delivery
|
||||
|
||||
Thanks to [jsdelivr.com](https://www.jsdelivr.com/package/npm/@mlc-ai/web-llm), WebLLM can be imported directly through URL and work out-of-the-box on cloud development platforms like [jsfiddle.net](https://jsfiddle.net/), [Codepen.io](https://codepen.io/), and [Scribbler](https://scribbler.live):
|
||||
|
||||
```javascript
|
||||
import * as webllm from "https://esm.run/@mlc-ai/web-llm";
|
||||
```
|
||||
|
||||
It can also be dynamically imported as:
|
||||
|
||||
```javascript
|
||||
const webllm = await import("https://esm.run/@mlc-ai/web-llm");
|
||||
```
|
||||
|
||||
### Create MLCEngine
|
||||
|
||||
Most operations in WebLLM are invoked through the `MLCEngine` interface. You can create an `MLCEngine` instance and loading the model by calling the `CreateMLCEngine()` factory function.
|
||||
|
||||
(Note that loading models requires downloading and it can take a significant amount of time for the very first run without caching previously. You should properly handle this asynchronous call.)
|
||||
|
||||
```typescript
|
||||
import { CreateMLCEngine } from "@mlc-ai/web-llm";
|
||||
|
||||
// Callback function to update model loading progress
|
||||
const initProgressCallback = (initProgress) => {
|
||||
console.log(initProgress);
|
||||
};
|
||||
const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
|
||||
|
||||
const engine = await CreateMLCEngine(
|
||||
selectedModel,
|
||||
{ initProgressCallback: initProgressCallback }, // engineConfig
|
||||
);
|
||||
```
|
||||
|
||||
Under the hood, this factory function does the following steps for first creating an engine instance (synchronous) and then loading the model (asynchronous). You can also do them separately in your application.
|
||||
|
||||
```typescript
|
||||
import { MLCEngine } from "@mlc-ai/web-llm";
|
||||
|
||||
// This is a synchronous call that returns immediately
|
||||
const engine = new MLCEngine({
|
||||
initProgressCallback: initProgressCallback,
|
||||
});
|
||||
|
||||
// This is an asynchronous call and can take a long time to finish
|
||||
await engine.reload(selectedModel);
|
||||
```
|
||||
|
||||
### Cache Backend Policy
|
||||
|
||||
WebLLM supports four cache backends through `AppConfig.cacheBackend`:
|
||||
|
||||
- `"cache"`: browser [Cache API](https://developer.mozilla.org/en-US/docs/Web/API/Cache) (default).
|
||||
- `"indexeddb"`: browser [IndexedDB](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API).
|
||||
- `"opfs"`: browser [Origin Private File System (OPFS)](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system).
|
||||
- `"cross-origin"`: experimental Chrome [Cross-Origin Storage API](https://github.com/WICG/cross-origin-storage) extension backend. Install the [Cross-Origin Storage extension](https://chromewebstore.google.com/detail/cross-origin-storage/denpnpcgjgikjpoglpjefakmdcbmlgih) to use it. (If the extension isn't installed, WebLLM falls back to the default cache automatically.)
|
||||
|
||||
Example:
|
||||
|
||||
```typescript
|
||||
import { CreateMLCEngine, prebuiltAppConfig } from "@mlc-ai/web-llm";
|
||||
|
||||
const appConfig = { ...prebuiltAppConfig, cacheBackend: "cross-origin" };
|
||||
const engine = await CreateMLCEngine("Llama-3.1-8B-Instruct-q4f32_1-MLC", {
|
||||
appConfig,
|
||||
});
|
||||
```
|
||||
|
||||
Notes:
|
||||
- If `"opfs"` is selected in an environment without OPFS support, cache operations fail with an OPFS availability error.
|
||||
- When using `"opfs"`, `appConfig.opfsAccessMode` can be set to `"auto"` to use OPFS sync access handles where supported, or `"sync"` to require sync access handles. The default is `"async"`.
|
||||
- The `"cross-origin"` backend requires installing and enabling a compatible browser extension.
|
||||
- Cross-origin backend currently does not support programmatic tensor-cache deletion; clearing is extension-managed.
|
||||
|
||||
### Chat Completion
|
||||
|
||||
After successfully initializing the engine, you can now invoke chat completions using OpenAI style chat APIs through the `engine.chat.completions` interface. For the full list of parameters and their descriptions, check [section below](#full-openai-compatibility) and [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create).
|
||||
|
||||
(Note: The `model` parameter is not supported and will be ignored here. Instead, call `CreateMLCEngine(model)` or `engine.reload(model)` instead as shown in the [Create MLCEngine](#create-mlcengine) above.)
|
||||
|
||||
```typescript
|
||||
const messages = [
|
||||
{ role: "system", content: "You are a helpful AI assistant." },
|
||||
{ role: "user", content: "Hello!" },
|
||||
];
|
||||
|
||||
const reply = await engine.chat.completions.create({
|
||||
messages,
|
||||
});
|
||||
console.log(reply.choices[0].message);
|
||||
console.log(reply.usage);
|
||||
```
|
||||
|
||||
### Streaming
|
||||
|
||||
WebLLM also supports streaming chat completion generating. To use it, simply pass `stream: true` to the `engine.chat.completions.create` call.
|
||||
|
||||
```typescript
|
||||
const messages = [
|
||||
{ role: "system", content: "You are a helpful AI assistant." },
|
||||
{ role: "user", content: "Hello!" },
|
||||
];
|
||||
|
||||
// Chunks is an AsyncGenerator object
|
||||
const chunks = await engine.chat.completions.create({
|
||||
messages,
|
||||
temperature: 1,
|
||||
stream: true, // <-- Enable streaming
|
||||
stream_options: { include_usage: true },
|
||||
});
|
||||
|
||||
let reply = "";
|
||||
for await (const chunk of chunks) {
|
||||
reply += chunk.choices[0]?.delta.content || "";
|
||||
console.log(reply);
|
||||
if (chunk.usage) {
|
||||
console.log(chunk.usage); // only last chunk has usage
|
||||
}
|
||||
}
|
||||
|
||||
const fullReply = await engine.getMessage();
|
||||
console.log(fullReply);
|
||||
```
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Using Workers
|
||||
|
||||
You can put the heavy computation in a worker script to optimize your application performance. To do so, you need to:
|
||||
|
||||
1. Create a handler in the worker thread that communicates with the frontend while handling the requests.
|
||||
2. Create a Worker Engine in your main application, which under the hood sends messages to the handler in the worker thread.
|
||||
|
||||
For detailed implementations of different kinds of Workers, check the following sections.
|
||||
|
||||
#### Dedicated Web Worker
|
||||
|
||||
WebLLM comes with API support for WebWorker so you can hook
|
||||
the generation process into a separate worker thread so that
|
||||
the computing in the worker thread won't disrupt the UI.
|
||||
|
||||
We create a handler in the worker thread that communicates with the frontend while handling the requests.
|
||||
|
||||
```typescript
|
||||
// worker.ts
|
||||
import { WebWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
|
||||
|
||||
// A handler that resides in the worker thread
|
||||
const handler = new WebWorkerMLCEngineHandler();
|
||||
self.onmessage = (msg: MessageEvent) => {
|
||||
handler.onmessage(msg);
|
||||
};
|
||||
```
|
||||
|
||||
In the main logic, we create a `WebWorkerMLCEngine` that
|
||||
implements the same `MLCEngineInterface`. The rest of the logic remains the same.
|
||||
|
||||
```typescript
|
||||
// main.ts
|
||||
import { CreateWebWorkerMLCEngine } from "@mlc-ai/web-llm";
|
||||
|
||||
async function main() {
|
||||
// Use a WebWorkerMLCEngine instead of MLCEngine here
|
||||
const engine = await CreateWebWorkerMLCEngine(
|
||||
new Worker(new URL("./worker.ts", import.meta.url), {
|
||||
type: "module",
|
||||
}),
|
||||
selectedModel,
|
||||
{ initProgressCallback }, // engineConfig
|
||||
);
|
||||
|
||||
// everything else remains the same
|
||||
}
|
||||
```
|
||||
|
||||
### Use Service Worker
|
||||
|
||||
WebLLM comes with API support for ServiceWorker so you can hook the generation process
|
||||
into a service worker to avoid reloading the model in every page visit and optimize
|
||||
your application's offline experience.
|
||||
|
||||
(Note, Service Worker's life cycle is managed by the browser and can be killed any time without notifying the webapp. `ServiceWorkerMLCEngine` will try to keep the service worker thread alive by periodically sending heartbeat events, but your application should also include proper error handling. Check `keepAliveMs` and `missedHeatbeat` in [`ServiceWorkerMLCEngine`](https://github.com/mlc-ai/web-llm/blob/main/src/service_worker.ts#L234) for more details.)
|
||||
|
||||
We create a handler in the worker thread that communicates with the frontend while handling the requests.
|
||||
|
||||
```typescript
|
||||
// sw.ts
|
||||
import { ServiceWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
|
||||
|
||||
let handler: ServiceWorkerMLCEngineHandler;
|
||||
|
||||
self.addEventListener("activate", function (event) {
|
||||
handler = new ServiceWorkerMLCEngineHandler();
|
||||
console.log("Service Worker is ready");
|
||||
});
|
||||
```
|
||||
|
||||
Then in the main logic, we register the service worker and create the engine using
|
||||
`CreateServiceWorkerMLCEngine` function. The rest of the logic remains the same.
|
||||
|
||||
```typescript
|
||||
// main.ts
|
||||
import {
|
||||
MLCEngineInterface,
|
||||
CreateServiceWorkerMLCEngine,
|
||||
} from "@mlc-ai/web-llm";
|
||||
|
||||
if ("serviceWorker" in navigator) {
|
||||
navigator.serviceWorker.register(
|
||||
new URL("sw.ts", import.meta.url), // worker script
|
||||
{ type: "module" },
|
||||
);
|
||||
}
|
||||
|
||||
const engine: MLCEngineInterface = await CreateServiceWorkerMLCEngine(
|
||||
selectedModel,
|
||||
{ initProgressCallback }, // engineConfig
|
||||
);
|
||||
```
|
||||
|
||||
You can find a complete example on how to run WebLLM in service worker in [examples/service-worker](examples/service-worker/).
|
||||
|
||||
### Chrome Extension
|
||||
|
||||
You can also find examples of building Chrome extension with WebLLM in [examples/chrome-extension](examples/chrome-extension/) and [examples/chrome-extension-webgpu-service-worker](examples/chrome-extension-webgpu-service-worker/). The latter one leverages service worker, so the extension is persistent in the background. Additionally, you can explore another full project of a Chrome extension, WebLLM Assistant, which leverages WebLLM [here](https://github.com/mlc-ai/web-llm-assistant).
|
||||
|
||||
## Full OpenAI Compatibility
|
||||
|
||||
WebLLM is designed to be fully compatible with [OpenAI API](https://platform.openai.com/docs/api-reference/chat). Thus, besides building a simple chatbot, you can also have the following functionalities with WebLLM:
|
||||
|
||||
- [streaming](examples/streaming): return output as chunks in real-time in the form of an AsyncGenerator
|
||||
- [json-mode](examples/json-mode): efficiently ensure output is in JSON format, see [OpenAI Reference](https://platform.openai.com/docs/guides/text-generation/chat-completions-api) for more.
|
||||
- [seed-to-reproduce](examples/seed-to-reproduce): use seeding to ensure a reproducible output with fields `seed`.
|
||||
- [function-calling](examples/function-calling) (WIP): function calling with fields `tools` and `tool_choice` (with preliminary support); or manual function calling without `tools` or `tool_choice` (keeps the most flexibility).
|
||||
|
||||
## Integrity Verification
|
||||
|
||||
WebLLM supports optional integrity verification for model artifacts using
|
||||
[SRI (Subresource Integrity)](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) hashes.
|
||||
When the `integrity` field is set on a `ModelRecord`, WebLLM will verify the downloaded config,
|
||||
WASM, and tokenizer files against the provided hashes before loading.
|
||||
|
||||
```typescript
|
||||
import { CreateMLCEngine } from "@mlc-ai/web-llm";
|
||||
|
||||
const appConfig = {
|
||||
model_list: [
|
||||
{
|
||||
model: "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_1-MLC",
|
||||
model_id: "Llama-3.2-1B-Instruct-q4f16_1-MLC",
|
||||
model_lib:
|
||||
"https://raw.githubusercontent.com/user/model-libs/main/model.wasm",
|
||||
integrity: {
|
||||
config: "sha256-<base64-hash-of-mlc-chat-config.json>",
|
||||
model_lib: "sha256-<base64-hash-of-wasm-file>",
|
||||
tokenizer: {
|
||||
"tokenizer.json": "sha256-<base64-hash-of-tokenizer.json>",
|
||||
},
|
||||
onFailure: "error", // "error" (default) throws IntegrityError, "warn" logs and continues
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const engine = await CreateMLCEngine("Llama-3.2-1B-Instruct-q4f16_1-MLC", {
|
||||
appConfig,
|
||||
});
|
||||
```
|
||||
|
||||
You can generate SRI hashes for model files with:
|
||||
|
||||
```bash
|
||||
# SHA-256
|
||||
openssl dgst -sha256 -binary <file> | openssl base64 -A | sed 's/^/sha256-/'
|
||||
# SHA-384
|
||||
openssl dgst -sha384 -binary <file> | openssl base64 -A | sed 's/^/sha384-/'
|
||||
# SHA-512
|
||||
openssl dgst -sha512 -binary <file> | openssl base64 -A | sed 's/^/sha512-/'
|
||||
```
|
||||
|
||||
> The `openssl` commands require a Unix-like shell (macOS/Linux). On Windows, run `openssl` via [Git Bash](https://gitforwindows.org/) or [WSL](https://learn.microsoft.com/en-us/windows/wsl/).
|
||||
|
||||
If a hash does not match, an `IntegrityError` is thrown (or a warning is logged when `onFailure: "warn"`).
|
||||
All fields in `integrity` are optional — only specified artifacts will be verified.
|
||||
When the `integrity` field is omitted entirely, WebLLM behaves exactly as before (no verification).
|
||||
|
||||
See the [integrity-verification example](examples/integrity-verification/) for a complete working demo.
|
||||
|
||||
## Custom Models
|
||||
|
||||
WebLLM works as a companion project of [MLC LLM](https://github.com/mlc-ai/mlc-llm) and it supports custom models in MLC format.
|
||||
It reuses the model artifact and builds the flow of MLC LLM. To compile and use your own models with WebLLM, please check out
|
||||
[MLC LLM document](https://llm.mlc.ai/docs/deploy/webllm.html)
|
||||
on how to compile and deploy new model weights and libraries to WebLLM.
|
||||
|
||||
Here, we go over the high-level idea. There are two elements of the WebLLM package that enable new models and weight variants.
|
||||
|
||||
- `model`: Contains a URL to model artifacts, such as weights and meta-data.
|
||||
- `model_lib`: A URL to the web assembly library (i.e. wasm file) that contains the executables to accelerate the model computations.
|
||||
|
||||
Both are customizable in the WebLLM.
|
||||
|
||||
```typescript
|
||||
import { CreateMLCEngine } from "@mlc-ai/web-llm";
|
||||
|
||||
async main() {
|
||||
const appConfig = {
|
||||
"model_list": [
|
||||
{
|
||||
"model": "/url/to/my/llama",
|
||||
"model_id": "MyLlama-3b-v1-q4f32_0",
|
||||
"model_lib": "/url/to/myllama3b.wasm",
|
||||
}
|
||||
],
|
||||
};
|
||||
// override default
|
||||
const chatOpts = {
|
||||
"repetition_penalty": 1.01
|
||||
};
|
||||
|
||||
// load a prebuilt model
|
||||
// with a chat option override and app config
|
||||
// under the hood, it will load the model from myLlamaUrl
|
||||
// and cache it in the browser cache
|
||||
// The chat will also load the model library from "/url/to/myllama3b.wasm",
|
||||
// assuming that it is compatible to the model in myLlamaUrl.
|
||||
const engine = await CreateMLCEngine(
|
||||
"MyLlama-3b-v1-q4f32_0",
|
||||
{ appConfig }, // engineConfig
|
||||
chatOpts,
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
In many cases, we only want to supply the model weight variant, but
|
||||
not necessarily a new model (e.g. `NeuralHermes-Mistral` can reuse `Mistral`'s
|
||||
model library). For examples of how a model library can be shared by different model variants,
|
||||
see `webllm.prebuiltAppConfig`.
|
||||
|
||||
## Build WebLLM Package From Source
|
||||
|
||||
NOTE: you don't need to build from source unless you would like to modify the WebLLM package.
|
||||
To use the npm, simply follow [Get Started](#get-started) or any of the [examples](examples) instead.
|
||||
|
||||
To build from source, simply run:
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
Then, to test the effects of your code change in an example, inside `examples/get-started/package.json`, change from `"@mlc-ai/web-llm": "^0.2.84"` to `"@mlc-ai/web-llm": ../..`.
|
||||
|
||||
Then run:
|
||||
|
||||
```bash
|
||||
cd examples/get-started
|
||||
npm install
|
||||
npm start
|
||||
```
|
||||
|
||||
Note that sometimes you would need to switch between `file:../..` and `../..` to trigger npm to recognize new changes. In the worst case, you can run:
|
||||
|
||||
```bash
|
||||
cd examples/get-started
|
||||
rm -rf node_modules dist package-lock.json .parcel-cache
|
||||
npm install
|
||||
npm start
|
||||
```
|
||||
|
||||
### In case you need to build TVMjs from source
|
||||
|
||||
WebLLM's runtime largely depends on TVMjs: https://github.com/apache/tvm/tree/main/web
|
||||
|
||||
While it is also available as an npm package: https://www.npmjs.com/package/@mlc-ai/web-runtime, you can build it from source if needed by following the steps below.
|
||||
|
||||
1. Install [emscripten](https://emscripten.org). It is an LLVM-based compiler that compiles C/C++ source code to WebAssembly.
|
||||
- Follow the [installation instruction](https://emscripten.org/docs/getting_started/downloads.html#installation-instructions-using-the-emsdk-recommended) to install the latest emsdk.
|
||||
- Source `emsdk_env.sh` by `source path/to/emsdk_env.sh`, so that `emcc` is reachable from PATH and the command `emcc` works.
|
||||
|
||||
We can verify the successful installation by trying out `emcc` terminal.
|
||||
|
||||
Note: We recently found that using the latest `emcc` version may run into issues during runtime. Use `./emsdk install 3.1.56` instead of `./emsdk install latest` for now as a workaround. The error may look like
|
||||
|
||||
```
|
||||
Init error, LinkError: WebAssembly.instantiate(): Import #6 module="wasi_snapshot_preview1"
|
||||
function="proc_exit": function import requires a callable
|
||||
```
|
||||
|
||||
2. In `./package.json`, change from `"@mlc-ai/web-runtime": "0.18.0-dev2",` to `"@mlc-ai/web-runtime": "file:./tvm_home/web",`.
|
||||
|
||||
3. Setup necessary environment
|
||||
|
||||
Prepare all the necessary dependencies for web build:
|
||||
|
||||
```shell
|
||||
./scripts/prep_deps.sh
|
||||
```
|
||||
|
||||
In this step, if `$TVM_SOURCE_DIR` is not defined in the environment, we will execute the following line to build `tvmjs` dependency:
|
||||
|
||||
```shell
|
||||
git clone https://github.com/mlc-ai/relax 3rdparty/tvm-unity --recursive
|
||||
```
|
||||
|
||||
This clones the current HEAD of `mlc-ai/relax`. However, it may not always be the correct branch or commit to clone. To build a specific npm version from source, refer to the version bump PR, which states which branch (i.e. `mlc-ai/relax` or `apache/tvm`) and which commit the current WebLLM version depends on. For instance, version 0.2.52, according to its version bump PR https://github.com/mlc-ai/web-llm/pull/521, is built by checking out the following commit https://github.com/apache/tvm/commit/e6476847753c80e054719ac47bc2091c888418b6 in `apache/tvm`, rather than the HEAD of `mlc-ai/relax`.
|
||||
|
||||
Besides, `--recursive` is necessary and important. Otherwise, you may encounter errors like `fatal error: 'dlpack/dlpack.h' file not found`.
|
||||
|
||||
4. Build WebLLM Package
|
||||
|
||||
```shell
|
||||
npm run build
|
||||
```
|
||||
|
||||
5. Validate some of the sub-packages
|
||||
|
||||
You can then go to the subfolders in [examples](examples) to validate some of the sub-packages.
|
||||
We use Parcelv2 for bundling. Although Parcel is not very good at tracking parent directory
|
||||
changes sometimes. When you make a change in the WebLLM package, try to edit the `package.json`
|
||||
of the subfolder and save it, which will trigger Parcel to rebuild.
|
||||
|
||||
## Links
|
||||
|
||||
- [Demo App: WebLLM Chat](https://chat.webllm.ai/)
|
||||
- If you want to run LLM on native runtime, check out [MLC-LLM](https://github.com/mlc-ai/mlc-llm)
|
||||
- You might also be interested in [Web Stable Diffusion](https://github.com/mlc-ai/web-stable-diffusion/).
|
||||
|
||||
## Acknowledgement
|
||||
|
||||
This project is initiated by members from CMU Catalyst, UW SAMPL, SJTU, OctoML, and the MLC community. We would love to continue developing and supporting the open-source ML community.
|
||||
|
||||
This project is only possible thanks to the shoulders open-source ecosystems that we stand on. We want to thank the Apache TVM community and developers of the TVM Unity effort. The open-source ML community members made these models publicly available. PyTorch and Hugging Face communities make these models accessible. We would like to thank the teams behind Vicuna, SentencePiece, LLaMA, and Alpaca. We also would like to thank the WebAssembly, Emscripten, and WebGPU communities. Finally, thanks to Dawn and WebGPU developers.
|
||||
|
||||
## Citation
|
||||
|
||||
If you find this project to be useful, please cite:
|
||||
|
||||
```
|
||||
@misc{ruan2026webllmhighperformanceinbrowserllm,
|
||||
title={WebLLM: A High-Performance In-Browser LLM Inference Engine},
|
||||
author={Charlie F. Ruan and Yucheng Qin and Akaash R. Parthasarathy and Xun Zhou and Ruihang Lai and Hongyi Jin and Yixin Dong and Bohan Hou and Meng-Shiun Yu and Yiyan Zhai and Sudeep Agarwal and Hangrui Cao and Siyuan Feng and Tianqi Chen},
|
||||
year={2026},
|
||||
eprint={2412.15803},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.LG},
|
||||
url={https://arxiv.org/abs/2412.15803},
|
||||
}
|
||||
```
|
||||
|
||||
## Contributors
|
||||
|
||||
<a href="https://github.com/mlc-ai/web-llm/graphs/contributors">
|
||||
<img alt="contributors" src="https://contrib.rocks/image?repo=mlc-ai/web-llm"/>
|
||||
</a>
|
||||
|
||||
<p align="right">
|
||||
<a href="#top">⬆ Back to Top ⬆</a>
|
||||
</p>
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`mlc-ai/web-llm`
|
||||
- 原始仓库：https://github.com/mlc-ai/web-llm
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
@@ -0,0 +1,5 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
For security concerns or vulnerability reports, please send email to `mlc-llm-private@googlegroups.com`.
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
# Remove instances of string "const{createRequire:createRequire}=await import('module');"
|
||||
# This is required to allow background workers packaged with Parcel for the chrome extension
|
||||
# to run the `ChatModule`.
|
||||
sed -e s/"const{createRequire:createRequire}=await import('module');"//g -i.backup lib/index.js
|
||||
sed -e s/"const{createRequire:createRequire}=await import('module');"//g -i.backup lib/index.js.map
|
||||
|
||||
# Replace scriptDirectory init that Parcel cannot resolve ("new URL('./', import.meta.url)") with a plain relative string
|
||||
sed -e s~"require(\\\"url\\\").fileURLToPath(new URL(\\\"\\.\\/\\\",import.meta.url))"~"\\\"./\\\""~g -i.backup lib/index.js
|
||||
sed -e s~"require(\\\"url\\\").fileURLToPath(new URL(\\\"\\.\\/\\\",import.meta.url))"~'\\\".\\\"'~g -i.backup lib/index.js.map
|
||||
|
||||
# Replace string "new (require('u' + 'rl').URL)('file:' + __filename).href" with "MLC_DUMMY_PATH"
|
||||
# This is required for building nextJS projects -- its compile time would complain about `require()`
|
||||
# See https://github.com/mlc-ai/web-llm/issues/383 and the fixing PR's description for more.
|
||||
sed -e s/"new (require('u' + 'rl').URL)('file:' + __filename).href"/"\"MLC_DUMMY_PATH\""/g -i.backup lib/index.js
|
||||
# Replace rollup's newer output "require('u' + 'rl').pathToFileURL(__filename).href" with "MLC_DUMMY_PATH"
|
||||
# This avoids `require is not defined` in ESM SSR contexts (e.g. SvelteKit/Astro with bun/node).
|
||||
sed -e s/"require('u' + 'rl')\\.pathToFileURL(__filename)\\.href"/"\"MLC_DUMMY_PATH\""/g -i.backup lib/index.js
|
||||
# Replace with \"MLC_DUMMY_PATH\"
|
||||
sed -e s/"new (require('u' + 'rl').URL)('file:' + __filename).href"/'\\\"MLC_DUMMY_PATH\\\"'/g -i.backup lib/index.js.map
|
||||
sed -e s/"require('u' + 'rl')\\.pathToFileURL(__filename)\\.href"/'\\\"MLC_DUMMY_PATH\\\"'/g -i.backup lib/index.js.map
|
||||
|
||||
# Replace "import require$$N from 'perf_hooks';" with "const require$$N = "MLC_DUMMY_REQUIRE_VAR""
|
||||
# This is to prevent `perf_hooks` not found error
|
||||
# For more see https://github.com/mlc-ai/web-llm/issues/258 and https://github.com/mlc-ai/web-llm/issues/127
|
||||
# Use a regex to match any variable number (require$$0, require$$1, etc.)
|
||||
sed -E -i.backup "s/import (require\\\$\\\$[0-9]+) from 'perf_hooks';/const \1 = \"MLC_DUMMY_REQUIRE_VAR\"/g" lib/index.js
|
||||
# Similarly replace `const performanceNode = require(\"perf_hooks\")` with `const performanceNode = \"MLC_DUMMY_REQUIRE_VAR\"`
|
||||
sed -e s/'require(\\\"perf_hooks\\\")'/'\\\"MLC_DUMMY_REQUIRE_VAR\\\"'/g -i.backup lib/index.js.map
|
||||
|
||||
# Below is added when we include dependency @mlc-ai/web-runtime, rather than using local tvm_home
|
||||
# Replace "import require$$N from 'ws'" with "const require$$N = "MLC_DUMMY_REQUIRE_VAR""
|
||||
# This is to prevent error `Cannot find module 'ws'`
|
||||
sed -E -i.backup "s/import (require\\\$\\\$[0-9]+) from 'ws';/const \1 = \"MLC_DUMMY_REQUIRE_VAR\"/g" lib/index.js
|
||||
# Similarly replace `const WebSocket = require(\"ws\")` with `const WebSocket = \"MLC_DUMMY_REQUIRE_VAR\"`
|
||||
sed -e s/'require(\\\"ws\\\")'/'\\\"MLC_DUMMY_REQUIRE_VAR\\\"'/g -i.backup lib/index.js.map
|
||||
|
||||
# Cleanup backup files
|
||||
rm -f lib/index.js.backup
|
||||
rm -f lib/index.js.map.backup
|
||||
@@ -0,0 +1,20 @@
|
||||
# Minimal makefile for Sphinx documentation
|
||||
#
|
||||
|
||||
# You can set these variables from the command line, and also
|
||||
# from the environment for the first two.
|
||||
SPHINXOPTS ?=
|
||||
SPHINXBUILD ?= python -m sphinx
|
||||
SOURCEDIR = .
|
||||
BUILDDIR = _build
|
||||
|
||||
# Put it first so that "make" without argument is like "make help".
|
||||
help:
|
||||
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
||||
.PHONY: help Makefile
|
||||
|
||||
# Catch-all target: route all unknown targets to Sphinx using the new
|
||||
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
|
||||
%: Makefile
|
||||
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
@@ -0,0 +1,30 @@
|
||||
# WebLLM Documentation
|
||||
|
||||
The documentation was built upon [Sphinx](https://www.sphinx-doc.org/en/master/).
|
||||
|
||||
## Dependencies
|
||||
|
||||
Run the following command in this directory to install dependencies first:
|
||||
|
||||
```bash
|
||||
pip3 install -r requirements.txt
|
||||
```
|
||||
|
||||
## Build the Documentation
|
||||
|
||||
Then you can build the documentation by running:
|
||||
|
||||
```bash
|
||||
make html
|
||||
```
|
||||
|
||||
## View the Documentation
|
||||
|
||||
Run the following command to start a simple HTTP server:
|
||||
|
||||
```bash
|
||||
cd _build/html
|
||||
python3 -m http.server
|
||||
```
|
||||
|
||||
Then you can view the documentation in your browser at `http://localhost:8000` (the port can be customized by appending ` -p PORT_NUMBER` in the python command above).
|
||||
@@ -0,0 +1,87 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<svg
|
||||
xmlns:i="&#38;ns_ai;"
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:cc="http://creativecommons.org/ns#"
|
||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:svg="http://www.w3.org/2000/svg"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
version="1.1"
|
||||
id="Layer_1"
|
||||
x="0px"
|
||||
y="0px"
|
||||
viewBox="0 0 523.56958 171.35398"
|
||||
xml:space="preserve"
|
||||
sodipodi:docname="mlc-logo-with-text-landscape.svg"
|
||||
width="523.56958"
|
||||
height="171.35399"
|
||||
inkscape:version="1.0.1 (1.0.1+r75)"><metadata
|
||||
id="metadata23"><rdf:RDF><cc:Work
|
||||
rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
|
||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" /></cc:Work></rdf:RDF></metadata><defs
|
||||
id="defs21" /><sodipodi:namedview
|
||||
pagecolor="#ffffff"
|
||||
bordercolor="#666666"
|
||||
borderopacity="1"
|
||||
objecttolerance="10"
|
||||
gridtolerance="10"
|
||||
guidetolerance="10"
|
||||
inkscape:pageopacity="0"
|
||||
inkscape:pageshadow="2"
|
||||
inkscape:window-width="1853"
|
||||
inkscape:window-height="1025"
|
||||
id="namedview19"
|
||||
showgrid="false"
|
||||
inkscape:zoom="1.42"
|
||||
inkscape:cx="201.79468"
|
||||
inkscape:cy="119.53208"
|
||||
inkscape:window-x="67"
|
||||
inkscape:window-y="27"
|
||||
inkscape:window-maximized="1"
|
||||
inkscape:current-layer="Layer_1"
|
||||
inkscape:document-rotation="0" />
|
||||
<style
|
||||
type="text/css"
|
||||
id="style2">
|
||||
.st0{fill-rule:evenodd;clip-rule:evenodd;fill:#062578;}
|
||||
.st1{fill:#062578;}
|
||||
</style>
|
||||
<switch
|
||||
id="switch16"
|
||||
transform="translate(19.160879,-130.46791)">
|
||||
<foreignObject
|
||||
requiredExtensions="http://ns.adobe.com/AdobeIllustrator/10.0/"
|
||||
x="0"
|
||||
y="0"
|
||||
width="1"
|
||||
height="1">
|
||||
|
||||
</foreignObject>
|
||||
<g
|
||||
i:extraneous="self"
|
||||
id="g14">
|
||||
<g
|
||||
id="g12">
|
||||
<path
|
||||
class="st0"
|
||||
d="M 124.8,208.2 H 82.7 c -1.4,0 -2.6,1.2 -2.6,2.6 v 7.5 7.5 3.4 c 1.7,-0.2 3.3,0.1 4.7,0.9 v -11.8 -5.4 h 37.9 v 49.9 H 84.8 v -4.6 -11.1 c -1.1,0.8 -2.3,1.4 -3.4,2 -0.4,0.2 -0.8,0.4 -1.2,0.5 v 9.5 3.6 2.1 c 0,1.4 1.2,2.6 2.6,2.6 h 42.1 c 1.4,0 2.6,-1.1 2.6,-2.6 v -54.2 c -0.1,-1.2 -1.3,-2.4 -2.7,-2.4 z m -44.6,38.5 c -3,1.4 -8.3,2.7 -11.5,3.4 -3.6,0.8 -14.2,3.2 -15.1,-2.5 -0.7,-4.7 11.6,-9.9 14.8,-11.3 2.9,-1.3 5.9,-2.4 8.8,-3.6 3.2,-1.2 6.6,-1.7 8.2,2.3 0.7,1.7 0.9,3.2 0.9,5 v 0.2 0.2 c -0.4,3.1 -3.4,5.1 -6.1,6.3 z m 10.6,-51.3 v 0 c 1.4,-0.1 2.7,0.9 2.8,2.4 l 0.4,4.9 c 0.1,1.4 -0.9,2.7 -2.4,2.8 v 0 c -1.4,0.1 -2.7,-0.9 -2.8,-2.4 l -0.4,-4.9 c -0.1,-1.4 1,-2.6 2.4,-2.8 z m 18.8,10.9 c 0,-2.9 -0.2,-5.7 -0.4,-8.4 v 0 c -0.1,-0.9 -0.2,-1.8 -0.3,-2.7 v -0.1 -0.2 c -0.5,-3.3 -1.1,-6.6 -1.9,-10 -0.8,-3.3 -3.7,-5.4 -7,-5.5 -6.1,-0.1 -12.3,-0.2 -18.4,0 -1,-1.9 -4.2,-3.1 -7.9,-2.9 L 69.5,167 c 0.8,-0.7 1.2,-1.8 1.1,-3 -0.2,-2 -1.9,-3.5 -3.9,-3.3 -2,0.2 -3.5,1.9 -3.3,3.9 0.2,1.9 1.7,3.3 3.6,3.3 l 4,9 c -2.6,0.7 -4.6,2.2 -5.1,3.9 -5.8,0.8 -11.5,2 -17.2,3.1 -0.7,0.1 -1.3,0.4 -1.9,0.6 -6.4,1.6 -13,5.1 -13,5.1 -0.1,0.8 -0.2,1.7 -0.2,2.5 0.3,-0.1 0.6,-0.1 1,-0.2 5,-0.4 9.6,4.7 10.2,11.5 0.6,6.8 -3,12.7 -8,13.1 -0.4,0 -0.8,0 -1.1,0 0.2,0.9 0.4,1.8 0.7,2.6 4.2,2.3 9.9,3.8 13.4,4.6 v 0 c 0.8,0.3 1.7,0.5 2.6,0.5 8.6,0.2 17.3,0.3 25.9,-0.5 h 0.3 v -6.3 c -5.8,0.5 -11.8,0.5 -20.8,0.3 -1.7,0 -2.9,-1.1 -3.4,-2.5 -2,-7.2 -2.5,-14.8 -2,-22.9 0.1,-1.5 1.2,-2.8 2.9,-3.1 1.6,-0.3 3.2,-0.6 4.6,-0.9 0.7,-0.1 2.5,-0.5 3.8,-0.7 v 0 c 1,-0.2 2.1,-0.4 3.1,-0.6 3.3,0.6 5.6,5.4 10.5,4.6 5,-0.1 6.4,-5.2 9.5,-6.3 2.3,0 4.6,0 6.9,0.1 1.1,0 2.1,0 3.9,0.1 1.7,0 3,1 3.4,2.5 0.4,1.6 0.7,3.2 1,4.8 0.6,5.4 0.9,9.8 0.8,13.2 h 6.8 z m -75.1,-7.8 c 2,-0.2 3.8,2.5 4.1,6 0.3,3.5 -1,6.4 -3,6.6 -1,0.1 -1.9,-0.5 -2.6,-1.5 0.4,0.3 0.9,0.5 1.3,0.4 1.5,-0.1 2.6,-2.5 2.3,-5.3 -0.2,-2.8 -1.7,-5 -3.2,-4.8 -0.5,0 -0.9,0.3 -1.2,0.7 0.5,-1.3 1.3,-2.1 2.3,-2.1 z M 77,182.4 c 1.6,0 2.9,1.3 2.9,2.9 0,1.6 -1.3,2.9 -2.9,2.9 -1.6,0 -2.9,-1.3 -2.9,-2.9 -0.1,-1.6 1.3,-2.9 2.9,-2.9 z m -10.3,15.2 v 0 c 1.4,-0.1 2.7,0.9 2.8,2.4 l 0.4,4.9 c 0.1,1.4 -0.9,2.7 -2.4,2.8 v 0 c -1.4,0.1 -2.7,-0.9 -2.8,-2.4 l -0.4,-4.9 c -0.1,-1.4 1,-2.7 2.4,-2.8 z m -32.2,-3.1 c 3.7,-0.3 7.1,3.9 7.6,9.4 0.5,5.5 -2.1,10.2 -5.9,10.6 -3.7,0.3 -7.1,-3.9 -7.6,-9.4 -0.5,-5.6 2.1,-10.3 5.9,-10.6 z m 43.7,66.1 c -3.9,-1 -7.3,-3.6 -8.8,-7.9 2.7,-0.6 5.9,-1.3 8.8,-2.3 z m -10.9,-26.8 -0.7,-7.3 11.6,-0.6 v 3.6 c -0.6,0.2 -1.3,0.4 -1.9,0.6 -3,1.1 -6,2.3 -8.9,3.6 z m -13.1,-0.2 c 3.5,-2.6 7.9,-2.5 9.8,0.1 0.3,0.4 0.5,0.8 0.6,1.3 -4.3,1.9 -10.3,5 -12.6,8.8 -0.3,-0.2 -0.6,-0.5 -0.8,-0.8 -1.9,-2.6 -0.6,-6.8 3,-9.4 z m 48.6,10.7 c 0.7,-0.2 1.4,-0.1 2.1,0 l 0.8,-1.4 0.3,0.1 c 0.7,0.2 1.3,0.6 1.9,1.1 l 0.3,0.2 -0.8,1.4 c 0.2,0.3 0.4,0.5 0.6,0.8 0.2,0.3 0.3,0.6 0.4,0.9 h 1.6 l 0.1,0.4 c 0.1,0.7 0.1,1.5 0,2.2 l -0.1,0.4 h -1.6 c -0.2,0.7 -0.6,1.3 -1,1.8 l 0.8,1.4 -0.3,0.2 c -0.3,0.2 -0.6,0.5 -0.9,0.6 -0.3,0.2 -0.6,0.3 -1,0.5 l -0.3,0.1 -0.8,-1.4 c -0.7,0.1 -1.4,0.1 -2.1,0 l -0.8,1.4 -0.3,-0.1 c -0.7,-0.3 -1.3,-0.6 -1.9,-1.1 l -0.3,-0.2 0.8,-1.4 c -0.2,-0.3 -0.4,-0.5 -0.6,-0.8 -0.2,-0.3 -0.3,-0.6 -0.4,-0.9 h -1.6 l -0.1,-0.4 c -0.1,-0.7 -0.1,-1.5 0,-2.2 l 0.1,-0.4 h 1.6 c 0.2,-0.7 0.6,-1.3 1,-1.8 l -0.8,-1.4 0.3,-0.2 c 0.3,-0.2 0.6,-0.5 0.9,-0.6 0.3,-0.2 0.6,-0.3 1,-0.5 l 0.3,-0.1 z M 88.4,217.9 h 29.9 v 3.2 H 88.4 Z m 0,7.2 h 12.1 v 3.2 H 88.4 Z m 0,7.3 h 9.7 v 3.2 h -9.7 l -0.1,-0.3 v -2.9 z m 25.7,-4.8 c 0.9,0.3 1.7,0.8 2.5,1.4 l 1.9,-1.1 0.3,0.4 c 0.7,0.8 1.2,1.7 1.5,2.6 l 0.2,0.5 -1.9,1.1 c 0.1,0.5 0.2,0.9 0.2,1.4 0,0.5 -0.1,1 -0.2,1.4 l 1.9,1.1 -0.2,0.5 c -0.3,0.9 -0.9,1.8 -1.5,2.6 l -0.3,0.4 -1.9,-1.1 c -0.7,0.6 -1.5,1.1 -2.5,1.4 v 2.2 l -0.5,0.1 c -0.5,0.1 -1,0.1 -1.5,0.1 -0.5,0 -1,0 -1.5,-0.1 l -0.5,-0.1 v -2.2 c -0.9,-0.3 -1.7,-0.8 -2.5,-1.4 l -1.9,1.1 -0.3,-0.4 c -0.6,-0.8 -1.2,-1.7 -1.5,-2.6 l -0.2,-0.5 1.9,-1.1 c -0.1,-0.5 -0.2,-0.9 -0.2,-1.4 0,-0.5 0.1,-0.9 0.2,-1.4 l -1.9,-1.1 0.2,-0.5 c 0.3,-0.9 0.9,-1.8 1.5,-2.6 l 0.3,-0.4 1.9,1.1 c 0.7,-0.6 1.5,-1.1 2.5,-1.4 v -2.2 l 0.5,-0.1 c 0.5,-0.1 1,-0.1 1.5,-0.1 0.5,0 1,0 1.5,0.1 l 0.5,0.1 z m -2,3 c -1.8,0 -3.3,1.5 -3.3,3.3 0,1.8 1.5,3.3 3.3,3.3 1.8,0 3.3,-1.5 3.3,-3.3 0,-1.8 -1.5,-3.3 -3.3,-3.3 z m -9.5,16.3 c -1.2,0.7 -1.6,2.2 -0.9,3.3 0.7,1.2 2.2,1.6 3.3,0.9 1.2,-0.7 1.6,-2.2 0.9,-3.3 -0.7,-1.2 -2.1,-1.6 -3.3,-0.9 z"
|
||||
id="path4" />
|
||||
<path
|
||||
class="st1"
|
||||
d="m 172,209.4 v 22.9 h -4.7 v -13.2 c 0,-0.3 0,-0.6 0,-1 0,-0.4 0,-0.7 0.1,-1.1 l -6.1,11.8 c -0.2,0.4 -0.4,0.6 -0.8,0.8 -0.3,0.2 -0.7,0.3 -1.1,0.3 h -0.7 c -0.4,0 -0.8,-0.1 -1.1,-0.3 -0.3,-0.2 -0.6,-0.5 -0.8,-0.8 L 150.7,217 c 0,0.4 0.1,0.7 0.1,1.1 0,0.4 0,0.7 0,1 v 13.2 h -4.7 v -22.9 h 4.1 c 0.2,0 0.4,0 0.6,0 0.2,0 0.3,0 0.5,0.1 0.1,0.1 0.3,0.1 0.4,0.2 0.1,0.1 0.2,0.3 0.3,0.5 l 5.9,11.6 c 0.2,0.4 0.4,0.8 0.6,1.2 0.2,0.4 0.4,0.9 0.6,1.3 0.2,-0.5 0.4,-0.9 0.6,-1.4 0.2,-0.4 0.4,-0.9 0.6,-1.3 l 5.9,-11.6 c 0.1,-0.2 0.2,-0.4 0.3,-0.5 0.1,-0.1 0.2,-0.2 0.4,-0.2 0.1,-0.1 0.3,-0.1 0.5,-0.1 0.2,0 0.4,0 0.6,0 h 4 z m 16.6,14.1 -2.1,-6.3 c -0.2,-0.4 -0.3,-0.9 -0.5,-1.4 -0.2,-0.5 -0.4,-1.1 -0.5,-1.8 -0.2,0.6 -0.3,1.2 -0.5,1.8 -0.2,0.5 -0.3,1 -0.5,1.4 l -2.1,6.2 h 6.2 z m 8.4,8.8 h -4.1 c -0.5,0 -0.8,-0.1 -1.1,-0.3 -0.3,-0.2 -0.5,-0.5 -0.6,-0.8 l -1.4,-4 h -8.7 l -1.4,4 c -0.1,0.3 -0.3,0.6 -0.6,0.8 -0.3,0.2 -0.7,0.4 -1.1,0.4 h -4.2 l 8.9,-22.9 h 5.4 z m 16.5,-6 c 0.1,0 0.3,0 0.4,0.1 0.1,0 0.2,0.1 0.4,0.2 l 2.1,2.2 c -0.9,1.2 -2.1,2.1 -3.5,2.7 -1.4,0.6 -3,0.9 -4.9,0.9 -1.7,0 -3.3,-0.3 -4.7,-0.9 -1.4,-0.6 -2.5,-1.4 -3.5,-2.5 -1,-1 -1.7,-2.3 -2.2,-3.7 -0.5,-1.4 -0.8,-3 -0.8,-4.7 0,-1.7 0.3,-3.3 0.8,-4.7 0.6,-1.4 1.3,-2.7 2.3,-3.7 1,-1 2.2,-1.8 3.6,-2.4 1.4,-0.6 3,-0.9 4.6,-0.9 0.9,0 1.7,0.1 2.4,0.2 0.8,0.2 1.5,0.4 2.1,0.6 0.7,0.3 1.3,0.6 1.8,1 0.6,0.4 1,0.8 1.5,1.2 l -1.8,2.4 c -0.1,0.1 -0.3,0.3 -0.4,0.4 -0.2,0.1 -0.4,0.2 -0.7,0.2 -0.2,0 -0.4,0 -0.5,-0.1 -0.2,-0.1 -0.3,-0.2 -0.5,-0.3 -0.2,-0.1 -0.4,-0.3 -0.6,-0.4 -0.2,-0.1 -0.5,-0.3 -0.8,-0.4 -0.3,-0.1 -0.7,-0.2 -1.1,-0.3 -0.4,-0.1 -0.9,-0.1 -1.5,-0.1 -0.9,0 -1.7,0.2 -2.4,0.5 -0.7,0.3 -1.4,0.8 -1.9,1.4 -0.5,0.6 -0.9,1.4 -1.2,2.3 -0.3,0.9 -0.4,1.9 -0.4,3.1 0,1.2 0.2,2.2 0.5,3.1 0.3,0.9 0.8,1.7 1.3,2.3 0.6,0.6 1.2,1.1 1.9,1.4 0.7,0.3 1.5,0.5 2.4,0.5 0.5,0 0.9,0 1.3,-0.1 0.4,0 0.8,-0.1 1.1,-0.2 0.3,-0.1 0.7,-0.3 1,-0.4 0.3,-0.2 0.6,-0.4 0.9,-0.7 0.1,-0.1 0.3,-0.2 0.4,-0.3 0.3,0.2 0.5,0.1 0.6,0.1 z m 25.2,-16.9 v 22.9 h -5.3 v -9.7 h -9.3 v 9.7 h -5.4 v -22.9 h 5.4 v 9.6 h 9.3 v -9.6 z m 9.8,22.9 h -5.4 v -22.9 h 5.4 z M 273,209.4 v 22.9 h -2.8 c -0.4,0 -0.8,-0.1 -1,-0.2 -0.3,-0.1 -0.6,-0.4 -0.8,-0.7 l -10.8,-13.7 c 0,0.4 0.1,0.8 0.1,1.2 0,0.4 0,0.7 0,1.1 v 12.3 H 253 v -22.9 h 2.8 c 0.2,0 0.4,0 0.6,0 0.2,0 0.3,0.1 0.4,0.1 0.1,0.1 0.2,0.1 0.4,0.2 0.1,0.1 0.2,0.2 0.4,0.4 l 10.9,13.8 c -0.1,-0.4 -0.1,-0.9 -0.1,-1.3 0,-0.4 0,-0.8 0,-1.2 v -12.1 h 4.6 z m 9.8,4 v 5.4 h 7.2 v 3.9 h -7.2 v 5.5 h 9.4 v 4.1 h -14.8 v -22.9 h 14.8 v 4.1 h -9.4 z"
|
||||
id="path6" />
|
||||
<path
|
||||
class="st1"
|
||||
d="m 316.5,228 v 4.2 H 302.7 V 209.3 H 308 V 228 Z m 8.1,-14.6 v 5.4 h 7.2 v 3.9 h -7.2 v 5.5 h 9.4 v 4.1 H 319.2 V 209.4 H 334 v 4.1 h -9.4 z m 24.8,10.1 -2.1,-6.3 c -0.2,-0.4 -0.3,-0.9 -0.5,-1.4 -0.2,-0.5 -0.4,-1.1 -0.5,-1.8 -0.2,0.6 -0.3,1.2 -0.5,1.8 -0.2,0.5 -0.3,1 -0.5,1.4 l -2.1,6.2 h 6.2 z m 8.4,8.8 h -4.1 c -0.5,0 -0.8,-0.1 -1.1,-0.3 -0.3,-0.2 -0.5,-0.5 -0.6,-0.8 l -1.4,-4 h -8.7 l -1.4,4 c -0.1,0.3 -0.3,0.6 -0.6,0.8 -0.3,0.2 -0.7,0.4 -1.1,0.4 h -4.2 l 8.9,-22.9 h 5.4 z m 9.3,-12.2 c 0.7,0 1.3,-0.1 1.9,-0.3 0.5,-0.2 0.9,-0.4 1.2,-0.8 0.3,-0.3 0.6,-0.7 0.7,-1.1 0.1,-0.4 0.2,-0.9 0.2,-1.4 0,-1 -0.3,-1.8 -1,-2.4 -0.7,-0.6 -1.7,-0.8 -3,-0.8 H 365 v 6.8 z m 11.3,12.2 h -4.8 c -0.9,0 -1.5,-0.3 -1.9,-1 l -3.8,-6.7 c -0.2,-0.3 -0.4,-0.5 -0.6,-0.6 -0.2,-0.1 -0.5,-0.2 -0.9,-0.2 H 365 v 8.5 h -5.3 v -22.9 h 7.5 c 1.7,0 3.1,0.2 4.2,0.5 1.2,0.3 2.1,0.8 2.9,1.4 0.7,0.6 1.3,1.3 1.6,2.2 0.3,0.8 0.5,1.7 0.5,2.7 0,0.7 -0.1,1.4 -0.3,2.1 -0.2,0.7 -0.5,1.3 -0.9,1.8 -0.4,0.6 -0.9,1.1 -1.4,1.5 -0.6,0.4 -1.2,0.8 -2,1.1 0.3,0.2 0.7,0.4 1,0.7 0.3,0.3 0.6,0.6 0.8,1 z m 22,-22.9 v 22.9 h -2.8 c -0.4,0 -0.8,-0.1 -1,-0.2 -0.3,-0.1 -0.6,-0.4 -0.8,-0.7 L 385,217.7 c 0,0.4 0.1,0.8 0.1,1.2 0,0.4 0,0.7 0,1.1 v 12.3 h -4.7 v -22.9 h 2.8 c 0.2,0 0.4,0 0.6,0 0.2,0 0.3,0.1 0.4,0.1 0.1,0.1 0.2,0.1 0.4,0.2 0.1,0.1 0.2,0.2 0.4,0.4 l 10.9,13.8 c -0.1,-0.4 -0.1,-0.9 -0.1,-1.3 0,-0.4 0,-0.8 0,-1.2 v -12.1 h 4.6 z m 9.7,22.9 h -5.4 v -22.9 h 5.4 z m 24.5,-22.9 v 22.9 h -2.8 c -0.4,0 -0.7,-0.1 -1,-0.2 -0.3,-0.1 -0.6,-0.4 -0.8,-0.7 l -10.8,-13.7 c 0,0.4 0.1,0.8 0.1,1.2 0,0.4 0,0.7 0,1.1 v 12.3 h -4.7 v -22.9 h 2.8 c 0.2,0 0.4,0 0.6,0 0.2,0 0.3,0.1 0.4,0.1 0.1,0.1 0.2,0.1 0.4,0.2 0.1,0.1 0.2,0.2 0.4,0.4 l 10.9,13.8 c -0.1,-0.4 -0.1,-0.9 -0.1,-1.3 0,-0.4 0,-0.8 0,-1.2 v -12.1 h 4.6 z m 15.5,11 h 8.2 v 9.7 c -1.2,0.9 -2.4,1.5 -3.8,1.9 -1.3,0.4 -2.7,0.6 -4.2,0.6 -1.9,0 -3.6,-0.3 -5.2,-0.9 -1.5,-0.6 -2.9,-1.4 -4,-2.5 -1.1,-1 -2,-2.3 -2.5,-3.7 -0.6,-1.4 -0.9,-3 -0.9,-4.7 0,-1.7 0.3,-3.3 0.9,-4.7 0.6,-1.4 1.4,-2.7 2.4,-3.7 1.1,-1 2.3,-1.8 3.8,-2.4 1.5,-0.6 3.2,-0.9 5,-0.9 1,0 1.9,0.1 2.7,0.2 0.8,0.2 1.6,0.4 2.3,0.6 0.7,0.3 1.4,0.6 1.9,1 0.6,0.4 1.1,0.8 1.6,1.2 l -1.5,2.3 c -0.2,0.4 -0.6,0.6 -0.9,0.7 -0.4,0.1 -0.8,0 -1.2,-0.3 -0.4,-0.3 -0.8,-0.5 -1.2,-0.7 -0.4,-0.2 -0.8,-0.3 -1.1,-0.4 -0.4,-0.1 -0.8,-0.2 -1.2,-0.3 -0.4,-0.1 -0.9,-0.1 -1.4,-0.1 -1,0 -1.9,0.2 -2.7,0.5 -0.8,0.4 -1.5,0.9 -2,1.5 -0.6,0.6 -1,1.4 -1.3,2.3 -0.3,0.9 -0.5,1.9 -0.5,3 0,1.2 0.2,2.3 0.5,3.2 0.3,0.9 0.8,1.7 1.4,2.4 0.6,0.7 1.3,1.1 2.2,1.5 0.9,0.3 1.8,0.5 2.8,0.5 0.6,0 1.2,-0.1 1.7,-0.2 0.5,-0.1 1,-0.3 1.5,-0.5 V 224 h -2.3 c -0.3,0 -0.6,-0.1 -0.8,-0.3 -0.2,-0.2 -0.3,-0.4 -0.3,-0.7 v -2.6 z"
|
||||
id="path8" />
|
||||
<path
|
||||
class="st1"
|
||||
d="m 155.6,253.7 c 0.1,0 0.2,0 0.3,0.1 0.1,0 0.2,0.1 0.3,0.2 l 1.5,1.6 c -0.7,0.9 -1.5,1.5 -2.5,2 -1,0.4 -2.2,0.7 -3.6,0.7 -1.3,0 -2.4,-0.2 -3.4,-0.7 -1,-0.4 -1.9,-1 -2.6,-1.8 -0.7,-0.8 -1.2,-1.7 -1.6,-2.7 -0.4,-1 -0.6,-2.2 -0.6,-3.4 0,-1.2 0.2,-2.4 0.6,-3.4 0.4,-1 1,-1.9 1.7,-2.7 0.7,-0.8 1.6,-1.3 2.6,-1.8 1,-0.4 2.2,-0.6 3.4,-0.6 0.6,0 1.2,0.1 1.8,0.2 0.6,0.1 1.1,0.3 1.6,0.5 0.5,0.2 0.9,0.4 1.3,0.7 0.4,0.3 0.8,0.6 1.1,0.9 l -1.3,1.8 c -0.1,0.1 -0.2,0.2 -0.3,0.3 -0.1,0.1 -0.3,0.1 -0.5,0.1 -0.1,0 -0.3,0 -0.4,-0.1 -0.1,-0.1 -0.3,-0.1 -0.4,-0.2 -0.1,-0.1 -0.3,-0.2 -0.5,-0.3 -0.2,-0.1 -0.4,-0.2 -0.6,-0.3 -0.2,-0.1 -0.5,-0.2 -0.8,-0.2 -0.3,-0.1 -0.7,-0.1 -1.1,-0.1 -0.6,0 -1.2,0.1 -1.8,0.4 -0.5,0.2 -1,0.6 -1.4,1 -0.4,0.5 -0.7,1 -0.9,1.7 -0.2,0.7 -0.3,1.4 -0.3,2.3 0,0.9 0.1,1.6 0.4,2.3 0.2,0.7 0.6,1.2 1,1.7 0.4,0.5 0.9,0.8 1.4,1 0.5,0.2 1.1,0.4 1.7,0.4 0.4,0 0.7,0 1,-0.1 0.3,0 0.6,-0.1 0.8,-0.2 0.3,-0.1 0.5,-0.2 0.7,-0.3 0.2,-0.1 0.4,-0.3 0.7,-0.5 0.1,-0.1 0.2,-0.1 0.3,-0.2 0.2,-0.3 0.3,-0.3 0.4,-0.3 z m 36.1,-4.1 c 0,1.2 -0.2,2.3 -0.6,3.4 -0.4,1 -1,1.9 -1.8,2.7 -0.8,0.8 -1.7,1.4 -2.7,1.8 -1.1,0.4 -2.3,0.7 -3.6,0.7 -1.3,0 -2.5,-0.2 -3.6,-0.7 -1.1,-0.4 -2,-1 -2.7,-1.8 -0.8,-0.8 -1.4,-1.7 -1.8,-2.7 -0.4,-1 -0.6,-2.2 -0.6,-3.4 0,-1.2 0.2,-2.3 0.6,-3.4 0.4,-1 1,-1.9 1.8,-2.7 0.8,-0.8 1.7,-1.4 2.7,-1.8 1.1,-0.4 2.3,-0.7 3.6,-0.7 1.3,0 2.5,0.2 3.6,0.7 1.1,0.4 2,1 2.7,1.8 0.8,0.8 1.3,1.7 1.8,2.7 0.4,1.1 0.6,2.2 0.6,3.4 z m -3.9,0 c 0,-0.8 -0.1,-1.6 -0.3,-2.2 -0.2,-0.7 -0.5,-1.2 -0.9,-1.7 -0.4,-0.5 -0.9,-0.8 -1.5,-1.1 -0.6,-0.2 -1.2,-0.4 -2,-0.4 -0.8,0 -1.4,0.1 -2,0.4 -0.6,0.2 -1.1,0.6 -1.5,1.1 -0.4,0.5 -0.7,1 -0.9,1.7 -0.2,0.7 -0.3,1.4 -0.3,2.2 0,0.8 0.1,1.6 0.3,2.2 0.2,0.7 0.5,1.2 0.9,1.7 0.4,0.5 0.9,0.8 1.5,1 0.6,0.2 1.3,0.4 2,0.4 0.7,0 1.4,-0.1 2,-0.4 0.6,-0.2 1.1,-0.6 1.5,-1 0.4,-0.5 0.7,-1 0.9,-1.7 0.2,-0.6 0.3,-1.3 0.3,-2.2 z m 41,-8.3 V 258 h -3.4 v -9.6 c 0,-0.2 0,-0.5 0,-0.7 0,-0.3 0,-0.5 0.1,-0.8 l -4.4,8.6 c -0.1,0.3 -0.3,0.5 -0.6,0.6 -0.2,0.1 -0.5,0.2 -0.8,0.2 h -0.5 c -0.3,0 -0.6,-0.1 -0.8,-0.2 -0.2,-0.1 -0.4,-0.3 -0.6,-0.6 l -4.4,-8.6 c 0,0.3 0,0.5 0.1,0.8 0,0.3 0,0.5 0,0.7 v 9.6 h -3.4 v -16.7 h 3 c 0.2,0 0.3,0 0.4,0 0.1,0 0.2,0 0.3,0.1 0.1,0 0.2,0.1 0.3,0.2 0.1,0.1 0.2,0.2 0.2,0.3 l 4.3,8.5 c 0.2,0.3 0.3,0.6 0.4,0.9 0.1,0.3 0.3,0.6 0.4,1 0.1,-0.3 0.3,-0.7 0.4,-1 0.1,-0.3 0.3,-0.6 0.5,-0.9 l 4.3,-8.5 c 0.1,-0.1 0.2,-0.3 0.2,-0.3 0.1,-0.1 0.2,-0.1 0.3,-0.2 0.1,0 0.2,-0.1 0.3,-0.1 0.1,0 0.3,0 0.4,0 z m 25,8.2 c 1,0 1.7,-0.2 2.2,-0.7 0.4,-0.5 0.7,-1.2 0.7,-2 0,-0.4 -0.1,-0.7 -0.2,-1 -0.1,-0.3 -0.3,-0.6 -0.5,-0.8 -0.2,-0.2 -0.5,-0.4 -0.9,-0.5 -0.4,-0.1 -0.8,-0.2 -1.3,-0.2 h -2 v 5.3 h 2 z m 0,-8.2 c 1.2,0 2.2,0.1 3,0.4 0.8,0.3 1.5,0.7 2.1,1.2 0.5,0.5 0.9,1.1 1.2,1.7 0.3,0.7 0.4,1.4 0.4,2.2 0,0.8 -0.1,1.6 -0.4,2.3 -0.3,0.7 -0.7,1.3 -1.2,1.8 -0.6,0.5 -1.2,0.9 -2.1,1.2 -0.8,0.3 -1.8,0.4 -3,0.4 h -2 v 5.6 h -3.9 v -16.7 h 5.9 z m 27.9,16.7 h -3.9 v -16.7 h 3.9 z m 29.3,-3.1 v 3.1 h -10.1 v -16.7 h 3.9 v 13.6 z m 25.2,-3.3 -1.6,-4.6 c -0.1,-0.3 -0.2,-0.6 -0.4,-1 -0.1,-0.4 -0.3,-0.8 -0.4,-1.3 -0.1,0.5 -0.2,0.9 -0.4,1.3 -0.1,0.4 -0.3,0.7 -0.4,1 l -1.5,4.6 z m 6.2,6.4 h -3 c -0.3,0 -0.6,-0.1 -0.8,-0.2 -0.2,-0.2 -0.4,-0.4 -0.5,-0.6 l -1,-2.9 h -6.4 l -1,2.9 c -0.1,0.2 -0.2,0.4 -0.5,0.6 -0.2,0.2 -0.5,0.3 -0.8,0.3 h -3 l 6.5,-16.7 h 4 z m 27.8,-13.6 h -4.7 V 258 h -3.9 v -13.7 h -4.7 v -3.1 h 13.3 z m 21.1,13.6 h -3.9 v -16.7 h 3.9 z m 35.5,-8.4 c 0,1.2 -0.2,2.3 -0.6,3.4 -0.4,1 -1,1.9 -1.8,2.7 -0.8,0.8 -1.7,1.4 -2.7,1.8 -1.1,0.4 -2.3,0.7 -3.6,0.7 -1.3,0 -2.5,-0.2 -3.6,-0.7 -1.1,-0.4 -2,-1 -2.7,-1.8 -0.8,-0.8 -1.4,-1.7 -1.8,-2.7 -0.4,-1 -0.6,-2.2 -0.6,-3.4 0,-1.2 0.2,-2.3 0.6,-3.4 0.4,-1 1,-1.9 1.8,-2.7 0.8,-0.8 1.7,-1.4 2.7,-1.8 1.1,-0.4 2.3,-0.7 3.6,-0.7 1.3,0 2.5,0.2 3.6,0.7 1.1,0.4 2,1 2.7,1.8 0.8,0.8 1.3,1.7 1.8,2.7 0.4,1.1 0.6,2.2 0.6,3.4 z m -3.9,0 c 0,-0.8 -0.1,-1.6 -0.3,-2.2 -0.2,-0.7 -0.5,-1.2 -0.9,-1.7 -0.4,-0.5 -0.9,-0.8 -1.5,-1.1 -0.6,-0.2 -1.2,-0.4 -2,-0.4 -0.8,0 -1.4,0.1 -2,0.4 -0.6,0.2 -1.1,0.6 -1.5,1.1 -0.4,0.5 -0.7,1 -0.9,1.7 -0.2,0.7 -0.3,1.4 -0.3,2.2 0,0.8 0.1,1.6 0.3,2.2 0.2,0.7 0.5,1.2 0.9,1.7 0.4,0.5 0.9,0.8 1.5,1 0.6,0.2 1.3,0.4 2,0.4 0.7,0 1.4,-0.1 2,-0.4 0.6,-0.2 1.1,-0.6 1.5,-1 0.4,-0.5 0.7,-1 0.9,-1.7 0.2,-0.6 0.3,-1.3 0.3,-2.2 z m 36.8,-8.3 V 258 h -2 c -0.3,0 -0.5,-0.1 -0.8,-0.1 -0.2,-0.1 -0.4,-0.3 -0.6,-0.5 l -7.9,-10 c 0,0.3 0.1,0.6 0.1,0.9 0,0.3 0,0.5 0,0.8 v 9 h -3.4 v -16.7 h 2 c 0.2,0 0.3,0 0.4,0 0.1,0 0.2,0 0.3,0.1 0.1,0 0.2,0.1 0.3,0.2 0.1,0.1 0.2,0.2 0.3,0.3 l 8,10.1 c 0,-0.3 -0.1,-0.6 -0.1,-0.9 0,-0.3 0,-0.6 0,-0.9 v -8.9 h 3.4 z"
|
||||
id="path10" />
|
||||
</g>
|
||||
</g>
|
||||
</switch>
|
||||
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 16 KiB |
+102
@@ -0,0 +1,102 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import sys
|
||||
|
||||
import tlcpack_sphinx_addon
|
||||
|
||||
# -- General configuration ------------------------------------------------
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../python"))
|
||||
sys.path.insert(0, os.path.abspath("../"))
|
||||
autodoc_mock_imports = ["torch"]
|
||||
|
||||
# General information about the project.
|
||||
project = "web-llm"
|
||||
author = "WebLLM Contributors"
|
||||
copyright = "2023, %s" % author
|
||||
|
||||
# Version information.
|
||||
|
||||
version = "0.2.84"
|
||||
release = "0.2.84"
|
||||
|
||||
extensions = [
|
||||
"sphinx_tabs.tabs",
|
||||
"sphinx_toolbox.collapse",
|
||||
"sphinxcontrib.httpdomain",
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.napoleon",
|
||||
"sphinx_reredirects",
|
||||
]
|
||||
|
||||
redirects = {"get_started/try_out": "../index.html#getting-started"}
|
||||
|
||||
source_suffix = [".rst"]
|
||||
|
||||
language = "en"
|
||||
|
||||
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
|
||||
|
||||
# The name of the Pygments (syntax highlighting) style to use.
|
||||
pygments_style = "sphinx"
|
||||
|
||||
# A list of ignored prefixes for module index sorting.
|
||||
# If true, `todo` and `todoList` produce output, else they produce nothing.
|
||||
todo_include_todos = False
|
||||
|
||||
# -- Options for HTML output ----------------------------------------------
|
||||
|
||||
# The theme is set by the make target
|
||||
import sphinx_rtd_theme
|
||||
|
||||
html_theme = "sphinx_rtd_theme"
|
||||
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
|
||||
|
||||
templates_path = []
|
||||
|
||||
html_static_path = []
|
||||
|
||||
footer_copyright = "© 2023 MLC LLM"
|
||||
footer_note = " "
|
||||
|
||||
html_logo = "_static/img/mlc-logo-with-text-landscape.svg"
|
||||
|
||||
html_theme_options = {
|
||||
"logo_only": True,
|
||||
}
|
||||
|
||||
header_links = [
|
||||
("Home", "https://webllm.mlc.ai/"),
|
||||
("GitHub", "https://github.com/mlc-ai/web-llm"),
|
||||
("Discord", "https://discord.gg/9Xpy2HGBuD"),
|
||||
]
|
||||
|
||||
header_dropdown = {
|
||||
"name": "Other Resources",
|
||||
"items": [
|
||||
("WebLLM Chat", "https://chat.webllm.ai/"),
|
||||
("MLC Course", "https://mlc.ai/"),
|
||||
("MLC Blog", "https://blog.mlc.ai/"),
|
||||
("MLC LLM", "https://llm.mlc.ai/"),
|
||||
],
|
||||
}
|
||||
|
||||
html_context = {
|
||||
"footer_copyright": footer_copyright,
|
||||
"footer_note": footer_note,
|
||||
"header_links": header_links,
|
||||
"header_dropdown": header_dropdown,
|
||||
"display_github": True,
|
||||
"github_user": "mlc-ai",
|
||||
"github_repo": "web-llm",
|
||||
"github_version": "main/docs/",
|
||||
"theme_vcs_pageview_mode": "edit",
|
||||
# "header_logo": "/path/to/logo",
|
||||
# "header_logo_link": "",
|
||||
# "version_selecter": "",
|
||||
}
|
||||
|
||||
|
||||
# add additional overrides
|
||||
templates_path += [tlcpack_sphinx_addon.get_templates_path()]
|
||||
html_static_path += [tlcpack_sphinx_addon.get_static_path()]
|
||||
@@ -0,0 +1,6 @@
|
||||
Adding Models
|
||||
=============
|
||||
|
||||
WebLLM allows you to compile custom language models using `MLC-LLM <https://llm.mlc.ai/>`_ and then serve the compiled model through WebLLM.
|
||||
|
||||
For instructions on how to compile and add custom models to WebLLM, please refer to the `MLC-LLM documentation <https://llm.mlc.ai/docs/deploy/webllm.html>`_.
|
||||
@@ -0,0 +1,35 @@
|
||||
Building From Source
|
||||
====================
|
||||
|
||||
Clone the Repository
|
||||
---------------------
|
||||
.. code-block:: bash
|
||||
|
||||
git clone https://github.com/mlc-ai/web-llm.git
|
||||
cd web-llm
|
||||
|
||||
Install Dependencies
|
||||
---------------------
|
||||
.. code-block:: bash
|
||||
|
||||
npm install
|
||||
|
||||
Build the Project
|
||||
-----------------
|
||||
.. code-block:: bash
|
||||
|
||||
npm run build
|
||||
|
||||
Test Changes
|
||||
------------
|
||||
|
||||
To test your changes, you can reuse an existing example or create a new example that specifically tests the new functionality you wish to provide.
|
||||
|
||||
To test the effects of your code change in an example, inside ``examples/<example>/package.json``, change ``"@mlc-ai/web-llm": "^0.2.xx"`` to ``"@mlc-ai/web-llm": "../.."`` to let it reference your local code. Note that sometimes you may need to switch between ``"file:../.."`` and ``"../.."`` to trigger npm to recognize new changes.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
cd examples/<example>
|
||||
# Modify package.json as described
|
||||
npm install
|
||||
npm start
|
||||
@@ -0,0 +1,35 @@
|
||||
👋 Welcome to WebLLM
|
||||
====================
|
||||
|
||||
`GitHub <https://github.com/mlc-ai/web-llm>`_ | `WebLLM Chat <https://chat.webllm.ai/>`_ | `NPM <https://www.npmjs.com/package/@mlc-ai/web-llm>`_ | `Discord <https://discord.gg/9Xpy2HGBuD>`_
|
||||
|
||||
WebLLM is a high-performance in-browser language model inference engine that brings large language models (LLMs) to web browsers with hardware acceleration. With WebGPU support, it allows developers to build AI-powered applications directly within the browser environment, removing the need for server-side processing and ensuring privacy.
|
||||
|
||||
It provides a specialized runtime for the web backend of MLCEngine, leverages
|
||||
`WebGPU <https://www.w3.org/TR/webgpu/>`_ for local acceleration, offers OpenAI-compatible API,
|
||||
and provides built-in support for web workers to separate heavy computation from the UI flow.
|
||||
|
||||
Key Features
|
||||
------------
|
||||
- 🌐 In-Browser Inference: Run LLMs directly in the browser
|
||||
- 🚀 WebGPU Acceleration: Leverage hardware acceleration for optimal performance
|
||||
- 🔄 OpenAI API Compatibility: Seamless integration with standard AI workflows
|
||||
- 📦 Multiple Model Support: Works with Llama, Phi, Gemma, Mistral, and more
|
||||
|
||||
Start exploring WebLLM by `chatting with WebLLM Chat <https://chat.webllm.ai/>`_, and start building webapps with high-performance local LLM inference with the following guides and tutorials.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: User Guide
|
||||
|
||||
user/get_started.rst
|
||||
user/basic_usage.rst
|
||||
user/advanced_usage.rst
|
||||
user/api_reference.rst
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Developer Guide
|
||||
|
||||
developer/building_from_source.rst
|
||||
developer/add_models.rst
|
||||
@@ -0,0 +1,35 @@
|
||||
@ECHO OFF
|
||||
|
||||
pushd %~dp0
|
||||
|
||||
REM Command file for Sphinx documentation
|
||||
|
||||
if "%SPHINXBUILD%" == "" (
|
||||
set SPHINXBUILD=sphinx-build
|
||||
)
|
||||
set SOURCEDIR=.
|
||||
set BUILDDIR=_build
|
||||
|
||||
%SPHINXBUILD% >NUL 2>NUL
|
||||
if errorlevel 9009 (
|
||||
echo.
|
||||
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
|
||||
echo.installed, then set the SPHINXBUILD environment variable to point
|
||||
echo.to the full path of the 'sphinx-build' executable. Alternatively you
|
||||
echo.may add the Sphinx directory to PATH.
|
||||
echo.
|
||||
echo.If you don't have Sphinx installed, grab it from
|
||||
echo.https://www.sphinx-doc.org/
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
if "%1" == "" goto help
|
||||
|
||||
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
|
||||
goto end
|
||||
|
||||
:help
|
||||
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
|
||||
|
||||
:end
|
||||
popd
|
||||
@@ -0,0 +1,8 @@
|
||||
sphinx-tabs == 3.4.1
|
||||
sphinx-rtd-theme
|
||||
sphinx == 5.2.3
|
||||
sphinx-toolbox == 3.4.0
|
||||
tlcpack-sphinx-addon==0.2.2
|
||||
sphinxcontrib_httpdomain==1.8.1
|
||||
sphinxcontrib-napoleon==0.7
|
||||
sphinx-reredirects==0.1.2
|
||||
@@ -0,0 +1,153 @@
|
||||
Advanced Use Cases
|
||||
==================
|
||||
|
||||
Using Workers
|
||||
-------------
|
||||
|
||||
You can put the heavy computation in a worker script to optimize your application performance. To do so, you need to:
|
||||
|
||||
Create a handler in the worker thread that communicates with the frontend while handling the requests.
|
||||
Create a worker engine in your main application that sends messages to the handler in the worker thread under the hood.
|
||||
For detailed implementations of different kinds of workers, look at the following sections.
|
||||
|
||||
Using Web Workers
|
||||
^^^^^^^^^^^^^^^^^
|
||||
WebLLM comes with API support for `Web Workers <https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers>`_ so you can offload the computation-heavy generation work into a separate worker thread. WebLLM has implemented cross-thread communication through messages under the hood, so manual implementation is not required.
|
||||
|
||||
In the worker script, import and instantiate a ``WebWorkerMLCEngineHandler``, which handles communication with other scripts and processes incoming requests.
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
// worker.ts
|
||||
import { WebWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
|
||||
|
||||
const handler = new WebWorkerMLCEngineHandler();
|
||||
self.onmessage = (msg: MessageEvent) => {
|
||||
handler.onmessage(msg);
|
||||
};
|
||||
|
||||
In the main script, import and instantiate a ``WebWorkerMLCEngine`` that implements the same ``MLCEngineInterface`` and exposes the same APIs. Then, simply use it as you would a normal ``MLCEngine``.
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
import { CreateWebWorkerMLCEngine } from "@mlc-ai/web-llm";
|
||||
|
||||
async function runWorker() {
|
||||
const engine = await CreateWebWorkerMLCEngine(
|
||||
new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }),
|
||||
"Llama-3.1-8B-Instruct"
|
||||
);
|
||||
|
||||
const messages = [{ role: "user", content: "How does WebLLM use workers?" }];
|
||||
const reply = await engine.chat.completions.create({ messages });
|
||||
console.log(reply.choices[0].message.content);
|
||||
}
|
||||
|
||||
runWorker();
|
||||
|
||||
|
||||
Under the hood, ``WebWorkerMLCEngine`` does **not** perform any computation. It translates all calls into messages and sends them to the ``WebWorkerMLCEngineHandler`` for processing. The worker thread receives these messages and processes the actual computation using a hidden engine, and returns the result to the main thread using messages.
|
||||
|
||||
Service Workers
|
||||
^^^^^^^^^^^^^^^
|
||||
WebLLM also supports offloading computation using `Service Workers <https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API>`_. This allows you to avoid reloading the model between page refreshes and optimize your application's offline experience.
|
||||
|
||||
(Note, the lifecycle of a Service Worker is managed by the browser and can be killed any time without notifying the web application. WebLLM's ``ServiceWorkerMLCEngine`` attempts to keep the service worker thread alive by periodically sending heartbeat events. However, the script could still be killed at any time by Chrome, and your application should include proper error handling. Check `keepAliveMs` and `missedHeartbeat` in `ServiceWorkerMLCEngine <https://github.com/mlc-ai/web-llm/blob/main/src/service_worker.ts#L218>`_ for more details.)
|
||||
|
||||
In the worker script, import and instantiate ``ServiceWorkerMLCEngineHandler``, which handles communication with page scripts and processes incoming requests.
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
// sw.ts
|
||||
import { ServiceWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
|
||||
|
||||
self.addEventListener("activate", () => {
|
||||
const handler = new ServiceWorkerMLCEngineHandler();
|
||||
console.log("Service Worker activated!");
|
||||
});
|
||||
|
||||
|
||||
Then, in the main page script, register the service worker and instantiate the engine using the ``CreateServiceWorkerMLCEngine`` factory function that implements the same ``MLCEngineInterface`` and exposes the same APIs. Then, simply use it as you would a normal ``MLCEngine``.
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
// main.ts
|
||||
import { MLCEngineInterface, CreateServiceWorkerMLCEngine } from "@mlc-ai/web-llm";
|
||||
|
||||
if ("serviceWorker" in navigator) {
|
||||
navigator.serviceWorker.register(
|
||||
new URL("sw.ts", import.meta.url), // worker script
|
||||
{ type: "module" },
|
||||
);
|
||||
}
|
||||
|
||||
const engine: MLCEngineInterface =
|
||||
await CreateServiceWorkerMLCEngine(
|
||||
selectedModel,
|
||||
{ initProgressCallback }, // engineConfig
|
||||
);
|
||||
|
||||
Similar to the ``WebWorkerMLCEngine`` above, the ``ServiceWorkerMLCEngine`` is also a proxy and does not perform any actual computation. Instead, it forwards all calls to the service worker thread and receives the result through messages.
|
||||
|
||||
Chrome Extension
|
||||
----------------
|
||||
|
||||
WebLLM can be used in Chrome extensions to empower local LLM inference. You can find examples of building Chrome extension using WebLLM in `examples/chrome-extension <https://github.com/mlc-ai/web-llm/blob/main/examples/chrome-extension>`_ and `examples/chrome-extension-webgpu-service-worker <https://github.com/mlc-ai/web-llm/blob/main/examples/chrome-extension-webgpu-service-worker>`_. The latter leverages Service Worker, so the extension is persistent in the background.
|
||||
|
||||
Additionally, we have a full Chrome extension project, `WebLLM Assistant <https://github.com/mlc-ai/web-llm-assistant>`_, which leverages WebLLM to provide a personal web browsing copilot assistant experience. Feel free to check it out and contribute if you are interested.
|
||||
|
||||
|
||||
Additional Customization
|
||||
------------------------
|
||||
|
||||
Using IndexedDB Cache
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
By default, WebLLM caches model artifacts using the `Cache API <https://developer.mozilla.org/en-US/docs/Web/API/Cache>`_ for faster subsequent model loads. You can alternatively use `IndexedDB caching <https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API>`_ by setting ``appConfig.cacheBackend = "indexeddb"``. When changing only the cache backend, preserve the prebuilt model list by spreading ``prebuiltAppConfig``.
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
import { AppConfig, CreateMLCEngine, prebuiltAppConfig } from "@mlc-ai/web-llm";
|
||||
|
||||
const appConfig: AppConfig = {
|
||||
...prebuiltAppConfig,
|
||||
cacheBackend: "indexeddb",
|
||||
};
|
||||
|
||||
const engine = await CreateMLCEngine("Llama-3.1-8B-Instruct-q4f32_1-MLC", {
|
||||
appConfig,
|
||||
});
|
||||
|
||||
Using Cross-Origin Storage Cache
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
WebLLM also supports caching model artifacts across different origins using the experimental Cross-Origin Storage API. You can enable this cache backend by setting ``appConfig.cacheBackend = "cross-origin"``. For users with the `Cross-Origin Storage browser extension <https://chromewebstore.google.com/detail/cross-origin-storage/denpnpcgjgikjpoglpjefakmdcbmlgih>`_ installed, resources will then be cached and shared across origins. This means two independent apps opted into this cache backend using the same AI model will download and cache the required resources only once. See the `cache usage example <https://github.com/mlc-ai/web-llm/tree/main/examples/cache-usage>`_ for more details. If Cross-Origin Storage isn't available, WebLLM will automatically fall back to using the default cache.
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
import { AppConfig, CreateMLCEngine, prebuiltAppConfig } from "@mlc-ai/web-llm";
|
||||
|
||||
const appConfig: AppConfig = {
|
||||
...prebuiltAppConfig,
|
||||
cacheBackend: "cross-origin",
|
||||
};
|
||||
|
||||
const engine = await CreateMLCEngine("Llama-3.1-8B-Instruct-q4f32_1-MLC", {
|
||||
appConfig,
|
||||
});
|
||||
|
||||
Customizing Token Behavior
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
You can modify `logit_bias` in `GenerationConfig` to control token likelihood. Setting a token's bias to a positive value increases its likelihood of being generated, while a negative value decreases it. A large negative value (e.g., -100) can effectively prevent the token from being generated.
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
const messages = [
|
||||
{ role: "user", content: "Describe WebLLM in detail." },
|
||||
];
|
||||
|
||||
const response = await engine.chatCompletion({
|
||||
messages,
|
||||
logit_bias: { "50256": -100 }, // Example: Prevent specific token generation
|
||||
});
|
||||
@@ -0,0 +1,239 @@
|
||||
.. _api-reference:
|
||||
|
||||
WebLLM API Reference
|
||||
====================
|
||||
|
||||
The ``MLCEngine`` class is the core interface of WebLLM. It enables model loading, chat completions, embeddings, and other operations. Below, we document its methods, along with the associated configuration interfaces.
|
||||
|
||||
Interfaces
|
||||
----------
|
||||
|
||||
The following interfaces are used as parameters or configurations within ``MLCEngine`` methods. They are linked to their respective methods for reference.
|
||||
|
||||
MLCEngineConfig
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
Optional configurations for ``CreateMLCEngine()`` and ``CreateWebWorkerMLCEngine()``.
|
||||
|
||||
|
||||
- **Fields**:
|
||||
- ``appConfig``: Configure the app, including the list of models and whether to use IndexedDB cache.
|
||||
- ``initProgressCallback``: A callback for showing model loading progress.
|
||||
- ``logitProcessorRegistry``: A registry for stateful logit processors (see ``webllm.LogitProcessor``).
|
||||
|
||||
|
||||
- **Usage**:
|
||||
- ``appConfig``: Contains application-specific settings, including:
|
||||
- Model configurations.
|
||||
- IndexedDB caching preferences.
|
||||
- ``initProgressCallback``: Allows developers to visualize model loading progress by implementing a callback.
|
||||
- ``logitProcessorRegistry``: A ``Map`` object for registering custom logit processors. Only applies to ``MLCEngine``.
|
||||
|
||||
|
||||
.. note:: All fields are optional, and ``logitProcessorRegistry`` is only used in ``MLCEngine``.
|
||||
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
const engine = await CreateMLCEngine("Llama-3.1-8B-Instruct", {
|
||||
appConfig: { /* app-specific config */ },
|
||||
initProgressCallback: (progress) => console.log(progress),
|
||||
});
|
||||
|
||||
|
||||
GenerationConfig
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
Configurations for a single generation task, primarily used in chat completions.
|
||||
|
||||
- **Fields**:
|
||||
- ``repetition_penalty``, ``ignore_eos``: Parameters specific to MLC models.
|
||||
- ``top_p``, ``temperature``, ``max_tokens``, ``stop``: Common parameters shared with OpenAI APIs.
|
||||
- ``frequency_penalty``, ``presence_penalty``: Tune repetition behavior following OpenAI semantics.
|
||||
- ``logit_bias``, ``n``, ``logprobs``, ``top_logprobs``: Advanced sampling controls.
|
||||
- ``response_format``, ``enable_thinking``, ``enable_latency_breakdown``: Additional OpenAI-style request features.
|
||||
|
||||
- **Usage**:
|
||||
- Fields like ``repetition_penalty`` and ``ignore_eos`` give explicit control over repetition handling and whether the model stops at the EOS token, respectively.
|
||||
- Common parameters shared with OpenAI APIs (e.g., ``temperature``, ``top_p``) ensure compatibility while still falling back to the values configured during ``MLCEngine.reload()`` when omitted.
|
||||
- ``frequency_penalty`` and ``presence_penalty`` mirror OpenAI's bounds ``[-2, 2]``; providing only one will default the other to ``0``.
|
||||
- ``response_format`` (for JSON or other schema outputs), ``enable_thinking``, and ``enable_latency_breakdown`` pass through directly to the engine and surface enhanced telemetry or structured responses when the underlying model supports them.
|
||||
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
const messages = [
|
||||
{ role: "system", content: "You are a helpful assistant." },
|
||||
{ role: "user", content: "Explain WebLLM." },
|
||||
];
|
||||
|
||||
const response = await engine.chatCompletion({
|
||||
messages,
|
||||
top_p: 0.9,
|
||||
temperature: 0.8,
|
||||
max_tokens: 150,
|
||||
});
|
||||
|
||||
ChatConfig
|
||||
^^^^^^^^^^
|
||||
|
||||
Model's baseline configuration loaded from ``mlc-chat-config.json`` when ``MLCEngine.reload()`` runs. ``ChatOptions`` (and therefore the ``chatOpts`` argument to ``reload``) can override any subset of these fields.
|
||||
|
||||
- **Fields** (subset):
|
||||
- ``tokenizer_files``, ``tokenizer_info``: Files and parameters required to initialize the tokenizer.
|
||||
- ``conv_template``, ``conv_config``: Conversation templates that define prompts, separators, and role formatting.
|
||||
- ``context_window_size``, ``sliding_window_size``, ``attention_sink_size``: KV-cache and memory settings.
|
||||
- Default generation knobs such as ``repetition_penalty``, ``frequency_penalty``, ``presence_penalty``, ``top_p``, and ``temperature``.
|
||||
|
||||
- **Usage**:
|
||||
- Loaded automatically for each model; provides defaults that ``GenerationConfig`` falls back to when fields are omitted.
|
||||
- Override selected values per model load by supplying ``chatOpts`` (``Partial<ChatConfig>``) to ``MLCEngine.reload()``.
|
||||
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
await engine.reload("Llama-3.1-8B-Instruct", {
|
||||
temperature: 0.7,
|
||||
repetition_penalty: 1.1,
|
||||
context_window_size: 4096,
|
||||
});
|
||||
|
||||
ChatCompletionRequest
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Defines the structure for chat completion requests.
|
||||
|
||||
- **Base Interface**: ``ChatCompletionRequestBase``
|
||||
- Contains parameters such as ``messages``, ``stream``, ``frequency_penalty``, and ``presence_penalty``.
|
||||
- **Sub-interfaces**:
|
||||
- ``ChatCompletionRequestNonStreaming``: For non-streaming completions.
|
||||
- ``ChatCompletionRequestStreaming``: For streaming completions.
|
||||
|
||||
- **Usage**:
|
||||
- Combines settings from ``GenerationConfig`` and ``ChatCompletionRequestBase`` to provide complete control over chat behavior.
|
||||
- The ``stream`` parameter enables streaming responses, improving interactivity in conversational agents.
|
||||
- The ``logit_bias`` feature allows controlling token generation probabilities, providing a mechanism to restrict or encourage specific outputs.
|
||||
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
const response = await engine.chatCompletion({
|
||||
messages: [
|
||||
{ role: "user", content: "Tell me about WebLLM." },
|
||||
],
|
||||
stream: true,
|
||||
});
|
||||
|
||||
Model Loading
|
||||
-------------
|
||||
|
||||
``MLCEngine.reload(modelId: string | string[], chatOpts?: ChatOptions | ChatOptions[]): Promise<void>``
|
||||
|
||||
Loads the specified model(s) into the engine. Uses ``MLCEngineConfig`` during initialization.
|
||||
|
||||
- Parameters:
|
||||
- ``modelId``: Identifier(s) for the model(s) to load.
|
||||
- ``chatOpts``: Configuration for generation (see ``ChatConfig``).
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
await engine.reload(["Llama-3.1-8B", "Gemma-2B"], [
|
||||
{ temperature: 0.7 },
|
||||
{ top_p: 0.9 },
|
||||
]);
|
||||
|
||||
``MLCEngine.unload(): Promise<void>``
|
||||
|
||||
Unloads all loaded models and clears their associated configurations.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
await engine.unload();
|
||||
|
||||
---
|
||||
|
||||
Chat Completions
|
||||
----------------
|
||||
|
||||
``MLCEngine.chat.completions.create(request: ChatCompletionRequest): Promise<ChatCompletion | AsyncIterable<ChatCompletionChunk>>``
|
||||
|
||||
Generates chat-based completions using a specified request configuration.
|
||||
|
||||
- Parameters:
|
||||
- ``request``: A ``ChatCompletionRequest`` instance.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
const response = await engine.chat.completions.create({
|
||||
messages: [
|
||||
{ role: "system", content: "You are a helpful AI assistant." },
|
||||
{ role: "user", content: "What is WebLLM?" },
|
||||
],
|
||||
temperature: 0.8,
|
||||
stream: false,
|
||||
});
|
||||
|
||||
---
|
||||
|
||||
Utility Methods
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
``MLCEngine.getMessage(modelId?: string): Promise<string>``
|
||||
|
||||
Retrieves the current output message from the specified model.
|
||||
|
||||
- Parameters:
|
||||
- ``modelId``: (Optional) Identifier of model to query. Omitting modelId only works when the engine currently has a single model loaded.
|
||||
|
||||
``MLCEngine.resetChat(keepStats?: boolean, modelId?: string): Promise<void>``
|
||||
|
||||
Resets the chat history and optionally retains usage statistics.
|
||||
|
||||
- Parameters:
|
||||
- ``keepStats``: (Optional) If true, retains usage statistics.
|
||||
- ``modelId``: (Optional) Identifier of the model to reset. Omitting modelId only works when the engine currently has a single model loaded.
|
||||
|
||||
GPU Information
|
||||
----------------
|
||||
|
||||
The following methods provide detailed information about the GPU used for WebLLM computations.
|
||||
|
||||
``MLCEngine.getGPUVendor(): Promise<string>``
|
||||
|
||||
Retrieves the vendor name of the GPU used for computations. This is useful for understanding hardware capabilities during inference.
|
||||
|
||||
- **Returns**: A string indicating the GPU vendor (e.g., "Intel", "NVIDIA").
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
const gpuVendor = await engine.getGPUVendor();
|
||||
console.log(``GPU Vendor: ${gpuVendor}``);
|
||||
|
||||
``MLCEngine.getMaxStorageBufferBindingSize(): Promise<number>``
|
||||
|
||||
Returns the maximum storage buffer size supported by the GPU. This is important when working with larger models that require significant memory for processing.
|
||||
|
||||
- **Returns**: A number representing the maximum size in bytes.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
const maxBufferSize = await engine.getMaxStorageBufferBindingSize();
|
||||
console.log(``Max Storage Buffer Binding Size: ${maxBufferSize}``);
|
||||
@@ -0,0 +1,120 @@
|
||||
Basic Usage
|
||||
================
|
||||
|
||||
Model Records in WebLLM
|
||||
-----------------------
|
||||
|
||||
Each of the model available WebLLM is registered as an instance of
|
||||
``ModelRecord`` and can be accessed at
|
||||
`webllm.prebuiltAppConfig.model_list <https://github.com/mlc-ai/web-llm/blob/main/src/config.ts#L313>`__.
|
||||
|
||||
Creating an MLCEngine
|
||||
---------------------
|
||||
|
||||
WebLLM APIs are exposed through the ``MLCEngine`` interface. You can create an ``MLCEngine`` instance and load the model by calling the CreateMLCEngine() factory function.
|
||||
|
||||
(Note that loading models requires downloading and it can take a significant amount of time for the very first run without previous caching. You should properly handle this asynchronous call.)
|
||||
|
||||
``MLCEngine`` can be instantiated in two ways:
|
||||
1. Using the factory function ``CreateMLCEngine``.
|
||||
2. Instantiating the ``MLCEngine`` class directly and using ``reload()`` to load models.
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
import { CreateMLCEngine, MLCEngine } from "@mlc-ai/web-llm";
|
||||
|
||||
// Initialize with a progress callback
|
||||
const initProgressCallback = (progress) => {
|
||||
console.log("Model loading progress:", progress);
|
||||
};
|
||||
|
||||
// Using CreateMLCEngine
|
||||
const engine = await CreateMLCEngine("Llama-3.1-8B-Instruct", { initProgressCallback });
|
||||
|
||||
// Direct instantiation
|
||||
const engineInstance = new MLCEngine({ initProgressCallback });
|
||||
await engineInstance.reload("Llama-3.1-8B-Instruct");
|
||||
|
||||
Under the hood, this factory function ``CreateMLCEngine`` does the following steps for first creating an engine instance (synchronous) and then loading the model (asynchronous). You can also do them separately in your application.
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
import { MLCEngine } from "@mlc-ai/web-llm";
|
||||
|
||||
// This is a synchronous call that returns immediately
|
||||
const engine = new MLCEngine({
|
||||
initProgressCallback: initProgressCallback
|
||||
});
|
||||
|
||||
// This is an asynchronous call and can take a long time to finish
|
||||
await engine.reload(selectedModel);
|
||||
|
||||
|
||||
Chat Completion
|
||||
---------------
|
||||
|
||||
Chat completions can be invoked using OpenAI style chat APIs through the ``engine.chat.completions`` interface of an initialized ``MLCEngine``. For the full list of parameters and their descriptions, check :ref:`api-reference` for full list of parameters.
|
||||
|
||||
(Note: Since the model is determined during ``MLCEngine`` instantiation, the ``model`` parameter is not supported and will be **ignored**. Instead, call ``CreateMLCEngine(model)`` or ``engine.reload(model)`` to reinitialize the engine to use a specific model.)
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
const messages = [
|
||||
{ role: "system", content: "You are a helpful AI assistant." },
|
||||
{ role: "user", content: "Hello!" }
|
||||
];
|
||||
|
||||
const reply = await engine.chat.completions.create({
|
||||
messages,
|
||||
});
|
||||
|
||||
console.log(reply.choices[0].message);
|
||||
console.log(reply.usage);
|
||||
|
||||
|
||||
Streaming Chat Completion
|
||||
-------------------------
|
||||
|
||||
Streaming chat completion could be enabled by passsing ``stream: true`` parameter to the `engine.chat.completions.create` call configuration. Check :ref:`api-reference` for full list of parameters.
|
||||
|
||||
.. code-block:: typescript
|
||||
|
||||
const messages = [
|
||||
{ role: "system", content: "You are a helpful AI assistant." },
|
||||
{ role: "user", content: "Hello!" },
|
||||
]
|
||||
|
||||
// chunks is an AsyncGenerator object
|
||||
const chunks = await engine.chat.completions.create({
|
||||
messages,
|
||||
temperature: 1,
|
||||
stream: true, // <-- Enable streaming
|
||||
stream_options: { include_usage: true },
|
||||
});
|
||||
|
||||
let reply = "";
|
||||
for await (const chunk of chunks) {
|
||||
reply += chunk.choices[0]?.delta.content || "";
|
||||
console.log(reply);
|
||||
if (chunk.usage) {
|
||||
console.log(chunk.usage); // only last chunk has usage
|
||||
}
|
||||
}
|
||||
|
||||
const fullReply = await engine.getMessage();
|
||||
console.log(fullReply);
|
||||
|
||||
|
||||
Chatbot Examples
|
||||
----------------
|
||||
|
||||
Learn how to use WebLLM to integrate large language models into your applications and generate chat completions through this simple Chatbot example:
|
||||
|
||||
- `Example in JSFiddle <https://jsfiddle.net/neetnestor/4nmgvsa2/>`_
|
||||
- `Example in CodePen <https://codepen.io/neetnestor/pen/vYwgZaG>`_
|
||||
|
||||
For an advanced example of a larger, more complicated project, look at `WebLLM Chat <https://github.com/mlc-ai/web-llm-chat/blob/main/app/client/webllm.ts>`_.
|
||||
|
||||
More examples for different use cases are available in the `WebLLM examples folder <https://github.com/mlc-ai/web-llm/tree/main/examples>`_.
|
||||
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
Getting Started with WebLLM
|
||||
===========================
|
||||
|
||||
This guide will help you set up WebLLM in your project, install necessary dependencies, and verify your setup.
|
||||
|
||||
|
||||
WebLLM Chat
|
||||
-----------
|
||||
|
||||
If you want to experience AI Chat supported by local LLM inference and understand how WebLLM works, try out `WebLLM Chat <https://chat.webllm.ai/>`__, which provides a great example
|
||||
of integrating WebLLM into a full web application.
|
||||
|
||||
A WebGPU-compatible browser is needed to run WebLLM-powered web applications.
|
||||
You can download the latest Google Chrome and use `WebGPU Report <https://webgpureport.org/>`__
|
||||
to verify the functionality of WebGPU on your browser.
|
||||
|
||||
Installation
|
||||
------------
|
||||
|
||||
WebLLM offers a minimalist and modular interface to access the chatbot in the browser. The package is designed in a modular way to hook to any of the UI components.
|
||||
|
||||
WebLLM is available as an `npm package <https://www.npmjs.com/package/@mlc-ai/web-llm>`_ and is also CDN-delivered. Therefore, you can install WebLLM using Node.js package managers like npm, yarn, or pnpm, or directly import the pacakge via CDN.
|
||||
|
||||
Using Package Managers
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
Install WebLLM via your preferred package manager:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# npm
|
||||
npm install @mlc-ai/web-llm
|
||||
# yarn
|
||||
yarn add @mlc-ai/web-llm
|
||||
# pnpm
|
||||
pnpm install @mlc-ai/web-llm
|
||||
|
||||
Import WebLLM into your project:
|
||||
|
||||
.. code-block:: javascript
|
||||
|
||||
// Import everything
|
||||
import * as webllm from "@mlc-ai/web-llm";
|
||||
|
||||
// Or only import what you need
|
||||
import { CreateMLCEngine } from "@mlc-ai/web-llm";
|
||||
|
||||
Using CDN
|
||||
^^^^^^^^^
|
||||
Thanks to `jsdelivr.com <https://www.jsdelivr.com/package/npm/@mlc-ai/web-llm>`_, WebLLM can be imported directly through URL and work out-of-the-box on cloud development platforms like `jsfiddle.net <https://jsfiddle.net/>`_, `Codepen.io <https://codepen.io/>`_, and `Scribbler <https://scribbler.live/>`_:
|
||||
|
||||
.. code-block:: javascript
|
||||
|
||||
import * as webllm from "https://esm.run/@mlc-ai/web-llm";
|
||||
|
||||
This method is especially useful for online environments like CodePen, JSFiddle, or local experiments.
|
||||
|
||||
Verifying Installation
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
Run the following script to verify the installation:
|
||||
|
||||
.. code-block:: javascript
|
||||
|
||||
import { CreateMLCEngine } from "@mlc-ai/web-llm";
|
||||
console.log("WebLLM loaded successfully!");
|
||||
|
||||
|
||||
Online IDE Sandbox
|
||||
------------------
|
||||
|
||||
Instead of setting WebLLM locally, you can also try it on online Javascript IDE sandboxes like:
|
||||
|
||||
- `Example in JSFiddle <https://jsfiddle.net/neetnestor/4nmgvsa2/>`_
|
||||
- `Example in CodePen <https://codepen.io/neetnestor/pen/vYwgZaG>`_
|
||||
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
const {
|
||||
defineConfig,
|
||||
globalIgnores,
|
||||
} = require("eslint/config");
|
||||
|
||||
const tsParser = require("@typescript-eslint/parser");
|
||||
const typescriptEslint = require("@typescript-eslint/eslint-plugin");
|
||||
const js = require("@eslint/js");
|
||||
|
||||
const {
|
||||
FlatCompat,
|
||||
} = require("@eslint/eslintrc");
|
||||
|
||||
const compat = new FlatCompat({
|
||||
baseDirectory: __dirname,
|
||||
recommendedConfig: js.configs.recommended,
|
||||
allConfig: js.configs.all
|
||||
});
|
||||
|
||||
module.exports = defineConfig([{
|
||||
extends: compat.extends(
|
||||
"eslint:recommended",
|
||||
"plugin:@typescript-eslint/recommended",
|
||||
"plugin:prettier/recommended",
|
||||
),
|
||||
|
||||
languageOptions: {
|
||||
parser: tsParser,
|
||||
},
|
||||
|
||||
plugins: {
|
||||
"@typescript-eslint": typescriptEslint,
|
||||
},
|
||||
|
||||
rules: {
|
||||
"@typescript-eslint/no-explicit-any": "off",
|
||||
"@typescript-eslint/no-empty-function": "off",
|
||||
"@typescript-eslint/no-non-null-assertion": "off",
|
||||
},
|
||||
}, {
|
||||
files: ["examples/**/*.js", "examples/**/*.ts"],
|
||||
|
||||
"rules": {
|
||||
"no-undef": "off",
|
||||
"@typescript-eslint/no-unused-vars": "off",
|
||||
},
|
||||
}, globalIgnores([
|
||||
"**/dist",
|
||||
"**/debug",
|
||||
"**/lib",
|
||||
"**/build",
|
||||
"**/node_modules",
|
||||
"**/3rdparty",
|
||||
"**/.eslintrc.cjs",
|
||||
"**/.next",
|
||||
])]);
|
||||
@@ -0,0 +1 @@
|
||||
package-lock.json
|
||||
@@ -0,0 +1,57 @@
|
||||
# Awesome WebLLM
|
||||
|
||||
This page contains a curated list of examples, tutorials, blogs about WebLLM usecases.
|
||||
Please send a pull request if you find things that belong here.
|
||||
|
||||
## Example Projects
|
||||
|
||||
Note that all examples below run in-browser and use WebGPU as a backend.
|
||||
|
||||
#### Project List
|
||||
|
||||
- [get-started](get-started): minimum get started example with chat completion.
|
||||
|
||||
[](https://jsfiddle.net/neetnestor/yac9gbwf/)
|
||||
[](https://codepen.io/neetnestor/pen/NWVdgey)
|
||||
|
||||
- [simple-chat-js](simple-chat-js): a mininum and complete chat bot app in vanilla JavaScript.
|
||||
|
||||
[](https://jsfiddle.net/neetnestor/4nmgvsa2/)
|
||||
[](https://codepen.io/neetnestor/pen/vYwgZaG)
|
||||
|
||||
- [simple-chat-ts](simple-chat-ts): a mininum and complete chat bot app in TypeScript.
|
||||
- [get-started-web-worker](get-started-web-worker): same as get-started, but using web worker.
|
||||
- [next-simple-chat](next-simple-chat): a mininum and complete chat bot app with [Next.js](https://nextjs.org/).
|
||||
- [subgroups-usage](subgroups-usage): capability-based routing between baseline and subgroup WebGPU WASM builds.
|
||||
- [multi-round-chat](multi-round-chat): while APIs are functional, we internally optimize so that multi round chat usage can reuse KV cache
|
||||
- [text-completion](text-completion): demonstrates API `engine.completions.create()`, which is pure text completion with no conversation, as opposed to `engine.chat.completions.create()`
|
||||
- [embeddings](embeddings): demonstrates API `engine.embeddings.create()`, integration with `EmbeddingsInterface` and `MemoryVectorStore` of [Langchain.js](https://js.langchain.com), and RAG with Langchain.js using WebLLM for both LLM and Embedding in a single engine
|
||||
- [multi-models](multi-models): demonstrates loading multiple models in a single engine concurrently
|
||||
|
||||
#### Advanced OpenAI API Capabilities
|
||||
|
||||
These examples demonstrate various capabilities via WebLLM's OpenAI-like API.
|
||||
|
||||
- [streaming](streaming): return output as chunks in real-time in the form of an AsyncGenerator
|
||||
- [json-mode](json-mode): efficiently ensure output is in json format, see [OpenAI Reference](https://platform.openai.com/docs/guides/text-generation/chat-completions-api) for more.
|
||||
- [json-schema](json-schema): besides guaranteeing output to be in JSON, ensure output to adhere to a specific JSON schema specified the user
|
||||
- [seed-to-reproduce](seed-to-reproduce): use seeding to ensure reproducible output with fields `seed`.
|
||||
- [function-calling](function-calling) (WIP): function calling with fields `tools` and `tool_choice` (with preliminary support).
|
||||
- [vision-model](vision-model): process request with image input using Vision Language Model (e.g. Phi3.5-vision)
|
||||
|
||||
#### Chrome Extension
|
||||
|
||||
- [chrome-extension](chrome-extension): chrome extension that does not have a persistent background
|
||||
- [chrome-extension-webgpu-service-worker](chrome-extension-webgpu-service-worker): chrome extension using service worker, hence having a persistent background
|
||||
|
||||
#### Others
|
||||
|
||||
- [logit-processor](logit-processor): while `logit_bias` is supported, we additionally support stateful logit processing where users can specify their own rules. We also expose low-level API `forwardTokensAndSample()`.
|
||||
- [cache-usage](cache-usage): demonstrates how WebLLM supports multiple cache backends. Choose between the [Cache API](https://developer.mozilla.org/en-US/docs/Web/API/Cache), [IndexedDB cache](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API), [OPFS](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system), or the experimental Chrome [Cross-Origin Storage](https://github.com/WICG/cross-origin-storage) extension via `appConfig.cacheBackend`. Also demonstrates various cache utils such as checking
|
||||
whether a model is cached, deleting a model's weights from cache, deleting a model library wasm from cache, etc. Note: cross-origin backend currently does not support programmatic tensor-cache deletion.
|
||||
- [simple-chat-upload](simple-chat-upload): demonstrates how to upload local models to WebLLM instead of downloading via a URL link
|
||||
|
||||
## Demo Spaces
|
||||
|
||||
- [web-llm-embed](https://huggingface.co/spaces/matthoffner/web-llm-embed): document chat prototype using react-llm with transformers.js embeddings
|
||||
- [DeVinci](https://x6occ-biaaa-aaaai-acqzq-cai.icp0.io/): AI chat app based on WebLLM and hosted on decentralized cloud platform
|
||||
@@ -0,0 +1,13 @@
|
||||
# WebLLM Get Started App
|
||||
|
||||
This folder provides a demo for cancelling model fetching after calling `engine.reload()`.
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm start
|
||||
```
|
||||
|
||||
Note if you would like to hack WebLLM core package.
|
||||
You can change web-llm dependencies as `"file:../.."`, and follow the build from source
|
||||
instruction in the project to build webllm locally. This option is only recommended
|
||||
if you would like to hack WebLLM core package.
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "get-started",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "parcel src/get_started.html --port 8887",
|
||||
"build": "parcel build src/get_started.html --dist-dir lib"
|
||||
},
|
||||
"devDependencies": {
|
||||
"buffer": "^5.7.1",
|
||||
"parcel": "^2.8.3",
|
||||
"process": "^0.11.10",
|
||||
"tslib": "^2.3.1",
|
||||
"typescript": "^4.9.5",
|
||||
"url": "^0.11.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mlc-ai/web-llm": "^0.2.84"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<script>
|
||||
webLLMGlobal = {};
|
||||
</script>
|
||||
<body>
|
||||
<h2>WebLLM Test Page</h2>
|
||||
Open console to see output
|
||||
<br />
|
||||
<br />
|
||||
<label id="init-label"> </label>
|
||||
|
||||
<h3>Prompt</h3>
|
||||
<label id="prompt-label"> </label>
|
||||
|
||||
<h3>Response</h3>
|
||||
<label id="generate-label"> </label>
|
||||
<br />
|
||||
<label id="stats-label"> </label>
|
||||
|
||||
<script type="module" src="./get_started.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,32 @@
|
||||
import * as webllm from "@mlc-ai/web-llm";
|
||||
import { error } from "loglevel";
|
||||
|
||||
let engine;
|
||||
|
||||
function setLabel(id, text) {
|
||||
const label = document.getElementById(id);
|
||||
if (label == null) {
|
||||
throw Error("Cannot find label " + id);
|
||||
}
|
||||
label.innerText = text;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const initProgressCallback = (report) => {
|
||||
console.log(report.text);
|
||||
setLabel("init-label", report.text);
|
||||
};
|
||||
// Option 1: If we do not specify appConfig, we use `prebuiltAppConfig` defined in `config.ts`
|
||||
const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
|
||||
engine = new webllm.MLCEngine({
|
||||
initProgressCallback,
|
||||
});
|
||||
engine.reload(selectedModel);
|
||||
}
|
||||
main();
|
||||
setTimeout(() => {
|
||||
console.log("calling unload");
|
||||
engine.unload().catch((err) => {
|
||||
console.log(err);
|
||||
});
|
||||
}, 5000);
|
||||
@@ -0,0 +1,30 @@
|
||||
# WebLLM Cache Usage
|
||||
|
||||
WebLLM supports multiple persistent cache backends. You can pick the classic Cache API, IndexedDB, [Origin Private File System (OPFS)](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system), or the experimental Chrome [Cross-Origin Storage](https://github.com/WICG/cross-origin-storage) extension by
|
||||
setting `AppConfig.cacheBackend` to `"cache"`, `"indexeddb"`, `"opfs"`, or `"cross-origin"`.
|
||||
This folder provides an example on how different caches are used in WebLLM. We also
|
||||
demonstrate the utility cache functions such as deleting models, checking if models are in cache, etc.
|
||||
|
||||
> **Note:** The cross-origin backend requires installation of the [Cross-Origin Storage browser extension](https://chromewebstore.google.com/detail/cross-origin-storage/denpnpcgjgikjpoglpjefakmdcbmlgih) ([source code](https://github.com/web-ai-community/cross-origin-storage-extension)). This does not currently support programmatic tensor-cache deletion; deletion is extension-managed.
|
||||
|
||||
> **Note:** If `"opfs"` is selected in an environment without OPFS support, cache operations fail with an OPFS availability error.
|
||||
> Use `appConfig.opfsAccessMode = "auto"` to use OPFS sync access handles where supported, or `"sync"` to require sync access handles. The default is `"async"`.
|
||||
|
||||
For more information about Cache API and IndexedDB, see:
|
||||
https://developer.mozilla.org/en-US/docs/Web/API/Storage_API/Storage_quotas_and_eviction_criteria#what_technologies_store_data_in_the_browser.
|
||||
|
||||
To inspect the downloaded artifacts in your browser, open up developer console, go to application,
|
||||
and you will find the artifacts under `IndexedDB`, `Cache storage`, or OPFS (under the browser's origin-private file system). When `"cross-origin"` is selected,
|
||||
the extension displays origins and resource hashes.
|
||||
|
||||
To run the example, you can do the following steps under this folder
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm start
|
||||
```
|
||||
|
||||
Note if you would like to hack WebLLM core package.
|
||||
You can change web-llm dependencies as `"file:../.."`, and follow the build from source
|
||||
instruction in the project to build webllm locally. This option is only recommended
|
||||
if you would like to hack WebLLM core package.
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "cache-usage",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "parcel src/cache_usage.html --port 8889",
|
||||
"build": "parcel build src/cache_usage.html --dist-dir lib"
|
||||
},
|
||||
"devDependencies": {
|
||||
"buffer": "^5.7.1",
|
||||
"parcel": "2.8.3",
|
||||
"process": "^0.11.10",
|
||||
"tslib": "^2.3.1",
|
||||
"typescript": "^4.9.5",
|
||||
"url": "^0.11.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mlc-ai/web-llm": "^0.2.84"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<script>
|
||||
webLLMGlobal = {};
|
||||
</script>
|
||||
|
||||
<body>
|
||||
<h2>WebLLM Test Page</h2>
|
||||
Open console to see output
|
||||
<br />
|
||||
<br />
|
||||
<label id="init-label"> </label>
|
||||
|
||||
<h3>Prompt</h3>
|
||||
<label id="prompt-label"> </label>
|
||||
|
||||
<h3>Response</h3>
|
||||
<label id="generate-label"> </label>
|
||||
<br />
|
||||
<label id="stats-label"> </label>
|
||||
|
||||
<script type="module" src="./cache_usage.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,90 @@
|
||||
import * as webllm from "@mlc-ai/web-llm";
|
||||
|
||||
function setLabel(id: string, text: string) {
|
||||
const label = document.getElementById(id);
|
||||
if (label == null) {
|
||||
throw Error("Cannot find label " + id);
|
||||
}
|
||||
label.innerText = text;
|
||||
}
|
||||
|
||||
const initProgressCallback = (report: webllm.InitProgressReport) => {
|
||||
setLabel("init-label", report.text);
|
||||
};
|
||||
|
||||
async function main() {
|
||||
const appConfig = webllm.prebuiltAppConfig;
|
||||
// CHANGE THIS TO SEE THE EFFECTS OF EACH, CODE BELOW DOES NOT NEED TO CHANGE
|
||||
appConfig.cacheBackend = "cache"; // "indexeddb" or "cache" or "cross-origin" or "opfs"
|
||||
|
||||
const cacheBackend = appConfig.cacheBackend as string;
|
||||
if (cacheBackend === "indexeddb") {
|
||||
console.log("Using IndexedDB Cache");
|
||||
} else if (cacheBackend === "cache") {
|
||||
console.log("Using Cache API");
|
||||
} else if (cacheBackend === "cross-origin") {
|
||||
console.log("Using Cross-Origin Storage");
|
||||
} else if (cacheBackend === "opfs") {
|
||||
console.log("Using Origin Private File System");
|
||||
}
|
||||
|
||||
// 1. This triggers downloading and caching the model with either Cache or IndexedDB Cache
|
||||
const selectedModel = "Llama-3.2-1B-Instruct-q4f16_1-MLC";
|
||||
const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
|
||||
selectedModel,
|
||||
{ initProgressCallback: initProgressCallback, appConfig: appConfig },
|
||||
);
|
||||
|
||||
const request: webllm.ChatCompletionRequest = {
|
||||
stream: false,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: "Write an analogy between mathematics and a lighthouse.",
|
||||
},
|
||||
],
|
||||
n: 1,
|
||||
temperature: 0,
|
||||
};
|
||||
let reply = await engine.chat.completions.create(request);
|
||||
console.log(reply);
|
||||
|
||||
// 2. Check whether model weights are cached
|
||||
let modelCached = await webllm.hasModelInCache(selectedModel, appConfig);
|
||||
console.log("hasModelInCache: ", modelCached);
|
||||
if (!modelCached) {
|
||||
throw Error("Expect hasModelInCache() to be true, but got: " + modelCached);
|
||||
}
|
||||
|
||||
// 3. We reload, and we should see this time it is much faster because the weights are cached.
|
||||
console.log("Reload model start");
|
||||
await engine.reload(selectedModel);
|
||||
console.log("Reload model end");
|
||||
reply = await engine.chat.completions.create(request);
|
||||
console.log(reply);
|
||||
|
||||
// Cross-Origin Storage does not support deletion
|
||||
if (cacheBackend === "cross-origin") {
|
||||
return;
|
||||
}
|
||||
|
||||
// 4. Delete everything about this model from cache
|
||||
// You can also delete only the model library wasm, only the model weights, or only the config file
|
||||
await webllm.deleteModelAllInfoInCache(selectedModel, appConfig);
|
||||
modelCached = await webllm.hasModelInCache(selectedModel, appConfig);
|
||||
console.log("After deletion, hasModelInCache: ", modelCached);
|
||||
if (modelCached) {
|
||||
throw Error(
|
||||
"Expect hasModelInCache() to be false, but got: " + modelCached,
|
||||
);
|
||||
}
|
||||
|
||||
// 5. If we reload, we should expect the model to start downloading again
|
||||
console.log("Reload model start");
|
||||
await engine.reload(selectedModel);
|
||||
console.log("Reload model end");
|
||||
reply = await engine.chat.completions.create(request);
|
||||
console.log(reply);
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,29 @@
|
||||
# WebLLM Chrome Extension using WebGPU Running on Service Worker
|
||||
|
||||
|
||||
|
||||
> [!WARNING]
|
||||
> Service worker support in WebGPU is enabled by default in [Chrome 124](https://chromiumdash.appspot.com/commit/8d78510e4aca5ac3cd8ee4a33e96b404eaa43246).
|
||||
> If you are using Chrome 123, go to `chrome://flags/#enable-experimental-web-platform-features`, enable the `#enable-experimental-web-platform-features` flag, and **relaunch the browser**.
|
||||
|
||||
This example shows how we can create a Chrome extension using WebGPU and service worker.
|
||||
|
||||
- The project structure is as follows:
|
||||
- `manifest.json`: A required file that lists important information about the structure and behavior of that extension. Here we are using manifest V3.
|
||||
- `popup.ts`: Script of the extension pop-up window.
|
||||
- `background.ts`: Script of the service worker. An extension service worker is loaded when it is needed, and unloaded when it goes dormant.
|
||||
- `content.js`: Content script that interacts with DOM.
|
||||
- Run
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
This will create a new directory at `./dist/`. To load the extension into Chrome, go to Extensions > Manage Extensions and select Load Unpacked. Add the `./dist/` directory. You can now pin the extension to your toolbar and use it to chat with your favorite model!
|
||||
|
||||
**Note**: This example disables chatting using the contents of the active tab by default.
|
||||
To enable it, set `useContext` in `popup.ts` to `true`. More info about this feature can be found
|
||||
[here](https://github.com/mlc-ai/web-llm/pull/190).
|
||||
However, if the web content is too large, it might run into issues. We recommend using `example.html` to
|
||||
test this feature.
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "chrome-extension",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "parcel build src/manifest.json --config @parcel/config-webextension"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"devDependencies": {
|
||||
"@parcel/config-webextension": "^2.9.3",
|
||||
"@types/chrome": "^0.0.242",
|
||||
"buffer": "^6.0.3",
|
||||
"parcel": "^2.9.3",
|
||||
"process": "^0.11.10",
|
||||
"url": "^0.11.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mlc-ai/web-llm": "^0.2.84",
|
||||
"progressbar.js": "^1.1.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { ExtensionServiceWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
|
||||
|
||||
// Hookup an engine to a service worker handler
|
||||
let handler;
|
||||
|
||||
chrome.runtime.onConnect.addListener(function (port) {
|
||||
console.assert(port.name === "web_llm_service_worker");
|
||||
if (handler === undefined) {
|
||||
handler = new ExtensionServiceWorkerMLCEngineHandler(port);
|
||||
} else {
|
||||
handler.setPort(port);
|
||||
}
|
||||
port.onMessage.addListener(handler.onmessage.bind(handler));
|
||||
});
|
||||
@@ -0,0 +1,6 @@
|
||||
// Only the content script is able to access the DOM
|
||||
chrome.runtime.onConnect.addListener(function (port) {
|
||||
port.onMessage.addListener(function (msg) {
|
||||
port.postMessage({ contents: document.body.innerHTML });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
In the year 2154, humanity had colonized several planets in the distant reaches
|
||||
of the galaxy. The planet of Xylophia-IV was one of the most remote and
|
||||
inhospitable, with temperatures often dropping to -200 degrees Celsius. Despite
|
||||
these harsh conditions, a team of scientists had established a research station
|
||||
on the planet to study the unique geological formations and exotic flora and
|
||||
fauna. One day, while conducting a routine survey of the planet's surface, the
|
||||
team discovered an strange object buried deep in the ice. As they examined it
|
||||
closer, they realized it was a small, metallic capsule with a glowing blue
|
||||
symbol etched onto its surface. The team's leader, a brilliant scientist named
|
||||
Dr. Maria Rodriguez, was immediately intrigued by the capsule's mysterious
|
||||
origins. She ordered her team to bring it back to the research station for
|
||||
further analysis. After weeks of studying the capsule, the team finally cracked
|
||||
the code to the symbol etched onto its surface. It was a message from an alien
|
||||
race, warning Earth of an impending attack from an unknown threat. The team was
|
||||
shocked and dismayed by the news, but they knew they had to act quickly to warn
|
||||
the rest of humanity. They transmitted the message to the nearest space station,
|
||||
which relayed it to Earth's government. As the threat of attack loomed near, the
|
||||
team remained on high alert, ready to face whatever dangers lay ahead. They had
|
||||
uncovered a secrets of the universe, and now they were determined to protect
|
||||
their planet and its inhabitants at all costs.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 3.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 581 B |
Binary file not shown.
|
After Width: | Height: | Size: 1.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.9 KiB |
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"manifest_version": 3,
|
||||
"name": "MLCBot",
|
||||
"version": "0.1.0",
|
||||
"description": "Chat with your browser",
|
||||
"icons": {
|
||||
"16": "icons/icon-16.png",
|
||||
"32": "icons/icon-32.png",
|
||||
"64": "icons/icon-64.png",
|
||||
"128": "icons/icon-128.png"
|
||||
},
|
||||
"content_security_policy": {
|
||||
"extension_pages": "style-src-elem 'self' https://cdnjs.cloudflare.com; font-src 'self' https://cdnjs.cloudflare.com; script-src 'self' 'wasm-unsafe-eval'; default-src 'self' data:; connect-src 'self' data: http://localhost:8000 https://huggingface.co https://cdn-lfs.huggingface.co https://cdn-lfs-us-1.huggingface.co https://raw.githubusercontent.com https://cdn-lfs-us-1.hf.co https://cas-bridge.xethub.hf.co"
|
||||
},
|
||||
"action": {
|
||||
"default_title": "MLCBot",
|
||||
"default_popup": "popup.html"
|
||||
},
|
||||
"content_scripts": [
|
||||
{
|
||||
"matches": ["<all_urls>"],
|
||||
"js": ["content.js"]
|
||||
}
|
||||
],
|
||||
"background": {
|
||||
"service_worker": "background.ts",
|
||||
"type": "module"
|
||||
},
|
||||
"permissions": ["storage", "tabs", "webNavigation"]
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
*,
|
||||
*::before,
|
||||
*::after {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
html {
|
||||
font-family:
|
||||
-apple-system,
|
||||
BlinkMacSystemFont,
|
||||
Segoe UI,
|
||||
Helvetica,
|
||||
Arial,
|
||||
sans-serif;
|
||||
color: #222;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 0.5rem;
|
||||
background-color: #778da9;
|
||||
width: 320px;
|
||||
font-size: small;
|
||||
}
|
||||
|
||||
p {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
/* LOADING BAR */
|
||||
#loadingContainer {
|
||||
margin-bottom: 15px;
|
||||
width: 300px;
|
||||
height: 8px;
|
||||
}
|
||||
|
||||
/* INPUT AREA */
|
||||
#query-input {
|
||||
border: 1px solid #ccc;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.input-container {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.input-container input {
|
||||
width: 100%;
|
||||
outline: none;
|
||||
padding: 0.5rem;
|
||||
margin-right: 0.5rem;
|
||||
}
|
||||
|
||||
/* SUBMIT BUTTON */
|
||||
.btn {
|
||||
background-color: #1b263b;
|
||||
color: white;
|
||||
font-size: small;
|
||||
cursor: pointer;
|
||||
border-radius: 4px;
|
||||
border: none;
|
||||
padding: 0.5rem;
|
||||
}
|
||||
|
||||
.btn:hover {
|
||||
background-color: #d0d0d0;
|
||||
}
|
||||
|
||||
.btn:disabled {
|
||||
background-color: #a7a7a7;
|
||||
color: rgb(255, 255, 255);
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
.btn img {
|
||||
width: 1rem;
|
||||
height: 1rem;
|
||||
}
|
||||
|
||||
/* LOADING */
|
||||
|
||||
.stage {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
position: relative;
|
||||
margin: 0 -5%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
#loading-indicator {
|
||||
display: none;
|
||||
color: white;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
.dot-flashing {
|
||||
position: relative;
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 5px;
|
||||
background-color: #1b263b;
|
||||
color: #1b263b;
|
||||
animation: dot-flashing 0.4s infinite linear alternate;
|
||||
animation-delay: 0.2s;
|
||||
}
|
||||
|
||||
.dot-flashing::before,
|
||||
.dot-flashing::after {
|
||||
content: "";
|
||||
display: inline-block;
|
||||
position: absolute;
|
||||
top: 0;
|
||||
}
|
||||
|
||||
.dot-flashing::before {
|
||||
left: -15px;
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 5px;
|
||||
background-color: #1b263b;
|
||||
color: #1b263b;
|
||||
animation: dot-flashing 0.4s infinite alternate;
|
||||
animation-delay: 0s;
|
||||
}
|
||||
|
||||
.dot-flashing::after {
|
||||
left: 15px;
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 5px;
|
||||
background-color: #1b263b;
|
||||
color: #1b263b;
|
||||
animation: dot-flashing 0.4s infinite alternate;
|
||||
animation-delay: 0.4s;
|
||||
}
|
||||
|
||||
@keyframes dot-flashing {
|
||||
0% {
|
||||
background-color: #1b263b;
|
||||
}
|
||||
|
||||
50%,
|
||||
100% {
|
||||
background-color: #415a77;
|
||||
}
|
||||
}
|
||||
|
||||
/* ANSWERS */
|
||||
#queriesAnswersContainer {
|
||||
display: block;
|
||||
color: white;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
#answer {
|
||||
color: #333333;
|
||||
}
|
||||
|
||||
#answerWrapper {
|
||||
display: none;
|
||||
background-color: #ffd166;
|
||||
border-radius: 8px;
|
||||
padding: 0.5rem;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
.queriesAnswers {
|
||||
border-radius: 8px;
|
||||
background-color: #ffd166;
|
||||
padding: 0.5rem;
|
||||
color: #333333;
|
||||
}
|
||||
|
||||
#lastQuery {
|
||||
color: rgb(188, 188, 188);
|
||||
}
|
||||
|
||||
#lastAnswer {
|
||||
color: white;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
#lastRequest {
|
||||
padding: 0.5rem;
|
||||
margin-top: 0.5rem;
|
||||
background-color: #333333;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
/* ANSWER OPTIONS */
|
||||
.timeStamp {
|
||||
color: #9a8c98;
|
||||
}
|
||||
|
||||
.copyRow {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
align-items: end;
|
||||
justify-content: space-between;
|
||||
color: #a7a7a7;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
.copyText {
|
||||
display: none;
|
||||
color: #a7a7a7;
|
||||
margin-right: 0.5rem;
|
||||
}
|
||||
|
||||
.copyButton {
|
||||
color: #415a77;
|
||||
background-color: transparent;
|
||||
border: none;
|
||||
cursor: pointer;
|
||||
padding: 0;
|
||||
margin-left: 0.5rem;
|
||||
}
|
||||
|
||||
.copyButton:hover {
|
||||
color: #5e80a7;
|
||||
background-color: transparent;
|
||||
}
|
||||
|
||||
.removeButton {
|
||||
color: #415a77;
|
||||
background-color: transparent;
|
||||
border: none;
|
||||
cursor: pointer;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.removeButton:hover {
|
||||
color: #5e80a7;
|
||||
background-color: transparent;
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<title>Chatbot</title>
|
||||
<link rel="stylesheet" href="popup.css" />
|
||||
<link
|
||||
rel="stylesheet"
|
||||
href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css"
|
||||
/>
|
||||
</head>
|
||||
<body>
|
||||
<div id="loadingContainer"></div>
|
||||
|
||||
<div class="input-container form-group">
|
||||
<input
|
||||
type="search"
|
||||
id="query-input"
|
||||
placeholder="What's on your mind?"
|
||||
/>
|
||||
<button id="submit-button" class="btn">
|
||||
<i class="fa fa-comments"></i>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="stage">
|
||||
<div id="loading-indicator" class="dot-flashing"></div>
|
||||
</div>
|
||||
|
||||
<div id="answerWrapper">
|
||||
<div id="answer"></div>
|
||||
<div class="copyRow">
|
||||
<span id="timestamp"></span>
|
||||
<button
|
||||
id="copyAnswer"
|
||||
class="btn copyButton"
|
||||
title="Copy the Answer to the Clipboard"
|
||||
>
|
||||
<i class="fa-solid fa-copy fa-lg"></i>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script type="module" src="./popup.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,167 @@
|
||||
"use strict";
|
||||
|
||||
// This code is partially adapted from the openai-chatgpt-chrome-extension repo:
|
||||
// https://github.com/jessedi0n/openai-chatgpt-chrome-extension
|
||||
|
||||
import "./popup.css";
|
||||
|
||||
import {
|
||||
ChatCompletionMessageParam,
|
||||
CreateExtensionServiceWorkerMLCEngine,
|
||||
MLCEngineInterface,
|
||||
InitProgressReport,
|
||||
} from "@mlc-ai/web-llm";
|
||||
import { ProgressBar, Line } from "progressbar.js";
|
||||
|
||||
/***************** UI elements *****************/
|
||||
// Whether or not to use the content from the active tab as the context
|
||||
const useContext = false;
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
const queryInput = document.getElementById("query-input")!;
|
||||
const submitButton = document.getElementById("submit-button")!;
|
||||
|
||||
let isLoadingParams = false;
|
||||
|
||||
(<HTMLButtonElement>submitButton).disabled = true;
|
||||
|
||||
const progressBar: ProgressBar = new Line("#loadingContainer", {
|
||||
strokeWidth: 4,
|
||||
easing: "easeInOut",
|
||||
duration: 1400,
|
||||
color: "#ffd166",
|
||||
trailColor: "#eee",
|
||||
trailWidth: 1,
|
||||
svgStyle: { width: "100%", height: "100%" },
|
||||
});
|
||||
|
||||
/***************** Web-LLM MLCEngine Configuration *****************/
|
||||
const initProgressCallback = (report: InitProgressReport) => {
|
||||
progressBar.animate(report.progress, {
|
||||
duration: 50,
|
||||
});
|
||||
if (report.progress == 1.0) {
|
||||
enableInputs();
|
||||
}
|
||||
};
|
||||
|
||||
const engine: MLCEngineInterface = await CreateExtensionServiceWorkerMLCEngine(
|
||||
"Qwen2-0.5B-Instruct-q4f16_1-MLC",
|
||||
{ initProgressCallback: initProgressCallback },
|
||||
);
|
||||
const chatHistory: ChatCompletionMessageParam[] = [];
|
||||
|
||||
isLoadingParams = true;
|
||||
|
||||
function enableInputs() {
|
||||
if (isLoadingParams) {
|
||||
sleep(500);
|
||||
(<HTMLButtonElement>submitButton).disabled = false;
|
||||
const loadingBarContainer = document.getElementById("loadingContainer")!;
|
||||
loadingBarContainer.remove();
|
||||
queryInput.focus();
|
||||
isLoadingParams = false;
|
||||
}
|
||||
}
|
||||
|
||||
/***************** Event Listeners *****************/
|
||||
|
||||
// Disable submit button if input field is empty
|
||||
queryInput.addEventListener("keyup", () => {
|
||||
if ((<HTMLInputElement>queryInput).value === "") {
|
||||
(<HTMLButtonElement>submitButton).disabled = true;
|
||||
} else {
|
||||
(<HTMLButtonElement>submitButton).disabled = false;
|
||||
}
|
||||
});
|
||||
|
||||
// If user presses enter, click submit button
|
||||
queryInput.addEventListener("keyup", (event) => {
|
||||
if (event.code === "Enter") {
|
||||
event.preventDefault();
|
||||
submitButton.click();
|
||||
}
|
||||
});
|
||||
|
||||
// Listen for clicks on submit button
|
||||
async function handleClick() {
|
||||
// Get the message from the input field
|
||||
const message = (<HTMLInputElement>queryInput).value;
|
||||
console.log("message", message);
|
||||
chatHistory.push({ role: "user", content: message });
|
||||
|
||||
// Clear the answer
|
||||
document.getElementById("answer")!.innerHTML = "";
|
||||
// Hide the answer
|
||||
document.getElementById("answerWrapper")!.style.display = "none";
|
||||
// Show the loading indicator
|
||||
document.getElementById("loading-indicator")!.style.display = "block";
|
||||
|
||||
// Send the chat completion message to the engine
|
||||
let curMessage = "";
|
||||
const completion = await engine.chat.completions.create({
|
||||
stream: true,
|
||||
messages: chatHistory,
|
||||
});
|
||||
|
||||
// Update the answer as the model generates more text
|
||||
for await (const chunk of completion) {
|
||||
const curDelta = chunk.choices[0].delta.content;
|
||||
if (curDelta) {
|
||||
curMessage += curDelta;
|
||||
}
|
||||
updateAnswer(curMessage);
|
||||
}
|
||||
chatHistory.push({ role: "assistant", content: await engine.getMessage() });
|
||||
}
|
||||
|
||||
submitButton.addEventListener("click", handleClick);
|
||||
|
||||
function updateAnswer(answer: string) {
|
||||
// Show answer
|
||||
document.getElementById("answerWrapper")!.style.display = "block";
|
||||
const answerWithBreaks = answer.replace(/\n/g, "<br>");
|
||||
document.getElementById("answer")!.innerHTML = answerWithBreaks;
|
||||
// Add event listener to copy button
|
||||
document.getElementById("copyAnswer")!.addEventListener("click", () => {
|
||||
// Get the answer text
|
||||
const answerText = answer;
|
||||
// Copy the answer text to the clipboard
|
||||
navigator.clipboard
|
||||
.writeText(answerText)
|
||||
.then(() => console.log("Answer text copied to clipboard"))
|
||||
.catch((err) => console.error("Could not copy text: ", err));
|
||||
});
|
||||
const options: Intl.DateTimeFormatOptions = {
|
||||
month: "short",
|
||||
day: "2-digit",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
};
|
||||
const time = new Date().toLocaleString("en-US", options);
|
||||
// Update timestamp
|
||||
document.getElementById("timestamp")!.innerText = time;
|
||||
// Hide loading indicator
|
||||
document.getElementById("loading-indicator")!.style.display = "none";
|
||||
}
|
||||
|
||||
function fetchPageContents() {
|
||||
chrome.tabs.query({ currentWindow: true, active: true }, function (tabs) {
|
||||
if (tabs[0]?.id) {
|
||||
const port = chrome.tabs.connect(tabs[0].id, { name: "channelName" });
|
||||
port.postMessage({});
|
||||
port.onMessage.addListener(function (msg) {
|
||||
console.log("Page contents:", msg.contents);
|
||||
chrome.runtime.sendMessage({ context: msg.contents });
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Grab the page contents when the popup is opened
|
||||
window.onload = function () {
|
||||
if (useContext) {
|
||||
fetchPageContents();
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,12 @@
|
||||
# WebLLM Chrome Extension
|
||||
|
||||
|
||||
|
||||
To run the extension, do the following steps under this folder
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
This will create a new directory at `chrome-extension/dist/`. To load the extension into Chrome, go to Extensions > Manage Extensions and select Load Unpacked. Add the `chrome-extension/dist/` directory. You can now pin the extension to your toolbar and use the drop-down menu to chat with your favorite model!
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "chrome-extension",
|
||||
"version": "1.0.1",
|
||||
"description": "",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "parcel build src/manifest.json --config @parcel/config-webextension"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"devDependencies": {
|
||||
"@parcel/config-webextension": "^2.9.3",
|
||||
"@types/chrome": "^0.0.242",
|
||||
"buffer": "^6.0.3",
|
||||
"parcel": "^2.9.3",
|
||||
"process": "^0.11.10",
|
||||
"url": "^0.11.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mlc-ai/web-llm": "^0.2.84",
|
||||
"progressbar.js": "^1.1.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
// Only the content script is able to access the DOM
|
||||
chrome.runtime.onConnect.addListener(function (port) {
|
||||
port.onMessage.addListener(function (msg) {
|
||||
port.postMessage({ contents: document.body.innerText });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
In the year 2154, humanity had colonized several planets in the distant reaches
|
||||
of the galaxy. The planet of Xylophia-IV was one of the most remote and
|
||||
inhospitable, with temperatures often dropping to -200 degrees Celsius. Despite
|
||||
these harsh conditions, a team of scientists had established a research station
|
||||
on the planet to study the unique geological formations and exotic flora and
|
||||
fauna. One day, while conducting a routine survey of the planet's surface, the
|
||||
team discovered an strange object buried deep in the ice. As they examined it
|
||||
closer, they realized it was a small, metallic capsule with a glowing blue
|
||||
symbol etched onto its surface. The team's leader, a brilliant scientist named
|
||||
Dr. Maria Rodriguez, was immediately intrigued by the capsule's mysterious
|
||||
origins. She ordered her team to bring it back to the research station for
|
||||
further analysis. After weeks of studying the capsule, the team finally cracked
|
||||
the code to the symbol etched onto its surface. It was a message from an alien
|
||||
race, warning Earth of an impending attack from an unknown threat. The team was
|
||||
shocked and dismayed by the news, but they knew they had to act quickly to warn
|
||||
the rest of humanity. They transmitted the message to the nearest space station,
|
||||
which relayed it to Earth's government. As the threat of attack loomed near, the
|
||||
team remained on high alert, ready to face whatever dangers lay ahead. They had
|
||||
uncovered a secrets of the universe, and now they were determined to protect
|
||||
their planet and its inhabitants at all costs.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 3.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 581 B |
Binary file not shown.
|
After Width: | Height: | Size: 1.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.9 KiB |
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"manifest_version": 3,
|
||||
"name": "MLCBot",
|
||||
"version": "0.1.1",
|
||||
"description": "Chat with your browser",
|
||||
"icons": {
|
||||
"16": "icons/icon-16.png",
|
||||
"32": "icons/icon-32.png",
|
||||
"64": "icons/icon-64.png",
|
||||
"128": "icons/icon-128.png"
|
||||
},
|
||||
"content_security_policy": {
|
||||
"extension_pages": "style-src-elem 'self' https://cdnjs.cloudflare.com; font-src 'self' https://cdnjs.cloudflare.com; script-src 'self' 'wasm-unsafe-eval'; default-src 'self' data:; connect-src 'self' data: http://localhost:8000 https://huggingface.co https://cdn-lfs.huggingface.co https://cdn-lfs-us-1.huggingface.co https://raw.githubusercontent.com https://cdn-lfs-us-1.hf.co https://cas-bridge.xethub.hf.co"
|
||||
},
|
||||
"action": {
|
||||
"default_title": "MLCBot",
|
||||
"default_popup": "popup.html"
|
||||
},
|
||||
"content_scripts": [
|
||||
{
|
||||
"matches": ["<all_urls>"],
|
||||
"js": ["content.js"]
|
||||
}
|
||||
],
|
||||
"permissions": ["storage", "tabs", "webNavigation", "activeTab", "scripting"],
|
||||
"host_permissions": ["http://*/", "https://*/"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"manifest_version": 2,
|
||||
"name": "MLCBot",
|
||||
"version": "0.1.0",
|
||||
"description": "Chat with your browser",
|
||||
"icons": {
|
||||
"16": "icons/icon-16.png",
|
||||
"32": "icons/icon-32.png",
|
||||
"64": "icons/icon-64.png",
|
||||
"128": "icons/icon-128.png"
|
||||
},
|
||||
"content_security_policy": "style-src-elem 'self' https://cdnjs.cloudflare.com; font-src 'self' https://cdnjs.cloudflare.com; script-src 'self' 'unsafe-eval' 'wasm-unsafe-eval'; default-src 'self' data:; connect-src 'self' data: http://localhost:8000 https://huggingface.co https://cdn-lfs.huggingface.co https://raw.githubusercontent.com https://cdn-lfs-us-1.hf.co https://cas-bridge.xethub.hf.co",
|
||||
"browser_action": {
|
||||
"default_popup": "popup.html"
|
||||
},
|
||||
"content_scripts": [
|
||||
{
|
||||
"matches": ["<all_urls>"],
|
||||
"js": ["content.js"]
|
||||
}
|
||||
],
|
||||
"permissions": ["storage", "tabs", "webNavigation", "activeTab"]
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
*,
|
||||
*::before,
|
||||
*::after {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
html {
|
||||
font-family:
|
||||
-apple-system,
|
||||
BlinkMacSystemFont,
|
||||
Segoe UI,
|
||||
Helvetica,
|
||||
Arial,
|
||||
sans-serif;
|
||||
color: #222;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 0.5rem;
|
||||
background-color: #778da9;
|
||||
width: 335px;
|
||||
font-size: small;
|
||||
}
|
||||
|
||||
p {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
/* LOADING BAR */
|
||||
#loadingContainer {
|
||||
margin-bottom: 15px;
|
||||
width: 315px;
|
||||
height: 8px;
|
||||
}
|
||||
|
||||
/* INPUT AREA */
|
||||
#query-input {
|
||||
border: 1px solid #ccc;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.input-container {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.input-container input {
|
||||
width: 100%;
|
||||
outline: none;
|
||||
padding: 0.5rem;
|
||||
margin-right: 0.5rem;
|
||||
}
|
||||
|
||||
/* BUTTON */
|
||||
.btn {
|
||||
background-color: #1b263b;
|
||||
color: white;
|
||||
font-size: small;
|
||||
cursor: pointer;
|
||||
border-radius: 4px;
|
||||
border: none;
|
||||
padding: 0.5rem;
|
||||
}
|
||||
|
||||
.btn:hover {
|
||||
background-color: #d0d0d0;
|
||||
}
|
||||
|
||||
.btn:disabled {
|
||||
background-color: #a7a7a7;
|
||||
color: rgb(255, 255, 255);
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
.btn img {
|
||||
width: 1rem;
|
||||
height: 1rem;
|
||||
}
|
||||
|
||||
/* LOADING */
|
||||
|
||||
.stage {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
position: relative;
|
||||
margin: 0 -5%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
#loading-indicator {
|
||||
display: none;
|
||||
color: white;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
.dot-flashing {
|
||||
position: relative;
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 5px;
|
||||
background-color: #1b263b;
|
||||
color: #1b263b;
|
||||
animation: dot-flashing 0.4s infinite linear alternate;
|
||||
animation-delay: 0.2s;
|
||||
}
|
||||
|
||||
.dot-flashing::before,
|
||||
.dot-flashing::after {
|
||||
content: "";
|
||||
display: inline-block;
|
||||
position: absolute;
|
||||
top: 0;
|
||||
}
|
||||
|
||||
.dot-flashing::before {
|
||||
left: -15px;
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 5px;
|
||||
background-color: #1b263b;
|
||||
color: #1b263b;
|
||||
animation: dot-flashing 0.4s infinite alternate;
|
||||
animation-delay: 0s;
|
||||
}
|
||||
|
||||
.dot-flashing::after {
|
||||
left: 15px;
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 5px;
|
||||
background-color: #1b263b;
|
||||
color: #1b263b;
|
||||
animation: dot-flashing 0.4s infinite alternate;
|
||||
animation-delay: 0.4s;
|
||||
}
|
||||
|
||||
@keyframes dot-flashing {
|
||||
0% {
|
||||
background-color: #1b263b;
|
||||
}
|
||||
|
||||
50%,
|
||||
100% {
|
||||
background-color: #415a77;
|
||||
}
|
||||
}
|
||||
|
||||
/* ANSWERS */
|
||||
#queriesAnswersContainer {
|
||||
display: block;
|
||||
color: white;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
#answer {
|
||||
color: #333333;
|
||||
}
|
||||
|
||||
#answerWrapper {
|
||||
display: none;
|
||||
background-color: #ffd166;
|
||||
border-radius: 8px;
|
||||
padding: 0.5rem;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
.queriesAnswers {
|
||||
border-radius: 8px;
|
||||
background-color: #ffd166;
|
||||
padding: 0.5rem;
|
||||
color: #333333;
|
||||
}
|
||||
|
||||
#lastQuery {
|
||||
color: rgb(188, 188, 188);
|
||||
}
|
||||
|
||||
#lastAnswer {
|
||||
color: white;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
#lastRequest {
|
||||
padding: 0.5rem;
|
||||
margin-top: 0.5rem;
|
||||
background-color: #333333;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
/* ANSWER OPTIONS */
|
||||
.timeStamp {
|
||||
color: #9a8c98;
|
||||
}
|
||||
|
||||
.copyRow {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
align-items: end;
|
||||
justify-content: space-between;
|
||||
color: #a7a7a7;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
.copyText {
|
||||
display: none;
|
||||
color: #a7a7a7;
|
||||
margin-right: 0.5rem;
|
||||
}
|
||||
|
||||
.copyButton {
|
||||
color: #415a77;
|
||||
background-color: transparent;
|
||||
border: none;
|
||||
cursor: pointer;
|
||||
padding: 0;
|
||||
margin-left: 0.5rem;
|
||||
}
|
||||
|
||||
.copyButton:hover {
|
||||
color: #5e80a7;
|
||||
background-color: transparent;
|
||||
}
|
||||
|
||||
.removeButton {
|
||||
color: #415a77;
|
||||
background-color: transparent;
|
||||
border: none;
|
||||
cursor: pointer;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.removeButton:hover {
|
||||
color: #5e80a7;
|
||||
background-color: transparent;
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<title>Chatbot</title>
|
||||
<link rel="stylesheet" href="popup.css" />
|
||||
<link
|
||||
rel="stylesheet"
|
||||
href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css"
|
||||
/>
|
||||
</head>
|
||||
<body>
|
||||
<select id="model-selection"></select>
|
||||
<div id="loadingBox">
|
||||
<p id="init-label">Initializing model...</p>
|
||||
<div id="loadingContainer"></div>
|
||||
</div>
|
||||
<p id="model-name"></p>
|
||||
<div class="input-container form-group">
|
||||
<input
|
||||
type="search"
|
||||
id="query-input"
|
||||
placeholder="What's on your mind?"
|
||||
/>
|
||||
<button id="submit-button" class="btn">
|
||||
<i class="fa fa-comments"></i>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="stage">
|
||||
<div id="loading-indicator" class="dot-flashing"></div>
|
||||
</div>
|
||||
|
||||
<div id="answerWrapper">
|
||||
<div id="answer"></div>
|
||||
<div class="copyRow">
|
||||
<span id="timestamp"></span>
|
||||
<button
|
||||
id="copyAnswer"
|
||||
class="btn copyButton"
|
||||
title="Copy the Answer to the Clipboard"
|
||||
>
|
||||
<i class="fa-solid fa-copy fa-lg"></i>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script type="module" src="./popup.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,298 @@
|
||||
"use strict";
|
||||
|
||||
// This code is partially adapted from the openai-chatgpt-chrome-extension repo:
|
||||
// https://github.com/jessedi0n/openai-chatgpt-chrome-extension
|
||||
|
||||
import "./popup.css";
|
||||
|
||||
import {
|
||||
MLCEngineInterface,
|
||||
InitProgressReport,
|
||||
CreateMLCEngine,
|
||||
ChatCompletionMessageParam,
|
||||
prebuiltAppConfig,
|
||||
} from "@mlc-ai/web-llm";
|
||||
import { ProgressBar, Line } from "progressbar.js";
|
||||
|
||||
// modified setLabel to not throw error
|
||||
function setLabel(id: string, text: string) {
|
||||
const label = document.getElementById(id);
|
||||
if (label != null) {
|
||||
label.innerText = text;
|
||||
}
|
||||
}
|
||||
|
||||
function getElementAndCheck(id: string): HTMLElement {
|
||||
const element = document.getElementById(id);
|
||||
if (element == null) {
|
||||
throw Error("Cannot find element " + id);
|
||||
}
|
||||
return element;
|
||||
}
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
const queryInput = getElementAndCheck("query-input")!;
|
||||
const submitButton = getElementAndCheck("submit-button")!;
|
||||
const modelName = getElementAndCheck("model-name");
|
||||
|
||||
let context = "";
|
||||
let modelDisplayName = "";
|
||||
|
||||
// throws runtime.lastError if you refresh extension AND try to access a webpage that is already open
|
||||
fetchPageContents();
|
||||
|
||||
(<HTMLButtonElement>submitButton).disabled = true;
|
||||
|
||||
let progressBar: ProgressBar = new Line("#loadingContainer", {
|
||||
strokeWidth: 4,
|
||||
easing: "easeInOut",
|
||||
duration: 1400,
|
||||
color: "#ffd166",
|
||||
trailColor: "#eee",
|
||||
trailWidth: 1,
|
||||
svgStyle: { width: "100%", height: "100%" },
|
||||
});
|
||||
|
||||
let isLoadingParams = true;
|
||||
|
||||
let initProgressCallback = (report: InitProgressReport) => {
|
||||
setLabel("init-label", report.text);
|
||||
progressBar.animate(report.progress, {
|
||||
duration: 50,
|
||||
});
|
||||
if (report.progress == 1.0) {
|
||||
enableInputs();
|
||||
}
|
||||
};
|
||||
|
||||
// initially selected model
|
||||
let selectedModel = "Qwen2-0.5B-Instruct-q4f16_1-MLC";
|
||||
|
||||
// populate model-selection
|
||||
const modelSelector = getElementAndCheck(
|
||||
"model-selection",
|
||||
) as HTMLSelectElement;
|
||||
for (let i = 0; i < prebuiltAppConfig.model_list.length; ++i) {
|
||||
const model = prebuiltAppConfig.model_list[i];
|
||||
const opt = document.createElement("option");
|
||||
opt.value = model.model_id;
|
||||
opt.innerHTML = model.model_id;
|
||||
opt.selected = false;
|
||||
|
||||
// set initial selection as the initially selected model
|
||||
if (model.model_id == selectedModel) {
|
||||
opt.selected = true;
|
||||
}
|
||||
|
||||
modelSelector.appendChild(opt);
|
||||
}
|
||||
|
||||
modelName.innerText = "Loading initial model...";
|
||||
const engine: MLCEngineInterface = await CreateMLCEngine(selectedModel, {
|
||||
initProgressCallback: initProgressCallback,
|
||||
});
|
||||
modelName.innerText = "Now chatting with " + modelDisplayName;
|
||||
|
||||
let chatHistory: ChatCompletionMessageParam[] = [];
|
||||
|
||||
function enableInputs() {
|
||||
if (isLoadingParams) {
|
||||
sleep(500);
|
||||
isLoadingParams = false;
|
||||
}
|
||||
|
||||
// remove loading bar and loading bar descriptors, if exists
|
||||
const initLabel = document.getElementById("init-label");
|
||||
initLabel?.remove();
|
||||
const loadingBarContainer = document.getElementById("loadingContainer")!;
|
||||
loadingBarContainer?.remove();
|
||||
queryInput.focus();
|
||||
|
||||
const modelNameArray = selectedModel.split("-");
|
||||
modelDisplayName = modelNameArray[0];
|
||||
let j = 1;
|
||||
while (j < modelNameArray.length && modelNameArray[j][0] != "q") {
|
||||
modelDisplayName = modelDisplayName + "-" + modelNameArray[j];
|
||||
j++;
|
||||
}
|
||||
}
|
||||
|
||||
let requestInProgress = false;
|
||||
|
||||
// Disable submit button if input field is empty
|
||||
queryInput.addEventListener("keyup", () => {
|
||||
if (
|
||||
(<HTMLInputElement>queryInput).value === "" ||
|
||||
requestInProgress ||
|
||||
isLoadingParams
|
||||
) {
|
||||
(<HTMLButtonElement>submitButton).disabled = true;
|
||||
} else {
|
||||
(<HTMLButtonElement>submitButton).disabled = false;
|
||||
}
|
||||
});
|
||||
|
||||
// If user presses enter, click submit button
|
||||
queryInput.addEventListener("keyup", (event) => {
|
||||
if (event.code === "Enter") {
|
||||
event.preventDefault();
|
||||
submitButton.click();
|
||||
}
|
||||
});
|
||||
|
||||
// Listen for clicks on submit button
|
||||
async function handleClick() {
|
||||
requestInProgress = true;
|
||||
(<HTMLButtonElement>submitButton).disabled = true;
|
||||
|
||||
// Get the message from the input field
|
||||
const message = (<HTMLInputElement>queryInput).value;
|
||||
console.log("message", message);
|
||||
// Clear the answer
|
||||
document.getElementById("answer")!.innerHTML = "";
|
||||
// Hide the answer
|
||||
document.getElementById("answerWrapper")!.style.display = "none";
|
||||
// Show the loading indicator
|
||||
document.getElementById("loading-indicator")!.style.display = "block";
|
||||
|
||||
// Generate response
|
||||
let inp = message;
|
||||
if (context.length > 0) {
|
||||
inp =
|
||||
"Use only the following context when answering the question at the end. Don't use any other knowledge.\n" +
|
||||
context +
|
||||
"\n\nQuestion: " +
|
||||
message +
|
||||
"\n\nHelpful Answer: ";
|
||||
}
|
||||
console.log("Input:", inp);
|
||||
chatHistory.push({ role: "user", content: inp });
|
||||
|
||||
let curMessage = "";
|
||||
const completion = await engine.chat.completions.create({
|
||||
stream: true,
|
||||
messages: chatHistory,
|
||||
});
|
||||
for await (const chunk of completion) {
|
||||
const curDelta = chunk.choices[0].delta.content;
|
||||
if (curDelta) {
|
||||
curMessage += curDelta;
|
||||
}
|
||||
updateAnswer(curMessage);
|
||||
}
|
||||
const response = await engine.getMessage();
|
||||
chatHistory.push({ role: "assistant", content: await engine.getMessage() });
|
||||
console.log("response", response);
|
||||
|
||||
requestInProgress = false;
|
||||
(<HTMLButtonElement>submitButton).disabled = false;
|
||||
}
|
||||
submitButton.addEventListener("click", handleClick);
|
||||
|
||||
// listen for changes in modelSelector
|
||||
async function handleSelectChange() {
|
||||
if (isLoadingParams) {
|
||||
return;
|
||||
}
|
||||
|
||||
modelName.innerText = "";
|
||||
|
||||
const initLabel = document.createElement("p");
|
||||
initLabel.id = "init-label";
|
||||
initLabel.innerText = "Initializing model...";
|
||||
const loadingContainer = document.createElement("div");
|
||||
loadingContainer.id = "loadingContainer";
|
||||
|
||||
const loadingBox = getElementAndCheck("loadingBox");
|
||||
loadingBox.appendChild(initLabel);
|
||||
loadingBox.appendChild(loadingContainer);
|
||||
|
||||
isLoadingParams = true;
|
||||
(<HTMLButtonElement>submitButton).disabled = true;
|
||||
|
||||
if (requestInProgress) {
|
||||
engine.interruptGenerate();
|
||||
}
|
||||
engine.resetChat();
|
||||
chatHistory = [];
|
||||
await engine.unload();
|
||||
|
||||
selectedModel = modelSelector.value;
|
||||
|
||||
progressBar = new Line("#loadingContainer", {
|
||||
strokeWidth: 4,
|
||||
easing: "easeInOut",
|
||||
duration: 1400,
|
||||
color: "#ffd166",
|
||||
trailColor: "#eee",
|
||||
trailWidth: 1,
|
||||
svgStyle: { width: "100%", height: "100%" },
|
||||
});
|
||||
|
||||
initProgressCallback = (report: InitProgressReport) => {
|
||||
setLabel("init-label", report.text);
|
||||
progressBar.animate(report.progress, {
|
||||
duration: 50,
|
||||
});
|
||||
if (report.progress == 1.0) {
|
||||
enableInputs();
|
||||
}
|
||||
};
|
||||
|
||||
engine.setInitProgressCallback(initProgressCallback);
|
||||
|
||||
requestInProgress = true;
|
||||
modelName.innerText = "Reloading with new model...";
|
||||
await engine.reload(selectedModel);
|
||||
requestInProgress = false;
|
||||
modelName.innerText = "Now chatting with " + modelDisplayName;
|
||||
}
|
||||
modelSelector.addEventListener("change", handleSelectChange);
|
||||
|
||||
// Listen for messages from the background script
|
||||
chrome.runtime.onMessage.addListener(({ answer, error }) => {
|
||||
if (answer) {
|
||||
updateAnswer(answer);
|
||||
}
|
||||
});
|
||||
|
||||
function updateAnswer(answer: string) {
|
||||
// Show answer
|
||||
document.getElementById("answerWrapper")!.style.display = "block";
|
||||
const answerWithBreaks = answer.replace(/\n/g, "<br>");
|
||||
document.getElementById("answer")!.innerHTML = answerWithBreaks;
|
||||
// Add event listener to copy button
|
||||
document.getElementById("copyAnswer")!.addEventListener("click", () => {
|
||||
// Get the answer text
|
||||
const answerText = answer;
|
||||
// Copy the answer text to the clipboard
|
||||
navigator.clipboard
|
||||
.writeText(answerText)
|
||||
.then(() => console.log("Answer text copied to clipboard"))
|
||||
.catch((err) => console.error("Could not copy text: ", err));
|
||||
});
|
||||
const options: Intl.DateTimeFormatOptions = {
|
||||
month: "short",
|
||||
day: "2-digit",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
};
|
||||
const time = new Date().toLocaleString("en-US", options);
|
||||
// Update timestamp
|
||||
document.getElementById("timestamp")!.innerText = time;
|
||||
// Hide loading indicator
|
||||
document.getElementById("loading-indicator")!.style.display = "none";
|
||||
}
|
||||
|
||||
function fetchPageContents() {
|
||||
chrome.tabs.query({ currentWindow: true, active: true }, function (tabs) {
|
||||
const port = chrome.tabs.connect(tabs[0].id, { name: "channelName" });
|
||||
port.postMessage({});
|
||||
port.onMessage.addListener(function (msg) {
|
||||
console.log("Page contents:", msg.contents);
|
||||
context = msg.contents;
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
# WebLLM Get Started App
|
||||
|
||||
This folder provides a minimum demo to show WebLLM API in a webapp setting.
|
||||
To try it out, you can do the following steps under this folder
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm start
|
||||
```
|
||||
|
||||
Note if you would like to hack WebLLM core package.
|
||||
You can change web-llm dependencies as `"file:../.."`, and follow the build from source
|
||||
instruction in the project to build webllm locally. This option is only recommended
|
||||
if you would like to hack WebLLM core package.
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "embeddings-example",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "parcel src/embeddings.html --port 8885",
|
||||
"build": "parcel build src/embeddings.html --dist-dir lib"
|
||||
},
|
||||
"devDependencies": {
|
||||
"buffer": "^5.7.1",
|
||||
"parcel": "^2.8.3",
|
||||
"process": "^0.11.10",
|
||||
"tslib": "^2.3.1",
|
||||
"typescript": "^4.9.5",
|
||||
"url": "^0.11.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mlc-ai/web-llm": "^0.2.84",
|
||||
"langchain": "^0.3.37"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<script>
|
||||
webLLMGlobal = {};
|
||||
</script>
|
||||
<body>
|
||||
<h2>WebLLM Test Page</h2>
|
||||
Open console to see output
|
||||
<br />
|
||||
<br />
|
||||
<label id="init-label"> </label>
|
||||
|
||||
<h3>Prompt</h3>
|
||||
<label id="prompt-label"> </label>
|
||||
|
||||
<h3>Response</h3>
|
||||
<label id="generate-label"> </label>
|
||||
<br />
|
||||
<label id="stats-label"> </label>
|
||||
|
||||
<script type="module" src="./embeddings.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,211 @@
|
||||
import * as webllm from "@mlc-ai/web-llm";
|
||||
import { MemoryVectorStore } from "langchain/vectorstores/memory";
|
||||
import type { EmbeddingsInterface } from "@langchain/core/embeddings";
|
||||
import type { Document } from "@langchain/core/documents";
|
||||
import { formatDocumentsAsString } from "langchain/util/document";
|
||||
import { PromptTemplate } from "@langchain/core/prompts";
|
||||
import {
|
||||
RunnableSequence,
|
||||
RunnablePassthrough,
|
||||
} from "@langchain/core/runnables";
|
||||
|
||||
function setLabel(id: string, text: string) {
|
||||
const label = document.getElementById(id);
|
||||
if (label == null) {
|
||||
throw Error("Cannot find label " + id);
|
||||
}
|
||||
label.innerText = text;
|
||||
}
|
||||
|
||||
const initProgressCallback = (report: webllm.InitProgressReport) => {
|
||||
setLabel("init-label", report.text);
|
||||
};
|
||||
|
||||
// For integration with Langchain
|
||||
class WebLLMEmbeddings implements EmbeddingsInterface {
|
||||
engine: webllm.MLCEngineInterface;
|
||||
modelId: string;
|
||||
constructor(engine: webllm.MLCEngineInterface, modelId: string) {
|
||||
this.engine = engine;
|
||||
this.modelId = modelId;
|
||||
}
|
||||
|
||||
async _embed(texts: string[]): Promise<number[][]> {
|
||||
const reply = await this.engine.embeddings.create({
|
||||
input: texts,
|
||||
model: this.modelId,
|
||||
});
|
||||
const result: number[][] = [];
|
||||
for (let i = 0; i < texts.length; i++) {
|
||||
result.push(reply.data[i].embedding);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async embedQuery(document: string): Promise<number[]> {
|
||||
return this._embed([document]).then((embeddings) => embeddings[0]);
|
||||
}
|
||||
|
||||
async embedDocuments(documents: string[]): Promise<number[][]> {
|
||||
return this._embed(documents);
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare inputs
|
||||
const documents_og = ["The Data Cloud!", "Mexico City of Course!"];
|
||||
const queries_og = ["what is snowflake?", "Where can I get the best tacos?"];
|
||||
const documents: string[] = [];
|
||||
const queries: string[] = [];
|
||||
const query_prefix =
|
||||
"Represent this sentence for searching relevant passages: ";
|
||||
// Process according to Snowflake model
|
||||
documents_og.forEach(function (item, index) {
|
||||
documents[index] = `[CLS] ${item} [SEP]`;
|
||||
});
|
||||
queries_og.forEach(function (item, index) {
|
||||
queries[index] = `[CLS] ${query_prefix}${item} [SEP]`;
|
||||
});
|
||||
console.log("Formatted documents: ", documents);
|
||||
console.log("Formatted queries: ", queries);
|
||||
|
||||
// Using webllm's API
|
||||
async function webllmAPI() {
|
||||
// b4 means the max batch size is compiled as 4. That is, the model can process 4 inputs in a
|
||||
// batch. If given more than 4, the model will forward multiple times. The larger the max batch
|
||||
// size, the more memory it consumes.
|
||||
// const selectedModel = "snowflake-arctic-embed-m-q0f32-MLC-b32";
|
||||
const selectedModel = "snowflake-arctic-embed-m-q0f32-MLC-b4";
|
||||
const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
|
||||
selectedModel,
|
||||
{
|
||||
initProgressCallback: initProgressCallback,
|
||||
logLevel: "INFO", // specify the log level
|
||||
},
|
||||
);
|
||||
|
||||
const docReply = await engine.embeddings.create({ input: documents });
|
||||
console.log(docReply);
|
||||
console.log(docReply.usage);
|
||||
|
||||
const queryReply = await engine.embeddings.create({ input: queries });
|
||||
console.log(queryReply);
|
||||
console.log(queryReply.usage);
|
||||
|
||||
// Calculate similarity (we use langchain here, but any method works)
|
||||
const vectorStore = await MemoryVectorStore.fromExistingIndex(
|
||||
new WebLLMEmbeddings(engine, selectedModel),
|
||||
);
|
||||
// See score
|
||||
for (let i = 0; i < queries_og.length; i++) {
|
||||
console.log(`Similarity with: ${queries_og[i]}`);
|
||||
for (let j = 0; j < documents_og.length; j++) {
|
||||
const similarity = vectorStore.similarity(
|
||||
queryReply.data[i].embedding,
|
||||
docReply.data[j].embedding,
|
||||
);
|
||||
console.log(`${documents_og[j]}: ${similarity}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Alternatively, integrating with Langchain's API
|
||||
async function langchainAPI() {
|
||||
// b4 means the max batch size is compiled as 4. That is, the model can process 4 inputs in a
|
||||
// batch. If given more than 4, the model will forward multiple times. The larger the max batch
|
||||
// size, the more memory it consumes.
|
||||
// const selectedModel = "snowflake-arctic-embed-m-q0f32-MLC-b32";
|
||||
const selectedModel = "snowflake-arctic-embed-m-q0f32-MLC-b4";
|
||||
const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
|
||||
selectedModel,
|
||||
{
|
||||
initProgressCallback: initProgressCallback,
|
||||
logLevel: "INFO", // specify the log level
|
||||
},
|
||||
);
|
||||
|
||||
const vectorStore = await MemoryVectorStore.fromExistingIndex(
|
||||
new WebLLMEmbeddings(engine, selectedModel),
|
||||
);
|
||||
const document0: Document = {
|
||||
pageContent: documents[0],
|
||||
metadata: {},
|
||||
};
|
||||
const document1: Document = {
|
||||
pageContent: documents[1],
|
||||
metadata: {},
|
||||
};
|
||||
await vectorStore.addDocuments([document0, document1]);
|
||||
|
||||
const similaritySearchResults0 = await vectorStore.similaritySearch(
|
||||
queries[0],
|
||||
1,
|
||||
);
|
||||
for (const doc of similaritySearchResults0) {
|
||||
console.log(`* ${doc.pageContent}`);
|
||||
}
|
||||
|
||||
const similaritySearchResults1 = await vectorStore.similaritySearch(
|
||||
queries[1],
|
||||
1,
|
||||
);
|
||||
for (const doc of similaritySearchResults1) {
|
||||
console.log(`* ${doc.pageContent}`);
|
||||
}
|
||||
}
|
||||
|
||||
// RAG with Langchain.js using WebLLM for both LLM and Embedding in a single engine
|
||||
// Followed https://js.langchain.com/v0.1/docs/expression_language/cookbook/retrieval/
|
||||
// There are many possible ways to achieve RAG (e.g. degree of integration with Langchain,
|
||||
// using WebWorker, etc.). We provide a minimal example here.
|
||||
async function simpleRAG() {
|
||||
// 0. Load both embedding model and LLM to a single WebLLM Engine
|
||||
const embeddingModelId = "snowflake-arctic-embed-m-q0f32-MLC-b4";
|
||||
const llmModelId = "gemma-2-2b-it-q4f32_1-MLC-1k";
|
||||
const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
|
||||
[embeddingModelId, llmModelId],
|
||||
{
|
||||
initProgressCallback: initProgressCallback,
|
||||
logLevel: "INFO", // specify the log level
|
||||
},
|
||||
);
|
||||
|
||||
const vectorStore = await MemoryVectorStore.fromTexts(
|
||||
["mitochondria is the powerhouse of the cell"],
|
||||
[{ id: 1 }],
|
||||
new WebLLMEmbeddings(engine, embeddingModelId),
|
||||
);
|
||||
const retriever = vectorStore.asRetriever();
|
||||
|
||||
const prompt =
|
||||
PromptTemplate.fromTemplate(`Answer the question based only on the following context:
|
||||
{context}
|
||||
|
||||
Question: {question}`);
|
||||
|
||||
const chain = RunnableSequence.from([
|
||||
{
|
||||
context: retriever.pipe(formatDocumentsAsString),
|
||||
question: new RunnablePassthrough(),
|
||||
},
|
||||
prompt,
|
||||
]);
|
||||
|
||||
const formattedPrompt = (
|
||||
await chain.invoke("What is the powerhouse of the cell?")
|
||||
).toString();
|
||||
const reply = await engine.chat.completions.create({
|
||||
messages: [{ role: "user", content: formattedPrompt }],
|
||||
model: llmModelId,
|
||||
});
|
||||
|
||||
console.log(reply.choices[0].message.content);
|
||||
|
||||
/*
|
||||
"The powerhouse of the cell is the mitochondria."
|
||||
*/
|
||||
}
|
||||
|
||||
// Select one to run
|
||||
// webllmAPI();
|
||||
// langchainAPI();
|
||||
simpleRAG();
|
||||
@@ -0,0 +1,11 @@
|
||||
### OpenAI API Demos - Function calling
|
||||
|
||||
This folder contains two main ways of using function calling with WebLLM.
|
||||
|
||||
`function-calling-manual` demonstrates how you can use function calling with Llama3.1 and Hermes2
|
||||
without using the `tools`, `tool_choice`, and `tool_call` fields. This is the most flexible way and you can follow
|
||||
the instruction given by the model releaser and iterate yourself on top of that. However, you need to do parsing on your own, which differs for each model. For instance, Hermes2 models use `<tool_call>` and `</tool_call>` to wrap around a tool call, which may be very different from other models' format.
|
||||
|
||||
`function-calling-openai` conforms to the OpenAI function calling usage, leveraging `tools`, `tool_choice`, and `tool_call`
|
||||
fields. This is more usable, but sacrifices the flexibility since we have pre-defined system prompt
|
||||
for this.
|
||||
@@ -0,0 +1,8 @@
|
||||
### Demos - Function calling
|
||||
|
||||
Run `npm install` first, followed by `npm start`.
|
||||
|
||||
Note if you would like to hack WebLLM core package,
|
||||
you can change web-llm dependencies as `"file:../../.."`, and follow the build from source
|
||||
instruction in the project to build webllm locally. This option is only recommended
|
||||
if you would like to hack WebLLM core package.
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "openai-api",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "parcel src/function_calling_manual.html --port 8888",
|
||||
"build": "parcel build src/function_calling_manual.html --dist-dir lib"
|
||||
},
|
||||
"devDependencies": {
|
||||
"buffer": "^5.7.1",
|
||||
"parcel": "^2.8.3",
|
||||
"process": "^0.11.10",
|
||||
"tslib": "^2.3.1",
|
||||
"typescript": "^4.9.5",
|
||||
"url": "^0.11.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mlc-ai/web-llm": "^0.2.84"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<script>
|
||||
webLLMGlobal = {};
|
||||
</script>
|
||||
|
||||
<body>
|
||||
<h2>WebLLM Test Page</h2>
|
||||
Open console to see output
|
||||
<br />
|
||||
<br />
|
||||
<label id="init-label"> </label>
|
||||
<label id="generate-label"> </label>
|
||||
|
||||
<script type="module" src="./function_calling_manual.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,235 @@
|
||||
/* eslint-disable no-useless-escape */
|
||||
import * as webllm from "@mlc-ai/web-llm";
|
||||
|
||||
// Common helper methods
|
||||
function setLabel(id: string, text: string) {
|
||||
const label = document.getElementById(id);
|
||||
if (label == null) {
|
||||
throw Error("Cannot find label " + id);
|
||||
}
|
||||
label.innerText = text;
|
||||
}
|
||||
|
||||
const initProgressCallback = (report: webllm.InitProgressReport) => {
|
||||
setLabel("init-label", report.text);
|
||||
};
|
||||
|
||||
// Same example as https://huggingface.co/NousResearch/Hermes-2-Theta-Llama-3-8B#prompt-format-for-function-calling
|
||||
async function hermes2_example() {
|
||||
// 0. Setups
|
||||
// Most manual function calling models specify the tools inside the system prompt
|
||||
const system_prompt = `You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: <tools> {"type": "function", "function": {"name": "get_stock_fundamentals", "description": "get_stock_fundamentals(symbol: str) -> dict - Get fundamental data for a given stock symbol using yfinance API.\\n\\n Args:\\n symbol (str): The stock symbol.\\n\\n Returns:\\n dict: A dictionary containing fundamental data.\\n Keys:\\n - \'symbol\': The stock symbol.\\n - \'company_name\': The long name of the company.\\n - \'sector\': The sector to which the company belongs.\\n - \'industry\': The industry to which the company belongs.\\n - \'market_cap\': The market capitalization of the company.\\n - \'pe_ratio\': The forward price-to-earnings ratio.\\n - \'pb_ratio\': The price-to-book ratio.\\n - \'dividend_yield\': The dividend yield.\\n - \'eps\': The trailing earnings per share.\\n - \'beta\': The beta value of the stock.\\n - \'52_week_high\': The 52-week high price of the stock.\\n - \'52_week_low\': The 52-week low price of the stock.", "parameters": {"type": "object", "properties": {"symbol": {"type": "string"}}, "required": ["symbol"]}}} </tools> Use the following pydantic model json schema for each tool call you will make: {"properties": {"arguments": {"title": "Arguments", "type": "object"}, "name": {"title": "Name", "type": "string"}}, "required": ["arguments", "name"], "title": "FunctionCall", "type": "object"} For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:\n<tool_call>\n{"arguments": <args-dict>, "name": <function-name>}\n</tool_call>`;
|
||||
// Same formatting for Hermes-2-Pro-Llama-3, Hermes-2-Theta-Llama-3
|
||||
// const selectedModel = "Hermes-2-Theta-Llama-3-8B-q4f16_1-MLC";
|
||||
const selectedModel = "Hermes-2-Pro-Llama-3-8B-q4f16_1-MLC";
|
||||
const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
|
||||
selectedModel,
|
||||
{ initProgressCallback: initProgressCallback, logLevel: "INFO" },
|
||||
);
|
||||
const seed = 0;
|
||||
|
||||
// 1. First request, expect to generate tool call
|
||||
const messages: webllm.ChatCompletionMessageParam[] = [
|
||||
{ role: "system", content: system_prompt },
|
||||
{
|
||||
role: "user",
|
||||
content: "Fetch the stock fundamentals data for Tesla (TSLA)",
|
||||
},
|
||||
];
|
||||
const request1: webllm.ChatCompletionRequest = {
|
||||
stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
|
||||
messages: messages,
|
||||
seed: seed,
|
||||
};
|
||||
const reply1 = await engine.chat.completions.create(request1);
|
||||
const response1 = reply1.choices[0].message.content;
|
||||
console.log(reply1.usage);
|
||||
console.log("Response 1: " + response1);
|
||||
messages.push({ role: "assistant", content: response1 });
|
||||
// <tool_call>\n{"arguments": {"symbol": "TSLA"}, "name": "get_stock_fundamentals"}\n</tool_call>
|
||||
|
||||
// 2. Call function on your own to get tool response
|
||||
const tool_response = `<tool_response>\n{"name": "get_stock_fundamentals", "content": {'symbol': 'TSLA', 'company_name': 'Tesla, Inc.', 'sector': 'Consumer Cyclical', 'industry': 'Auto Manufacturers', 'market_cap': 611384164352, 'pe_ratio': 49.604652, 'pb_ratio': 9.762013, 'dividend_yield': None, 'eps': 4.3, 'beta': 2.427, '52_week_high': 299.29, '52_week_low': 152.37}}\n</tool_response>`;
|
||||
messages.push({ role: "tool", content: tool_response, tool_call_id: "0" });
|
||||
|
||||
// 3. Get natural language response
|
||||
const request2: webllm.ChatCompletionRequest = {
|
||||
stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
|
||||
messages: messages,
|
||||
seed: seed,
|
||||
};
|
||||
const reply2 = await engine.chat.completions.create(request2);
|
||||
const response2 = reply2.choices[0].message.content;
|
||||
messages.push({ role: "assistant", content: response2 });
|
||||
console.log(reply2.usage);
|
||||
console.log("Response 2: " + response2);
|
||||
|
||||
// 4. Another function call
|
||||
messages.push({
|
||||
role: "user",
|
||||
content: "Now do another one with NVIDIA, symbol being NVDA.",
|
||||
});
|
||||
const request3: webllm.ChatCompletionRequest = {
|
||||
stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
|
||||
messages: messages,
|
||||
seed: seed,
|
||||
};
|
||||
const reply3 = await engine.chat.completions.create(request3);
|
||||
const response3 = reply3.choices[0].message.content;
|
||||
messages.push({ role: "assistant", content: response3 });
|
||||
console.log(reply3.usage);
|
||||
console.log("Response 3: " + response3);
|
||||
// <tool_call>\n{"arguments": {"symbol": "NVDA"}, "name": "get_stock_fundamentals"}\n</tool_call>
|
||||
}
|
||||
|
||||
// Similar example to https://llama.meta.com/docs/model-cards-and-prompt-formats/llama3_1/#user-defined-custom-tool-calling
|
||||
async function llama3_1_example() {
|
||||
// Follows example, but tweaks the formatting with <function>
|
||||
const system_prompt = `Cutting Knowledge Date: December 2023
|
||||
Today Date: 23 Jul 2024
|
||||
# Tool Instructions
|
||||
- When looking for real time information use relevant functions if available
|
||||
You have access to the following functions:
|
||||
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_temperature",
|
||||
"description": "Get the current temperature at a location.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The location to get the temperature for, in the format \"City, Country\""
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"location"
|
||||
]
|
||||
},
|
||||
"return": {
|
||||
"type": "number",
|
||||
"description": "The current temperature at the specified location in the specified units, as a float."
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "send_message",
|
||||
"description": "Send a message to a recipient.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"recipient": {
|
||||
"type": "string",
|
||||
"description": "Name of the recipient of the message"
|
||||
}
|
||||
"content": {
|
||||
"type": "string",
|
||||
"description": "Content of the message"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"recipient",
|
||||
"content"
|
||||
]
|
||||
},
|
||||
"return": {
|
||||
"type": "None"
|
||||
}
|
||||
}
|
||||
}
|
||||
If a you choose to call a function ONLY reply in the following format:
|
||||
<function>{"name": function name, "parameters": dictionary of argument name and its value}</function>
|
||||
Here is an example,
|
||||
<function>{"name": "example_function_name", "parameters": {"example_name": "example_value"}}</function>
|
||||
Reminder:
|
||||
- Function calls MUST follow the specified format and use BOTH <function> and </function>
|
||||
- Required parameters MUST be specified
|
||||
- Only call one function at a time
|
||||
- When calling a function, do NOT add any other words, ONLY the function calling
|
||||
- Put the entire function call reply on one line
|
||||
- Always add your sources when using search results to answer the user query
|
||||
You are a helpful Assistant.`;
|
||||
|
||||
const selectedModel = "Llama-3.1-8B-Instruct-q4f16_1-MLC";
|
||||
const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
|
||||
selectedModel,
|
||||
{ initProgressCallback: initProgressCallback, logLevel: "INFO" },
|
||||
);
|
||||
const seed = 0;
|
||||
|
||||
// 1. First request, expect to generate tool call to get temperature of Paris
|
||||
const messages: webllm.ChatCompletionMessageParam[] = [
|
||||
{ role: "system", content: system_prompt },
|
||||
{
|
||||
role: "user",
|
||||
content: "Hey, what's the temperature in Paris right now?",
|
||||
},
|
||||
];
|
||||
const request1: webllm.ChatCompletionRequest = {
|
||||
stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
|
||||
messages: messages,
|
||||
seed: seed,
|
||||
};
|
||||
const reply1 = await engine.chat.completions.create(request1);
|
||||
const response1 = reply1.choices[0].message.content;
|
||||
console.log(reply1.usage);
|
||||
console.log("Response 1: " + response1);
|
||||
messages.push({ role: "assistant", content: response1 });
|
||||
// <function>{"name": "get_current_temperature", "parameters": {"location": "Paris, France"}}</function>
|
||||
|
||||
// 2. Call function on your own to get tool response
|
||||
const tool_response = `{"output": 22.5}`;
|
||||
messages.push({ role: "tool", content: tool_response, tool_call_id: "0" });
|
||||
|
||||
// 3. Get natural language response
|
||||
const request2: webllm.ChatCompletionRequest = {
|
||||
stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
|
||||
messages: messages,
|
||||
seed: seed,
|
||||
};
|
||||
const reply2 = await engine.chat.completions.create(request2);
|
||||
const response2 = reply2.choices[0].message.content;
|
||||
messages.push({ role: "assistant", content: response2 });
|
||||
console.log(reply2.usage);
|
||||
console.log("Response 2: " + response2);
|
||||
// The current temperature in Paris is 22.5°C.
|
||||
|
||||
// 4. Make another request, expect model to call `send_message`
|
||||
messages.push({
|
||||
role: "user",
|
||||
content: "Send a message to Tom to tell him this information.",
|
||||
});
|
||||
const request3: webllm.ChatCompletionRequest = {
|
||||
stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
|
||||
messages: messages,
|
||||
seed: seed,
|
||||
};
|
||||
const reply3 = await engine.chat.completions.create(request3);
|
||||
const response3 = reply3.choices[0].message.content;
|
||||
messages.push({ role: "assistant", content: response3 });
|
||||
console.log(reply3.usage);
|
||||
console.log("Response 3: " + response3);
|
||||
// <function>{"name": "send_message", "parameters": {"recipient": "Tom", "content": "The current temperature in Paris is 22.5°C."}}</function>
|
||||
|
||||
// 5. Call API, which has no return value, so simply prompt model again
|
||||
const tool_response2 = `{"output": None}`;
|
||||
messages.push({ role: "tool", content: tool_response2, tool_call_id: "1" });
|
||||
const request4: webllm.ChatCompletionRequest = {
|
||||
stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
|
||||
messages: messages,
|
||||
seed: seed,
|
||||
};
|
||||
const reply4 = await engine.chat.completions.create(request4);
|
||||
const response4 = reply4.choices[0].message.content;
|
||||
console.log(reply4.usage);
|
||||
console.log("Response 4: " + response4);
|
||||
// The message has been sent to Tom.
|
||||
}
|
||||
|
||||
// Pick one to run
|
||||
// hermes2_example();
|
||||
llama3_1_example();
|
||||
@@ -0,0 +1,8 @@
|
||||
### Demos - Function calling
|
||||
|
||||
Run `npm install` first, followed by `npm start`.
|
||||
|
||||
Note if you would like to hack WebLLM core package,
|
||||
you can change web-llm dependencies as `"file:../../.."`, and follow the build from source
|
||||
instruction in the project to build webllm locally. This option is only recommended
|
||||
if you would like to hack WebLLM core package.
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "openai-api",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "parcel src/function_calling_openai.html --port 8888",
|
||||
"build": "parcel build src/function_calling_openai.html --dist-dir lib"
|
||||
},
|
||||
"devDependencies": {
|
||||
"buffer": "^5.7.1",
|
||||
"parcel": "^2.8.3",
|
||||
"process": "^0.11.10",
|
||||
"tslib": "^2.3.1",
|
||||
"typescript": "^4.9.5",
|
||||
"url": "^0.11.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mlc-ai/web-llm": "^0.2.84"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<script>
|
||||
webLLMGlobal = {};
|
||||
</script>
|
||||
|
||||
<body>
|
||||
<h2>WebLLM Test Page</h2>
|
||||
Open console to see output
|
||||
<br />
|
||||
<br />
|
||||
<label id="init-label"> </label>
|
||||
<label id="generate-label"> </label>
|
||||
|
||||
<script type="module" src="./function_calling_openai.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,80 @@
|
||||
import * as webllm from "@mlc-ai/web-llm";
|
||||
|
||||
function setLabel(id: string, text: string) {
|
||||
const label = document.getElementById(id);
|
||||
if (label == null) {
|
||||
throw Error("Cannot find label " + id);
|
||||
}
|
||||
label.innerText = text;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const initProgressCallback = (report: webllm.InitProgressReport) => {
|
||||
setLabel("init-label", report.text);
|
||||
};
|
||||
const selectedModel = "Hermes-2-Pro-Llama-3-8B-q4f16_1-MLC";
|
||||
const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
|
||||
selectedModel,
|
||||
{ initProgressCallback: initProgressCallback },
|
||||
);
|
||||
|
||||
const tools: Array<webllm.ChatCompletionTool> = [
|
||||
{
|
||||
type: "function",
|
||||
function: {
|
||||
name: "get_current_weather",
|
||||
description: "Get the current weather in a given location",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
location: {
|
||||
type: "string",
|
||||
description: "The city and state, e.g. San Francisco, CA",
|
||||
},
|
||||
unit: { type: "string", enum: ["celsius", "fahrenheit"] },
|
||||
},
|
||||
required: ["location"],
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const request: webllm.ChatCompletionRequest = {
|
||||
stream: true, // works with stream as well, where the last chunk returns tool_calls
|
||||
stream_options: { include_usage: true },
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content:
|
||||
"What is the current weather in celsius in Pittsburgh and Tokyo?",
|
||||
},
|
||||
],
|
||||
tool_choice: "auto",
|
||||
tools: tools,
|
||||
};
|
||||
|
||||
if (!request.stream) {
|
||||
const reply0 = await engine.chat.completions.create(request);
|
||||
console.log(reply0.choices[0]);
|
||||
console.log(reply0.usage);
|
||||
} else {
|
||||
// If streaming, the last chunk returns tool calls
|
||||
const asyncChunkGenerator = await engine.chat.completions.create(request);
|
||||
let message = "";
|
||||
let lastChunk: webllm.ChatCompletionChunk | undefined;
|
||||
let usageChunk: webllm.ChatCompletionChunk | undefined;
|
||||
for await (const chunk of asyncChunkGenerator) {
|
||||
console.log(chunk);
|
||||
message += chunk.choices[0]?.delta?.content || "";
|
||||
setLabel("generate-label", message);
|
||||
if (!chunk.usage) {
|
||||
lastChunk = chunk;
|
||||
}
|
||||
usageChunk = chunk;
|
||||
}
|
||||
console.log(lastChunk!.choices[0].delta);
|
||||
console.log(usageChunk!.usage);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,15 @@
|
||||
# WebLLM Get Started App
|
||||
|
||||
This folder provides a minimum demo to show WebLLM API in a webapp setting with
|
||||
collection of latency statistics for individual token sampling steps.
|
||||
To try it out, you can do the following steps under this folder
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm start
|
||||
```
|
||||
|
||||
Note if you would like to hack WebLLM core package.
|
||||
You can change web-llm dependencies as `"file:../.."`, and follow the build from source
|
||||
instruction in the project to build webllm locally. This option is only recommended
|
||||
if you would like to hack WebLLM core package.
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "get-started-latency-breakdown",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "parcel src/get_started_latency_breakdown.html --port 8888",
|
||||
"build": "parcel build src/get_started_latency_breakdown.html --dist-dir lib"
|
||||
},
|
||||
"devDependencies": {
|
||||
"buffer": "^5.7.1",
|
||||
"parcel": "^2.8.3",
|
||||
"process": "^0.11.10",
|
||||
"tslib": "^2.3.1",
|
||||
"typescript": "^4.9.5",
|
||||
"url": "^0.11.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mlc-ai/web-llm": "^0.2.84"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<script>
|
||||
webLLMGlobal = {};
|
||||
</script>
|
||||
<body>
|
||||
<h2>WebLLM Test Page</h2>
|
||||
Open console to see output
|
||||
<br />
|
||||
<br />
|
||||
<label id="init-label"> </label>
|
||||
|
||||
<h3>Prompt</h3>
|
||||
<label id="prompt-label"> </label>
|
||||
|
||||
<h3>Response</h3>
|
||||
<label id="generate-label"> </label>
|
||||
<br />
|
||||
<label id="stats-label"> </label>
|
||||
|
||||
<script type="module" src="./get_started_latency_breakdown.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,135 @@
|
||||
import * as webllm from "@mlc-ai/web-llm";
|
||||
|
||||
function setLabel(id: string, text: string) {
|
||||
const label = document.getElementById(id);
|
||||
if (label == null) {
|
||||
throw Error("Cannot find label " + id);
|
||||
}
|
||||
label.innerText = text;
|
||||
}
|
||||
|
||||
type LatencyBreakdown = {
|
||||
logitProcessorTime: number[];
|
||||
logitBiasTime: number[];
|
||||
penaltyTime: number[];
|
||||
sampleTime: number[];
|
||||
totalTime: number[];
|
||||
grammarBitmaskTime: number[];
|
||||
};
|
||||
function computeStats(
|
||||
latency_breakdown: LatencyBreakdown,
|
||||
): Record<string, any> {
|
||||
function _computeStats(arr: number[]) {
|
||||
if (!arr.length) return undefined;
|
||||
const sorted = [...arr].sort((a, b) => a - b);
|
||||
const sum = arr.reduce((a, b) => a + b, 0);
|
||||
const avg = sum / arr.length;
|
||||
const min = sorted[0];
|
||||
const max = sorted[sorted.length - 1];
|
||||
const p99 = sorted[Math.floor(0.99 * (sorted.length - 1))];
|
||||
return { avg, min, max, p99 };
|
||||
}
|
||||
|
||||
const latencyStats: Record<string, any> = {};
|
||||
for (const key of Object.keys(latency_breakdown)) {
|
||||
const arr = (latency_breakdown as any)[key];
|
||||
if (Array.isArray(arr) && arr.length > 0) {
|
||||
latencyStats[key] = _computeStats(arr);
|
||||
}
|
||||
}
|
||||
return latencyStats;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const initProgressCallback = (report: webllm.InitProgressReport) => {
|
||||
setLabel("init-label", report.text);
|
||||
};
|
||||
// Option 1: If we do not specify appConfig, we use `prebuiltAppConfig` defined in `config.ts`
|
||||
const selectedModel = "Qwen3-0.6B-q0f32-MLC";
|
||||
const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
|
||||
selectedModel,
|
||||
{
|
||||
initProgressCallback: initProgressCallback,
|
||||
logLevel: "INFO", // specify the log level
|
||||
},
|
||||
// customize kv cache, use either context_window_size or sliding_window_size (with attention sink)
|
||||
{
|
||||
context_window_size: 2048,
|
||||
// sliding_window_size: 1024,
|
||||
// attention_sink_size: 4,
|
||||
},
|
||||
);
|
||||
|
||||
const latencyBreakdown: LatencyBreakdown = {
|
||||
logitProcessorTime: [],
|
||||
logitBiasTime: [],
|
||||
penaltyTime: [],
|
||||
sampleTime: [],
|
||||
totalTime: [],
|
||||
grammarBitmaskTime: [],
|
||||
};
|
||||
|
||||
const decodeTokensPerS: number[] = [];
|
||||
const completionTokens: number[] = [];
|
||||
const e2eLatencyS: number[] = [];
|
||||
const timePerOutputTokenS: number[] = [];
|
||||
|
||||
const numTrials = 20;
|
||||
for (let i = 0; i < numTrials; i++) {
|
||||
console.log(`Trial ${i + 1} / ${numTrials}`);
|
||||
const reply0 = await engine.chat.completions.create({
|
||||
messages: [{ role: "user", content: "List twenty US states." }],
|
||||
// below configurations are all optional
|
||||
n: 1,
|
||||
temperature: 0,
|
||||
max_tokens: 2048,
|
||||
// 46510 and 7188 are "California", and 8421 and 51325 are "Texas" in Llama-3.1-8B-Instruct
|
||||
// So we would have a higher chance of seeing the latter two, but never the first in the answer
|
||||
// logit_bias: {
|
||||
// "46510": -100,
|
||||
// "7188": -100,
|
||||
// "8421": 5,
|
||||
// "41325": 5,
|
||||
// },
|
||||
top_p: 0.8,
|
||||
logprobs: true,
|
||||
top_logprobs: 2,
|
||||
frequency_penalty: 1.2,
|
||||
presence_penalty: 1.0,
|
||||
repetition_penalty: 1.1,
|
||||
});
|
||||
|
||||
const logitProcessorTime =
|
||||
reply0.usage?.extra.latencyBreakdown?.logitProcessorTime;
|
||||
const logitBiasTime = reply0.usage?.extra.latencyBreakdown?.logitBiasTime;
|
||||
const penaltyTime = reply0.usage?.extra.latencyBreakdown?.penaltyTime;
|
||||
const sampleTime = reply0.usage?.extra.latencyBreakdown?.sampleTime;
|
||||
const totalTime = reply0.usage?.extra.latencyBreakdown?.totalTime;
|
||||
const grammarBitmaskTime =
|
||||
reply0.usage?.extra.latencyBreakdown?.grammarBitmaskTime;
|
||||
|
||||
latencyBreakdown.logitProcessorTime.push(...(logitProcessorTime || []));
|
||||
latencyBreakdown.logitBiasTime.push(...(logitBiasTime || []));
|
||||
latencyBreakdown.penaltyTime.push(...(penaltyTime || []));
|
||||
latencyBreakdown.sampleTime.push(...(sampleTime || []));
|
||||
latencyBreakdown.totalTime.push(...(totalTime || []));
|
||||
latencyBreakdown.grammarBitmaskTime.push(...(grammarBitmaskTime || []));
|
||||
|
||||
decodeTokensPerS.push(reply0.usage?.extra.decode_tokens_per_s || 0);
|
||||
e2eLatencyS.push(reply0.usage?.extra.e2e_latency_s || 0);
|
||||
timePerOutputTokenS.push(reply0.usage?.extra.time_per_output_token_s || 0);
|
||||
completionTokens.push(reply0.usage?.completion_tokens || 0);
|
||||
}
|
||||
|
||||
const latencyStats: { [key: string]: number } =
|
||||
computeStats(latencyBreakdown);
|
||||
console.log("Latency stats: ", latencyStats);
|
||||
console.log("Decode tokens per second: ", decodeTokensPerS);
|
||||
console.log("Completion tokens: ", completionTokens);
|
||||
console.log("E2E latency (s): ", e2eLatencyS);
|
||||
console.log("Time per output token (s): ", timePerOutputTokenS);
|
||||
|
||||
// To change model, either create a new engine via `CreateMLCEngine()`, or call `engine.reload(modelId)`
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,18 @@
|
||||
# WebLLM Get Started with WebWorker
|
||||
|
||||
This folder provides a minimum demo to show WebLLM API using
|
||||
[WebWorker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers).
|
||||
The main benefit of web worker is that all ML workloads runs on a separate thread as a result
|
||||
will less likely block the UI.
|
||||
|
||||
To try it out, you can do the following steps under this folder
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm start
|
||||
```
|
||||
|
||||
Note if you would like to hack WebLLM core package.
|
||||
You can change web-llm dependencies as `"file:../.."`, and follow the build from source
|
||||
instruction in the project to build webllm locally. This option is only recommended
|
||||
if you would like to hack WebLLM core package.
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "get-started-web-worker",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "parcel src/get_started.html --port 8885",
|
||||
"build": "parcel build src/get_started.html --dist-dir lib"
|
||||
},
|
||||
"devDependencies": {
|
||||
"buffer": "^6.0.3",
|
||||
"parcel": "^2.8.3",
|
||||
"process": "^0.11.10",
|
||||
"tslib": "^2.3.1",
|
||||
"typescript": "^4.9.5",
|
||||
"url": "^0.11.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mlc-ai/web-llm": "^0.2.84"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<script>
|
||||
webLLMGlobal = {};
|
||||
</script>
|
||||
<body>
|
||||
<h2>WebLLM Test Page</h2>
|
||||
Open console to see output
|
||||
<br />
|
||||
<br />
|
||||
<label id="init-label"> </label>
|
||||
|
||||
<h3>Prompt</h3>
|
||||
<label id="prompt-label"> </label>
|
||||
|
||||
<h3>Response</h3>
|
||||
<label id="generate-label"> </label>
|
||||
<br />
|
||||
<label id="stats-label"> </label>
|
||||
|
||||
<script type="module" src="./main.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,102 @@
|
||||
import * as webllm from "@mlc-ai/web-llm";
|
||||
|
||||
function setLabel(id: string, text: string) {
|
||||
const label = document.getElementById(id);
|
||||
if (label == null) {
|
||||
throw Error("Cannot find label " + id);
|
||||
}
|
||||
label.innerText = text;
|
||||
}
|
||||
|
||||
// There are two demonstrations, pick one to run
|
||||
|
||||
/**
|
||||
* Chat completion (OpenAI style) without streaming, where we get the entire response at once.
|
||||
*/
|
||||
async function mainNonStreaming() {
|
||||
const initProgressCallback = (report: webllm.InitProgressReport) => {
|
||||
setLabel("init-label", report.text);
|
||||
};
|
||||
const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
|
||||
|
||||
const engine: webllm.MLCEngineInterface =
|
||||
await webllm.CreateWebWorkerMLCEngine(
|
||||
new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }),
|
||||
selectedModel,
|
||||
{ initProgressCallback: initProgressCallback },
|
||||
);
|
||||
|
||||
const request: webllm.ChatCompletionRequest = {
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content:
|
||||
"You are a helpful, respectful and honest assistant. " +
|
||||
"Be as happy as you can when speaking please. ",
|
||||
},
|
||||
{ role: "user", content: "Provide me three US states." },
|
||||
{ role: "assistant", content: "California, New York, Pennsylvania." },
|
||||
{ role: "user", content: "Two more please!" },
|
||||
],
|
||||
n: 3,
|
||||
temperature: 1.5,
|
||||
max_tokens: 256,
|
||||
};
|
||||
|
||||
const reply0 = await engine.chat.completions.create(request);
|
||||
console.log(reply0);
|
||||
|
||||
console.log(reply0.usage);
|
||||
}
|
||||
|
||||
/**
|
||||
* Chat completion (OpenAI style) with streaming, where delta is sent while generating response.
|
||||
*/
|
||||
async function mainStreaming() {
|
||||
const initProgressCallback = (report: webllm.InitProgressReport) => {
|
||||
setLabel("init-label", report.text);
|
||||
};
|
||||
const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
|
||||
|
||||
const engine: webllm.MLCEngineInterface =
|
||||
await webllm.CreateWebWorkerMLCEngine(
|
||||
new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }),
|
||||
selectedModel,
|
||||
{ initProgressCallback: initProgressCallback },
|
||||
);
|
||||
|
||||
const request: webllm.ChatCompletionRequest = {
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content:
|
||||
"You are a helpful, respectful and honest assistant. " +
|
||||
"Be as happy as you can when speaking please. ",
|
||||
},
|
||||
{ role: "user", content: "Provide me three US states." },
|
||||
{ role: "assistant", content: "California, New York, Pennsylvania." },
|
||||
{ role: "user", content: "Two more please!" },
|
||||
],
|
||||
temperature: 1.5,
|
||||
max_tokens: 256,
|
||||
};
|
||||
|
||||
const asyncChunkGenerator = await engine.chat.completions.create(request);
|
||||
let message = "";
|
||||
for await (const chunk of asyncChunkGenerator) {
|
||||
console.log(chunk);
|
||||
message += chunk.choices[0]?.delta?.content || "";
|
||||
setLabel("generate-label", message);
|
||||
if (chunk.usage) {
|
||||
console.log(chunk.usage); // only last chunk has usage
|
||||
}
|
||||
// engine.interruptGenerate(); // works with interrupt as well
|
||||
}
|
||||
console.log("Final message:\n", await engine.getMessage()); // the concatenated message
|
||||
}
|
||||
|
||||
// Run one of the function below
|
||||
// mainNonStreaming();
|
||||
mainStreaming();
|
||||
@@ -0,0 +1,7 @@
|
||||
import { WebWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
|
||||
|
||||
// Hookup an engine to a worker handler
|
||||
const handler = new WebWorkerMLCEngineHandler();
|
||||
self.onmessage = (msg: MessageEvent) => {
|
||||
handler.onmessage(msg);
|
||||
};
|
||||
@@ -0,0 +1,14 @@
|
||||
# WebLLM Get Started App
|
||||
|
||||
This folder provides a minimum demo to show WebLLM API in a webapp setting.
|
||||
To try it out, you can do the following steps under this folder
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm start
|
||||
```
|
||||
|
||||
Note if you would like to hack WebLLM core package.
|
||||
You can change web-llm dependencies as `"file:../.."`, and follow the build from source
|
||||
instruction in the project to build webllm locally. This option is only recommended
|
||||
if you would like to hack WebLLM core package.
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "get-started",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "parcel src/get_started.html --port 8888",
|
||||
"build": "parcel build src/get_started.html --dist-dir lib"
|
||||
},
|
||||
"devDependencies": {
|
||||
"buffer": "^5.7.1",
|
||||
"parcel": "^2.8.3",
|
||||
"process": "^0.11.10",
|
||||
"tslib": "^2.3.1",
|
||||
"typescript": "^4.9.5",
|
||||
"url": "^0.11.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mlc-ai/web-llm": "^0.2.84"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<script>
|
||||
webLLMGlobal = {};
|
||||
</script>
|
||||
<body>
|
||||
<h2>WebLLM Test Page</h2>
|
||||
Open console to see output
|
||||
<br />
|
||||
<br />
|
||||
<label id="init-label"> </label>
|
||||
|
||||
<h3>Prompt</h3>
|
||||
<label id="prompt-label"> </label>
|
||||
|
||||
<h3>Response</h3>
|
||||
<label id="generate-label"> </label>
|
||||
<br />
|
||||
<label id="stats-label"> </label>
|
||||
|
||||
<script type="module" src="./get_started.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,82 @@
|
||||
import * as webllm from "@mlc-ai/web-llm";
|
||||
|
||||
function setLabel(id: string, text: string) {
|
||||
const label = document.getElementById(id);
|
||||
if (label == null) {
|
||||
throw Error("Cannot find label " + id);
|
||||
}
|
||||
label.innerText = text;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const initProgressCallback = (report: webllm.InitProgressReport) => {
|
||||
setLabel("init-label", report.text);
|
||||
};
|
||||
// Option 1: If we do not specify appConfig, we use `prebuiltAppConfig` defined in `config.ts`
|
||||
const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
|
||||
const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
|
||||
selectedModel,
|
||||
{
|
||||
initProgressCallback: initProgressCallback,
|
||||
logLevel: "INFO", // specify the log level
|
||||
},
|
||||
// customize kv cache, use either context_window_size or sliding_window_size (with attention sink)
|
||||
{
|
||||
context_window_size: 2048,
|
||||
// sliding_window_size: 1024,
|
||||
// attention_sink_size: 4,
|
||||
},
|
||||
);
|
||||
|
||||
// Option 2: Specify your own model other than the prebuilt ones
|
||||
// const appConfig: webllm.AppConfig = {
|
||||
// model_list: [
|
||||
// {
|
||||
// model: "https://huggingface.co/mlc-ai/Llama-3.1-8B-Instruct-q4f32_1-MLC",
|
||||
// model_id: "Llama-3.1-8B-Instruct-q4f32_1-MLC",
|
||||
// model_lib:
|
||||
// webllm.modelLibURLPrefix +
|
||||
// webllm.modelVersion +
|
||||
// "/Llama-3_1-8B-Instruct-q4f32_1-ctx4k_cs1k-webgpu.wasm",
|
||||
// overrides: {
|
||||
// context_window_size: 2048,
|
||||
// },
|
||||
// },
|
||||
// ],
|
||||
// };
|
||||
// const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
|
||||
// selectedModel,
|
||||
// { appConfig: appConfig, initProgressCallback: initProgressCallback },
|
||||
// );
|
||||
|
||||
// Option 3: Instantiate MLCEngine() and call reload() separately
|
||||
// const engine: webllm.MLCEngineInterface = new webllm.MLCEngine({
|
||||
// appConfig: appConfig, // if do not specify, we use webllm.prebuiltAppConfig
|
||||
// initProgressCallback: initProgressCallback,
|
||||
// });
|
||||
// await engine.reload(selectedModel);
|
||||
|
||||
const reply0 = await engine.chat.completions.create({
|
||||
messages: [{ role: "user", content: "List three US states." }],
|
||||
// below configurations are all optional
|
||||
n: 3,
|
||||
temperature: 1.5,
|
||||
max_tokens: 256,
|
||||
// 46510 and 7188 are "California", and 8421 and 51325 are "Texas" in Llama-3.1-8B-Instruct
|
||||
// So we would have a higher chance of seeing the latter two, but never the first in the answer
|
||||
logit_bias: {
|
||||
"46510": -100,
|
||||
"7188": -100,
|
||||
"8421": 5,
|
||||
"51325": 5,
|
||||
},
|
||||
logprobs: true,
|
||||
top_logprobs: 2,
|
||||
});
|
||||
console.log(reply0);
|
||||
console.log(reply0.usage);
|
||||
|
||||
// To change model, either create a new engine via `CreateMLCEngine()`, or call `engine.reload(modelId)`
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "integrity-verification",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "parcel src/integrity_verification.html --port 8888",
|
||||
"build": "parcel build src/integrity_verification.html --dist-dir lib"
|
||||
},
|
||||
"devDependencies": {
|
||||
"buffer": "^5.7.1",
|
||||
"parcel": "^2.8.3",
|
||||
"process": "^0.11.10",
|
||||
"tslib": "^2.3.1",
|
||||
"typescript": "^4.9.5",
|
||||
"url": "^0.11.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mlc-ai/web-llm": "^0.2.84"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<body>
|
||||
<h2>WebLLM Integrity Verification Example</h2>
|
||||
<p>
|
||||
This example demonstrates how to use the optional
|
||||
<code>integrity</code> field on <code>ModelRecord</code> to verify
|
||||
downloaded artifacts with SRI hashes.
|
||||
</p>
|
||||
<div id="status">Loading...</div>
|
||||
<script type="module" src="./integrity_verification.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,89 @@
|
||||
import * as webllm from "@mlc-ai/web-llm";
|
||||
|
||||
const statusEl = document.getElementById("status")!;
|
||||
|
||||
function setStatus(msg: string) {
|
||||
console.log(msg);
|
||||
statusEl.innerText = msg;
|
||||
}
|
||||
|
||||
/**
|
||||
* Example: Basic integrity verification using the native `integrity` field.
|
||||
*
|
||||
* When `integrity` is set on a ModelRecord, WebLLM will verify the downloaded
|
||||
* config, WASM, and tokenizer files against the provided SRI hashes before
|
||||
* loading the model. If a hash does not match, an IntegrityError is thrown
|
||||
* (or a warning is logged if `onFailure: "warn"` is set).
|
||||
*
|
||||
* Hash-generation commands are documented in README.md under "Integrity Verification".
|
||||
* This example uses the same commands:
|
||||
* - `openssl dgst -sha256 -binary <file> | openssl base64 -A | sed 's/^/sha256-/'`
|
||||
* - `openssl dgst -sha384 -binary <file> | openssl base64 -A | sed 's/^/sha384-/'`
|
||||
* - `openssl dgst -sha512 -binary <file> | openssl base64 -A | sed 's/^/sha512-/'`
|
||||
*/
|
||||
async function main() {
|
||||
setStatus("Initializing...");
|
||||
|
||||
// Example model configuration with integrity hashes.
|
||||
// Replace the placeholder hashes below with real SRI hashes for your model.
|
||||
const appConfig: webllm.AppConfig = {
|
||||
model_list: [
|
||||
{
|
||||
model:
|
||||
"https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_1-MLC",
|
||||
model_id: "Llama-3.2-1B-Instruct-q4f16_1-MLC",
|
||||
model_lib:
|
||||
webllm.modelLibURLPrefix +
|
||||
webllm.modelVersion +
|
||||
"/Llama-3.2-1B-Instruct-q4f16_1-ctx4k_cs1k-webgpu.wasm",
|
||||
low_resource_required: true,
|
||||
vram_required_MB: 879.04,
|
||||
overrides: {
|
||||
context_window_size: 4096,
|
||||
},
|
||||
// Uncomment and fill in real SRI hashes to enable verification:
|
||||
// integrity: {
|
||||
// config: "sha256-<base64-hash-of-mlc-chat-config.json>",
|
||||
// model_lib: "sha256-<base64-hash-of-wasm-file>",
|
||||
// tokenizer: {
|
||||
// "tokenizer.json": "sha256-<base64-hash-of-tokenizer.json>",
|
||||
// },
|
||||
// onFailure: "error", // or "warn" to log and continue
|
||||
// },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
try {
|
||||
const engine = await webllm.CreateMLCEngine(
|
||||
"Llama-3.2-1B-Instruct-q4f16_1-MLC",
|
||||
{
|
||||
appConfig,
|
||||
initProgressCallback: (report: webllm.InitProgressReport) => {
|
||||
setStatus(report.text);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
setStatus("Model loaded! Generating response...");
|
||||
|
||||
const reply = await engine.chat.completions.create({
|
||||
messages: [{ role: "user", content: "Hello! What can you do?" }],
|
||||
});
|
||||
|
||||
setStatus("Response: " + reply.choices[0].message.content);
|
||||
} catch (error) {
|
||||
if (error instanceof webllm.IntegrityError) {
|
||||
setStatus(
|
||||
`Integrity verification failed!\n` +
|
||||
`URL: ${error.url}\n` +
|
||||
`Expected: ${error.expected}\n` +
|
||||
`Got: ${error.actual}`,
|
||||
);
|
||||
} else {
|
||||
setStatus("Error: " + error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,8 @@
|
||||
### OpenAI API Demos
|
||||
|
||||
Run `npm install` first, followed by `npm start`.
|
||||
|
||||
Note if you would like to hack WebLLM core package,
|
||||
you can change web-llm dependencies as `"file:../.."`, and follow the build from source
|
||||
instruction in the project to build webllm locally. This option is only recommended
|
||||
if you would like to hack WebLLM core package.
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "openai-api",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "parcel src/json_mode.html --port 8888",
|
||||
"build": "parcel build src/json_mode.html --dist-dir lib"
|
||||
},
|
||||
"devDependencies": {
|
||||
"buffer": "^5.7.1",
|
||||
"parcel": "^2.8.3",
|
||||
"process": "^0.11.10",
|
||||
"tslib": "^2.3.1",
|
||||
"typescript": "^4.9.5",
|
||||
"url": "^0.11.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mlc-ai/web-llm": "^0.2.84"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<script>
|
||||
webLLMGlobal = {};
|
||||
</script>
|
||||
|
||||
<body>
|
||||
<h2>WebLLM Test Page</h2>
|
||||
Open console to see output.
|
||||
<br />
|
||||
<br />
|
||||
<label id="init-label"> </label>
|
||||
|
||||
<script type="module" src="./json_mode.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user