chore: import upstream snapshot with attribution

examples/.gitignore

@@ -0,0 +1 @@
+package-lock.json

examples/README.md

@@ -0,0 +1,57 @@
+# Awesome WebLLM
+
+This page contains a curated list of examples, tutorials, blogs about WebLLM usecases.
+Please send a pull request if you find things that belong here.
+
+## Example Projects
+
+Note that all examples below run in-browser and use WebGPU as a backend.
+
+#### Project List
+
+- [get-started](get-started): minimum get started example with chat completion.
+
+  [![Open in JSFiddle](https://img.shields.io/badge/open-JSFiddle-blue?logo=jsfiddle&logoColor=white)](https://jsfiddle.net/neetnestor/yac9gbwf/)
+  [![Open in Codepen](https://img.shields.io/badge/open-codepen-gainsboro?logo=codepen)](https://codepen.io/neetnestor/pen/NWVdgey)
+
+- [simple-chat-js](simple-chat-js): a mininum and complete chat bot app in vanilla JavaScript.
+
+  [![Open in JSFiddle](https://img.shields.io/badge/open-JSFiddle-blue?logo=jsfiddle&logoColor=white)](https://jsfiddle.net/neetnestor/4nmgvsa2/)
+  [![Open in Codepen](https://img.shields.io/badge/open-codepen-gainsboro?logo=codepen)](https://codepen.io/neetnestor/pen/vYwgZaG)
+
+- [simple-chat-ts](simple-chat-ts): a mininum and complete chat bot app in TypeScript.
+- [get-started-web-worker](get-started-web-worker): same as get-started, but using web worker.
+- [next-simple-chat](next-simple-chat): a mininum and complete chat bot app with [Next.js](https://nextjs.org/).
+- [subgroups-usage](subgroups-usage): capability-based routing between baseline and subgroup WebGPU WASM builds.
+- [multi-round-chat](multi-round-chat): while APIs are functional, we internally optimize so that multi round chat usage can reuse KV cache
+- [text-completion](text-completion): demonstrates API `engine.completions.create()`, which is pure text completion with no conversation, as opposed to `engine.chat.completions.create()`
+- [embeddings](embeddings): demonstrates API `engine.embeddings.create()`, integration with `EmbeddingsInterface` and `MemoryVectorStore` of [Langchain.js](https://js.langchain.com), and RAG with Langchain.js using WebLLM for both LLM and Embedding in a single engine
+- [multi-models](multi-models): demonstrates loading multiple models in a single engine concurrently
+
+#### Advanced OpenAI API Capabilities
+
+These examples demonstrate various capabilities via WebLLM's OpenAI-like API.
+
+- [streaming](streaming): return output as chunks in real-time in the form of an AsyncGenerator
+- [json-mode](json-mode): efficiently ensure output is in json format, see [OpenAI Reference](https://platform.openai.com/docs/guides/text-generation/chat-completions-api) for more.
+- [json-schema](json-schema): besides guaranteeing output to be in JSON, ensure output to adhere to a specific JSON schema specified the user
+- [seed-to-reproduce](seed-to-reproduce): use seeding to ensure reproducible output with fields `seed`.
+- [function-calling](function-calling) (WIP): function calling with fields `tools` and `tool_choice` (with preliminary support).
+- [vision-model](vision-model): process request with image input using Vision Language Model (e.g. Phi3.5-vision)
+
+#### Chrome Extension
+
+- [chrome-extension](chrome-extension): chrome extension that does not have a persistent background
+- [chrome-extension-webgpu-service-worker](chrome-extension-webgpu-service-worker): chrome extension using service worker, hence having a persistent background
+
+#### Others
+
+- [logit-processor](logit-processor): while `logit_bias` is supported, we additionally support stateful logit processing where users can specify their own rules. We also expose low-level API `forwardTokensAndSample()`.
+- [cache-usage](cache-usage): demonstrates how WebLLM supports multiple cache backends. Choose between the [Cache API](https://developer.mozilla.org/en-US/docs/Web/API/Cache), [IndexedDB cache](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API), [OPFS](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system), or the experimental Chrome [Cross-Origin Storage](https://github.com/WICG/cross-origin-storage) extension via `appConfig.cacheBackend`. Also demonstrates various cache utils such as checking
+  whether a model is cached, deleting a model's weights from cache, deleting a model library wasm from cache, etc. Note: cross-origin backend currently does not support programmatic tensor-cache deletion.
+- [simple-chat-upload](simple-chat-upload): demonstrates how to upload local models to WebLLM instead of downloading via a URL link
+
+## Demo Spaces
+
+- [web-llm-embed](https://huggingface.co/spaces/matthoffner/web-llm-embed): document chat prototype using react-llm with transformers.js embeddings
+- [DeVinci](https://x6occ-biaaa-aaaai-acqzq-cai.icp0.io/): AI chat app based on WebLLM and hosted on decentralized cloud platform

examples/abort-reload/README.md

@@ -0,0 +1,13 @@
+# WebLLM Get Started App
+
+This folder provides a demo for cancelling model fetching after calling `engine.reload()`.
+
+```bash
+npm install
+npm start
+```
+
+Note if you would like to hack WebLLM core package.
+You can change web-llm dependencies as `"file:../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.

examples/abort-reload/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "get-started",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/get_started.html  --port 8887",
+    "build": "parcel build src/get_started.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}

examples/abort-reload/src/get_started.html

@@ -0,0 +1,23 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output
+    <br />
+    <br />
+    <label id="init-label"> </label>
+
+    <h3>Prompt</h3>
+    <label id="prompt-label"> </label>
+
+    <h3>Response</h3>
+    <label id="generate-label"> </label>
+    <br />
+    <label id="stats-label"> </label>
+
+    <script type="module" src="./get_started.js"></script>
+  </body>
+</html>

examples/abort-reload/src/get_started.js

@@ -0,0 +1,32 @@
+import * as webllm from "@mlc-ai/web-llm";
+import { error } from "loglevel";
+
+let engine;
+
+function setLabel(id, text) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+async function main() {
+  const initProgressCallback = (report) => {
+    console.log(report.text);
+    setLabel("init-label", report.text);
+  };
+  // Option 1: If we do not specify appConfig, we use `prebuiltAppConfig` defined in `config.ts`
+  const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
+  engine = new webllm.MLCEngine({
+    initProgressCallback,
+  });
+  engine.reload(selectedModel);
+}
+main();
+setTimeout(() => {
+  console.log("calling unload");
+  engine.unload().catch((err) => {
+    console.log(err);
+  });
+}, 5000);

examples/cache-usage/README.md

@@ -0,0 +1,30 @@
+# WebLLM Cache Usage
+
+WebLLM supports multiple persistent cache backends. You can pick the classic Cache API, IndexedDB, [Origin Private File System (OPFS)](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system), or the experimental Chrome [Cross-Origin Storage](https://github.com/WICG/cross-origin-storage) extension by
+setting `AppConfig.cacheBackend` to `"cache"`, `"indexeddb"`, `"opfs"`, or `"cross-origin"`.
+This folder provides an example on how different caches are used in WebLLM. We also
+demonstrate the utility cache functions such as deleting models, checking if models are in cache, etc.
+
+> **Note:** The cross-origin backend requires installation of the [Cross-Origin Storage browser extension](https://chromewebstore.google.com/detail/cross-origin-storage/denpnpcgjgikjpoglpjefakmdcbmlgih) ([source code](https://github.com/web-ai-community/cross-origin-storage-extension)). This does not currently support programmatic tensor-cache deletion; deletion is extension-managed.
+
+> **Note:** If `"opfs"` is selected in an environment without OPFS support, cache operations fail with an OPFS availability error.
+> Use `appConfig.opfsAccessMode = "auto"` to use OPFS sync access handles where supported, or `"sync"` to require sync access handles. The default is `"async"`.
+
+For more information about Cache API and IndexedDB, see:
+https://developer.mozilla.org/en-US/docs/Web/API/Storage_API/Storage_quotas_and_eviction_criteria#what_technologies_store_data_in_the_browser.
+
+To inspect the downloaded artifacts in your browser, open up developer console, go to application,
+and you will find the artifacts under `IndexedDB`, `Cache storage`, or OPFS (under the browser's origin-private file system). When `"cross-origin"` is selected,
+the extension displays origins and resource hashes.
+
+To run the example, you can do the following steps under this folder
+
+```bash
+npm install
+npm start
+```
+
+Note if you would like to hack WebLLM core package.
+You can change web-llm dependencies as `"file:../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.

examples/cache-usage/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "cache-usage",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/cache_usage.html  --port 8889",
+    "build": "parcel build src/cache_usage.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}

examples/cache-usage/src/cache_usage.html

@@ -0,0 +1,24 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output
+    <br />
+    <br />
+    <label id="init-label"> </label>
+
+    <h3>Prompt</h3>
+    <label id="prompt-label"> </label>
+
+    <h3>Response</h3>
+    <label id="generate-label"> </label>
+    <br />
+    <label id="stats-label"> </label>
+
+    <script type="module" src="./cache_usage.ts"></script>
+  </body>
+</html>

examples/cache-usage/src/cache_usage.ts

@@ -0,0 +1,90 @@
+import * as webllm from "@mlc-ai/web-llm";
+
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+const initProgressCallback = (report: webllm.InitProgressReport) => {
+  setLabel("init-label", report.text);
+};
+
+async function main() {
+  const appConfig = webllm.prebuiltAppConfig;
+  // CHANGE THIS TO SEE THE EFFECTS OF EACH, CODE BELOW DOES NOT NEED TO CHANGE
+  appConfig.cacheBackend = "cache"; // "indexeddb" or "cache" or "cross-origin" or "opfs"
+
+  const cacheBackend = appConfig.cacheBackend as string;
+  if (cacheBackend === "indexeddb") {
+    console.log("Using IndexedDB Cache");
+  } else if (cacheBackend === "cache") {
+    console.log("Using Cache API");
+  } else if (cacheBackend === "cross-origin") {
+    console.log("Using Cross-Origin Storage");
+  } else if (cacheBackend === "opfs") {
+    console.log("Using Origin Private File System");
+  }
+
+  // 1. This triggers downloading and caching the model with either Cache or IndexedDB Cache
+  const selectedModel = "Llama-3.2-1B-Instruct-q4f16_1-MLC";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    { initProgressCallback: initProgressCallback, appConfig: appConfig },
+  );
+
+  const request: webllm.ChatCompletionRequest = {
+    stream: false,
+    messages: [
+      {
+        role: "user",
+        content: "Write an analogy between mathematics and a lighthouse.",
+      },
+    ],
+    n: 1,
+    temperature: 0,
+  };
+  let reply = await engine.chat.completions.create(request);
+  console.log(reply);
+
+  // 2. Check whether model weights are cached
+  let modelCached = await webllm.hasModelInCache(selectedModel, appConfig);
+  console.log("hasModelInCache: ", modelCached);
+  if (!modelCached) {
+    throw Error("Expect hasModelInCache() to be true, but got: " + modelCached);
+  }
+
+  // 3. We reload, and we should see this time it is much faster because the weights are cached.
+  console.log("Reload model start");
+  await engine.reload(selectedModel);
+  console.log("Reload model end");
+  reply = await engine.chat.completions.create(request);
+  console.log(reply);
+
+  // Cross-Origin Storage does not support deletion
+  if (cacheBackend === "cross-origin") {
+    return;
+  }
+
+  // 4. Delete everything about this model from cache
+  // You can also delete only the model library wasm, only the model weights, or only the config file
+  await webllm.deleteModelAllInfoInCache(selectedModel, appConfig);
+  modelCached = await webllm.hasModelInCache(selectedModel, appConfig);
+  console.log("After deletion, hasModelInCache: ", modelCached);
+  if (modelCached) {
+    throw Error(
+      "Expect hasModelInCache() to be false, but got: " + modelCached,
+    );
+  }
+
+  // 5. If we reload, we should expect the model to start downloading again
+  console.log("Reload model start");
+  await engine.reload(selectedModel);
+  console.log("Reload model end");
+  reply = await engine.chat.completions.create(request);
+  console.log(reply);
+}
+
+main();

examples/chrome-extension-webgpu-service-worker/README.md

@@ -0,0 +1,29 @@
+# WebLLM Chrome Extension using WebGPU Running on Service Worker
+
+![Chrome Extension](https://github.com/mlc-ai/mlc-llm/assets/11940172/0d94cc73-eff1-4128-a6e4-70dc879f04e0)
+
+> [!WARNING]  
+> Service worker support in WebGPU is enabled by default in [Chrome 124](https://chromiumdash.appspot.com/commit/8d78510e4aca5ac3cd8ee4a33e96b404eaa43246).
+> If you are using Chrome 123, go to `chrome://flags/#enable-experimental-web-platform-features`, enable the `#enable-experimental-web-platform-features` flag, and **relaunch the browser**.
+
+This example shows how we can create a Chrome extension using WebGPU and service worker.
+
+- The project structure is as follows:
+  - `manifest.json`: A required file that lists important information about the structure and behavior of that extension. Here we are using manifest V3.
+  - `popup.ts`: Script of the extension pop-up window.
+  - `background.ts`: Script of the service worker. An extension service worker is loaded when it is needed, and unloaded when it goes dormant.
+  - `content.js`: Content script that interacts with DOM.
+- Run
+
+  ```bash
+  npm install
+  npm run build
+  ```
+
+  This will create a new directory at `./dist/`. To load the extension into Chrome, go to Extensions > Manage Extensions and select Load Unpacked. Add the `./dist/` directory. You can now pin the extension to your toolbar and use it to chat with your favorite model!
+
+**Note**: This example disables chatting using the contents of the active tab by default.
+To enable it, set `useContext` in `popup.ts` to `true`. More info about this feature can be found
+[here](https://github.com/mlc-ai/web-llm/pull/190).
+However, if the web content is too large, it might run into issues. We recommend using `example.html` to
+test this feature.

examples/chrome-extension-webgpu-service-worker/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "chrome-extension",
+  "version": "1.0.0",
+  "description": "",
+  "private": true,
+  "scripts": {
+    "build": "parcel build src/manifest.json --config @parcel/config-webextension"
+  },
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@parcel/config-webextension": "^2.9.3",
+    "@types/chrome": "^0.0.242",
+    "buffer": "^6.0.3",
+    "parcel": "^2.9.3",
+    "process": "^0.11.10",
+    "url": "^0.11.1"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84",
+    "progressbar.js": "^1.1.0"
+  }
+}

examples/chrome-extension-webgpu-service-worker/src/background.ts

@@ -0,0 +1,14 @@
+import { ExtensionServiceWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
+
+// Hookup an engine to a service worker handler
+let handler;
+
+chrome.runtime.onConnect.addListener(function (port) {
+  console.assert(port.name === "web_llm_service_worker");
+  if (handler === undefined) {
+    handler = new ExtensionServiceWorkerMLCEngineHandler(port);
+  } else {
+    handler.setPort(port);
+  }
+  port.onMessage.addListener(handler.onmessage.bind(handler));
+});

examples/chrome-extension-webgpu-service-worker/src/content.js

@@ -0,0 +1,6 @@
+// Only the content script is able to access the DOM
+chrome.runtime.onConnect.addListener(function (port) {
+  port.onMessage.addListener(function (msg) {
+    port.postMessage({ contents: document.body.innerHTML });
+  });
+});

examples/chrome-extension-webgpu-service-worker/src/example.html

@@ -0,0 +1,20 @@
+In the year 2154, humanity had colonized several planets in the distant reaches
+of the galaxy. The planet of Xylophia-IV was one of the most remote and
+inhospitable, with temperatures often dropping to -200 degrees Celsius. Despite
+these harsh conditions, a team of scientists had established a research station
+on the planet to study the unique geological formations and exotic flora and
+fauna. One day, while conducting a routine survey of the planet's surface, the
+team discovered an strange object buried deep in the ice. As they examined it
+closer, they realized it was a small, metallic capsule with a glowing blue
+symbol etched onto its surface. The team's leader, a brilliant scientist named
+Dr. Maria Rodriguez, was immediately intrigued by the capsule's mysterious
+origins. She ordered her team to bring it back to the research station for
+further analysis. After weeks of studying the capsule, the team finally cracked
+the code to the symbol etched onto its surface. It was a message from an alien
+race, warning Earth of an impending attack from an unknown threat. The team was
+shocked and dismayed by the news, but they knew they had to act quickly to warn
+the rest of humanity. They transmitted the message to the nearest space station,
+which relayed it to Earth's government. As the threat of attack loomed near, the
+team remained on high alert, ready to face whatever dangers lay ahead. They had
+uncovered a secrets of the universe, and now they were determined to protect
+their planet and its inhabitants at all costs.

examples/chrome-extension-webgpu-service-worker/src/manifest.json

@@ -0,0 +1,30 @@
+{
+  "manifest_version": 3,
+  "name": "MLCBot",
+  "version": "0.1.0",
+  "description": "Chat with your browser",
+  "icons": {
+    "16": "icons/icon-16.png",
+    "32": "icons/icon-32.png",
+    "64": "icons/icon-64.png",
+    "128": "icons/icon-128.png"
+  },
+  "content_security_policy": {
+    "extension_pages": "style-src-elem 'self' https://cdnjs.cloudflare.com; font-src 'self' https://cdnjs.cloudflare.com; script-src 'self' 'wasm-unsafe-eval'; default-src 'self' data:; connect-src 'self' data: http://localhost:8000 https://huggingface.co https://cdn-lfs.huggingface.co https://cdn-lfs-us-1.huggingface.co https://raw.githubusercontent.com https://cdn-lfs-us-1.hf.co https://cas-bridge.xethub.hf.co"
+  },
+  "action": {
+    "default_title": "MLCBot",
+    "default_popup": "popup.html"
+  },
+  "content_scripts": [
+    {
+      "matches": ["<all_urls>"],
+      "js": ["content.js"]
+    }
+  ],
+  "background": {
+    "service_worker": "background.ts",
+    "type": "module"
+  },
+  "permissions": ["storage", "tabs", "webNavigation"]
+}

examples/chrome-extension-webgpu-service-worker/src/popup.css

@@ -0,0 +1,240 @@
+*,
+*::before,
+*::after {
+  margin: 0;
+  padding: 0;
+  box-sizing: border-box;
+}
+
+html {
+  font-family:
+    -apple-system,
+    BlinkMacSystemFont,
+    Segoe UI,
+    Helvetica,
+    Arial,
+    sans-serif;
+  color: #222;
+}
+
+body {
+  margin: 0;
+  padding: 0.5rem;
+  background-color: #778da9;
+  width: 320px;
+  font-size: small;
+}
+
+p {
+  margin: 0;
+}
+
+/* LOADING BAR */
+#loadingContainer {
+  margin-bottom: 15px;
+  width: 300px;
+  height: 8px;
+}
+
+/* INPUT AREA */
+#query-input {
+  border: 1px solid #ccc;
+  border-radius: 4px;
+}
+
+.input-container {
+  display: flex;
+  flex-direction: row;
+  align-items: center;
+}
+
+.input-container input {
+  width: 100%;
+  outline: none;
+  padding: 0.5rem;
+  margin-right: 0.5rem;
+}
+
+/* SUBMIT BUTTON */
+.btn {
+  background-color: #1b263b;
+  color: white;
+  font-size: small;
+  cursor: pointer;
+  border-radius: 4px;
+  border: none;
+  padding: 0.5rem;
+}
+
+.btn:hover {
+  background-color: #d0d0d0;
+}
+
+.btn:disabled {
+  background-color: #a7a7a7;
+  color: rgb(255, 255, 255);
+  cursor: default;
+}
+
+.btn img {
+  width: 1rem;
+  height: 1rem;
+}
+
+/* LOADING */
+
+.stage {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  position: relative;
+  margin: 0 -5%;
+  overflow: hidden;
+}
+
+#loading-indicator {
+  display: none;
+  color: white;
+  margin-top: 0.5rem;
+}
+
+.dot-flashing {
+  position: relative;
+  width: 10px;
+  height: 10px;
+  border-radius: 5px;
+  background-color: #1b263b;
+  color: #1b263b;
+  animation: dot-flashing 0.4s infinite linear alternate;
+  animation-delay: 0.2s;
+}
+
+.dot-flashing::before,
+.dot-flashing::after {
+  content: "";
+  display: inline-block;
+  position: absolute;
+  top: 0;
+}
+
+.dot-flashing::before {
+  left: -15px;
+  width: 10px;
+  height: 10px;
+  border-radius: 5px;
+  background-color: #1b263b;
+  color: #1b263b;
+  animation: dot-flashing 0.4s infinite alternate;
+  animation-delay: 0s;
+}
+
+.dot-flashing::after {
+  left: 15px;
+  width: 10px;
+  height: 10px;
+  border-radius: 5px;
+  background-color: #1b263b;
+  color: #1b263b;
+  animation: dot-flashing 0.4s infinite alternate;
+  animation-delay: 0.4s;
+}
+
+@keyframes dot-flashing {
+  0% {
+    background-color: #1b263b;
+  }
+
+  50%,
+  100% {
+    background-color: #415a77;
+  }
+}
+
+/* ANSWERS */
+#queriesAnswersContainer {
+  display: block;
+  color: white;
+  margin-top: 0.5rem;
+}
+
+#answer {
+  color: #333333;
+}
+
+#answerWrapper {
+  display: none;
+  background-color: #ffd166;
+  border-radius: 8px;
+  padding: 0.5rem;
+  margin-top: 0.5rem;
+}
+
+.queriesAnswers {
+  border-radius: 8px;
+  background-color: #ffd166;
+  padding: 0.5rem;
+  color: #333333;
+}
+
+#lastQuery {
+  color: rgb(188, 188, 188);
+}
+
+#lastAnswer {
+  color: white;
+  margin-top: 0.5rem;
+}
+
+#lastRequest {
+  padding: 0.5rem;
+  margin-top: 0.5rem;
+  background-color: #333333;
+  border-radius: 4px;
+}
+
+/* ANSWER OPTIONS */
+.timeStamp {
+  color: #9a8c98;
+}
+
+.copyRow {
+  display: flex;
+  flex-direction: row;
+  align-items: end;
+  justify-content: space-between;
+  color: #a7a7a7;
+  margin-top: 0.5rem;
+}
+
+.copyText {
+  display: none;
+  color: #a7a7a7;
+  margin-right: 0.5rem;
+}
+
+.copyButton {
+  color: #415a77;
+  background-color: transparent;
+  border: none;
+  cursor: pointer;
+  padding: 0;
+  margin-left: 0.5rem;
+}
+
+.copyButton:hover {
+  color: #5e80a7;
+  background-color: transparent;
+}
+
+.removeButton {
+  color: #415a77;
+  background-color: transparent;
+  border: none;
+  cursor: pointer;
+  padding: 0;
+}
+
+.removeButton:hover {
+  color: #5e80a7;
+  background-color: transparent;
+}

examples/chrome-extension-webgpu-service-worker/src/popup.html

@@ -0,0 +1,46 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>Chatbot</title>
+    <link rel="stylesheet" href="popup.css" />
+    <link
+      rel="stylesheet"
+      href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css"
+    />
+  </head>
+  <body>
+    <div id="loadingContainer"></div>
+
+    <div class="input-container form-group">
+      <input
+        type="search"
+        id="query-input"
+        placeholder="What's on your mind?"
+      />
+      <button id="submit-button" class="btn">
+        <i class="fa fa-comments"></i>
+      </button>
+    </div>
+
+    <div class="stage">
+      <div id="loading-indicator" class="dot-flashing"></div>
+    </div>
+
+    <div id="answerWrapper">
+      <div id="answer"></div>
+      <div class="copyRow">
+        <span id="timestamp"></span>
+        <button
+          id="copyAnswer"
+          class="btn copyButton"
+          title="Copy the Answer to the Clipboard"
+        >
+          <i class="fa-solid fa-copy fa-lg"></i>
+        </button>
+      </div>
+    </div>
+
+    <script type="module" src="./popup.ts"></script>
+  </body>
+</html>

examples/chrome-extension-webgpu-service-worker/src/popup.ts

@@ -0,0 +1,167 @@
+"use strict";
+
+// This code is partially adapted from the openai-chatgpt-chrome-extension repo:
+// https://github.com/jessedi0n/openai-chatgpt-chrome-extension
+
+import "./popup.css";
+
+import {
+  ChatCompletionMessageParam,
+  CreateExtensionServiceWorkerMLCEngine,
+  MLCEngineInterface,
+  InitProgressReport,
+} from "@mlc-ai/web-llm";
+import { ProgressBar, Line } from "progressbar.js";
+
+/***************** UI elements *****************/
+// Whether or not to use the content from the active tab as the context
+const useContext = false;
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+const queryInput = document.getElementById("query-input")!;
+const submitButton = document.getElementById("submit-button")!;
+
+let isLoadingParams = false;
+
+(<HTMLButtonElement>submitButton).disabled = true;
+
+const progressBar: ProgressBar = new Line("#loadingContainer", {
+  strokeWidth: 4,
+  easing: "easeInOut",
+  duration: 1400,
+  color: "#ffd166",
+  trailColor: "#eee",
+  trailWidth: 1,
+  svgStyle: { width: "100%", height: "100%" },
+});
+
+/***************** Web-LLM MLCEngine Configuration *****************/
+const initProgressCallback = (report: InitProgressReport) => {
+  progressBar.animate(report.progress, {
+    duration: 50,
+  });
+  if (report.progress == 1.0) {
+    enableInputs();
+  }
+};
+
+const engine: MLCEngineInterface = await CreateExtensionServiceWorkerMLCEngine(
+  "Qwen2-0.5B-Instruct-q4f16_1-MLC",
+  { initProgressCallback: initProgressCallback },
+);
+const chatHistory: ChatCompletionMessageParam[] = [];
+
+isLoadingParams = true;
+
+function enableInputs() {
+  if (isLoadingParams) {
+    sleep(500);
+    (<HTMLButtonElement>submitButton).disabled = false;
+    const loadingBarContainer = document.getElementById("loadingContainer")!;
+    loadingBarContainer.remove();
+    queryInput.focus();
+    isLoadingParams = false;
+  }
+}
+
+/***************** Event Listeners *****************/
+
+// Disable submit button if input field is empty
+queryInput.addEventListener("keyup", () => {
+  if ((<HTMLInputElement>queryInput).value === "") {
+    (<HTMLButtonElement>submitButton).disabled = true;
+  } else {
+    (<HTMLButtonElement>submitButton).disabled = false;
+  }
+});
+
+// If user presses enter, click submit button
+queryInput.addEventListener("keyup", (event) => {
+  if (event.code === "Enter") {
+    event.preventDefault();
+    submitButton.click();
+  }
+});
+
+// Listen for clicks on submit button
+async function handleClick() {
+  // Get the message from the input field
+  const message = (<HTMLInputElement>queryInput).value;
+  console.log("message", message);
+  chatHistory.push({ role: "user", content: message });
+
+  // Clear the answer
+  document.getElementById("answer")!.innerHTML = "";
+  // Hide the answer
+  document.getElementById("answerWrapper")!.style.display = "none";
+  // Show the loading indicator
+  document.getElementById("loading-indicator")!.style.display = "block";
+
+  // Send the chat completion message to the engine
+  let curMessage = "";
+  const completion = await engine.chat.completions.create({
+    stream: true,
+    messages: chatHistory,
+  });
+
+  // Update the answer as the model generates more text
+  for await (const chunk of completion) {
+    const curDelta = chunk.choices[0].delta.content;
+    if (curDelta) {
+      curMessage += curDelta;
+    }
+    updateAnswer(curMessage);
+  }
+  chatHistory.push({ role: "assistant", content: await engine.getMessage() });
+}
+
+submitButton.addEventListener("click", handleClick);
+
+function updateAnswer(answer: string) {
+  // Show answer
+  document.getElementById("answerWrapper")!.style.display = "block";
+  const answerWithBreaks = answer.replace(/\n/g, "<br>");
+  document.getElementById("answer")!.innerHTML = answerWithBreaks;
+  // Add event listener to copy button
+  document.getElementById("copyAnswer")!.addEventListener("click", () => {
+    // Get the answer text
+    const answerText = answer;
+    // Copy the answer text to the clipboard
+    navigator.clipboard
+      .writeText(answerText)
+      .then(() => console.log("Answer text copied to clipboard"))
+      .catch((err) => console.error("Could not copy text: ", err));
+  });
+  const options: Intl.DateTimeFormatOptions = {
+    month: "short",
+    day: "2-digit",
+    hour: "2-digit",
+    minute: "2-digit",
+    second: "2-digit",
+  };
+  const time = new Date().toLocaleString("en-US", options);
+  // Update timestamp
+  document.getElementById("timestamp")!.innerText = time;
+  // Hide loading indicator
+  document.getElementById("loading-indicator")!.style.display = "none";
+}
+
+function fetchPageContents() {
+  chrome.tabs.query({ currentWindow: true, active: true }, function (tabs) {
+    if (tabs[0]?.id) {
+      const port = chrome.tabs.connect(tabs[0].id, { name: "channelName" });
+      port.postMessage({});
+      port.onMessage.addListener(function (msg) {
+        console.log("Page contents:", msg.contents);
+        chrome.runtime.sendMessage({ context: msg.contents });
+      });
+    }
+  });
+}
+
+// Grab the page contents when the popup is opened
+window.onload = function () {
+  if (useContext) {
+    fetchPageContents();
+  }
+};

examples/chrome-extension/README.md

@@ -0,0 +1,12 @@
+# WebLLM Chrome Extension
+
+![Chrome Extension](https://github.com/mlc-ai/mlc-llm/assets/11940172/0d94cc73-eff1-4128-a6e4-70dc879f04e0)
+
+To run the extension, do the following steps under this folder
+
+```bash
+npm install
+npm run build
+```
+
+This will create a new directory at `chrome-extension/dist/`. To load the extension into Chrome, go to Extensions > Manage Extensions and select Load Unpacked. Add the `chrome-extension/dist/` directory. You can now pin the extension to your toolbar and use the drop-down menu to chat with your favorite model!

examples/chrome-extension/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "chrome-extension",
+  "version": "1.0.1",
+  "description": "",
+  "private": true,
+  "scripts": {
+    "build": "parcel build src/manifest.json --config @parcel/config-webextension"
+  },
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@parcel/config-webextension": "^2.9.3",
+    "@types/chrome": "^0.0.242",
+    "buffer": "^6.0.3",
+    "parcel": "^2.9.3",
+    "process": "^0.11.10",
+    "url": "^0.11.1"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84",
+    "progressbar.js": "^1.1.0"
+  }
+}

examples/chrome-extension/src/content.js

@@ -0,0 +1,6 @@
+// Only the content script is able to access the DOM
+chrome.runtime.onConnect.addListener(function (port) {
+  port.onMessage.addListener(function (msg) {
+    port.postMessage({ contents: document.body.innerText });
+  });
+});

examples/chrome-extension/src/example.html

@@ -0,0 +1,20 @@
+In the year 2154, humanity had colonized several planets in the distant reaches
+of the galaxy. The planet of Xylophia-IV was one of the most remote and
+inhospitable, with temperatures often dropping to -200 degrees Celsius. Despite
+these harsh conditions, a team of scientists had established a research station
+on the planet to study the unique geological formations and exotic flora and
+fauna. One day, while conducting a routine survey of the planet's surface, the
+team discovered an strange object buried deep in the ice. As they examined it
+closer, they realized it was a small, metallic capsule with a glowing blue
+symbol etched onto its surface. The team's leader, a brilliant scientist named
+Dr. Maria Rodriguez, was immediately intrigued by the capsule's mysterious
+origins. She ordered her team to bring it back to the research station for
+further analysis. After weeks of studying the capsule, the team finally cracked
+the code to the symbol etched onto its surface. It was a message from an alien
+race, warning Earth of an impending attack from an unknown threat. The team was
+shocked and dismayed by the news, but they knew they had to act quickly to warn
+the rest of humanity. They transmitted the message to the nearest space station,
+which relayed it to Earth's government. As the threat of attack loomed near, the
+team remained on high alert, ready to face whatever dangers lay ahead. They had
+uncovered a secrets of the universe, and now they were determined to protect
+their planet and its inhabitants at all costs.

examples/chrome-extension/src/manifest.json

@@ -0,0 +1,27 @@
+{
+  "manifest_version": 3,
+  "name": "MLCBot",
+  "version": "0.1.1",
+  "description": "Chat with your browser",
+  "icons": {
+    "16": "icons/icon-16.png",
+    "32": "icons/icon-32.png",
+    "64": "icons/icon-64.png",
+    "128": "icons/icon-128.png"
+  },
+  "content_security_policy": {
+    "extension_pages": "style-src-elem 'self' https://cdnjs.cloudflare.com; font-src 'self' https://cdnjs.cloudflare.com; script-src 'self' 'wasm-unsafe-eval'; default-src 'self' data:; connect-src 'self' data: http://localhost:8000 https://huggingface.co https://cdn-lfs.huggingface.co https://cdn-lfs-us-1.huggingface.co https://raw.githubusercontent.com https://cdn-lfs-us-1.hf.co https://cas-bridge.xethub.hf.co"
+  },
+  "action": {
+    "default_title": "MLCBot",
+    "default_popup": "popup.html"
+  },
+  "content_scripts": [
+    {
+      "matches": ["<all_urls>"],
+      "js": ["content.js"]
+    }
+  ],
+  "permissions": ["storage", "tabs", "webNavigation", "activeTab", "scripting"],
+  "host_permissions": ["http://*/", "https://*/"]
+}

examples/chrome-extension/src/manifest_v2.json

@@ -0,0 +1,23 @@
+{
+  "manifest_version": 2,
+  "name": "MLCBot",
+  "version": "0.1.0",
+  "description": "Chat with your browser",
+  "icons": {
+    "16": "icons/icon-16.png",
+    "32": "icons/icon-32.png",
+    "64": "icons/icon-64.png",
+    "128": "icons/icon-128.png"
+  },
+  "content_security_policy": "style-src-elem 'self' https://cdnjs.cloudflare.com; font-src 'self' https://cdnjs.cloudflare.com; script-src 'self' 'unsafe-eval' 'wasm-unsafe-eval'; default-src 'self' data:; connect-src 'self' data: http://localhost:8000 https://huggingface.co https://cdn-lfs.huggingface.co https://raw.githubusercontent.com https://cdn-lfs-us-1.hf.co https://cas-bridge.xethub.hf.co",
+  "browser_action": {
+    "default_popup": "popup.html"
+  },
+  "content_scripts": [
+    {
+      "matches": ["<all_urls>"],
+      "js": ["content.js"]
+    }
+  ],
+  "permissions": ["storage", "tabs", "webNavigation", "activeTab"]
+}

examples/chrome-extension/src/popup.css

@@ -0,0 +1,240 @@
+*,
+*::before,
+*::after {
+  margin: 0;
+  padding: 0;
+  box-sizing: border-box;
+}
+
+html {
+  font-family:
+    -apple-system,
+    BlinkMacSystemFont,
+    Segoe UI,
+    Helvetica,
+    Arial,
+    sans-serif;
+  color: #222;
+}
+
+body {
+  margin: 0;
+  padding: 0.5rem;
+  background-color: #778da9;
+  width: 335px;
+  font-size: small;
+}
+
+p {
+  margin: 0;
+}
+
+/* LOADING BAR */
+#loadingContainer {
+  margin-bottom: 15px;
+  width: 315px;
+  height: 8px;
+}
+
+/* INPUT AREA */
+#query-input {
+  border: 1px solid #ccc;
+  border-radius: 4px;
+}
+
+.input-container {
+  display: flex;
+  flex-direction: row;
+  align-items: center;
+}
+
+.input-container input {
+  width: 100%;
+  outline: none;
+  padding: 0.5rem;
+  margin-right: 0.5rem;
+}
+
+/* BUTTON */
+.btn {
+  background-color: #1b263b;
+  color: white;
+  font-size: small;
+  cursor: pointer;
+  border-radius: 4px;
+  border: none;
+  padding: 0.5rem;
+}
+
+.btn:hover {
+  background-color: #d0d0d0;
+}
+
+.btn:disabled {
+  background-color: #a7a7a7;
+  color: rgb(255, 255, 255);
+  cursor: default;
+}
+
+.btn img {
+  width: 1rem;
+  height: 1rem;
+}
+
+/* LOADING */
+
+.stage {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  position: relative;
+  margin: 0 -5%;
+  overflow: hidden;
+}
+
+#loading-indicator {
+  display: none;
+  color: white;
+  margin-top: 0.5rem;
+}
+
+.dot-flashing {
+  position: relative;
+  width: 10px;
+  height: 10px;
+  border-radius: 5px;
+  background-color: #1b263b;
+  color: #1b263b;
+  animation: dot-flashing 0.4s infinite linear alternate;
+  animation-delay: 0.2s;
+}
+
+.dot-flashing::before,
+.dot-flashing::after {
+  content: "";
+  display: inline-block;
+  position: absolute;
+  top: 0;
+}
+
+.dot-flashing::before {
+  left: -15px;
+  width: 10px;
+  height: 10px;
+  border-radius: 5px;
+  background-color: #1b263b;
+  color: #1b263b;
+  animation: dot-flashing 0.4s infinite alternate;
+  animation-delay: 0s;
+}
+
+.dot-flashing::after {
+  left: 15px;
+  width: 10px;
+  height: 10px;
+  border-radius: 5px;
+  background-color: #1b263b;
+  color: #1b263b;
+  animation: dot-flashing 0.4s infinite alternate;
+  animation-delay: 0.4s;
+}
+
+@keyframes dot-flashing {
+  0% {
+    background-color: #1b263b;
+  }
+
+  50%,
+  100% {
+    background-color: #415a77;
+  }
+}
+
+/* ANSWERS */
+#queriesAnswersContainer {
+  display: block;
+  color: white;
+  margin-top: 0.5rem;
+}
+
+#answer {
+  color: #333333;
+}
+
+#answerWrapper {
+  display: none;
+  background-color: #ffd166;
+  border-radius: 8px;
+  padding: 0.5rem;
+  margin-top: 0.5rem;
+}
+
+.queriesAnswers {
+  border-radius: 8px;
+  background-color: #ffd166;
+  padding: 0.5rem;
+  color: #333333;
+}
+
+#lastQuery {
+  color: rgb(188, 188, 188);
+}
+
+#lastAnswer {
+  color: white;
+  margin-top: 0.5rem;
+}
+
+#lastRequest {
+  padding: 0.5rem;
+  margin-top: 0.5rem;
+  background-color: #333333;
+  border-radius: 4px;
+}
+
+/* ANSWER OPTIONS */
+.timeStamp {
+  color: #9a8c98;
+}
+
+.copyRow {
+  display: flex;
+  flex-direction: row;
+  align-items: end;
+  justify-content: space-between;
+  color: #a7a7a7;
+  margin-top: 0.5rem;
+}
+
+.copyText {
+  display: none;
+  color: #a7a7a7;
+  margin-right: 0.5rem;
+}
+
+.copyButton {
+  color: #415a77;
+  background-color: transparent;
+  border: none;
+  cursor: pointer;
+  padding: 0;
+  margin-left: 0.5rem;
+}
+
+.copyButton:hover {
+  color: #5e80a7;
+  background-color: transparent;
+}
+
+.removeButton {
+  color: #415a77;
+  background-color: transparent;
+  border: none;
+  cursor: pointer;
+  padding: 0;
+}
+
+.removeButton:hover {
+  color: #5e80a7;
+  background-color: transparent;
+}

examples/chrome-extension/src/popup.html

@@ -0,0 +1,50 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>Chatbot</title>
+    <link rel="stylesheet" href="popup.css" />
+    <link
+      rel="stylesheet"
+      href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css"
+    />
+  </head>
+  <body>
+    <select id="model-selection"></select>
+    <div id="loadingBox">
+      <p id="init-label">Initializing model...</p>
+      <div id="loadingContainer"></div>
+    </div>
+    <p id="model-name"></p>
+    <div class="input-container form-group">
+      <input
+        type="search"
+        id="query-input"
+        placeholder="What's on your mind?"
+      />
+      <button id="submit-button" class="btn">
+        <i class="fa fa-comments"></i>
+      </button>
+    </div>
+
+    <div class="stage">
+      <div id="loading-indicator" class="dot-flashing"></div>
+    </div>
+
+    <div id="answerWrapper">
+      <div id="answer"></div>
+      <div class="copyRow">
+        <span id="timestamp"></span>
+        <button
+          id="copyAnswer"
+          class="btn copyButton"
+          title="Copy the Answer to the Clipboard"
+        >
+          <i class="fa-solid fa-copy fa-lg"></i>
+        </button>
+      </div>
+    </div>
+
+    <script type="module" src="./popup.ts"></script>
+  </body>
+</html>

examples/chrome-extension/src/popup.ts

@@ -0,0 +1,298 @@
+"use strict";
+
+// This code is partially adapted from the openai-chatgpt-chrome-extension repo:
+// https://github.com/jessedi0n/openai-chatgpt-chrome-extension
+
+import "./popup.css";
+
+import {
+  MLCEngineInterface,
+  InitProgressReport,
+  CreateMLCEngine,
+  ChatCompletionMessageParam,
+  prebuiltAppConfig,
+} from "@mlc-ai/web-llm";
+import { ProgressBar, Line } from "progressbar.js";
+
+// modified setLabel to not throw error
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label != null) {
+    label.innerText = text;
+  }
+}
+
+function getElementAndCheck(id: string): HTMLElement {
+  const element = document.getElementById(id);
+  if (element == null) {
+    throw Error("Cannot find element " + id);
+  }
+  return element;
+}
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+const queryInput = getElementAndCheck("query-input")!;
+const submitButton = getElementAndCheck("submit-button")!;
+const modelName = getElementAndCheck("model-name");
+
+let context = "";
+let modelDisplayName = "";
+
+// throws runtime.lastError if you refresh extension AND try to access a webpage that is already open
+fetchPageContents();
+
+(<HTMLButtonElement>submitButton).disabled = true;
+
+let progressBar: ProgressBar = new Line("#loadingContainer", {
+  strokeWidth: 4,
+  easing: "easeInOut",
+  duration: 1400,
+  color: "#ffd166",
+  trailColor: "#eee",
+  trailWidth: 1,
+  svgStyle: { width: "100%", height: "100%" },
+});
+
+let isLoadingParams = true;
+
+let initProgressCallback = (report: InitProgressReport) => {
+  setLabel("init-label", report.text);
+  progressBar.animate(report.progress, {
+    duration: 50,
+  });
+  if (report.progress == 1.0) {
+    enableInputs();
+  }
+};
+
+// initially selected model
+let selectedModel = "Qwen2-0.5B-Instruct-q4f16_1-MLC";
+
+// populate model-selection
+const modelSelector = getElementAndCheck(
+  "model-selection",
+) as HTMLSelectElement;
+for (let i = 0; i < prebuiltAppConfig.model_list.length; ++i) {
+  const model = prebuiltAppConfig.model_list[i];
+  const opt = document.createElement("option");
+  opt.value = model.model_id;
+  opt.innerHTML = model.model_id;
+  opt.selected = false;
+
+  // set initial selection as the initially selected model
+  if (model.model_id == selectedModel) {
+    opt.selected = true;
+  }
+
+  modelSelector.appendChild(opt);
+}
+
+modelName.innerText = "Loading initial model...";
+const engine: MLCEngineInterface = await CreateMLCEngine(selectedModel, {
+  initProgressCallback: initProgressCallback,
+});
+modelName.innerText = "Now chatting with " + modelDisplayName;
+
+let chatHistory: ChatCompletionMessageParam[] = [];
+
+function enableInputs() {
+  if (isLoadingParams) {
+    sleep(500);
+    isLoadingParams = false;
+  }
+
+  // remove loading bar and loading bar descriptors, if exists
+  const initLabel = document.getElementById("init-label");
+  initLabel?.remove();
+  const loadingBarContainer = document.getElementById("loadingContainer")!;
+  loadingBarContainer?.remove();
+  queryInput.focus();
+
+  const modelNameArray = selectedModel.split("-");
+  modelDisplayName = modelNameArray[0];
+  let j = 1;
+  while (j < modelNameArray.length && modelNameArray[j][0] != "q") {
+    modelDisplayName = modelDisplayName + "-" + modelNameArray[j];
+    j++;
+  }
+}
+
+let requestInProgress = false;
+
+// Disable submit button if input field is empty
+queryInput.addEventListener("keyup", () => {
+  if (
+    (<HTMLInputElement>queryInput).value === "" ||
+    requestInProgress ||
+    isLoadingParams
+  ) {
+    (<HTMLButtonElement>submitButton).disabled = true;
+  } else {
+    (<HTMLButtonElement>submitButton).disabled = false;
+  }
+});
+
+// If user presses enter, click submit button
+queryInput.addEventListener("keyup", (event) => {
+  if (event.code === "Enter") {
+    event.preventDefault();
+    submitButton.click();
+  }
+});
+
+// Listen for clicks on submit button
+async function handleClick() {
+  requestInProgress = true;
+  (<HTMLButtonElement>submitButton).disabled = true;
+
+  // Get the message from the input field
+  const message = (<HTMLInputElement>queryInput).value;
+  console.log("message", message);
+  // Clear the answer
+  document.getElementById("answer")!.innerHTML = "";
+  // Hide the answer
+  document.getElementById("answerWrapper")!.style.display = "none";
+  // Show the loading indicator
+  document.getElementById("loading-indicator")!.style.display = "block";
+
+  // Generate response
+  let inp = message;
+  if (context.length > 0) {
+    inp =
+      "Use only the following context when answering the question at the end. Don't use any other knowledge.\n" +
+      context +
+      "\n\nQuestion: " +
+      message +
+      "\n\nHelpful Answer: ";
+  }
+  console.log("Input:", inp);
+  chatHistory.push({ role: "user", content: inp });
+
+  let curMessage = "";
+  const completion = await engine.chat.completions.create({
+    stream: true,
+    messages: chatHistory,
+  });
+  for await (const chunk of completion) {
+    const curDelta = chunk.choices[0].delta.content;
+    if (curDelta) {
+      curMessage += curDelta;
+    }
+    updateAnswer(curMessage);
+  }
+  const response = await engine.getMessage();
+  chatHistory.push({ role: "assistant", content: await engine.getMessage() });
+  console.log("response", response);
+
+  requestInProgress = false;
+  (<HTMLButtonElement>submitButton).disabled = false;
+}
+submitButton.addEventListener("click", handleClick);
+
+// listen for changes in modelSelector
+async function handleSelectChange() {
+  if (isLoadingParams) {
+    return;
+  }
+
+  modelName.innerText = "";
+
+  const initLabel = document.createElement("p");
+  initLabel.id = "init-label";
+  initLabel.innerText = "Initializing model...";
+  const loadingContainer = document.createElement("div");
+  loadingContainer.id = "loadingContainer";
+
+  const loadingBox = getElementAndCheck("loadingBox");
+  loadingBox.appendChild(initLabel);
+  loadingBox.appendChild(loadingContainer);
+
+  isLoadingParams = true;
+  (<HTMLButtonElement>submitButton).disabled = true;
+
+  if (requestInProgress) {
+    engine.interruptGenerate();
+  }
+  engine.resetChat();
+  chatHistory = [];
+  await engine.unload();
+
+  selectedModel = modelSelector.value;
+
+  progressBar = new Line("#loadingContainer", {
+    strokeWidth: 4,
+    easing: "easeInOut",
+    duration: 1400,
+    color: "#ffd166",
+    trailColor: "#eee",
+    trailWidth: 1,
+    svgStyle: { width: "100%", height: "100%" },
+  });
+
+  initProgressCallback = (report: InitProgressReport) => {
+    setLabel("init-label", report.text);
+    progressBar.animate(report.progress, {
+      duration: 50,
+    });
+    if (report.progress == 1.0) {
+      enableInputs();
+    }
+  };
+
+  engine.setInitProgressCallback(initProgressCallback);
+
+  requestInProgress = true;
+  modelName.innerText = "Reloading with new model...";
+  await engine.reload(selectedModel);
+  requestInProgress = false;
+  modelName.innerText = "Now chatting with " + modelDisplayName;
+}
+modelSelector.addEventListener("change", handleSelectChange);
+
+// Listen for messages from the background script
+chrome.runtime.onMessage.addListener(({ answer, error }) => {
+  if (answer) {
+    updateAnswer(answer);
+  }
+});
+
+function updateAnswer(answer: string) {
+  // Show answer
+  document.getElementById("answerWrapper")!.style.display = "block";
+  const answerWithBreaks = answer.replace(/\n/g, "<br>");
+  document.getElementById("answer")!.innerHTML = answerWithBreaks;
+  // Add event listener to copy button
+  document.getElementById("copyAnswer")!.addEventListener("click", () => {
+    // Get the answer text
+    const answerText = answer;
+    // Copy the answer text to the clipboard
+    navigator.clipboard
+      .writeText(answerText)
+      .then(() => console.log("Answer text copied to clipboard"))
+      .catch((err) => console.error("Could not copy text: ", err));
+  });
+  const options: Intl.DateTimeFormatOptions = {
+    month: "short",
+    day: "2-digit",
+    hour: "2-digit",
+    minute: "2-digit",
+    second: "2-digit",
+  };
+  const time = new Date().toLocaleString("en-US", options);
+  // Update timestamp
+  document.getElementById("timestamp")!.innerText = time;
+  // Hide loading indicator
+  document.getElementById("loading-indicator")!.style.display = "none";
+}
+
+function fetchPageContents() {
+  chrome.tabs.query({ currentWindow: true, active: true }, function (tabs) {
+    const port = chrome.tabs.connect(tabs[0].id, { name: "channelName" });
+    port.postMessage({});
+    port.onMessage.addListener(function (msg) {
+      console.log("Page contents:", msg.contents);
+      context = msg.contents;
+    });
+  });
+}

examples/embeddings/README.md

@@ -0,0 +1,14 @@
+# WebLLM Get Started App
+
+This folder provides a minimum demo to show WebLLM API in a webapp setting.
+To try it out, you can do the following steps under this folder
+
+```bash
+npm install
+npm start
+```
+
+Note if you would like to hack WebLLM core package.
+You can change web-llm dependencies as `"file:../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.

examples/embeddings/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "embeddings-example",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/embeddings.html  --port 8885",
+    "build": "parcel build src/embeddings.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84",
+    "langchain": "^0.3.37"
+  }
+}

examples/embeddings/src/embeddings.html

@@ -0,0 +1,23 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output
+    <br />
+    <br />
+    <label id="init-label"> </label>
+
+    <h3>Prompt</h3>
+    <label id="prompt-label"> </label>
+
+    <h3>Response</h3>
+    <label id="generate-label"> </label>
+    <br />
+    <label id="stats-label"> </label>
+
+    <script type="module" src="./embeddings.ts"></script>
+  </body>
+</html>

examples/embeddings/src/embeddings.ts

@@ -0,0 +1,211 @@
+import * as webllm from "@mlc-ai/web-llm";
+import { MemoryVectorStore } from "langchain/vectorstores/memory";
+import type { EmbeddingsInterface } from "@langchain/core/embeddings";
+import type { Document } from "@langchain/core/documents";
+import { formatDocumentsAsString } from "langchain/util/document";
+import { PromptTemplate } from "@langchain/core/prompts";
+import {
+  RunnableSequence,
+  RunnablePassthrough,
+} from "@langchain/core/runnables";
+
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+const initProgressCallback = (report: webllm.InitProgressReport) => {
+  setLabel("init-label", report.text);
+};
+
+// For integration with Langchain
+class WebLLMEmbeddings implements EmbeddingsInterface {
+  engine: webllm.MLCEngineInterface;
+  modelId: string;
+  constructor(engine: webllm.MLCEngineInterface, modelId: string) {
+    this.engine = engine;
+    this.modelId = modelId;
+  }
+
+  async _embed(texts: string[]): Promise<number[][]> {
+    const reply = await this.engine.embeddings.create({
+      input: texts,
+      model: this.modelId,
+    });
+    const result: number[][] = [];
+    for (let i = 0; i < texts.length; i++) {
+      result.push(reply.data[i].embedding);
+    }
+    return result;
+  }
+
+  async embedQuery(document: string): Promise<number[]> {
+    return this._embed([document]).then((embeddings) => embeddings[0]);
+  }
+
+  async embedDocuments(documents: string[]): Promise<number[][]> {
+    return this._embed(documents);
+  }
+}
+
+// Prepare inputs
+const documents_og = ["The Data Cloud!", "Mexico City of Course!"];
+const queries_og = ["what is snowflake?", "Where can I get the best tacos?"];
+const documents: string[] = [];
+const queries: string[] = [];
+const query_prefix =
+  "Represent this sentence for searching relevant passages: ";
+// Process according to Snowflake model
+documents_og.forEach(function (item, index) {
+  documents[index] = `[CLS] ${item} [SEP]`;
+});
+queries_og.forEach(function (item, index) {
+  queries[index] = `[CLS] ${query_prefix}${item} [SEP]`;
+});
+console.log("Formatted documents: ", documents);
+console.log("Formatted queries: ", queries);
+
+// Using webllm's API
+async function webllmAPI() {
+  // b4 means the max batch size is compiled as 4. That is, the model can process 4 inputs in a
+  // batch. If given more than 4, the model will forward multiple times. The larger the max batch
+  // size, the more memory it consumes.
+  // const selectedModel = "snowflake-arctic-embed-m-q0f32-MLC-b32";
+  const selectedModel = "snowflake-arctic-embed-m-q0f32-MLC-b4";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    {
+      initProgressCallback: initProgressCallback,
+      logLevel: "INFO", // specify the log level
+    },
+  );
+
+  const docReply = await engine.embeddings.create({ input: documents });
+  console.log(docReply);
+  console.log(docReply.usage);
+
+  const queryReply = await engine.embeddings.create({ input: queries });
+  console.log(queryReply);
+  console.log(queryReply.usage);
+
+  // Calculate similarity (we use langchain here, but any method works)
+  const vectorStore = await MemoryVectorStore.fromExistingIndex(
+    new WebLLMEmbeddings(engine, selectedModel),
+  );
+  // See score
+  for (let i = 0; i < queries_og.length; i++) {
+    console.log(`Similarity with: ${queries_og[i]}`);
+    for (let j = 0; j < documents_og.length; j++) {
+      const similarity = vectorStore.similarity(
+        queryReply.data[i].embedding,
+        docReply.data[j].embedding,
+      );
+      console.log(`${documents_og[j]}: ${similarity}`);
+    }
+  }
+}
+
+// Alternatively, integrating with Langchain's API
+async function langchainAPI() {
+  // b4 means the max batch size is compiled as 4. That is, the model can process 4 inputs in a
+  // batch. If given more than 4, the model will forward multiple times. The larger the max batch
+  // size, the more memory it consumes.
+  // const selectedModel = "snowflake-arctic-embed-m-q0f32-MLC-b32";
+  const selectedModel = "snowflake-arctic-embed-m-q0f32-MLC-b4";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    {
+      initProgressCallback: initProgressCallback,
+      logLevel: "INFO", // specify the log level
+    },
+  );
+
+  const vectorStore = await MemoryVectorStore.fromExistingIndex(
+    new WebLLMEmbeddings(engine, selectedModel),
+  );
+  const document0: Document = {
+    pageContent: documents[0],
+    metadata: {},
+  };
+  const document1: Document = {
+    pageContent: documents[1],
+    metadata: {},
+  };
+  await vectorStore.addDocuments([document0, document1]);
+
+  const similaritySearchResults0 = await vectorStore.similaritySearch(
+    queries[0],
+    1,
+  );
+  for (const doc of similaritySearchResults0) {
+    console.log(`* ${doc.pageContent}`);
+  }
+
+  const similaritySearchResults1 = await vectorStore.similaritySearch(
+    queries[1],
+    1,
+  );
+  for (const doc of similaritySearchResults1) {
+    console.log(`* ${doc.pageContent}`);
+  }
+}
+
+// RAG with Langchain.js using WebLLM for both LLM and Embedding in a single engine
+// Followed https://js.langchain.com/v0.1/docs/expression_language/cookbook/retrieval/
+// There are many possible ways to achieve RAG (e.g. degree of integration with Langchain,
+// using WebWorker, etc.). We provide a minimal example here.
+async function simpleRAG() {
+  // 0. Load both embedding model and LLM to a single WebLLM Engine
+  const embeddingModelId = "snowflake-arctic-embed-m-q0f32-MLC-b4";
+  const llmModelId = "gemma-2-2b-it-q4f32_1-MLC-1k";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    [embeddingModelId, llmModelId],
+    {
+      initProgressCallback: initProgressCallback,
+      logLevel: "INFO", // specify the log level
+    },
+  );
+
+  const vectorStore = await MemoryVectorStore.fromTexts(
+    ["mitochondria is the powerhouse of the cell"],
+    [{ id: 1 }],
+    new WebLLMEmbeddings(engine, embeddingModelId),
+  );
+  const retriever = vectorStore.asRetriever();
+
+  const prompt =
+    PromptTemplate.fromTemplate(`Answer the question based only on the following context:
+  {context}
+  
+  Question: {question}`);
+
+  const chain = RunnableSequence.from([
+    {
+      context: retriever.pipe(formatDocumentsAsString),
+      question: new RunnablePassthrough(),
+    },
+    prompt,
+  ]);
+
+  const formattedPrompt = (
+    await chain.invoke("What is the powerhouse of the cell?")
+  ).toString();
+  const reply = await engine.chat.completions.create({
+    messages: [{ role: "user", content: formattedPrompt }],
+    model: llmModelId,
+  });
+
+  console.log(reply.choices[0].message.content);
+
+  /*
+    "The powerhouse of the cell is the mitochondria."
+  */
+}
+
+// Select one to run
+// webllmAPI();
+// langchainAPI();
+simpleRAG();

examples/function-calling/README.md

@@ -0,0 +1,11 @@
+### OpenAI API Demos - Function calling
+
+This folder contains two main ways of using function calling with WebLLM.
+
+`function-calling-manual` demonstrates how you can use function calling with Llama3.1 and Hermes2
+without using the `tools`, `tool_choice`, and `tool_call` fields. This is the most flexible way and you can follow
+the instruction given by the model releaser and iterate yourself on top of that. However, you need to do parsing on your own, which differs for each model. For instance, Hermes2 models use `<tool_call>` and `</tool_call>` to wrap around a tool call, which may be very different from other models' format.
+
+`function-calling-openai` conforms to the OpenAI function calling usage, leveraging `tools`, `tool_choice`, and `tool_call`
+fields. This is more usable, but sacrifices the flexibility since we have pre-defined system prompt
+for this.

examples/function-calling/function-calling-manual/README.md

@@ -0,0 +1,8 @@
+### Demos - Function calling
+
+Run `npm install` first, followed by `npm start`.
+
+Note if you would like to hack WebLLM core package,
+you can change web-llm dependencies as `"file:../../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.

examples/function-calling/function-calling-manual/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "openai-api",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/function_calling_manual.html  --port 8888",
+    "build": "parcel build src/function_calling_manual.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}

examples/function-calling/function-calling-manual/src/function_calling_manual.html

@@ -0,0 +1,17 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output
+    <br />
+    <br />
+    <label id="init-label"> </label>
+    <label id="generate-label"> </label>
+
+    <script type="module" src="./function_calling_manual.ts"></script>
+  </body>
+</html>

examples/function-calling/function-calling-manual/src/function_calling_manual.ts

@@ -0,0 +1,235 @@
+/* eslint-disable no-useless-escape */
+import * as webllm from "@mlc-ai/web-llm";
+
+// Common helper methods
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+const initProgressCallback = (report: webllm.InitProgressReport) => {
+  setLabel("init-label", report.text);
+};
+
+// Same example as https://huggingface.co/NousResearch/Hermes-2-Theta-Llama-3-8B#prompt-format-for-function-calling
+async function hermes2_example() {
+  // 0. Setups
+  // Most manual function calling models specify the tools inside the system prompt
+  const system_prompt = `You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: <tools> {"type": "function", "function": {"name": "get_stock_fundamentals", "description": "get_stock_fundamentals(symbol: str) -> dict - Get fundamental data for a given stock symbol using yfinance API.\\n\\n    Args:\\n        symbol (str): The stock symbol.\\n\\n    Returns:\\n        dict: A dictionary containing fundamental data.\\n            Keys:\\n                - \'symbol\': The stock symbol.\\n                - \'company_name\': The long name of the company.\\n                - \'sector\': The sector to which the company belongs.\\n                - \'industry\': The industry to which the company belongs.\\n                - \'market_cap\': The market capitalization of the company.\\n                - \'pe_ratio\': The forward price-to-earnings ratio.\\n                - \'pb_ratio\': The price-to-book ratio.\\n                - \'dividend_yield\': The dividend yield.\\n                - \'eps\': The trailing earnings per share.\\n                - \'beta\': The beta value of the stock.\\n                - \'52_week_high\': The 52-week high price of the stock.\\n                - \'52_week_low\': The 52-week low price of the stock.", "parameters": {"type": "object", "properties": {"symbol": {"type": "string"}}, "required": ["symbol"]}}}  </tools> Use the following pydantic model json schema for each tool call you will make: {"properties": {"arguments": {"title": "Arguments", "type": "object"}, "name": {"title": "Name", "type": "string"}}, "required": ["arguments", "name"], "title": "FunctionCall", "type": "object"} For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:\n<tool_call>\n{"arguments": <args-dict>, "name": <function-name>}\n</tool_call>`;
+  // Same formatting for Hermes-2-Pro-Llama-3, Hermes-2-Theta-Llama-3
+  // const selectedModel = "Hermes-2-Theta-Llama-3-8B-q4f16_1-MLC";
+  const selectedModel = "Hermes-2-Pro-Llama-3-8B-q4f16_1-MLC";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    { initProgressCallback: initProgressCallback, logLevel: "INFO" },
+  );
+  const seed = 0;
+
+  // 1. First request, expect to generate tool call
+  const messages: webllm.ChatCompletionMessageParam[] = [
+    { role: "system", content: system_prompt },
+    {
+      role: "user",
+      content: "Fetch the stock fundamentals data for Tesla (TSLA)",
+    },
+  ];
+  const request1: webllm.ChatCompletionRequest = {
+    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
+    messages: messages,
+    seed: seed,
+  };
+  const reply1 = await engine.chat.completions.create(request1);
+  const response1 = reply1.choices[0].message.content;
+  console.log(reply1.usage);
+  console.log("Response 1: " + response1);
+  messages.push({ role: "assistant", content: response1 });
+  // <tool_call>\n{"arguments": {"symbol": "TSLA"}, "name": "get_stock_fundamentals"}\n</tool_call>
+
+  // 2. Call function on your own to get tool response
+  const tool_response = `<tool_response>\n{"name": "get_stock_fundamentals", "content": {'symbol': 'TSLA', 'company_name': 'Tesla, Inc.', 'sector': 'Consumer Cyclical', 'industry': 'Auto Manufacturers', 'market_cap': 611384164352, 'pe_ratio': 49.604652, 'pb_ratio': 9.762013, 'dividend_yield': None, 'eps': 4.3, 'beta': 2.427, '52_week_high': 299.29, '52_week_low': 152.37}}\n</tool_response>`;
+  messages.push({ role: "tool", content: tool_response, tool_call_id: "0" });
+
+  // 3. Get natural language response
+  const request2: webllm.ChatCompletionRequest = {
+    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
+    messages: messages,
+    seed: seed,
+  };
+  const reply2 = await engine.chat.completions.create(request2);
+  const response2 = reply2.choices[0].message.content;
+  messages.push({ role: "assistant", content: response2 });
+  console.log(reply2.usage);
+  console.log("Response 2: " + response2);
+
+  // 4. Another function call
+  messages.push({
+    role: "user",
+    content: "Now do another one with NVIDIA, symbol being NVDA.",
+  });
+  const request3: webllm.ChatCompletionRequest = {
+    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
+    messages: messages,
+    seed: seed,
+  };
+  const reply3 = await engine.chat.completions.create(request3);
+  const response3 = reply3.choices[0].message.content;
+  messages.push({ role: "assistant", content: response3 });
+  console.log(reply3.usage);
+  console.log("Response 3: " + response3);
+  // <tool_call>\n{"arguments": {"symbol": "NVDA"}, "name": "get_stock_fundamentals"}\n</tool_call>
+}
+
+// Similar example to https://llama.meta.com/docs/model-cards-and-prompt-formats/llama3_1/#user-defined-custom-tool-calling
+async function llama3_1_example() {
+  // Follows example, but tweaks the formatting with <function>
+  const system_prompt = `Cutting Knowledge Date: December 2023
+Today Date: 23 Jul 2024
+# Tool Instructions
+- When looking for real time information use relevant functions if available
+You have access to the following functions:
+
+{
+    "type": "function",
+    "function": {
+        "name": "get_current_temperature",
+        "description": "Get the current temperature at a location.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "location": {
+                    "type": "string",
+                    "description": "The location to get the temperature for, in the format \"City, Country\""
+                }
+            },
+            "required": [
+                "location"
+            ]
+        },
+        "return": {
+            "type": "number",
+            "description": "The current temperature at the specified location in the specified units, as a float."
+        }
+    }
+}
+{
+    "type": "function",
+    "function": {
+        "name": "send_message",
+        "description": "Send a message to a recipient.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "recipient": {
+                    "type": "string",
+                    "description": "Name of the recipient of the message"
+                }
+                "content": {
+                    "type": "string",
+                    "description": "Content of the message"
+                }
+            },
+            "required": [
+                "recipient",
+                "content"
+            ]
+        },
+        "return": {
+            "type": "None"
+        }
+    }
+}
+If a you choose to call a function ONLY reply in the following format:
+    <function>{"name": function name, "parameters": dictionary of argument name and its value}</function>
+Here is an example,
+    <function>{"name": "example_function_name", "parameters": {"example_name": "example_value"}}</function>
+Reminder:
+- Function calls MUST follow the specified format and use BOTH <function> and </function>
+- Required parameters MUST be specified
+- Only call one function at a time
+- When calling a function, do NOT add any other words, ONLY the function calling
+- Put the entire function call reply on one line
+- Always add your sources when using search results to answer the user query
+You are a helpful Assistant.`;
+
+  const selectedModel = "Llama-3.1-8B-Instruct-q4f16_1-MLC";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    { initProgressCallback: initProgressCallback, logLevel: "INFO" },
+  );
+  const seed = 0;
+
+  // 1. First request, expect to generate tool call to get temperature of Paris
+  const messages: webllm.ChatCompletionMessageParam[] = [
+    { role: "system", content: system_prompt },
+    {
+      role: "user",
+      content: "Hey, what's the temperature in Paris right now?",
+    },
+  ];
+  const request1: webllm.ChatCompletionRequest = {
+    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
+    messages: messages,
+    seed: seed,
+  };
+  const reply1 = await engine.chat.completions.create(request1);
+  const response1 = reply1.choices[0].message.content;
+  console.log(reply1.usage);
+  console.log("Response 1: " + response1);
+  messages.push({ role: "assistant", content: response1 });
+  // <function>{"name": "get_current_temperature", "parameters": {"location": "Paris, France"}}</function>
+
+  // 2. Call function on your own to get tool response
+  const tool_response = `{"output": 22.5}`;
+  messages.push({ role: "tool", content: tool_response, tool_call_id: "0" });
+
+  // 3. Get natural language response
+  const request2: webllm.ChatCompletionRequest = {
+    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
+    messages: messages,
+    seed: seed,
+  };
+  const reply2 = await engine.chat.completions.create(request2);
+  const response2 = reply2.choices[0].message.content;
+  messages.push({ role: "assistant", content: response2 });
+  console.log(reply2.usage);
+  console.log("Response 2: " + response2);
+  // The current temperature in Paris is 22.5°C.
+
+  // 4. Make another request, expect model to call `send_message`
+  messages.push({
+    role: "user",
+    content: "Send a message to Tom to tell him this information.",
+  });
+  const request3: webllm.ChatCompletionRequest = {
+    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
+    messages: messages,
+    seed: seed,
+  };
+  const reply3 = await engine.chat.completions.create(request3);
+  const response3 = reply3.choices[0].message.content;
+  messages.push({ role: "assistant", content: response3 });
+  console.log(reply3.usage);
+  console.log("Response 3: " + response3);
+  // <function>{"name": "send_message", "parameters": {"recipient": "Tom", "content": "The current temperature in Paris is 22.5°C."}}</function>
+
+  // 5. Call API, which has no return value, so simply prompt model again
+  const tool_response2 = `{"output": None}`;
+  messages.push({ role: "tool", content: tool_response2, tool_call_id: "1" });
+  const request4: webllm.ChatCompletionRequest = {
+    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
+    messages: messages,
+    seed: seed,
+  };
+  const reply4 = await engine.chat.completions.create(request4);
+  const response4 = reply4.choices[0].message.content;
+  console.log(reply4.usage);
+  console.log("Response 4: " + response4);
+  // The message has been sent to Tom.
+}
+
+// Pick one to run
+// hermes2_example();
+llama3_1_example();

examples/function-calling/function-calling-openai/README.md

@@ -0,0 +1,8 @@
+### Demos - Function calling
+
+Run `npm install` first, followed by `npm start`.
+
+Note if you would like to hack WebLLM core package,
+you can change web-llm dependencies as `"file:../../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.

examples/function-calling/function-calling-openai/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "openai-api",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/function_calling_openai.html  --port 8888",
+    "build": "parcel build src/function_calling_openai.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}

examples/function-calling/function-calling-openai/src/function_calling_openai.html

@@ -0,0 +1,17 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output
+    <br />
+    <br />
+    <label id="init-label"> </label>
+    <label id="generate-label"> </label>
+
+    <script type="module" src="./function_calling_openai.ts"></script>
+  </body>
+</html>

examples/function-calling/function-calling-openai/src/function_calling_openai.ts

@@ -0,0 +1,80 @@
+import * as webllm from "@mlc-ai/web-llm";
+
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+async function main() {
+  const initProgressCallback = (report: webllm.InitProgressReport) => {
+    setLabel("init-label", report.text);
+  };
+  const selectedModel = "Hermes-2-Pro-Llama-3-8B-q4f16_1-MLC";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    { initProgressCallback: initProgressCallback },
+  );
+
+  const tools: Array<webllm.ChatCompletionTool> = [
+    {
+      type: "function",
+      function: {
+        name: "get_current_weather",
+        description: "Get the current weather in a given location",
+        parameters: {
+          type: "object",
+          properties: {
+            location: {
+              type: "string",
+              description: "The city and state, e.g. San Francisco, CA",
+            },
+            unit: { type: "string", enum: ["celsius", "fahrenheit"] },
+          },
+          required: ["location"],
+        },
+      },
+    },
+  ];
+
+  const request: webllm.ChatCompletionRequest = {
+    stream: true, // works with stream as well, where the last chunk returns tool_calls
+    stream_options: { include_usage: true },
+    messages: [
+      {
+        role: "user",
+        content:
+          "What is the current weather in celsius in Pittsburgh and Tokyo?",
+      },
+    ],
+    tool_choice: "auto",
+    tools: tools,
+  };
+
+  if (!request.stream) {
+    const reply0 = await engine.chat.completions.create(request);
+    console.log(reply0.choices[0]);
+    console.log(reply0.usage);
+  } else {
+    // If streaming, the last chunk returns tool calls
+    const asyncChunkGenerator = await engine.chat.completions.create(request);
+    let message = "";
+    let lastChunk: webllm.ChatCompletionChunk | undefined;
+    let usageChunk: webllm.ChatCompletionChunk | undefined;
+    for await (const chunk of asyncChunkGenerator) {
+      console.log(chunk);
+      message += chunk.choices[0]?.delta?.content || "";
+      setLabel("generate-label", message);
+      if (!chunk.usage) {
+        lastChunk = chunk;
+      }
+      usageChunk = chunk;
+    }
+    console.log(lastChunk!.choices[0].delta);
+    console.log(usageChunk!.usage);
+  }
+}
+
+main();

examples/get-started-latency-breakdown/README.md

@@ -0,0 +1,15 @@
+# WebLLM Get Started App
+
+This folder provides a minimum demo to show WebLLM API in a webapp setting with
+collection of latency statistics for individual token sampling steps.
+To try it out, you can do the following steps under this folder
+
+```bash
+npm install
+npm start
+```
+
+Note if you would like to hack WebLLM core package.
+You can change web-llm dependencies as `"file:../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.

examples/get-started-latency-breakdown/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "get-started-latency-breakdown",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/get_started_latency_breakdown.html  --port 8888",
+    "build": "parcel build src/get_started_latency_breakdown.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}

examples/get-started-latency-breakdown/src/get_started_latency_breakdown.html

@@ -0,0 +1,23 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output
+    <br />
+    <br />
+    <label id="init-label"> </label>
+
+    <h3>Prompt</h3>
+    <label id="prompt-label"> </label>
+
+    <h3>Response</h3>
+    <label id="generate-label"> </label>
+    <br />
+    <label id="stats-label"> </label>
+
+    <script type="module" src="./get_started_latency_breakdown.ts"></script>
+  </body>
+</html>

examples/get-started-latency-breakdown/src/get_started_latency_breakdown.ts

@@ -0,0 +1,135 @@
+import * as webllm from "@mlc-ai/web-llm";
+
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+type LatencyBreakdown = {
+  logitProcessorTime: number[];
+  logitBiasTime: number[];
+  penaltyTime: number[];
+  sampleTime: number[];
+  totalTime: number[];
+  grammarBitmaskTime: number[];
+};
+function computeStats(
+  latency_breakdown: LatencyBreakdown,
+): Record<string, any> {
+  function _computeStats(arr: number[]) {
+    if (!arr.length) return undefined;
+    const sorted = [...arr].sort((a, b) => a - b);
+    const sum = arr.reduce((a, b) => a + b, 0);
+    const avg = sum / arr.length;
+    const min = sorted[0];
+    const max = sorted[sorted.length - 1];
+    const p99 = sorted[Math.floor(0.99 * (sorted.length - 1))];
+    return { avg, min, max, p99 };
+  }
+
+  const latencyStats: Record<string, any> = {};
+  for (const key of Object.keys(latency_breakdown)) {
+    const arr = (latency_breakdown as any)[key];
+    if (Array.isArray(arr) && arr.length > 0) {
+      latencyStats[key] = _computeStats(arr);
+    }
+  }
+  return latencyStats;
+}
+
+async function main() {
+  const initProgressCallback = (report: webllm.InitProgressReport) => {
+    setLabel("init-label", report.text);
+  };
+  // Option 1: If we do not specify appConfig, we use `prebuiltAppConfig` defined in `config.ts`
+  const selectedModel = "Qwen3-0.6B-q0f32-MLC";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    {
+      initProgressCallback: initProgressCallback,
+      logLevel: "INFO", // specify the log level
+    },
+    // customize kv cache, use either context_window_size or sliding_window_size (with attention sink)
+    {
+      context_window_size: 2048,
+      // sliding_window_size: 1024,
+      // attention_sink_size: 4,
+    },
+  );
+
+  const latencyBreakdown: LatencyBreakdown = {
+    logitProcessorTime: [],
+    logitBiasTime: [],
+    penaltyTime: [],
+    sampleTime: [],
+    totalTime: [],
+    grammarBitmaskTime: [],
+  };
+
+  const decodeTokensPerS: number[] = [];
+  const completionTokens: number[] = [];
+  const e2eLatencyS: number[] = [];
+  const timePerOutputTokenS: number[] = [];
+
+  const numTrials = 20;
+  for (let i = 0; i < numTrials; i++) {
+    console.log(`Trial ${i + 1} / ${numTrials}`);
+    const reply0 = await engine.chat.completions.create({
+      messages: [{ role: "user", content: "List twenty US states." }],
+      // below configurations are all optional
+      n: 1,
+      temperature: 0,
+      max_tokens: 2048,
+      // 46510 and 7188 are "California", and 8421 and 51325 are "Texas" in Llama-3.1-8B-Instruct
+      // So we would have a higher chance of seeing the latter two, but never the first in the answer
+      // logit_bias: {
+      //   "46510": -100,
+      //   "7188": -100,
+      //   "8421": 5,
+      //   "41325": 5,
+      // },
+      top_p: 0.8,
+      logprobs: true,
+      top_logprobs: 2,
+      frequency_penalty: 1.2,
+      presence_penalty: 1.0,
+      repetition_penalty: 1.1,
+    });
+
+    const logitProcessorTime =
+      reply0.usage?.extra.latencyBreakdown?.logitProcessorTime;
+    const logitBiasTime = reply0.usage?.extra.latencyBreakdown?.logitBiasTime;
+    const penaltyTime = reply0.usage?.extra.latencyBreakdown?.penaltyTime;
+    const sampleTime = reply0.usage?.extra.latencyBreakdown?.sampleTime;
+    const totalTime = reply0.usage?.extra.latencyBreakdown?.totalTime;
+    const grammarBitmaskTime =
+      reply0.usage?.extra.latencyBreakdown?.grammarBitmaskTime;
+
+    latencyBreakdown.logitProcessorTime.push(...(logitProcessorTime || []));
+    latencyBreakdown.logitBiasTime.push(...(logitBiasTime || []));
+    latencyBreakdown.penaltyTime.push(...(penaltyTime || []));
+    latencyBreakdown.sampleTime.push(...(sampleTime || []));
+    latencyBreakdown.totalTime.push(...(totalTime || []));
+    latencyBreakdown.grammarBitmaskTime.push(...(grammarBitmaskTime || []));
+
+    decodeTokensPerS.push(reply0.usage?.extra.decode_tokens_per_s || 0);
+    e2eLatencyS.push(reply0.usage?.extra.e2e_latency_s || 0);
+    timePerOutputTokenS.push(reply0.usage?.extra.time_per_output_token_s || 0);
+    completionTokens.push(reply0.usage?.completion_tokens || 0);
+  }
+
+  const latencyStats: { [key: string]: number } =
+    computeStats(latencyBreakdown);
+  console.log("Latency stats: ", latencyStats);
+  console.log("Decode tokens per second: ", decodeTokensPerS);
+  console.log("Completion tokens: ", completionTokens);
+  console.log("E2E latency (s): ", e2eLatencyS);
+  console.log("Time per output token (s): ", timePerOutputTokenS);
+
+  // To change model, either create a new engine via `CreateMLCEngine()`, or call `engine.reload(modelId)`
+}
+
+main();

examples/get-started-web-worker/README.md

@@ -0,0 +1,18 @@
+# WebLLM Get Started with WebWorker
+
+This folder provides a minimum demo to show WebLLM API using
+[WebWorker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers).
+The main benefit of web worker is that all ML workloads runs on a separate thread as a result
+will less likely block the UI.
+
+To try it out, you can do the following steps under this folder
+
+```bash
+npm install
+npm start
+```
+
+Note if you would like to hack WebLLM core package.
+You can change web-llm dependencies as `"file:../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.

examples/get-started-web-worker/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "get-started-web-worker",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/get_started.html  --port 8885",
+    "build": "parcel build src/get_started.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^6.0.3",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}

examples/get-started-web-worker/src/get_started.html

@@ -0,0 +1,23 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output
+    <br />
+    <br />
+    <label id="init-label"> </label>
+
+    <h3>Prompt</h3>
+    <label id="prompt-label"> </label>
+
+    <h3>Response</h3>
+    <label id="generate-label"> </label>
+    <br />
+    <label id="stats-label"> </label>
+
+    <script type="module" src="./main.ts"></script>
+  </body>
+</html>

examples/get-started-web-worker/src/main.ts

@@ -0,0 +1,102 @@
+import * as webllm from "@mlc-ai/web-llm";
+
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+// There are two demonstrations, pick one to run
+
+/**
+ * Chat completion (OpenAI style) without streaming, where we get the entire response at once.
+ */
+async function mainNonStreaming() {
+  const initProgressCallback = (report: webllm.InitProgressReport) => {
+    setLabel("init-label", report.text);
+  };
+  const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
+
+  const engine: webllm.MLCEngineInterface =
+    await webllm.CreateWebWorkerMLCEngine(
+      new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }),
+      selectedModel,
+      { initProgressCallback: initProgressCallback },
+    );
+
+  const request: webllm.ChatCompletionRequest = {
+    messages: [
+      {
+        role: "system",
+        content:
+          "You are a helpful, respectful and honest assistant. " +
+          "Be as happy as you can when speaking please. ",
+      },
+      { role: "user", content: "Provide me three US states." },
+      { role: "assistant", content: "California, New York, Pennsylvania." },
+      { role: "user", content: "Two more please!" },
+    ],
+    n: 3,
+    temperature: 1.5,
+    max_tokens: 256,
+  };
+
+  const reply0 = await engine.chat.completions.create(request);
+  console.log(reply0);
+
+  console.log(reply0.usage);
+}
+
+/**
+ * Chat completion (OpenAI style) with streaming, where delta is sent while generating response.
+ */
+async function mainStreaming() {
+  const initProgressCallback = (report: webllm.InitProgressReport) => {
+    setLabel("init-label", report.text);
+  };
+  const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
+
+  const engine: webllm.MLCEngineInterface =
+    await webllm.CreateWebWorkerMLCEngine(
+      new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }),
+      selectedModel,
+      { initProgressCallback: initProgressCallback },
+    );
+
+  const request: webllm.ChatCompletionRequest = {
+    stream: true,
+    stream_options: { include_usage: true },
+    messages: [
+      {
+        role: "system",
+        content:
+          "You are a helpful, respectful and honest assistant. " +
+          "Be as happy as you can when speaking please. ",
+      },
+      { role: "user", content: "Provide me three US states." },
+      { role: "assistant", content: "California, New York, Pennsylvania." },
+      { role: "user", content: "Two more please!" },
+    ],
+    temperature: 1.5,
+    max_tokens: 256,
+  };
+
+  const asyncChunkGenerator = await engine.chat.completions.create(request);
+  let message = "";
+  for await (const chunk of asyncChunkGenerator) {
+    console.log(chunk);
+    message += chunk.choices[0]?.delta?.content || "";
+    setLabel("generate-label", message);
+    if (chunk.usage) {
+      console.log(chunk.usage); // only last chunk has usage
+    }
+    // engine.interruptGenerate();  // works with interrupt as well
+  }
+  console.log("Final message:\n", await engine.getMessage()); // the concatenated message
+}
+
+// Run one of the function below
+// mainNonStreaming();
+mainStreaming();

examples/get-started-web-worker/src/worker.ts

@@ -0,0 +1,7 @@
+import { WebWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
+
+// Hookup an engine to a worker handler
+const handler = new WebWorkerMLCEngineHandler();
+self.onmessage = (msg: MessageEvent) => {
+  handler.onmessage(msg);
+};

examples/get-started/README.md

@@ -0,0 +1,14 @@
+# WebLLM Get Started App
+
+This folder provides a minimum demo to show WebLLM API in a webapp setting.
+To try it out, you can do the following steps under this folder
+
+```bash
+npm install
+npm start
+```
+
+Note if you would like to hack WebLLM core package.
+You can change web-llm dependencies as `"file:../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.

examples/get-started/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "get-started",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/get_started.html  --port 8888",
+    "build": "parcel build src/get_started.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}

examples/get-started/src/get_started.html

@@ -0,0 +1,23 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output
+    <br />
+    <br />
+    <label id="init-label"> </label>
+
+    <h3>Prompt</h3>
+    <label id="prompt-label"> </label>
+
+    <h3>Response</h3>
+    <label id="generate-label"> </label>
+    <br />
+    <label id="stats-label"> </label>
+
+    <script type="module" src="./get_started.ts"></script>
+  </body>
+</html>

examples/get-started/src/get_started.ts

@@ -0,0 +1,82 @@
+import * as webllm from "@mlc-ai/web-llm";
+
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+async function main() {
+  const initProgressCallback = (report: webllm.InitProgressReport) => {
+    setLabel("init-label", report.text);
+  };
+  // Option 1: If we do not specify appConfig, we use `prebuiltAppConfig` defined in `config.ts`
+  const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    {
+      initProgressCallback: initProgressCallback,
+      logLevel: "INFO", // specify the log level
+    },
+    // customize kv cache, use either context_window_size or sliding_window_size (with attention sink)
+    {
+      context_window_size: 2048,
+      // sliding_window_size: 1024,
+      // attention_sink_size: 4,
+    },
+  );
+
+  // Option 2: Specify your own model other than the prebuilt ones
+  // const appConfig: webllm.AppConfig = {
+  //   model_list: [
+  //     {
+  //       model: "https://huggingface.co/mlc-ai/Llama-3.1-8B-Instruct-q4f32_1-MLC",
+  //       model_id: "Llama-3.1-8B-Instruct-q4f32_1-MLC",
+  //       model_lib:
+  //         webllm.modelLibURLPrefix +
+  //         webllm.modelVersion +
+  //         "/Llama-3_1-8B-Instruct-q4f32_1-ctx4k_cs1k-webgpu.wasm",
+  //       overrides: {
+  //         context_window_size: 2048,
+  //       },
+  //     },
+  //   ],
+  // };
+  // const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+  //   selectedModel,
+  //   { appConfig: appConfig, initProgressCallback: initProgressCallback },
+  // );
+
+  // Option 3: Instantiate MLCEngine() and call reload() separately
+  // const engine: webllm.MLCEngineInterface = new webllm.MLCEngine({
+  //   appConfig: appConfig, // if do not specify, we use webllm.prebuiltAppConfig
+  //   initProgressCallback: initProgressCallback,
+  // });
+  // await engine.reload(selectedModel);
+
+  const reply0 = await engine.chat.completions.create({
+    messages: [{ role: "user", content: "List three US states." }],
+    // below configurations are all optional
+    n: 3,
+    temperature: 1.5,
+    max_tokens: 256,
+    // 46510 and 7188 are "California", and 8421 and 51325 are "Texas" in Llama-3.1-8B-Instruct
+    // So we would have a higher chance of seeing the latter two, but never the first in the answer
+    logit_bias: {
+      "46510": -100,
+      "7188": -100,
+      "8421": 5,
+      "51325": 5,
+    },
+    logprobs: true,
+    top_logprobs: 2,
+  });
+  console.log(reply0);
+  console.log(reply0.usage);
+
+  // To change model, either create a new engine via `CreateMLCEngine()`, or call `engine.reload(modelId)`
+}
+
+main();

examples/integrity-verification/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "integrity-verification",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/integrity_verification.html --port 8888",
+    "build": "parcel build src/integrity_verification.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}

examples/integrity-verification/src/integrity_verification.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html>
+  <body>
+    <h2>WebLLM Integrity Verification Example</h2>
+    <p>
+      This example demonstrates how to use the optional
+      <code>integrity</code> field on <code>ModelRecord</code> to verify
+      downloaded artifacts with SRI hashes.
+    </p>
+    <div id="status">Loading...</div>
+    <script type="module" src="./integrity_verification.ts"></script>
+  </body>
+</html>

examples/integrity-verification/src/integrity_verification.ts

@@ -0,0 +1,89 @@
+import * as webllm from "@mlc-ai/web-llm";
+
+const statusEl = document.getElementById("status")!;
+
+function setStatus(msg: string) {
+  console.log(msg);
+  statusEl.innerText = msg;
+}
+
+/**
+ * Example: Basic integrity verification using the native `integrity` field.
+ *
+ * When `integrity` is set on a ModelRecord, WebLLM will verify the downloaded
+ * config, WASM, and tokenizer files against the provided SRI hashes before
+ * loading the model. If a hash does not match, an IntegrityError is thrown
+ * (or a warning is logged if `onFailure: "warn"` is set).
+ *
+ * Hash-generation commands are documented in README.md under "Integrity Verification".
+ * This example uses the same commands:
+ * - `openssl dgst -sha256 -binary <file> | openssl base64 -A | sed 's/^/sha256-/'`
+ * - `openssl dgst -sha384 -binary <file> | openssl base64 -A | sed 's/^/sha384-/'`
+ * - `openssl dgst -sha512 -binary <file> | openssl base64 -A | sed 's/^/sha512-/'`
+ */
+async function main() {
+  setStatus("Initializing...");
+
+  // Example model configuration with integrity hashes.
+  // Replace the placeholder hashes below with real SRI hashes for your model.
+  const appConfig: webllm.AppConfig = {
+    model_list: [
+      {
+        model:
+          "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_1-MLC",
+        model_id: "Llama-3.2-1B-Instruct-q4f16_1-MLC",
+        model_lib:
+          webllm.modelLibURLPrefix +
+          webllm.modelVersion +
+          "/Llama-3.2-1B-Instruct-q4f16_1-ctx4k_cs1k-webgpu.wasm",
+        low_resource_required: true,
+        vram_required_MB: 879.04,
+        overrides: {
+          context_window_size: 4096,
+        },
+        // Uncomment and fill in real SRI hashes to enable verification:
+        // integrity: {
+        //   config: "sha256-<base64-hash-of-mlc-chat-config.json>",
+        //   model_lib: "sha256-<base64-hash-of-wasm-file>",
+        //   tokenizer: {
+        //     "tokenizer.json": "sha256-<base64-hash-of-tokenizer.json>",
+        //   },
+        //   onFailure: "error",  // or "warn" to log and continue
+        // },
+      },
+    ],
+  };
+
+  try {
+    const engine = await webllm.CreateMLCEngine(
+      "Llama-3.2-1B-Instruct-q4f16_1-MLC",
+      {
+        appConfig,
+        initProgressCallback: (report: webllm.InitProgressReport) => {
+          setStatus(report.text);
+        },
+      },
+    );
+
+    setStatus("Model loaded! Generating response...");
+
+    const reply = await engine.chat.completions.create({
+      messages: [{ role: "user", content: "Hello! What can you do?" }],
+    });
+
+    setStatus("Response: " + reply.choices[0].message.content);
+  } catch (error) {
+    if (error instanceof webllm.IntegrityError) {
+      setStatus(
+        `Integrity verification failed!\n` +
+          `URL: ${error.url}\n` +
+          `Expected: ${error.expected}\n` +
+          `Got: ${error.actual}`,
+      );
+    } else {
+      setStatus("Error: " + error);
+    }
+  }
+}
+
+main();

examples/json-mode/README.md

@@ -0,0 +1,8 @@
+### OpenAI API Demos
+
+Run `npm install` first, followed by `npm start`.
+
+Note if you would like to hack WebLLM core package,
+you can change web-llm dependencies as `"file:../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.

examples/json-mode/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "openai-api",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/json_mode.html  --port 8888",
+    "build": "parcel build src/json_mode.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}

examples/json-mode/src/json_mode.html

@@ -0,0 +1,16 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output.
+    <br />
+    <br />
+    <label id="init-label"> </label>
+
+    <script type="module" src="./json_mode.ts"></script>
+  </body>
+</html>

examples/json-mode/src/json_mode.ts

@@ -0,0 +1,44 @@
+import * as webllm from "@mlc-ai/web-llm";
+
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+async function main() {
+  const initProgressCallback = (report: webllm.InitProgressReport) => {
+    setLabel("init-label", report.text);
+  };
+  // Pick any one of these models to start trying -- most models in WebLLM support grammar
+  const selectedModel = "Llama-3.2-3B-Instruct-q4f16_1-MLC";
+  // const selectedModel = "Qwen2.5-1.5B-Instruct-q4f16_1-MLC";
+  // const selectedModel = "Phi-3.5-mini-instruct-q4f16_1-MLC";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    { initProgressCallback: initProgressCallback },
+  );
+  // Note that you'd need to prompt the model to answer in JSON either in
+  // user's message or the system prompt
+  const request: webllm.ChatCompletionRequest = {
+    stream: false, // works with streaming, logprobs, top_logprobs as well
+    messages: [
+      {
+        role: "user",
+        content: "Write a short JSON file introducing yourself.",
+      },
+    ],
+    n: 2,
+    max_tokens: 128,
+    response_format: { type: "json_object" } as webllm.ResponseFormat,
+  };
+
+  const reply0 = await engine.chatCompletion(request);
+  console.log(reply0);
+  console.log("First reply's last choice:\n" + (await engine.getMessage()));
+  console.log(reply0.usage);
+}
+
+main();

examples/json-schema/README.md

@@ -0,0 +1,8 @@
+### OpenAI API Demos
+
+Run `npm install` first, followed by `npm start`.
+
+Note if you would like to hack WebLLM core package,
+you can change web-llm dependencies as `"file:../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.

examples/json-schema/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "openai-api",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/json_schema.html  --port 8885",
+    "build": "parcel build src/json_schema.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}

examples/json-schema/src/json_schema.html

@@ -0,0 +1,16 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output.
+    <br />
+    <br />
+    <label id="init-label"> </label>
+
+    <script type="module" src="./json_schema.ts"></script>
+  </body>
+</html>

examples/json-schema/src/json_schema.ts

@@ -0,0 +1,288 @@
+import * as webllm from "@mlc-ai/web-llm";
+import { Type, Static } from "@sinclair/typebox";
+
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+async function simpleStructuredTextExample() {
+  // There are several options of providing such a schema
+  // 1. You can directly define a schema in string
+  const schema1 = `{
+        "properties": {
+            "size": {"title": "Size", "type": "integer"}, 
+            "is_accepted": {"title": "Is Accepted", "type": "boolean"}, 
+            "num": {"title": "Num", "type": "number"}
+        },
+        "required": ["size", "is_accepted", "num"], 
+        "title": "Schema", "type": "object"
+    }`;
+
+  // 2. You can use 3rdparty libraries like typebox to create a schema
+  const T = Type.Object({
+    size: Type.Integer(),
+    is_accepted: Type.Boolean(),
+    num: Type.Number(),
+  });
+  type T = Static<typeof T>;
+  const schema2 = JSON.stringify(T);
+  console.log(schema2);
+  // {"type":"object","properties":{"size":{"type":"integer"},"is_accepted":{"type":"boolean"},
+  // "num":{"type":"number"}},"required":["size","is_accepted","num"]}
+
+  const initProgressCallback = (report: webllm.InitProgressReport) => {
+    setLabel("init-label", report.text);
+  };
+
+  // Pick any one of these models to start trying -- most models in WebLLM support grammar
+  // const selectedModel = "Llama-3.2-3B-Instruct-q4f16_1-MLC";
+  // const selectedModel = "Qwen2.5-1.5B-Instruct-q4f16_1-MLC";
+  const selectedModel = "Phi-3.5-mini-instruct-q4f16_1-MLC";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    { initProgressCallback: initProgressCallback, logLevel: "INFO" },
+  );
+
+  // Note that you'd need to prompt the model to answer in JSON either in
+  // user's message or the system prompt
+  const request: webllm.ChatCompletionRequest = {
+    stream: false, // works with streaming, logprobs, top_logprobs as well
+    messages: [
+      {
+        role: "user",
+        content:
+          "Generate a json containing three fields: an integer field named size, a " +
+          "boolean field named is_accepted, and a float field named num.",
+      },
+    ],
+    max_tokens: 128,
+    response_format: {
+      type: "json_object",
+      schema: schema2,
+    } as webllm.ResponseFormat,
+  };
+
+  const reply0 = await engine.chatCompletion(request);
+  console.log(reply0);
+  console.log("Output:\n" + (await engine.getMessage()));
+  console.log(reply0.usage);
+}
+
+// The json schema and prompt is taken from
+// https://github.com/sgl-project/sglang/tree/main?tab=readme-ov-file#json-decoding
+async function harryPotterExample() {
+  const T = Type.Object({
+    name: Type.String(),
+    house: Type.Enum({
+      Gryffindor: "Gryffindor",
+      Hufflepuff: "Hufflepuff",
+      Ravenclaw: "Ravenclaw",
+      Slytherin: "Slytherin",
+    }),
+    blood_status: Type.Enum({
+      "Pure-blood": "Pure-blood",
+      "Half-blood": "Half-blood",
+      "Muggle-born": "Muggle-born",
+    }),
+    occupation: Type.Enum({
+      Student: "Student",
+      Professor: "Professor",
+      "Ministry of Magic": "Ministry of Magic",
+      Other: "Other",
+    }),
+    wand: Type.Object({
+      wood: Type.String(),
+      core: Type.String(),
+      length: Type.Number(),
+    }),
+    alive: Type.Boolean(),
+    patronus: Type.String(),
+  });
+
+  type T = Static<typeof T>;
+  const schema = JSON.stringify(T);
+  console.log(schema);
+
+  const initProgressCallback = (report: webllm.InitProgressReport) => {
+    setLabel("init-label", report.text);
+  };
+
+  // Pick any one of these models to start trying -- most models in WebLLM support grammar
+  const selectedModel = "Llama-3.2-3B-Instruct-q4f16_1-MLC";
+  // const selectedModel = "Qwen2.5-1.5B-Instruct-q4f16_1-MLC";
+  // const selectedModel = "Phi-3.5-mini-instruct-q4f16_1-MLC";
+
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    { initProgressCallback: initProgressCallback, logLevel: "INFO" },
+  );
+
+  // Note that you'd need to prompt the model to answer in JSON either in
+  // user's message or the system prompt
+  const request: webllm.ChatCompletionRequest = {
+    stream: false,
+    messages: [
+      {
+        role: "user",
+        content:
+          "Hermione Granger is a character in Harry Potter. Please fill in the following information about this character in JSON format." +
+          "Name is a string of character name. House is one of Gryffindor, Hufflepuff, Ravenclaw, Slytherin. Blood status is one of Pure-blood, Half-blood, Muggle-born. Occupation is one of Student, Professor, Ministry of Magic, Other. Wand is an object with wood, core, and length. Alive is a boolean. Patronus is a string.",
+      },
+    ],
+    max_tokens: 128,
+    response_format: {
+      type: "json_object",
+      schema: schema,
+    } as webllm.ResponseFormat,
+  };
+
+  const reply = await engine.chatCompletion(request);
+  console.log(reply);
+  console.log("Output:\n" + (await engine.getMessage()));
+  console.log(reply.usage);
+  console.log(reply.usage!.extra);
+}
+
+async function functionCallingExample() {
+  const T = Type.Object({
+    tool_calls: Type.Array(
+      Type.Object({
+        arguments: Type.Any(),
+        name: Type.String(),
+      }),
+    ),
+  });
+  type T = Static<typeof T>;
+  const schema = JSON.stringify(T);
+  console.log(schema);
+
+  const tools: Array<webllm.ChatCompletionTool> = [
+    {
+      type: "function",
+      function: {
+        name: "get_current_weather",
+        description: "Get the current weather in a given location",
+        parameters: {
+          type: "object",
+          properties: {
+            location: {
+              type: "string",
+              description: "The city and state, e.g. San Francisco, CA",
+            },
+            unit: { type: "string", enum: ["celsius", "fahrenheit"] },
+          },
+          required: ["location"],
+        },
+      },
+    },
+  ];
+
+  const initProgressCallback = (report: webllm.InitProgressReport) => {
+    setLabel("init-label", report.text);
+  };
+
+  const selectedModel = "Hermes-2-Pro-Llama-3-8B-q4f16_1-MLC";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    {
+      initProgressCallback: initProgressCallback,
+    },
+  );
+
+  const request: webllm.ChatCompletionRequest = {
+    stream: false,
+    messages: [
+      {
+        role: "system",
+        content: `You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: <tools> ${JSON.stringify(
+          tools,
+        )} </tools>. Do not stop calling functions until the task has been accomplished or you've reached max iteration of 10.
+      Calling multiple functions at once can overload the system and increase cost so call one function at a time please.
+      If you plan to continue with analysis, always call another function.
+      Return a valid json object (using double quotes) in the following schema: ${JSON.stringify(
+        schema,
+      )}.`,
+      },
+      {
+        role: "user",
+        content:
+          "What is the current weather in celsius in Pittsburgh and Tokyo?",
+      },
+    ],
+    response_format: {
+      type: "json_object",
+      schema: schema,
+    } as webllm.ResponseFormat,
+  };
+
+  const reply = await engine.chat.completions.create(request);
+  console.log(reply.choices[0].message.content);
+
+  console.log(reply.usage);
+}
+
+async function ebnfGrammarExample() {
+  // You can directly define an EBNFGrammar string with ResponseFormat.grammar
+  const jsonGrammarStr = String.raw`
+root ::= basic_array | basic_object
+basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object
+basic_integer ::= ("0" | "-"? [1-9] [0-9]*) ".0"?
+basic_number ::= ("0" | "-"? [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)?
+basic_string ::= (([\"] basic_string_1 [\"]))
+basic_string_1 ::= "" | [^"\\\x00-\x1F] basic_string_1 | "\\" escape basic_string_1
+escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9]
+basic_boolean ::= "true" | "false"
+basic_null ::= "null"
+basic_array ::= "[" ("" | ws basic_any (ws "," ws basic_any)*) ws "]"
+basic_object ::= "{" ("" | ws basic_string ws ":" ws basic_any ( ws "," ws basic_string ws ":" ws basic_any)*) ws "}"
+ws ::= [ \n\t]*
+`;
+
+  const initProgressCallback = (report: webllm.InitProgressReport) => {
+    setLabel("init-label", report.text);
+  };
+
+  // Pick any one of these models to start trying -- most models in WebLLM support grammar
+  const selectedModel = "Llama-3.2-3B-Instruct-q4f16_1-MLC";
+  // const selectedModel = "Qwen2.5-1.5B-Instruct-q4f16_1-MLC";
+  // const selectedModel = "Phi-3.5-mini-instruct-q4f16_1-MLC";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    { initProgressCallback: initProgressCallback, logLevel: "INFO" },
+  );
+
+  // Note that you'd need to prompt the model to answer in JSON either in
+  // user's message or the system prompt
+  const request: webllm.ChatCompletionRequest = {
+    stream: false, // works with streaming, logprobs, top_logprobs as well
+    messages: [
+      {
+        role: "user",
+        content: "Introduce yourself in JSON",
+      },
+    ],
+    max_tokens: 128,
+    response_format: {
+      type: "grammar",
+      grammar: jsonGrammarStr,
+    } as webllm.ResponseFormat,
+  };
+
+  const reply0 = await engine.chatCompletion(request);
+  console.log(reply0);
+  console.log("Output:\n" + (await engine.getMessage()));
+  console.log(reply0.usage);
+}
+
+async function main() {
+  // await simpleStructuredTextExample();
+  await harryPotterExample();
+  // await functionCallingExample();
+  // await ebnfGrammarExample();
+}
+
+main();

examples/logit-processor/README.md

@@ -0,0 +1,23 @@
+# WebLLM Logit Processor and Low-Level API Example
+
+This folder explains the usage of `LogitProcessor`, demonstrating how it can be used to
+manipulate the raw logits before sampling the token (e.g. setting certain tokens to `inf` or `-inf`).
+We demonstrate how to use it with and without a web worker, which can be toggled with `USE_WEB_WORKER`
+in `logit_processor.ts` (see `worker.ts` on how `LogitProcessor` plays a role there).
+
+We also demonstrate the usage of a low-level API `forwardTokenAndSample()`, which, unlike `chat.completions.create()`
+that assumes the usage is for autoregressive chatting, here we have more fine-grained control.
+
+See `my_logit_processor.ts` on how to customize your own logit processor. Here we make the logit
+of token 0 `100.0` manually, large enough that we should expect to always sample token 0, which
+is indeed the case if we observe the console log. We also demonstarte that a LogitProcessor can be
+stateful, and the state can also be cleaned with `LogitProcessor.resetState()`.
+
+To try it out, you can do the following steps under this folder
+
+```bash
+npm install
+npm start
+```
+
+Note if you would like to hack WebLLM core package, you can change web-llm dependencies as `"file:../.."`, and follow the build from source instruction in the project to build webllm locally. This option is only recommended if you would like to hack WebLLM core package.

examples/logit-processor/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "logit-processor",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/logit_processor.html  --port 8885",
+    "build": "parcel build src/logit_processor.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}

examples/logit-processor/src/logit_processor.html

@@ -0,0 +1,16 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+
+  <body>
+    <h2>WebLLM Logit Processor Test Page</h2>
+    Open console to see the effect of your logit processor.
+    <br />
+    <br />
+    <label id="init-label"> </label>
+
+    <script type="module" src="./logit_processor.ts"></script>
+  </body>
+</html>

examples/logit-processor/src/logit_processor.ts

@@ -0,0 +1,77 @@
+import * as webllm from "@mlc-ai/web-llm";
+import { MyLogitProcessor } from "./my_logit_processor";
+
+const USE_WEB_WORKER = true; // Toggle this to use Logit Processor without a web worker
+const AUTOREGRESS_LIMIT = 32; // How many tokens to generate for this test
+
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+async function main() {
+  const initProgressCallback = (report: webllm.InitProgressReport) => {
+    setLabel("init-label", report.text);
+  };
+  // Instantiate myLogitProcessor, registering in the logitProcessorRegistry
+  const myLogitProcessor = new MyLogitProcessor();
+  const logitProcessorRegistry = new Map<string, webllm.LogitProcessor>();
+  logitProcessorRegistry.set("phi-2-q4f32_1-MLC", myLogitProcessor);
+
+  let engine: webllm.MLCEngineInterface;
+
+  // Depending on whether we use a web worker, the code is slightly different
+  if (USE_WEB_WORKER) {
+    // see worker.ts on how LogitProcessor plays a role there
+    engine = await webllm.CreateWebWorkerMLCEngine(
+      new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }),
+      "phi-2-q4f32_1-MLC",
+      { initProgressCallback: initProgressCallback },
+    );
+  } else {
+    engine = await webllm.CreateMLCEngine("phi-2-q4f32_1-MLC", {
+      initProgressCallback: initProgressCallback,
+      logitProcessorRegistry: logitProcessorRegistry,
+    });
+  }
+
+  // Below we demonstrate the usage of a low-level API `forwardTokensAndSample()`
+  const prompt: Array<number> = [42];
+  let nextToken = await engine.forwardTokensAndSample(
+    prompt,
+    /*isPrefill=*/ true,
+  );
+  console.log(nextToken);
+
+  let counter = prompt.length;
+  while (counter < AUTOREGRESS_LIMIT) {
+    counter += 1;
+    nextToken = await engine.forwardTokensAndSample(
+      [nextToken],
+      /*isPrefill=*/ false,
+    );
+    console.log(nextToken);
+  }
+
+  // By calling `engine.resetChat()`, we triggers MyLogitProcessor.resetState()
+  engine.resetChat();
+  counter = prompt.length;
+  nextToken = await engine.forwardTokensAndSample(prompt, /*isPrefill=*/ true);
+  console.log(nextToken);
+  while (counter < AUTOREGRESS_LIMIT) {
+    counter += 1;
+    nextToken = await engine.forwardTokensAndSample(
+      [nextToken],
+      /*isPrefill=*/ false,
+    );
+    console.log(nextToken);
+  }
+
+  // `forwardTokensAndSample()` is made compatible with registering runtime stats.
+  console.log(await engine.runtimeStatsText());
+}
+
+main();

examples/logit-processor/src/my_logit_processor.ts

@@ -0,0 +1,21 @@
+import * as webllm from "@mlc-ai/web-llm";
+
+// Define LogitProcessor
+export class MyLogitProcessor implements webllm.LogitProcessor {
+  private tokenSequence: Array<number> = [];
+
+  processLogits(logits: Float32Array): Float32Array {
+    logits[0] = 100.0; // should be enough so that we always sample token 0 below
+    return logits;
+  }
+
+  processSampledToken(token: number): void {
+    this.tokenSequence.push(token);
+    console.log("processSampledToken: " + this.tokenSequence.length);
+  }
+
+  resetState(): void {
+    this.tokenSequence = [];
+    console.log("resetState");
+  }
+}

examples/logit-processor/src/worker.ts

@@ -0,0 +1,15 @@
+// Serve the chat workload through web worker
+import * as webllm from "@mlc-ai/web-llm";
+import { MyLogitProcessor } from "./my_logit_processor";
+
+console.log("Use web worker for logit processor");
+
+const myLogitProcessor = new MyLogitProcessor();
+const logitProcessorRegistry = new Map<string, webllm.LogitProcessor>();
+logitProcessorRegistry.set("phi-2-q4f32_1-MLC", myLogitProcessor);
+
+const handler = new webllm.WebWorkerMLCEngineHandler();
+handler.setLogitProcessorRegistry(logitProcessorRegistry);
+self.onmessage = (msg: MessageEvent) => {
+  handler.onmessage(msg);
+};

examples/multi-models/README.md

@@ -0,0 +1,14 @@
+# WebLLM Get Started App
+
+This folder provides a minimum demo to show WebLLM API in a webapp setting.
+To try it out, you can do the following steps under this folder
+
+```bash
+npm install
+npm start
+```
+
+Note if you would like to hack WebLLM core package.
+You can change web-llm dependencies as `"file:../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.

examples/multi-models/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "get-started",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/multi_models.html  --port 8888",
+    "build": "parcel build src/multi_models.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}

examples/multi-models/src/main.ts

@@ -0,0 +1,137 @@
+/**
+ * This example demonstrates loading multiple models in the same engine concurrently.
+ * sequentialGeneration() shows inference each model one at a time.
+ * parallelGeneration() shows inference both models at the same time.
+ * This example uses WebWorkerMLCEngine, but the same idea applies to MLCEngine and
+ * ServiceWorkerMLCEngine as well.
+ */
+
+import * as webllm from "@mlc-ai/web-llm";
+
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+const initProgressCallback = (report: webllm.InitProgressReport) => {
+  setLabel("init-label", report.text);
+};
+
+// Prepare request for each model, same for both methods
+const selectedModel1 = "Phi-3.5-mini-instruct-q4f32_1-MLC-1k";
+const selectedModel2 = "gemma-2-2b-it-q4f32_1-MLC-1k";
+const prompt1 = "Tell me about California in 3 short sentences.";
+const prompt2 = "Tell me about New York City in 3 short sentences.";
+setLabel("prompt-label-1", `(with model ${selectedModel1})\n` + prompt1);
+setLabel("prompt-label-2", `(with model ${selectedModel2})\n` + prompt2);
+
+const request1: webllm.ChatCompletionRequestStreaming = {
+  stream: true,
+  stream_options: { include_usage: true },
+  messages: [{ role: "user", content: prompt1 }],
+  model: selectedModel1, // without specifying it, error will throw due to ambiguity
+  max_tokens: 128,
+};
+
+const request2: webllm.ChatCompletionRequestStreaming = {
+  stream: true,
+  stream_options: { include_usage: true },
+  messages: [{ role: "user", content: prompt2 }],
+  model: selectedModel2, // without specifying it, error will throw due to ambiguity
+  max_tokens: 128,
+};
+
+/**
+ * Chat completion (OpenAI style) with streaming, with two models in the pipeline.
+ */
+async function sequentialGeneration() {
+  const engine = await webllm.CreateWebWorkerMLCEngine(
+    new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }),
+    [selectedModel1, selectedModel2],
+    { initProgressCallback: initProgressCallback },
+  );
+
+  const asyncChunkGenerator1 = await engine.chat.completions.create(request1);
+  let message1 = "";
+  for await (const chunk of asyncChunkGenerator1) {
+    // console.log(chunk);
+    message1 += chunk.choices[0]?.delta?.content || "";
+    setLabel("generate-label-1", message1);
+    if (chunk.usage) {
+      console.log(chunk.usage); // only last chunk has usage
+    }
+    // engine.interruptGenerate();  // works with interrupt as well
+  }
+  const asyncChunkGenerator2 = await engine.chat.completions.create(request2);
+  let message2 = "";
+  for await (const chunk of asyncChunkGenerator2) {
+    // console.log(chunk);
+    message2 += chunk.choices[0]?.delta?.content || "";
+    setLabel("generate-label-2", message2);
+    if (chunk.usage) {
+      console.log(chunk.usage); // only last chunk has usage
+    }
+    // engine.interruptGenerate();  // works with interrupt as well
+  }
+
+  // without specifying from which model to get message, error will throw due to ambiguity
+  console.log("Final message 1:\n", await engine.getMessage(selectedModel1));
+  console.log("Final message 2:\n", await engine.getMessage(selectedModel2));
+}
+
+/**
+ * Chat completion (OpenAI style) with streaming, with two models in the pipeline.
+ */
+async function parallelGeneration() {
+  const engine = await webllm.CreateWebWorkerMLCEngine(
+    new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }),
+    [selectedModel1, selectedModel2],
+    { initProgressCallback: initProgressCallback },
+  );
+
+  // We can serve the two requests concurrently
+  async function getModel1Response() {
+    let message1 = "";
+    const asyncChunkGenerator1 = await engine.chat.completions.create(request1);
+    for await (const chunk of asyncChunkGenerator1) {
+      // console.log(chunk);
+      message1 += chunk.choices[0]?.delta?.content || "";
+      setLabel("generate-label-1", message1);
+      if (chunk.usage) {
+        console.log(chunk.usage); // only last chunk has usage
+      }
+      // engine.interruptGenerate();  // works with interrupt as well
+    }
+  }
+
+  async function getModel2Response() {
+    let message2 = "";
+    const asyncChunkGenerator2 = await engine.chat.completions.create(request2);
+    for await (const chunk of asyncChunkGenerator2) {
+      // console.log(chunk);
+      message2 += chunk.choices[0]?.delta?.content || "";
+      setLabel("generate-label-2", message2);
+      if (chunk.usage) {
+        console.log(chunk.usage); // only last chunk has usage
+      }
+      // engine.interruptGenerate();  // works with interrupt as well
+    }
+  }
+
+  await Promise.all([getModel1Response(), getModel2Response()]);
+  // Note: concurrent requests to the same model are executed sequentially in FCFS,
+  // unlike to different models like above
+  // Fore more, see https://github.com/mlc-ai/web-llm/pull/549
+  // await Promise.all([getModel1Response(), getModel1Response()]);
+
+  // without specifying from which model to get message, error will throw due to ambiguity
+  console.log("Final message 1:\n", await engine.getMessage(selectedModel1));
+  console.log("Final message 2:\n", await engine.getMessage(selectedModel2));
+}
+
+// Pick one to run
+sequentialGeneration();
+// parallelGeneration();

examples/multi-models/src/multi_models.html

@@ -0,0 +1,30 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output
+    <br />
+    <br />
+    <label id="init-label"> </label>
+
+    <h3>Prompt 1</h3>
+    <label id="prompt-label-1"> </label>
+
+    <h3>Response from model 1</h3>
+    <label id="generate-label-1"> </label>
+    <br />
+
+    <h3>Prompt 2</h3>
+    <label id="prompt-label-2"> </label>
+
+    <h3>Response from model 2</h3>
+    <label id="generate-label-2"> </label>
+    <br />
+    <label id="stats-label"> </label>
+
+    <script type="module" src="./main.ts"></script>
+  </body>
+</html>

examples/multi-models/src/worker.ts

@@ -0,0 +1,7 @@
+import { WebWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
+
+// Hookup an engine to a worker handler
+const handler = new WebWorkerMLCEngineHandler();
+self.onmessage = (msg: MessageEvent) => {
+  handler.onmessage(msg);
+};

examples/multi-round-chat/README.md

@@ -0,0 +1,8 @@
+### OpenAI API Demos
+
+Run `npm install` first, followed by `npm start`.
+
+Note if you would like to hack WebLLM core package,
+you can change web-llm dependencies as `"file:../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.

examples/multi-round-chat/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "openai-api",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/multi_round_chat.html  --port 8888",
+    "build": "parcel build src/multi_round_chat.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}

examples/multi-round-chat/src/multi_round_chat.html

@@ -0,0 +1,16 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output
+    <br />
+    <br />
+    <label id="init-label"> </label>
+
+    <script type="module" src="./multi_round_chat.ts"></script>
+  </body>
+</html>

examples/multi-round-chat/src/multi_round_chat.ts

@@ -0,0 +1,82 @@
+import * as webllm from "@mlc-ai/web-llm";
+
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+/**
+ * We demonstrate multiround chatting. Though users are required to maintain chat history, internally
+ * we compare provided `messages` with the internal chat history. If it matches, we will reuse KVs
+ * and hence save computation -- essentially an implicit internal optimization.
+ */
+async function main() {
+  const initProgressCallback = (report: webllm.InitProgressReport) => {
+    setLabel("init-label", report.text);
+  };
+  const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    { initProgressCallback: initProgressCallback },
+  );
+
+  // Round 0
+  const messages: webllm.ChatCompletionMessageParam[] = [
+    {
+      role: "system",
+      content:
+        "You are a helpful, respectful and honest assistant. " +
+        "Be as happy as you can when speaking please. ",
+    },
+    { role: "user", content: "Provide me three US states." },
+  ];
+
+  const request0: webllm.ChatCompletionRequest = {
+    stream: false, // can be streaming, same behavior
+    messages: messages,
+  };
+
+  const reply0 = await engine.chat.completions.create(request0);
+  const replyMessage0 = await engine.getMessage();
+  console.log(reply0);
+  console.log(replyMessage0);
+  console.log(reply0.usage);
+
+  // Round 1
+  // Append generated response to messages
+  messages.push({ role: "assistant", content: replyMessage0 });
+  // Append new user input
+  messages.push({ role: "user", content: "Two more please!" });
+  // Below line would cause an internal reset (clear KV cache, etc.) since the history no longer
+  // matches the new request
+  // messages[0].content = "Another system prompt";
+
+  const request1: webllm.ChatCompletionRequest = {
+    stream: false, // can be streaming, same behavior
+    messages: messages,
+  };
+
+  const reply1 = await engine.chat.completions.create(request1);
+  const replyMessage1 = await engine.getMessage();
+  console.log(reply1);
+  console.log(replyMessage1);
+  console.log(reply1.usage);
+
+  // If we used multiround chat, request1 should only prefill a small number of tokens
+  const prefillTokens0 = reply0.usage?.prompt_tokens;
+  const prefillTokens1 = reply1.usage?.prompt_tokens;
+  console.log("Requset 0 prompt tokens: ", prefillTokens0);
+  console.log("Requset 1 prompt tokens: ", prefillTokens1);
+  if (
+    prefillTokens0 === undefined ||
+    prefillTokens1 === undefined ||
+    prefillTokens1 > prefillTokens0
+  ) {
+    throw Error("Multi-round chat is not triggered as expected.");
+  }
+}
+
+main();

examples/next-simple-chat/.gitignore

@@ -0,0 +1,35 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

examples/next-simple-chat/README.md

@@ -0,0 +1,17 @@
+This is a [Next.js](https://nextjs.org/) project using web-llm.
+
+## Getting Started
+
+First, install web-llm from source.
+
+Then, run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.

examples/next-simple-chat/next.config.js

@@ -0,0 +1,22 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  reactStrictMode: true,
+  outputFileTracingRoot: __dirname,
+
+  webpack: (config, { isServer }) => {
+    // Fixes npm packages that depend on `fs` module
+    if (!isServer) {
+      config.resolve.fallback = {
+        ...config.resolve.fallback, // if you miss it, all the other options in fallback, specified
+        // by next.js will be dropped. Doesn't make much sense, but how it is
+        fs: false, // the solution
+        module: false,
+        perf_hooks: false,
+      };
+    }
+
+    return config;
+  },
+};
+
+module.exports = nextConfig;

examples/next-simple-chat/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "next-simple-chat",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84",
+    "@types/node": "20.3.3",
+    "@types/react": "18.2.14",
+    "@types/react-dom": "18.2.6",
+    "autoprefixer": "10.4.14",
+    "eslint": "9.38.0",
+    "eslint-config-next": "15.5.15",
+    "next": "15.5.15",
+    "postcss": "^8.4.31",
+    "react": "18.2.0",
+    "react-dom": "18.2.0",
+    "tailwindcss": "3.3.2",
+    "typescript": "5.1.6"
+  }
+}

examples/next-simple-chat/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+};

examples/next-simple-chat/public/next.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

examples/next-simple-chat/public/vercel.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>

examples/next-simple-chat/src/pages/_app.tsx

@@ -0,0 +1,6 @@
+import "~/styles/globals.css";
+import type { AppProps } from "next/app";
+
+export default function App({ Component, pageProps }: AppProps) {
+  return <Component {...pageProps} />;
+}

examples/next-simple-chat/src/pages/_document.tsx

@@ -0,0 +1,13 @@
+import { Html, Head, Main, NextScript } from "next/document";
+
+export default function Document() {
+  return (
+    <Html lang="en">
+      <Head />
+      <body>
+        <Main />
+        <NextScript />
+      </body>
+    </Html>
+  );
+}

examples/next-simple-chat/src/pages/api/hello.ts

@@ -0,0 +1,13 @@
+// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
+import type { NextApiRequest, NextApiResponse } from "next";
+
+type Data = {
+  name: string;
+};
+
+export default function handler(
+  req: NextApiRequest,
+  res: NextApiResponse<Data>,
+) {
+  res.status(200).json({ name: "John Doe" });
+}

examples/next-simple-chat/src/pages/index.tsx

@@ -0,0 +1,28 @@
+import Head from "next/head";
+import dynamic from "next/dynamic";
+import { Inter } from "next/font/google";
+
+const inter = Inter({ subsets: ["latin"] });
+const ChatComponent = dynamic(() => import("~/utils/chat_component"), {
+  ssr: false,
+});
+
+export default function Home() {
+  return (
+    <>
+      <Head>
+        <title>Example App</title>
+        <meta
+          name="description"
+          content="Example app for web llm next compatibility"
+        />
+        <link rel="icon" href="/favicon.ico" />
+      </Head>
+      <main
+        className={`flex min-h-screen flex-col items-center justify-between p-24 ${inter.className}`}
+      >
+        <ChatComponent />
+      </main>
+    </>
+  );
+}