chore: import upstream snapshot with attribution

2026-07-13 12:42:51 +08:00
commit f73e710e38
276 changed files with 36598 additions and 0 deletions
@@ -0,0 +1,18 @@
+# WebLLM Get Started with WebWorker
+
+This folder provides a minimum demo to show WebLLM API using
+[WebWorker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers).
+The main benefit of web worker is that all ML workloads runs on a separate thread as a result
+will less likely block the UI.
+
+To try it out, you can do the following steps under this folder
+
+```bash
+npm install
+npm start
+```
+
+Note if you would like to hack WebLLM core package.
+You can change web-llm dependencies as `"file:../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.
@@ -0,0 +1,20 @@
+{
+  "name": "get-started-web-worker",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/get_started.html  --port 8885",
+    "build": "parcel build src/get_started.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^6.0.3",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}
@@ -0,0 +1,23 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output
+    <br />
+    <br />
+    <label id="init-label"> </label>
+
+    <h3>Prompt</h3>
+    <label id="prompt-label"> </label>
+
+    <h3>Response</h3>
+    <label id="generate-label"> </label>
+    <br />
+    <label id="stats-label"> </label>
+
+    <script type="module" src="./main.ts"></script>
+  </body>
+</html>
@@ -0,0 +1,102 @@
+import * as webllm from "@mlc-ai/web-llm";
+
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+// There are two demonstrations, pick one to run
+
+/**
+ * Chat completion (OpenAI style) without streaming, where we get the entire response at once.
+ */
+async function mainNonStreaming() {
+  const initProgressCallback = (report: webllm.InitProgressReport) => {
+    setLabel("init-label", report.text);
+  };
+  const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
+
+  const engine: webllm.MLCEngineInterface =
+    await webllm.CreateWebWorkerMLCEngine(
+      new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }),
+      selectedModel,
+      { initProgressCallback: initProgressCallback },
+    );
+
+  const request: webllm.ChatCompletionRequest = {
+    messages: [
+      {
+        role: "system",
+        content:
+          "You are a helpful, respectful and honest assistant. " +
+          "Be as happy as you can when speaking please. ",
+      },
+      { role: "user", content: "Provide me three US states." },
+      { role: "assistant", content: "California, New York, Pennsylvania." },
+      { role: "user", content: "Two more please!" },
+    ],
+    n: 3,
+    temperature: 1.5,
+    max_tokens: 256,
+  };
+
+  const reply0 = await engine.chat.completions.create(request);
+  console.log(reply0);
+
+  console.log(reply0.usage);
+}
+
+/**
+ * Chat completion (OpenAI style) with streaming, where delta is sent while generating response.
+ */
+async function mainStreaming() {
+  const initProgressCallback = (report: webllm.InitProgressReport) => {
+    setLabel("init-label", report.text);
+  };
+  const selectedModel = "Llama-3.1-8B-Instruct-q4f32_1-MLC";
+
+  const engine: webllm.MLCEngineInterface =
+    await webllm.CreateWebWorkerMLCEngine(
+      new Worker(new URL("./worker.ts", import.meta.url), { type: "module" }),
+      selectedModel,
+      { initProgressCallback: initProgressCallback },
+    );
+
+  const request: webllm.ChatCompletionRequest = {
+    stream: true,
+    stream_options: { include_usage: true },
+    messages: [
+      {
+        role: "system",
+        content:
+          "You are a helpful, respectful and honest assistant. " +
+          "Be as happy as you can when speaking please. ",
+      },
+      { role: "user", content: "Provide me three US states." },
+      { role: "assistant", content: "California, New York, Pennsylvania." },
+      { role: "user", content: "Two more please!" },
+    ],
+    temperature: 1.5,
+    max_tokens: 256,
+  };
+
+  const asyncChunkGenerator = await engine.chat.completions.create(request);
+  let message = "";
+  for await (const chunk of asyncChunkGenerator) {
+    console.log(chunk);
+    message += chunk.choices[0]?.delta?.content || "";
+    setLabel("generate-label", message);
+    if (chunk.usage) {
+      console.log(chunk.usage); // only last chunk has usage
+    }
+    // engine.interruptGenerate();  // works with interrupt as well
+  }
+  console.log("Final message:\n", await engine.getMessage()); // the concatenated message
+}
+
+// Run one of the function below
+// mainNonStreaming();
+mainStreaming();
@@ -0,0 +1,7 @@
+import { WebWorkerMLCEngineHandler } from "@mlc-ai/web-llm";
+
+// Hookup an engine to a worker handler
+const handler = new WebWorkerMLCEngineHandler();
+self.onmessage = (msg: MessageEvent) => {
+  handler.onmessage(msg);
+};