chore: import upstream snapshot with attribution

2026-07-13 12:42:51 +08:00
commit f73e710e38
276 changed files with 36598 additions and 0 deletions
@@ -0,0 +1,30 @@
+# WebLLM Cache Usage
+
+WebLLM supports multiple persistent cache backends. You can pick the classic Cache API, IndexedDB, [Origin Private File System (OPFS)](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system), or the experimental Chrome [Cross-Origin Storage](https://github.com/WICG/cross-origin-storage) extension by
+setting `AppConfig.cacheBackend` to `"cache"`, `"indexeddb"`, `"opfs"`, or `"cross-origin"`.
+This folder provides an example on how different caches are used in WebLLM. We also
+demonstrate the utility cache functions such as deleting models, checking if models are in cache, etc.
+
+> **Note:** The cross-origin backend requires installation of the [Cross-Origin Storage browser extension](https://chromewebstore.google.com/detail/cross-origin-storage/denpnpcgjgikjpoglpjefakmdcbmlgih) ([source code](https://github.com/web-ai-community/cross-origin-storage-extension)). This does not currently support programmatic tensor-cache deletion; deletion is extension-managed.
+
+> **Note:** If `"opfs"` is selected in an environment without OPFS support, cache operations fail with an OPFS availability error.
+> Use `appConfig.opfsAccessMode = "auto"` to use OPFS sync access handles where supported, or `"sync"` to require sync access handles. The default is `"async"`.
+
+For more information about Cache API and IndexedDB, see:
+https://developer.mozilla.org/en-US/docs/Web/API/Storage_API/Storage_quotas_and_eviction_criteria#what_technologies_store_data_in_the_browser.
+
+To inspect the downloaded artifacts in your browser, open up developer console, go to application,
+and you will find the artifacts under `IndexedDB`, `Cache storage`, or OPFS (under the browser's origin-private file system). When `"cross-origin"` is selected,
+the extension displays origins and resource hashes.
+
+To run the example, you can do the following steps under this folder
+
+```bash
+npm install
+npm start
+```
+
+Note if you would like to hack WebLLM core package.
+You can change web-llm dependencies as `"file:../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.
@@ -0,0 +1,20 @@
+{
+  "name": "cache-usage",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/cache_usage.html  --port 8889",
+    "build": "parcel build src/cache_usage.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}
@@ -0,0 +1,24 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output
+    <br />
+    <br />
+    <label id="init-label"> </label>
+
+    <h3>Prompt</h3>
+    <label id="prompt-label"> </label>
+
+    <h3>Response</h3>
+    <label id="generate-label"> </label>
+    <br />
+    <label id="stats-label"> </label>
+
+    <script type="module" src="./cache_usage.ts"></script>
+  </body>
+</html>
@@ -0,0 +1,90 @@
+import * as webllm from "@mlc-ai/web-llm";
+
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+const initProgressCallback = (report: webllm.InitProgressReport) => {
+  setLabel("init-label", report.text);
+};
+
+async function main() {
+  const appConfig = webllm.prebuiltAppConfig;
+  // CHANGE THIS TO SEE THE EFFECTS OF EACH, CODE BELOW DOES NOT NEED TO CHANGE
+  appConfig.cacheBackend = "cache"; // "indexeddb" or "cache" or "cross-origin" or "opfs"
+
+  const cacheBackend = appConfig.cacheBackend as string;
+  if (cacheBackend === "indexeddb") {
+    console.log("Using IndexedDB Cache");
+  } else if (cacheBackend === "cache") {
+    console.log("Using Cache API");
+  } else if (cacheBackend === "cross-origin") {
+    console.log("Using Cross-Origin Storage");
+  } else if (cacheBackend === "opfs") {
+    console.log("Using Origin Private File System");
+  }
+
+  // 1. This triggers downloading and caching the model with either Cache or IndexedDB Cache
+  const selectedModel = "Llama-3.2-1B-Instruct-q4f16_1-MLC";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    { initProgressCallback: initProgressCallback, appConfig: appConfig },
+  );
+
+  const request: webllm.ChatCompletionRequest = {
+    stream: false,
+    messages: [
+      {
+        role: "user",
+        content: "Write an analogy between mathematics and a lighthouse.",
+      },
+    ],
+    n: 1,
+    temperature: 0,
+  };
+  let reply = await engine.chat.completions.create(request);
+  console.log(reply);
+
+  // 2. Check whether model weights are cached
+  let modelCached = await webllm.hasModelInCache(selectedModel, appConfig);
+  console.log("hasModelInCache: ", modelCached);
+  if (!modelCached) {
+    throw Error("Expect hasModelInCache() to be true, but got: " + modelCached);
+  }
+
+  // 3. We reload, and we should see this time it is much faster because the weights are cached.
+  console.log("Reload model start");
+  await engine.reload(selectedModel);
+  console.log("Reload model end");
+  reply = await engine.chat.completions.create(request);
+  console.log(reply);
+
+  // Cross-Origin Storage does not support deletion
+  if (cacheBackend === "cross-origin") {
+    return;
+  }
+
+  // 4. Delete everything about this model from cache
+  // You can also delete only the model library wasm, only the model weights, or only the config file
+  await webllm.deleteModelAllInfoInCache(selectedModel, appConfig);
+  modelCached = await webllm.hasModelInCache(selectedModel, appConfig);
+  console.log("After deletion, hasModelInCache: ", modelCached);
+  if (modelCached) {
+    throw Error(
+      "Expect hasModelInCache() to be false, but got: " + modelCached,
+    );
+  }
+
+  // 5. If we reload, we should expect the model to start downloading again
+  console.log("Reload model start");
+  await engine.reload(selectedModel);
+  console.log("Reload model end");
+  reply = await engine.chat.completions.create(request);
+  console.log(reply);
+}
+
+main();