chore: import upstream snapshot with attribution

2026-07-13 12:42:51 +08:00
commit f73e710e38
276 changed files with 36598 additions and 0 deletions
@@ -0,0 +1,11 @@
+### OpenAI API Demos - Function calling
+
+This folder contains two main ways of using function calling with WebLLM.
+
+`function-calling-manual` demonstrates how you can use function calling with Llama3.1 and Hermes2
+without using the `tools`, `tool_choice`, and `tool_call` fields. This is the most flexible way and you can follow
+the instruction given by the model releaser and iterate yourself on top of that. However, you need to do parsing on your own, which differs for each model. For instance, Hermes2 models use `<tool_call>` and `</tool_call>` to wrap around a tool call, which may be very different from other models' format.
+
+`function-calling-openai` conforms to the OpenAI function calling usage, leveraging `tools`, `tool_choice`, and `tool_call`
+fields. This is more usable, but sacrifices the flexibility since we have pre-defined system prompt
+for this.
@@ -0,0 +1,8 @@
+### Demos - Function calling
+
+Run `npm install` first, followed by `npm start`.
+
+Note if you would like to hack WebLLM core package,
+you can change web-llm dependencies as `"file:../../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.
@@ -0,0 +1,20 @@
+{
+  "name": "openai-api",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/function_calling_manual.html  --port 8888",
+    "build": "parcel build src/function_calling_manual.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}
@@ -0,0 +1,17 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output
+    <br />
+    <br />
+    <label id="init-label"> </label>
+    <label id="generate-label"> </label>
+
+    <script type="module" src="./function_calling_manual.ts"></script>
+  </body>
+</html>
@@ -0,0 +1,235 @@
+/* eslint-disable no-useless-escape */
+import * as webllm from "@mlc-ai/web-llm";
+
+// Common helper methods
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+const initProgressCallback = (report: webllm.InitProgressReport) => {
+  setLabel("init-label", report.text);
+};
+
+// Same example as https://huggingface.co/NousResearch/Hermes-2-Theta-Llama-3-8B#prompt-format-for-function-calling
+async function hermes2_example() {
+  // 0. Setups
+  // Most manual function calling models specify the tools inside the system prompt
+  const system_prompt = `You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: <tools> {"type": "function", "function": {"name": "get_stock_fundamentals", "description": "get_stock_fundamentals(symbol: str) -> dict - Get fundamental data for a given stock symbol using yfinance API.\\n\\n    Args:\\n        symbol (str): The stock symbol.\\n\\n    Returns:\\n        dict: A dictionary containing fundamental data.\\n            Keys:\\n                - \'symbol\': The stock symbol.\\n                - \'company_name\': The long name of the company.\\n                - \'sector\': The sector to which the company belongs.\\n                - \'industry\': The industry to which the company belongs.\\n                - \'market_cap\': The market capitalization of the company.\\n                - \'pe_ratio\': The forward price-to-earnings ratio.\\n                - \'pb_ratio\': The price-to-book ratio.\\n                - \'dividend_yield\': The dividend yield.\\n                - \'eps\': The trailing earnings per share.\\n                - \'beta\': The beta value of the stock.\\n                - \'52_week_high\': The 52-week high price of the stock.\\n                - \'52_week_low\': The 52-week low price of the stock.", "parameters": {"type": "object", "properties": {"symbol": {"type": "string"}}, "required": ["symbol"]}}}  </tools> Use the following pydantic model json schema for each tool call you will make: {"properties": {"arguments": {"title": "Arguments", "type": "object"}, "name": {"title": "Name", "type": "string"}}, "required": ["arguments", "name"], "title": "FunctionCall", "type": "object"} For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:\n<tool_call>\n{"arguments": <args-dict>, "name": <function-name>}\n</tool_call>`;
+  // Same formatting for Hermes-2-Pro-Llama-3, Hermes-2-Theta-Llama-3
+  // const selectedModel = "Hermes-2-Theta-Llama-3-8B-q4f16_1-MLC";
+  const selectedModel = "Hermes-2-Pro-Llama-3-8B-q4f16_1-MLC";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    { initProgressCallback: initProgressCallback, logLevel: "INFO" },
+  );
+  const seed = 0;
+
+  // 1. First request, expect to generate tool call
+  const messages: webllm.ChatCompletionMessageParam[] = [
+    { role: "system", content: system_prompt },
+    {
+      role: "user",
+      content: "Fetch the stock fundamentals data for Tesla (TSLA)",
+    },
+  ];
+  const request1: webllm.ChatCompletionRequest = {
+    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
+    messages: messages,
+    seed: seed,
+  };
+  const reply1 = await engine.chat.completions.create(request1);
+  const response1 = reply1.choices[0].message.content;
+  console.log(reply1.usage);
+  console.log("Response 1: " + response1);
+  messages.push({ role: "assistant", content: response1 });
+  // <tool_call>\n{"arguments": {"symbol": "TSLA"}, "name": "get_stock_fundamentals"}\n</tool_call>
+
+  // 2. Call function on your own to get tool response
+  const tool_response = `<tool_response>\n{"name": "get_stock_fundamentals", "content": {'symbol': 'TSLA', 'company_name': 'Tesla, Inc.', 'sector': 'Consumer Cyclical', 'industry': 'Auto Manufacturers', 'market_cap': 611384164352, 'pe_ratio': 49.604652, 'pb_ratio': 9.762013, 'dividend_yield': None, 'eps': 4.3, 'beta': 2.427, '52_week_high': 299.29, '52_week_low': 152.37}}\n</tool_response>`;
+  messages.push({ role: "tool", content: tool_response, tool_call_id: "0" });
+
+  // 3. Get natural language response
+  const request2: webllm.ChatCompletionRequest = {
+    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
+    messages: messages,
+    seed: seed,
+  };
+  const reply2 = await engine.chat.completions.create(request2);
+  const response2 = reply2.choices[0].message.content;
+  messages.push({ role: "assistant", content: response2 });
+  console.log(reply2.usage);
+  console.log("Response 2: " + response2);
+
+  // 4. Another function call
+  messages.push({
+    role: "user",
+    content: "Now do another one with NVIDIA, symbol being NVDA.",
+  });
+  const request3: webllm.ChatCompletionRequest = {
+    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
+    messages: messages,
+    seed: seed,
+  };
+  const reply3 = await engine.chat.completions.create(request3);
+  const response3 = reply3.choices[0].message.content;
+  messages.push({ role: "assistant", content: response3 });
+  console.log(reply3.usage);
+  console.log("Response 3: " + response3);
+  // <tool_call>\n{"arguments": {"symbol": "NVDA"}, "name": "get_stock_fundamentals"}\n</tool_call>
+}
+
+// Similar example to https://llama.meta.com/docs/model-cards-and-prompt-formats/llama3_1/#user-defined-custom-tool-calling
+async function llama3_1_example() {
+  // Follows example, but tweaks the formatting with <function>
+  const system_prompt = `Cutting Knowledge Date: December 2023
+Today Date: 23 Jul 2024
+# Tool Instructions
+- When looking for real time information use relevant functions if available
+You have access to the following functions:
+
+{
+    "type": "function",
+    "function": {
+        "name": "get_current_temperature",
+        "description": "Get the current temperature at a location.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "location": {
+                    "type": "string",
+                    "description": "The location to get the temperature for, in the format \"City, Country\""
+                }
+            },
+            "required": [
+                "location"
+            ]
+        },
+        "return": {
+            "type": "number",
+            "description": "The current temperature at the specified location in the specified units, as a float."
+        }
+    }
+}
+{
+    "type": "function",
+    "function": {
+        "name": "send_message",
+        "description": "Send a message to a recipient.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "recipient": {
+                    "type": "string",
+                    "description": "Name of the recipient of the message"
+                }
+                "content": {
+                    "type": "string",
+                    "description": "Content of the message"
+                }
+            },
+            "required": [
+                "recipient",
+                "content"
+            ]
+        },
+        "return": {
+            "type": "None"
+        }
+    }
+}
+If a you choose to call a function ONLY reply in the following format:
+    <function>{"name": function name, "parameters": dictionary of argument name and its value}</function>
+Here is an example,
+    <function>{"name": "example_function_name", "parameters": {"example_name": "example_value"}}</function>
+Reminder:
+- Function calls MUST follow the specified format and use BOTH <function> and </function>
+- Required parameters MUST be specified
+- Only call one function at a time
+- When calling a function, do NOT add any other words, ONLY the function calling
+- Put the entire function call reply on one line
+- Always add your sources when using search results to answer the user query
+You are a helpful Assistant.`;
+
+  const selectedModel = "Llama-3.1-8B-Instruct-q4f16_1-MLC";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    { initProgressCallback: initProgressCallback, logLevel: "INFO" },
+  );
+  const seed = 0;
+
+  // 1. First request, expect to generate tool call to get temperature of Paris
+  const messages: webllm.ChatCompletionMessageParam[] = [
+    { role: "system", content: system_prompt },
+    {
+      role: "user",
+      content: "Hey, what's the temperature in Paris right now?",
+    },
+  ];
+  const request1: webllm.ChatCompletionRequest = {
+    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
+    messages: messages,
+    seed: seed,
+  };
+  const reply1 = await engine.chat.completions.create(request1);
+  const response1 = reply1.choices[0].message.content;
+  console.log(reply1.usage);
+  console.log("Response 1: " + response1);
+  messages.push({ role: "assistant", content: response1 });
+  // <function>{"name": "get_current_temperature", "parameters": {"location": "Paris, France"}}</function>
+
+  // 2. Call function on your own to get tool response
+  const tool_response = `{"output": 22.5}`;
+  messages.push({ role: "tool", content: tool_response, tool_call_id: "0" });
+
+  // 3. Get natural language response
+  const request2: webllm.ChatCompletionRequest = {
+    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
+    messages: messages,
+    seed: seed,
+  };
+  const reply2 = await engine.chat.completions.create(request2);
+  const response2 = reply2.choices[0].message.content;
+  messages.push({ role: "assistant", content: response2 });
+  console.log(reply2.usage);
+  console.log("Response 2: " + response2);
+  // The current temperature in Paris is 22.5°C.
+
+  // 4. Make another request, expect model to call `send_message`
+  messages.push({
+    role: "user",
+    content: "Send a message to Tom to tell him this information.",
+  });
+  const request3: webllm.ChatCompletionRequest = {
+    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
+    messages: messages,
+    seed: seed,
+  };
+  const reply3 = await engine.chat.completions.create(request3);
+  const response3 = reply3.choices[0].message.content;
+  messages.push({ role: "assistant", content: response3 });
+  console.log(reply3.usage);
+  console.log("Response 3: " + response3);
+  // <function>{"name": "send_message", "parameters": {"recipient": "Tom", "content": "The current temperature in Paris is 22.5°C."}}</function>
+
+  // 5. Call API, which has no return value, so simply prompt model again
+  const tool_response2 = `{"output": None}`;
+  messages.push({ role: "tool", content: tool_response2, tool_call_id: "1" });
+  const request4: webllm.ChatCompletionRequest = {
+    stream: false, // works with either streaming or non-streaming; code below assumes non-streaming
+    messages: messages,
+    seed: seed,
+  };
+  const reply4 = await engine.chat.completions.create(request4);
+  const response4 = reply4.choices[0].message.content;
+  console.log(reply4.usage);
+  console.log("Response 4: " + response4);
+  // The message has been sent to Tom.
+}
+
+// Pick one to run
+// hermes2_example();
+llama3_1_example();
@@ -0,0 +1,8 @@
+### Demos - Function calling
+
+Run `npm install` first, followed by `npm start`.
+
+Note if you would like to hack WebLLM core package,
+you can change web-llm dependencies as `"file:../../.."`, and follow the build from source
+instruction in the project to build webllm locally. This option is only recommended
+if you would like to hack WebLLM core package.
@@ -0,0 +1,20 @@
+{
+  "name": "openai-api",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/function_calling_openai.html  --port 8888",
+    "build": "parcel build src/function_calling_openai.html --dist-dir lib"
+  },
+  "devDependencies": {
+    "buffer": "^5.7.1",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "tslib": "^2.3.1",
+    "typescript": "^4.9.5",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.84"
+  }
+}
@@ -0,0 +1,17 @@
+<!doctype html>
+<html>
+  <script>
+    webLLMGlobal = {};
+  </script>
+
+  <body>
+    <h2>WebLLM Test Page</h2>
+    Open console to see output
+    <br />
+    <br />
+    <label id="init-label"> </label>
+    <label id="generate-label"> </label>
+
+    <script type="module" src="./function_calling_openai.ts"></script>
+  </body>
+</html>
@@ -0,0 +1,80 @@
+import * as webllm from "@mlc-ai/web-llm";
+
+function setLabel(id: string, text: string) {
+  const label = document.getElementById(id);
+  if (label == null) {
+    throw Error("Cannot find label " + id);
+  }
+  label.innerText = text;
+}
+
+async function main() {
+  const initProgressCallback = (report: webllm.InitProgressReport) => {
+    setLabel("init-label", report.text);
+  };
+  const selectedModel = "Hermes-2-Pro-Llama-3-8B-q4f16_1-MLC";
+  const engine: webllm.MLCEngineInterface = await webllm.CreateMLCEngine(
+    selectedModel,
+    { initProgressCallback: initProgressCallback },
+  );
+
+  const tools: Array<webllm.ChatCompletionTool> = [
+    {
+      type: "function",
+      function: {
+        name: "get_current_weather",
+        description: "Get the current weather in a given location",
+        parameters: {
+          type: "object",
+          properties: {
+            location: {
+              type: "string",
+              description: "The city and state, e.g. San Francisco, CA",
+            },
+            unit: { type: "string", enum: ["celsius", "fahrenheit"] },
+          },
+          required: ["location"],
+        },
+      },
+    },
+  ];
+
+  const request: webllm.ChatCompletionRequest = {
+    stream: true, // works with stream as well, where the last chunk returns tool_calls
+    stream_options: { include_usage: true },
+    messages: [
+      {
+        role: "user",
+        content:
+          "What is the current weather in celsius in Pittsburgh and Tokyo?",
+      },
+    ],
+    tool_choice: "auto",
+    tools: tools,
+  };
+
+  if (!request.stream) {
+    const reply0 = await engine.chat.completions.create(request);
+    console.log(reply0.choices[0]);
+    console.log(reply0.usage);
+  } else {
+    // If streaming, the last chunk returns tool calls
+    const asyncChunkGenerator = await engine.chat.completions.create(request);
+    let message = "";
+    let lastChunk: webllm.ChatCompletionChunk | undefined;
+    let usageChunk: webllm.ChatCompletionChunk | undefined;
+    for await (const chunk of asyncChunkGenerator) {
+      console.log(chunk);
+      message += chunk.choices[0]?.delta?.content || "";
+      setLabel("generate-label", message);
+      if (!chunk.usage) {
+        lastChunk = chunk;
+      }
+      usageChunk = chunk;
+    }
+    console.log(lastChunk!.choices[0].delta);
+    console.log(usageChunk!.usage);
+  }
+}
+
+main();