chore: import upstream snapshot with attribution

tools/server/bench/README.md

@@ -0,0 +1,119 @@
+### Server benchmark tools
+
+Benchmark is using [k6](https://k6.io/).
+
+##### Install k6 and sse extension
+
+SSE is not supported by default in k6, you have to build k6 with the [xk6-sse](https://github.com/phymbert/xk6-sse) extension.
+
+Example (assuming golang >= 1.21 is installed):
+```shell
+go install go.k6.io/xk6/cmd/xk6@latest
+$GOPATH/bin/xk6 build master \
+--with github.com/phymbert/xk6-sse
+```
+
+#### Download a dataset
+
+This dataset was originally proposed in [vLLM benchmarks](https://github.com/vllm-project/vllm/blob/main/benchmarks/README.md).
+
+```shell
+wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
+```
+
+#### Download a model
+Example for PHI-2
+
+```shell
+../../../scripts/hf.sh --repo ggml-org/models --file phi-2/ggml-model-q4_0.gguf
+```
+
+#### Start the server
+The server must answer OAI Chat completion requests on `http://localhost:8080/v1` or according to the environment variable `SERVER_BENCH_URL`.
+
+Example:
+```shell
+llama-server --host localhost --port 8080 \
+  --model ggml-model-q4_0.gguf \
+  --cont-batching \
+  --metrics \
+  --parallel 8 \
+  --batch-size 512 \
+  --ctx-size 4096 \
+  -ngl 33
+```
+
+#### Run the benchmark
+
+For 500 chat completions request with 8 concurrent users during maximum 10 minutes, run:
+```shell
+./k6 run script.js --duration 10m --iterations 500 --vus 8
+```
+
+The benchmark values can be overridden with:
+- `SERVER_BENCH_URL` server url prefix for chat completions, default `http://localhost:8080/v1`
+- `SERVER_BENCH_N_PROMPTS` total prompts to randomly select in the benchmark, default `480`
+- `SERVER_BENCH_MODEL_ALIAS` model alias to pass in the completion request, default `my-model`
+- `SERVER_BENCH_MAX_TOKENS` max tokens to predict, default: `512`
+- `SERVER_BENCH_DATASET` path to the benchmark dataset file
+- `SERVER_BENCH_MAX_PROMPT_TOKENS` maximum prompt tokens to filter out in the dataset: default `1024`
+- `SERVER_BENCH_MAX_CONTEXT` maximum context size of the completions request to filter out in the dataset: prompt + predicted tokens, default `2048`
+
+Note: the local tokenizer is just a string space split, real number of tokens will differ.
+
+Or with [k6 options](https://k6.io/docs/using-k6/k6-options/reference/):
+
+```shell
+SERVER_BENCH_N_PROMPTS=500 k6 run script.js --duration 10m --iterations 500 --vus 8
+```
+
+To [debug http request](https://k6.io/docs/using-k6/http-debugging/) use `--http-debug="full"`.
+
+#### Metrics
+
+Following metrics are available computed from the OAI chat completions response `usage`:
+- `llamacpp_tokens_second` Trend of `usage.total_tokens / request duration`
+- `llamacpp_prompt_tokens` Trend of `usage.prompt_tokens`
+- `llamacpp_prompt_tokens_total_counter` Counter of `usage.prompt_tokens`
+- `llamacpp_completion_tokens` Trend of `usage.completion_tokens`
+- `llamacpp_completion_tokens_total_counter` Counter of `usage.completion_tokens`
+- `llamacpp_completions_truncated_rate` Rate of completions truncated, i.e. if `finish_reason === 'length'`
+- `llamacpp_completions_stop_rate` Rate of completions stopped by the model, i.e. if `finish_reason === 'stop'`
+
+The script will fail if too many completions are truncated, see `llamacpp_completions_truncated_rate`.
+
+K6 metrics might be compared against [server metrics](../README.md), with:
+
+```shell
+curl http://localhost:8080/metrics
+```
+
+### Using the CI python script
+The `bench.py` script does several steps:
+- start the server
+- define good variable for k6
+- run k6 script
+- extract metrics from prometheus
+
+It aims to be used in the CI, but you can run it manually:
+
+```shell
+LLAMA_SERVER_BIN_PATH=../../../cmake-build-release/bin/llama-server python bench.py \
+              --runner-label local \
+              --name local \
+              --branch `git rev-parse --abbrev-ref HEAD` \
+              --commit `git rev-parse HEAD` \
+              --scenario script.js \
+              --duration 5m \
+              --hf-repo ggml-org/models	 \
+              --hf-file phi-2/ggml-model-q4_0.gguf \
+              --model-path-prefix models \
+              --parallel 4 \
+              -ngl 33 \
+              --batch-size 2048 \
+              --ubatch-size	256 \
+              --ctx-size 4096 \
+              --n-prompts 200 \
+              --max-prompt-tokens 256 \
+              --max-tokens 256
+```

tools/server/bench/bench.py

@@ -0,0 +1,325 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import signal
+import socket
+import subprocess
+import sys
+import threading
+import time
+import traceback
+from contextlib import closing
+from datetime import datetime
+
+import matplotlib
+import matplotlib.dates
+import matplotlib.pyplot as plt
+import requests
+from statistics import mean
+
+
+def main(args_in: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(description="Start server benchmark scenario")
+    parser.add_argument("--name", type=str, help="Bench name", required=True)
+    parser.add_argument("--runner-label", type=str, help="Runner label", required=True)
+    parser.add_argument("--branch", type=str, help="Branch name", default="detached")
+    parser.add_argument("--commit", type=str, help="Commit name", default="dirty")
+    parser.add_argument("--host", type=str, help="Server listen host", default="0.0.0.0")
+    parser.add_argument("--port", type=int, help="Server listen host", default="8080")
+    parser.add_argument("--model-path-prefix", type=str, help="Prefix where to store the model files", default="models")
+    parser.add_argument("--n-prompts", type=int,
+                        help="SERVER_BENCH_N_PROMPTS: total prompts to randomly select in the benchmark", required=True)
+    parser.add_argument("--max-prompt-tokens", type=int,
+                        help="SERVER_BENCH_MAX_PROMPT_TOKENS: maximum prompt tokens to filter out in the dataset",
+                        required=True)
+    parser.add_argument("--max-tokens", type=int,
+                        help="SERVER_BENCH_MAX_CONTEXT: maximum context size of the completions request to filter out in the dataset: prompt + predicted tokens",
+                        required=True)
+    parser.add_argument("--hf-repo", type=str, help="Hugging Face model repository", required=True)
+    parser.add_argument("--hf-file", type=str, help="Hugging Face model file", required=True)
+    parser.add_argument("--offline", action="store_true", default=False, help="Offline mode: forces use of cache, prevents network access")
+    parser.add_argument("-ngl", "--n-gpu-layers", type=int, help="layers to the GPU for computation", required=True)
+    parser.add_argument("--ctx-size", type=int, help="Set the size of the prompt context", required=True)
+    parser.add_argument("--parallel", type=int, help="Set the number of slots for process requests", required=True)
+    parser.add_argument("--batch-size", type=int, help="Set the batch size for prompt processing", required=True)
+    parser.add_argument("--ubatch-size", type=int, help="physical maximum batch size", required=True)
+    parser.add_argument("--scenario", type=str, help="Scenario to run", required=True)
+    parser.add_argument("--duration", type=str, help="Bench scenario", required=True)
+
+    args = parser.parse_args(args_in)
+
+    start_time = time.time()
+
+    # Start the server and performance scenario
+    try:
+        server_process = start_server(args)
+    except Exception:
+        print("bench: server start error :")
+        traceback.print_exc(file=sys.stdout)
+        sys.exit(1)
+
+    # start the benchmark
+    iterations = 0
+    data = {}
+    try:
+        start_benchmark(args)
+
+        with open("results.github.env", 'w') as github_env:
+            # parse output
+            with open('k6-results.json', 'r') as bench_results:
+                # Load JSON data from file
+                data = json.load(bench_results)
+                for metric_name in data['metrics']:
+                    for metric_metric in data['metrics'][metric_name]:
+                        value = data['metrics'][metric_name][metric_metric]
+                        if isinstance(value, float) or isinstance(value, int):
+                            value = round(value, 2)
+                            data['metrics'][metric_name][metric_metric]=value
+                            github_env.write(
+                                f"{escape_metric_name(metric_name)}_{escape_metric_name(metric_metric)}={value}\n")
+                iterations = data['root_group']['checks']['success completion']['passes']
+
+    except Exception:
+        print("bench: error :")
+        traceback.print_exc(file=sys.stdout)
+
+    # Stop the server
+    if server_process:
+        try:
+            print(f"bench: shutting down server pid={server_process.pid} ...")
+            if os.name == 'nt':
+                interrupt = signal.CTRL_C_EVENT
+            else:
+                interrupt = signal.SIGINT
+            server_process.send_signal(interrupt)
+            server_process.wait(0.5)
+
+        except subprocess.TimeoutExpired:
+            print(f"server still alive after 500ms, force-killing pid={server_process.pid} ...")
+            server_process.kill()  # SIGKILL
+            server_process.wait()
+
+        while is_server_listening(args.host, args.port):
+            time.sleep(0.1)
+
+    title = (f"llama.cpp {args.name} on {args.runner_label}\n "
+             f"duration={args.duration} {iterations} iterations")
+    xlabel = (f"{args.hf_repo}/{args.hf_file}\n"
+              f"parallel={args.parallel} ctx-size={args.ctx_size} ngl={args.n_gpu_layers} batch-size={args.batch_size} ubatch-size={args.ubatch_size} pp={args.max_prompt_tokens} pp+tg={args.max_tokens}\n"
+              f"branch={args.branch} commit={args.commit}")
+
+    # Prometheus
+    end_time = time.time()
+    prometheus_metrics = {}
+    if is_server_listening("0.0.0.0", 9090):
+        metrics = ['prompt_tokens_seconds', 'predicted_tokens_seconds',
+                   'kv_cache_usage_ratio', 'requests_processing', 'requests_deferred']
+
+        for metric in metrics:
+            resp = requests.get(f"http://localhost:9090/api/v1/query_range",
+                                params={'query': 'llamacpp:' + metric, 'start': start_time, 'end': end_time, 'step': 2})
+
+            with open(f"{metric}.json", 'w') as metric_json:
+                metric_json.write(resp.text)
+
+            if resp.status_code != 200:
+                print(f"bench: unable to extract prometheus metric {metric}: {resp.text}")
+            else:
+                metric_data = resp.json()
+                values = metric_data['data']['result'][0]['values']
+                timestamps, metric_values = zip(*values)
+                metric_values = [float(value) for value in metric_values]
+                prometheus_metrics[metric] = metric_values
+                timestamps_dt = [str(datetime.fromtimestamp(int(ts))) for ts in timestamps]
+                plt.figure(figsize=(16, 10), dpi=80)
+                plt.plot(timestamps_dt, metric_values, label=metric)
+                plt.xticks(rotation=0, fontsize=14, horizontalalignment='center', alpha=.7)
+                plt.yticks(fontsize=12, alpha=.7)
+
+                ylabel = f"llamacpp:{metric}"
+                plt.title(title,
+                          fontsize=14, wrap=True)
+                plt.grid(axis='both', alpha=.3)
+                plt.ylabel(ylabel, fontsize=22)
+                plt.xlabel(xlabel, fontsize=14, wrap=True)
+                plt.gca().xaxis.set_major_locator(matplotlib.dates.MinuteLocator())
+                plt.gca().xaxis.set_major_formatter(matplotlib.dates.DateFormatter("%Y-%m-%d %H:%M:%S"))
+                plt.gcf().autofmt_xdate()
+
+                # Remove borders
+                plt.gca().spines["top"].set_alpha(0.0)
+                plt.gca().spines["bottom"].set_alpha(0.3)
+                plt.gca().spines["right"].set_alpha(0.0)
+                plt.gca().spines["left"].set_alpha(0.3)
+
+                # Save the plot as a jpg image
+                plt.savefig(f'{metric}.jpg', dpi=60)
+                plt.close()
+
+                # Mermaid format in case images upload failed
+                with open(f"{metric}.mermaid", 'w') as mermaid_f:
+                    mermaid = (
+                    f"""---
+config:
+    xyChart:
+        titleFontSize: 12
+        width: 900
+        height: 600
+    themeVariables:
+        xyChart:
+            titleColor: "#000000"
+---
+xychart-beta
+    title "{title}"
+    y-axis "llamacpp:{metric}"
+    x-axis "llamacpp:{metric}" {int(min(timestamps))} --> {int(max(timestamps))}
+    line [{', '.join([str(round(float(value), 2)) for value in metric_values])}]
+                    """)
+                    mermaid_f.write(mermaid)
+
+    # 140 chars max for commit status description
+    bench_results = {
+        "i": iterations,
+        "req": {
+            "p95": round(data['metrics']["http_req_duration"]["p(95)"], 2),
+            "avg": round(data['metrics']["http_req_duration"]["avg"], 2),
+        },
+        "pp": {
+            "p95": round(data['metrics']["llamacpp_prompt_processing_second"]["p(95)"], 2),
+            "avg": round(data['metrics']["llamacpp_prompt_processing_second"]["avg"], 2),
+            "0": round(mean(prometheus_metrics['prompt_tokens_seconds']), 2) if 'prompt_tokens_seconds' in prometheus_metrics else 0,
+        },
+        "tg": {
+            "p95": round(data['metrics']["llamacpp_tokens_second"]["p(95)"], 2),
+            "avg": round(data['metrics']["llamacpp_tokens_second"]["avg"], 2),
+            "0": round(mean(prometheus_metrics['predicted_tokens_seconds']), 2) if 'predicted_tokens_seconds' in prometheus_metrics else 0,
+        },
+    }
+    with open("results.github.env", 'a') as github_env:
+        github_env.write(f"BENCH_RESULTS={json.dumps(bench_results, indent=None, separators=(',', ':') )}\n")
+        github_env.write(f"BENCH_ITERATIONS={iterations}\n")
+
+        title = title.replace('\n', ' ')
+        xlabel = xlabel.replace('\n', ' ')
+        github_env.write(f"BENCH_GRAPH_TITLE={title}\n")
+        github_env.write(f"BENCH_GRAPH_XLABEL={xlabel}\n")
+
+
+def start_benchmark(args):
+    k6_path = './k6'
+    if 'BENCH_K6_BIN_PATH' in os.environ:
+        k6_path = os.environ['BENCH_K6_BIN_PATH']
+    k6_args = [
+        'run', args.scenario,
+        '--no-color',
+        '--no-connection-reuse',
+        '--no-vu-connection-reuse',
+    ]
+    k6_args.extend(['--duration', args.duration])
+    k6_args.extend(['--iterations', args.n_prompts])
+    k6_args.extend(['--vus', args.parallel])
+    k6_args.extend(['--summary-export', 'k6-results.json'])
+    k6_args.extend(['--out', 'csv=k6-results.csv'])
+    args = f"SERVER_BENCH_N_PROMPTS={args.n_prompts} SERVER_BENCH_MAX_PROMPT_TOKENS={args.max_prompt_tokens} SERVER_BENCH_MAX_CONTEXT={args.max_tokens} "
+    args = args + ' '.join([str(arg) for arg in [k6_path, *k6_args]])
+    print(f"bench: starting k6 with: {args}")
+    k6_completed = subprocess.run(args, shell=True, stdout=sys.stdout, stderr=sys.stderr)
+    if k6_completed.returncode != 0:
+        raise Exception("bench: unable to run k6")
+
+
+def start_server(args):
+    server_process = start_server_background(args)
+
+    attempts = 0
+    max_attempts = 600
+    if 'GITHUB_ACTIONS' in os.environ:
+        max_attempts *= 2
+
+    while not is_server_listening(args.host, args.port):
+        attempts += 1
+        if attempts > max_attempts:
+            assert False, "server not started"
+        print(f"bench:     waiting for server to start ...")
+        time.sleep(0.5)
+
+    attempts = 0
+    while not is_server_ready(args.host, args.port):
+        attempts += 1
+        if attempts > max_attempts:
+            assert False, "server not ready"
+        print(f"bench:     waiting for server to be ready ...")
+        time.sleep(0.5)
+
+    print("bench: server started and ready.")
+    return server_process
+
+
+def start_server_background(args):
+    # Start the server
+    server_path = '../../../build/bin/llama-server'
+    if 'LLAMA_SERVER_BIN_PATH' in os.environ:
+        server_path = os.environ['LLAMA_SERVER_BIN_PATH']
+    server_args = [
+        '--host', args.host,
+        '--port', args.port,
+    ]
+    server_args.extend(['--hf-repo', args.hf_repo])
+    server_args.extend(['--hf-file', args.hf_file])
+    if args.offline:
+        server_args.append('--offline')
+    server_args.extend(['--n-gpu-layers', args.n_gpu_layers])
+    server_args.extend(['--ctx-size', args.ctx_size])
+    server_args.extend(['--parallel', args.parallel])
+    server_args.extend(['--batch-size', args.batch_size])
+    server_args.extend(['--ubatch-size', args.ubatch_size])
+    server_args.extend(['--n-predict', args.max_tokens * 2])
+    server_args.append('--cont-batching')
+    server_args.append('--metrics')
+    server_args.append('--flash-attn')
+    args = [str(arg) for arg in [server_path, *server_args]]
+    print(f"bench: starting server with: {' '.join(args)}")
+    pkwargs = {
+        'stdout': subprocess.PIPE,
+        'stderr': subprocess.PIPE
+    }
+    server_process = subprocess.Popen(
+        args,
+        **pkwargs)  # pyright: ignore[reportArgumentType, reportCallIssue] # ty: ignore[no-matching-overload]
+
+    def server_log(in_stream, out_stream):
+        for line in iter(in_stream.readline, b''):
+            print(line.decode('utf-8'), end='', file=out_stream)
+
+    thread_stdout = threading.Thread(target=server_log, args=(server_process.stdout, sys.stdout))
+    thread_stdout.start()
+    thread_stderr = threading.Thread(target=server_log, args=(server_process.stderr, sys.stderr))
+    thread_stderr.start()
+
+    return server_process
+
+
+def is_server_listening(server_fqdn, server_port):
+    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
+        result = sock.connect_ex((server_fqdn, server_port))
+        _is_server_listening = result == 0
+        if _is_server_listening:
+            print(f"server is listening on {server_fqdn}:{server_port}...")
+        return _is_server_listening
+
+
+def is_server_ready(server_fqdn, server_port):
+    url = f"http://{server_fqdn}:{server_port}/health"
+    response = requests.get(url)
+    return response.status_code == 200
+
+
+def escape_metric_name(metric_name):
+    return re.sub('[^A-Z0-9]', '_', metric_name.upper())
+
+
+if __name__ == '__main__':
+    main()

tools/server/bench/prometheus.yml

@@ -0,0 +1,9 @@
+global:
+  scrape_interval:     10s
+  external_labels:
+    llamacpp: 'server'
+
+scrape_configs:
+  - job_name: 'llama.cpp server'
+    static_configs:
+      - targets: ['localhost:8080']

tools/server/bench/requirements.txt

@@ -0,0 +1,2 @@
+matplotlib
+requests

tools/server/bench/script.js

@@ -0,0 +1,162 @@
+import sse from 'k6/x/sse'
+import {check, sleep} from 'k6'
+import {SharedArray} from 'k6/data'
+import {Counter, Rate, Trend} from 'k6/metrics'
+import exec from 'k6/execution';
+
+// Server chat completions prefix
+const server_url = __ENV.SERVER_BENCH_URL ? __ENV.SERVER_BENCH_URL : 'http://localhost:8080/v1'
+
+// Number of total prompts in the dataset - default 10m / 10 seconds/request * number of users
+const n_prompt = __ENV.SERVER_BENCH_N_PROMPTS ? parseInt(__ENV.SERVER_BENCH_N_PROMPTS) : 600 / 10 * 8
+
+// Model name to request
+const model = __ENV.SERVER_BENCH_MODEL_ALIAS ? __ENV.SERVER_BENCH_MODEL_ALIAS : 'my-model'
+
+// Dataset path
+const dataset_path = __ENV.SERVER_BENCH_DATASET ? __ENV.SERVER_BENCH_DATASET : './ShareGPT_V3_unfiltered_cleaned_split.json'
+
+// Max tokens to predict
+const max_tokens = __ENV.SERVER_BENCH_MAX_TOKENS ? parseInt(__ENV.SERVER_BENCH_MAX_TOKENS) : 512
+
+// Max prompt tokens
+const n_prompt_tokens = __ENV.SERVER_BENCH_MAX_PROMPT_TOKENS ? parseInt(__ENV.SERVER_BENCH_MAX_PROMPT_TOKENS) : 1024
+
+// Max slot context
+const n_ctx_slot = __ENV.SERVER_BENCH_MAX_CONTEXT ? parseInt(__ENV.SERVER_BENCH_MAX_CONTEXT) : 2048
+
+export function setup() {
+    console.info(`Benchmark config: server_url=${server_url} n_prompt=${n_prompt} model=${model} dataset_path=${dataset_path} max_tokens=${max_tokens}`)
+}
+
+const data = new SharedArray('conversations', function () {
+    const tokenizer = (message) => message.split(/[\s,'".?]/)
+
+    return JSON.parse(open(dataset_path))
+        // Filter out the conversations with less than 2 turns.
+        .filter(data => data["conversations"].length >= 2)
+        .filter(data => data["conversations"][0]["from"] === "human")
+        .map(data => {
+            return {
+                prompt: data["conversations"][0]["value"],
+                n_prompt_tokens: tokenizer(data["conversations"][0]["value"]).length,
+                n_completion_tokens: tokenizer(data["conversations"][1]["value"]).length,
+            }
+        })
+        // Filter out too short sequences
+        .filter(conv => conv.n_prompt_tokens >= 4 && conv.n_completion_tokens >= 4)
+        // Filter out too long sequences.
+        .filter(conv => conv.n_prompt_tokens <= n_prompt_tokens && conv.n_prompt_tokens + conv.n_completion_tokens <= n_ctx_slot)
+        // Keep only first n prompts
+        .slice(0, n_prompt)
+})
+
+const llamacpp_prompt_tokens = new Trend('llamacpp_prompt_tokens')
+const llamacpp_completion_tokens = new Trend('llamacpp_completion_tokens')
+
+const llamacpp_tokens_second = new Trend('llamacpp_tokens_second')
+const llamacpp_prompt_processing_second = new Trend('llamacpp_prompt_processing_second')
+const llamacpp_emit_first_token_second = new Trend('llamacpp_emit_first_token_second')
+
+const llamacpp_prompt_tokens_total_counter = new Counter('llamacpp_prompt_tokens_total_counter')
+const llamacpp_completion_tokens_total_counter = new Counter('llamacpp_completion_tokens_total_counter')
+
+const llamacpp_completions_truncated_rate = new Rate('llamacpp_completions_truncated_rate')
+const llamacpp_completions_stop_rate = new Rate('llamacpp_completions_stop_rate')
+
+export const options = {
+    thresholds: {
+        llamacpp_completions_truncated_rate: [
+            // more than 80% of truncated input will abort the test
+            {threshold: 'rate < 0.8', abortOnFail: true, delayAbortEval: '1m'},
+        ],
+    },
+    duration: '10m',
+    vus: 8,
+}
+
+export default function () {
+    const conversation = data[exec.scenario.iterationInInstance % data.length]
+    const payload = {
+        "messages": [
+            {
+                "role": "system",
+                "content": "You are ChatGPT, an AI assistant.",
+            },
+            {
+                "role": "user",
+                "content": conversation.prompt,
+            }
+        ],
+        "model": model,
+        "stream": true,
+        "stream_options": {
+          "include_usage": true, // False to be supported in llama.cpp server
+        },
+        "seed": 42,
+        "max_tokens": max_tokens,
+        "stop": ["<|im_end|>"] // This is temporary for phi-2 base (i.e. not instructed) since the server expects that the model always to emit BOS
+    }
+
+    const params = {method: 'POST', body: JSON.stringify(payload)};
+
+    const startTime = new Date()
+    let promptEvalEndTime = null
+    let prompt_tokens = 0
+    let completions_tokens = 0
+    let finish_reason = null
+    const res = sse.open(`${server_url}/chat/completions`, params, function (client) {
+        client.on('event', function (event) {
+            if (promptEvalEndTime == null) {
+                promptEvalEndTime = new Date()
+                llamacpp_emit_first_token_second.add((promptEvalEndTime - startTime) / 1.e3)
+            }
+
+            if (event.data === '[DONE]' || event.data === '') {
+                return
+            }
+
+            let chunk = JSON.parse(event.data)
+
+            if (chunk.choices && chunk.choices.length > 0) {
+                let choice = chunk.choices[0]
+                if (choice.finish_reason) {
+                    finish_reason = choice.finish_reason
+                }
+            }
+
+            if (chunk.usage) {
+                prompt_tokens = chunk.usage.prompt_tokens
+                llamacpp_prompt_tokens.add(prompt_tokens)
+                llamacpp_prompt_tokens_total_counter.add(prompt_tokens)
+
+                completions_tokens = chunk.usage.completion_tokens
+                llamacpp_completion_tokens.add(completions_tokens)
+                llamacpp_completion_tokens_total_counter.add(completions_tokens)
+            }
+        })
+
+        client.on('error', function (e) {
+            console.log('An unexpected error occurred: ', e.error());
+            throw e;
+        })
+    })
+
+    check(res, {'success completion': (r) => r.status === 200})
+
+    const endTime = new Date()
+
+    const promptEvalTime = promptEvalEndTime - startTime
+    if (promptEvalTime > 0) {
+        llamacpp_prompt_processing_second.add(prompt_tokens / (promptEvalEndTime - startTime) * 1.e3)
+    }
+
+    const completion_time = endTime - promptEvalEndTime
+    if (completions_tokens > 0 && completion_time > 0) {
+        llamacpp_tokens_second.add(completions_tokens / completion_time * 1.e3)
+    }
+    llamacpp_completions_truncated_rate.add(finish_reason === 'length')
+    llamacpp_completions_stop_rate.add(finish_reason === 'stop')
+
+    sleep(0.3)
+}

tools/server/bench/speed-bench/README.md

@@ -0,0 +1,117 @@
+# SPEED-Bench server benchmark
+
+A lightweight [SPEED-Bench](https://huggingface.co/datasets/nvidia/SPEED-Bench) client for benchmarking an already-running `llama-server` through its OpenAI-compatible API. It is primarily meant to evaluate speculative decoding (draft model, n-gram, MTP, EAGLE3, ...) by reporting per-category throughput, latency, and draft acceptance.
+
+The dataset handling follows the [aiperf SPEED-Bench tutorial](https://github.com/ai-dynamo/aiperf/blob/main/docs/tutorials/speed-bench.md), which also documents the dataset layout in more detail.
+
+## Install
+
+```bash
+pip install -r tools/server/bench/speed-bench/requirements.txt
+```
+
+## Start a server
+
+The client does not launch the server, so start `llama-server` yourself first. If you care about throughput numbers, set the client `--concurrency` to the server's slot count (`--np`):
+
+```bash
+llama-server \
+  -m target.gguf \
+  -c 8192 \
+  --port 8080 \
+  -ngl 99 -fa on \
+  --np 1 \
+  --jinja
+```
+
+For speculative decoding, start the server with the appropriate flags for your setup (e.g. a draft model with `-md`, or `--spec-type ngram-mod`). See the [speculative decoding doc](../../../../docs/speculative.md) for details.
+
+## Run
+
+```bash
+python tools/server/bench/speed-bench/speed_bench.py \
+  --url localhost:8080 \
+  --bench qualitative \
+  --category coding \
+  --osl 1024 \
+  --concurrency 1
+```
+
+## Options
+
+| Option | Default | Description |
+| --- | --- | --- |
+| `--url` | `localhost:8080` | Server URL. The scheme and `/v1` are optional and a trailing slash is fine, so `localhost:8080` and `http://localhost:8080/v1/` both work. |
+| `--model` | none | Optional `model` field sent in each request. |
+| `--bench` | `qualitative` | SPEED-Bench config, e.g. `qualitative`, `throughput_1k`. See [available dataset variants](https://github.com/ai-dynamo/aiperf/blob/main/docs/tutorials/speed-bench.md#available-dataset-variants). |
+| `--category` | `all` | Category filter within the bench; comma-separated list or `all`. For `qualitative` the categories are `coding`, `humanities`, `math`, `multilingual`, `qa`, `rag`, `reasoning`, `roleplay`, `stem`, `summarization`, `writing`. For the `throughput_{ISL}` splits they are `high_entropy`, `low_entropy`, `mixed`. |
+| `--osl` | `1024` | Output sequence length, mapped to `max_tokens`. |
+| `--extra-inputs` | `{"temperature":0}` | Extra request fields as a JSON object. |
+| `--concurrency` | `1` | Concurrent client requests; usually match `--np`. |
+| `--limit` | none | Max samples per category (handy for smoke tests). |
+| `--timeout` | `600` | Per-request timeout in seconds. |
+| `--output` | none | Save raw per-request results and the summary to JSON. |
+
+A few common ones:
+
+- `--category all` runs every category in the bench.
+- `--category coding,math` runs just those two.
+- `--bench throughput_8k` runs a fixed-input-length throughput split.
+- `--limit 8` keeps at most 8 samples per category, which is enough for a quick check.
+
+The `throughput_{ISL}` splits use fixed input lengths (1k - 32k), so they are handy for long-context testing and for comparing different `llama-server` batching settings (e.g. sweeping `-ub` / `--ubatch-size`) on prompts of a known size. Make sure the server `-c` is large enough for the chosen split. When raising `-ub`, also raise `-b` to at least the same value, since the physical ubatch cannot exceed the logical batch.
+
+When `--output` is given, the JSON file holds the run `config`, the `selected_samples` / `completed_samples` / `failed_samples` counts, the per-category `summary` rows, and the per-sample `results`.
+
+## Metrics
+
+The summary prints one row per category plus an `overall` row:
+
+- `samples` - how many samples finished successfully.
+- `avg_prompt_t/s` - prefill throughput from llama.cpp (`timings.prompt_per_second`), averaged over the category's samples.
+- `avg_pred_t/s` - decode throughput from llama.cpp (`timings.predicted_per_second`), averaged over the category's samples.
+- `avg_latency` - average end-to-end request latency seen by the client.
+- `accept_rate` - `accepted / draft_n` over the category, or `n/a` if nothing was drafted (`draft_n == 0`).
+
+## Baseline vs speculative decoding
+
+Save a run from each server with `--output`, then diff the two JSON files with `speed_bench_compare.py`.
+
+First, start a plain `llama-server` (no speculative decoding) and save a baseline:
+
+```bash
+python tools/server/bench/speed-bench/speed_bench.py \
+  --url localhost:8080 \
+  --bench qualitative \
+  --category all \
+  --osl 1024 \
+  --concurrency 1 \
+  --output baseline.json
+```
+
+Then restart `llama-server` with speculative decoding enabled and save another run:
+
+```bash
+python tools/server/bench/speed-bench/speed_bench.py \
+  --url localhost:8080 \
+  --bench qualitative \
+  --category all \
+  --osl 1024 \
+  --concurrency 1 \
+  --output spec.json
+```
+
+Finally compare the two:
+
+```bash
+python tools/server/bench/speed-bench/speed_bench_compare.py \
+  --baseline baseline.json \
+  --speculative spec.json
+```
+
+The comparison table adds:
+
+- `decode_speedup = spec_avg_pred_t/s / base_avg_pred_t/s`
+- `latency_speedup = base_avg_latency / spec_avg_latency`
+
+Keep `--bench`, `--category`, `--osl`, and `--limit` the same across both runs, otherwise they won't be using the same prompts.

tools/server/bench/speed-bench/requirements.txt

@@ -0,0 +1,3 @@
+datasets
+requests
+tqdm

tools/server/bench/speed-bench/speed_bench.py

@@ -0,0 +1,432 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import json
+import statistics
+import sys
+import time
+from dataclasses import asdict, dataclass
+from typing import Any
+from urllib.parse import urlparse
+
+import requests
+from datasets import get_dataset_config_names, load_dataset
+from tqdm import tqdm
+
+
+DATASET_REPO = "nvidia/SPEED-Bench"
+
+@dataclass
+class Sample:
+    id: str
+    category: str
+    turns: list[str]
+
+
+@dataclass
+class RequestResult:
+    id: str
+    category: str
+    ok: bool
+    turns: int
+    latency_s: float
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+    finish_reason: str | None
+    draft_n: int
+    draft_n_accepted: int
+    prompt_ms: float | None
+    predicted_ms: float | None
+    prompt_per_second: float | None
+    predicted_per_second: float | None
+    error: str | None
+
+
+def normalize_base_url(url: str) -> str:
+    url = url.strip().rstrip("/")
+    if not url:
+        raise ValueError("--url cannot be empty")
+    if "://" not in url:
+        url = "http://" + url
+    parsed = urlparse(url)
+    if not parsed.scheme or not parsed.netloc:
+        raise ValueError(f"invalid --url: {url}")
+    if not parsed.path.rstrip("/").endswith("/v1"):
+        url = url + "/v1"
+    return url.rstrip("/")
+
+
+def parse_extra_inputs(value: str) -> dict[str, Any]:
+    extra = json.loads(value)
+    if not isinstance(extra, dict):
+        raise ValueError("--extra-inputs must be a JSON object")
+    return extra
+
+
+def extract_turns(row: dict[str, Any]) -> list[str]:
+    turns = row.get("turns")
+    if isinstance(turns, list) and turns:
+        clean_turns = [str(turn).strip() for turn in turns if turn and str(turn).strip()]
+        if clean_turns:
+            return clean_turns
+    raise ValueError("missing or empty turns")
+
+
+def load_samples(args: argparse.Namespace) -> list[Sample]:
+    bench_names = get_dataset_config_names(DATASET_REPO)
+    if args.bench not in bench_names:
+        raise ValueError(
+            f"unknown --bench {args.bench!r}; available benches: {', '.join(bench_names)}"
+        )
+
+    dataset = load_dataset(DATASET_REPO, name=args.bench, split="test")
+    categories = list(dict.fromkeys(str(category) for category in dataset["category"]))
+    requested_categories = None
+    if args.category != "all":
+        requested_list = [category.strip() for category in args.category.split(",") if category.strip()]
+        if not requested_list:
+            raise ValueError(
+                f"--category must be 'all' or a comma-separated list; available categories: {', '.join(categories)}"
+            )
+        requested_categories = set(requested_list)
+        unknown_categories = [category for category in requested_list if category not in categories]
+        if unknown_categories:
+            unknown = ", ".join(unknown_categories)
+            raise ValueError(
+                f"unknown --category {unknown!r} for bench {args.bench!r}; "
+                f"available categories: all, {', '.join(categories)}"
+            )
+
+    samples: list[Sample] = []
+    samples_per_category: dict[str, int] = {}
+    skipped = 0
+    for index, row_raw in enumerate(dataset):
+        row = dict(row_raw)
+        category_raw = row.get("category")
+        if not isinstance(category_raw, str) or not category_raw.strip():
+            skipped += 1
+            continue
+        category = category_raw.strip()
+        if requested_categories is not None and category not in requested_categories:
+            continue
+        if args.limit is not None and samples_per_category.get(category, 0) >= args.limit:
+            continue
+
+        try:
+            turns = extract_turns(row)
+        except ValueError:
+            skipped += 1
+            continue
+        question_id = row.get("question_id")
+        if not isinstance(question_id, str) or not question_id.strip():
+            skipped += 1
+            continue
+        sample_id = question_id.strip()
+        samples.append(Sample(id=sample_id, category=category, turns=turns))
+        samples_per_category[category] = samples_per_category.get(category, 0) + 1
+
+    if not samples:
+        raise RuntimeError(f"no samples selected from bench={args.bench} category={args.category}")
+
+    if skipped:
+        print(f"speed_bench: skipped {skipped} rows without usable turns")
+    return samples
+
+
+def parse_completion_response(data: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any], str | None, str]:
+    usage = data.get("usage") or {}
+    timings = data.get("timings") or {}
+    finish_reason = None
+    content = ""
+    choices = data.get("choices")
+    if isinstance(choices, list) and choices and isinstance(choices[0], dict):
+        choice = choices[0]
+        finish_reason = choice.get("finish_reason")
+        message = choice.get("message")
+        if isinstance(message, dict) and isinstance(message.get("content"), str):
+            content = message["content"]
+        elif isinstance(choice.get("text"), str):
+            content = choice["text"]
+    return usage, timings, finish_reason, content
+
+
+def run_request(
+    endpoint: str,
+    model: str | None,
+    messages: list[dict[str, str]],
+    osl: int,
+    extra_inputs: dict[str, Any],
+    timeout: float,
+) -> tuple[dict[str, Any], float]:
+    payload: dict[str, Any] = {
+        "messages": messages,
+        "max_tokens": osl,
+        "stream": False,
+    }
+    if model:
+        payload["model"] = model
+    payload.update(extra_inputs)
+    payload["max_tokens"] = osl
+
+    start = time.perf_counter()
+    response = requests.post(endpoint, json=payload, timeout=timeout)
+    latency_s = time.perf_counter() - start
+    if response.status_code != 200:
+        body = response.text[:500].replace("\n", "\\n")
+        raise RuntimeError(f"HTTP {response.status_code}: {body}")
+    return response.json(), latency_s
+
+
+def run_one(
+    sample: Sample,
+    endpoint: str,
+    model: str | None,
+    osl: int,
+    extra_inputs: dict[str, Any],
+    timeout: float,
+) -> RequestResult:
+    selected_turns = sample.turns
+    messages: list[dict[str, str]] = []
+    total_latency_s = 0.0
+    prompt_tokens = 0
+    completion_tokens = 0
+    total_tokens = 0
+    draft_n = 0
+    draft_n_accepted = 0
+    prompt_ms = 0.0
+    predicted_ms = 0.0
+    prompt_per_second = None
+    predicted_per_second = None
+    finish_reason: str | None = None
+    try:
+        for turn in selected_turns:
+            messages.append({"role": "user", "content": turn})
+            data, latency_s = run_request(endpoint, model, messages, osl, extra_inputs, timeout)
+            total_latency_s += latency_s
+            usage, timings, finish_reason, assistant_text = parse_completion_response(data)
+
+            turn_prompt_tokens = int(usage.get("prompt_tokens") or timings.get("prompt_n") or 0)
+            turn_completion_tokens_count = int(usage.get("completion_tokens") or timings.get("predicted_n") or 0)
+            turn_total_tokens_count = int(usage.get("total_tokens") or (turn_prompt_tokens + turn_completion_tokens_count))
+            prompt_tokens += turn_prompt_tokens
+            completion_tokens += turn_completion_tokens_count
+            total_tokens += turn_total_tokens_count
+            draft_n += int(timings.get("draft_n") or 0)
+            draft_n_accepted += int(timings.get("draft_n_accepted") or 0)
+            prompt_ms += float(timings.get("prompt_ms") or 0)
+            predicted_ms += float(timings.get("predicted_ms") or 0)
+            if len(selected_turns) == 1 and isinstance(timings.get("prompt_per_second"), (int, float)):
+                prompt_per_second = float(timings["prompt_per_second"])
+            if len(selected_turns) == 1 and isinstance(timings.get("predicted_per_second"), (int, float)):
+                predicted_per_second = float(timings["predicted_per_second"])
+
+            messages.append({"role": "assistant", "content": assistant_text})
+
+        if total_tokens == 0:
+            total_tokens = prompt_tokens + completion_tokens
+        if len(selected_turns) > 1:
+            prompt_per_second = (prompt_tokens / (prompt_ms / 1000)) if prompt_ms > 0 else None
+            predicted_per_second = (completion_tokens / (predicted_ms / 1000)) if predicted_ms > 0 else None
+
+        return RequestResult(
+            id=sample.id,
+            category=sample.category,
+            ok=True,
+            turns=len(selected_turns),
+            latency_s=total_latency_s,
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=total_tokens,
+            finish_reason=finish_reason,
+            draft_n=draft_n,
+            draft_n_accepted=draft_n_accepted,
+            prompt_ms=prompt_ms if prompt_ms > 0 else None,
+            predicted_ms=predicted_ms if predicted_ms > 0 else None,
+            prompt_per_second=prompt_per_second,
+            predicted_per_second=predicted_per_second,
+            error=None,
+        )
+    except Exception as exc:
+        return RequestResult(
+            id=sample.id,
+            category=sample.category,
+            ok=False,
+            turns=len(selected_turns),
+            latency_s=total_latency_s,
+            prompt_tokens=0,
+            completion_tokens=0,
+            total_tokens=0,
+            finish_reason=None,
+            draft_n=0,
+            draft_n_accepted=0,
+            prompt_ms=None,
+            predicted_ms=None,
+            prompt_per_second=None,
+            predicted_per_second=None,
+            error=str(exc),
+        )
+
+
+def summarize_group(category: str, results: list[RequestResult]) -> dict[str, Any]:
+    ok_results = [result for result in results if result.ok]
+    latencies = [result.latency_s for result in ok_results]
+    server_prompt_speeds = [
+        result.prompt_per_second
+        for result in ok_results
+        if result.prompt_per_second is not None
+    ]
+    server_completion_speeds = [
+        result.predicted_per_second
+        for result in ok_results
+        if result.predicted_per_second is not None
+    ]
+    turns = sum(result.turns for result in ok_results)
+    draft_n = sum(result.draft_n for result in ok_results)
+    accepted = sum(result.draft_n_accepted for result in ok_results)
+
+    return {
+        "category": category,
+        "requests": len(ok_results),
+        "turns": turns,
+        "failed": len(results) - len(ok_results),
+        "avg_prompt_t_s": statistics.mean(server_prompt_speeds) if server_prompt_speeds else None,
+        "avg_pred_t_s": statistics.mean(server_completion_speeds) if server_completion_speeds else None,
+        "avg_latency": statistics.mean(latencies) if latencies else None,
+        "draft_n": draft_n,
+        "accepted": accepted,
+        "accept_rate": (accepted / draft_n) if draft_n > 0 else None,
+    }
+
+
+def fmt_value(value: Any, kind: str = "") -> str:
+    if value is None:
+        return "n/a"
+    if kind == "int":
+        return str(int(value))
+    if kind == "rate":
+        return f"{float(value):.4f}"
+    if kind == "seconds":
+        return f"{float(value):.3f}s"
+    if kind == "speed":
+        return f"{float(value):.2f}"
+    if kind == "speedup":
+        return f"{float(value):.2f}x"
+    return str(value)
+
+
+def print_table(rows: list[dict[str, Any]]) -> None:
+    columns = [
+        ("category", "category", ""),
+        ("samples", "requests", "int"),
+        ("avg_prompt_t/s", "avg_prompt_t_s", "speed"),
+        ("avg_pred_t/s", "avg_pred_t_s", "speed"),
+        ("avg_latency", "avg_latency", "seconds"),
+        ("accept_rate", "accept_rate", "rate"),
+    ]
+    print_rows(rows, columns)
+
+
+def print_rows(rows: list[dict[str, Any]], columns: list[tuple[str, str, str]]) -> None:
+    rendered_rows = []
+    for row in rows:
+        rendered_rows.append([fmt_value(row.get(key), kind) for _, key, kind in columns])
+
+    widths = [len(header) for header, _, _ in columns]
+    for rendered in rendered_rows:
+        for i, cell in enumerate(rendered):
+            widths[i] = max(widths[i], len(cell))
+
+    header = "  ".join(header.ljust(widths[i]) for i, (header, _, _) in enumerate(columns))
+    print(header)
+    print("  ".join("-" * width for width in widths))
+    for rendered in rendered_rows:
+        print("  ".join(cell.ljust(widths[i]) for i, cell in enumerate(rendered)))
+
+
+def save_output(path: str, args: argparse.Namespace, samples: list[Sample], results: list[RequestResult], summary: list[dict[str, Any]]) -> None:
+    payload = {
+        "config": {
+            "url": args.url,
+            "model": args.model,
+            "bench": args.bench,
+            "category": args.category,
+            "osl": args.osl,
+            "concurrency": args.concurrency,
+            "extra_inputs": args.extra_inputs,
+        },
+        "selected_samples": len(samples),
+        "completed_samples": sum(1 for result in results if result.ok),
+        "failed_samples": sum(1 for result in results if not result.ok),
+        "summary": summary,
+        "results": [asdict(result) for result in results],
+    }
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(payload, f, indent=2, sort_keys=True)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Run SPEED-Bench against an OpenAI-compatible llama-server.")
+    parser.add_argument("--url", default="localhost:8080", help="Server URL, for example localhost:8080 or http://localhost:8080/v1")
+    parser.add_argument("--model", default=None, help="Optional model name to send in OpenAI requests")
+    parser.add_argument("--bench", default="qualitative", help="SPEED-Bench config to run, for example qualitative or throughput_1k")
+    parser.add_argument("--category", default="all", help="Category to run within the selected bench; use all for no category filter")
+    parser.add_argument("--osl", type=int, default=4096, help="Output sequence length, mapped to max_tokens")
+    parser.add_argument("--extra-inputs", default='{"temperature":0}', help="Extra request fields as a JSON object")
+    parser.add_argument("--concurrency", type=int, default=1, help="Concurrent client requests; usually match llama-server --np")
+    parser.add_argument("--limit", type=int, default=None, help="Optional sample limit per category for smoke tests")
+    parser.add_argument("--timeout", type=float, default=600, help="Per-request timeout in seconds")
+    parser.add_argument("--output", default=None, help="Optional path to save raw results JSON")
+    args = parser.parse_args(argv)
+    try:
+        base_url = normalize_base_url(args.url)
+        endpoint = base_url + "/chat/completions"
+        extra_inputs = parse_extra_inputs(args.extra_inputs)
+        args.extra_inputs = extra_inputs
+        samples = load_samples(args)
+    except Exception as exc:
+        print(f"speed_bench: setup failed: {exc}", file=sys.stderr)
+        return 2
+
+    print(f"speed_bench: loaded {len(samples)} samples from bench={args.bench} category={args.category}")
+
+    results: list[RequestResult] = []
+    started = time.perf_counter()
+    with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as executor:
+        futures = [
+            executor.submit(run_one, sample, endpoint, args.model, args.osl, extra_inputs, args.timeout)
+            for sample in samples
+        ]
+        for future in tqdm(concurrent.futures.as_completed(futures), total=len(futures), desc="speed_bench", unit="sample"):
+            result = future.result()
+            results.append(result)
+
+    elapsed = time.perf_counter() - started
+    categories = list(dict.fromkeys(sample.category for sample in samples))
+    summary = [
+        summarize_group(category, [result for result in results if result.category == category])
+        for category in categories
+    ]
+    summary.append(summarize_group("overall", results))
+    print()
+    print(f"Summary (elapsed={elapsed:.2f}s)")
+    print_table(summary)
+
+    if args.output:
+        save_output(args.output, args, samples, results, summary)
+        print(f"\nspeed_bench: wrote {args.output}")
+
+    failed = sum(1 for result in results if not result.ok)
+    if failed:
+        print(f"\nspeed_bench: {failed} samples failed", file=sys.stderr)
+        first_error = next((result.error for result in results if result.error), None)
+        if first_error:
+            print(f"first error: {first_error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/server/bench/speed-bench/speed_bench_compare.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import Any
+
+from speed_bench import fmt_value, print_rows
+
+
+def load_summary(path: str) -> list[dict[str, Any]]:
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    summary = data.get("summary")
+    if not isinstance(summary, list):
+        raise ValueError(f"{path} does not contain a summary list")
+    return summary
+
+
+def compare_rows(baseline: list[dict[str, Any]], speculative: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    baseline_by_category = {row["category"]: row for row in baseline}
+    comparisons = []
+    for row in speculative:
+        base = baseline_by_category.get(row["category"])
+        if not base:
+            continue
+        base_speed = base.get("avg_pred_t_s")
+        spec_speed = row.get("avg_pred_t_s")
+        base_latency = base.get("avg_latency")
+        spec_latency = row.get("avg_latency")
+        comparisons.append(
+            {
+                "category": row["category"],
+                "base_avg_pred_t_s": base_speed,
+                "spec_avg_pred_t_s": spec_speed,
+                "decode_speedup": (spec_speed / base_speed) if base_speed and spec_speed else None,
+                "base_avg_latency": base_latency,
+                "spec_avg_latency": spec_latency,
+                "latency_speedup": (base_latency / spec_latency) if base_latency and spec_latency else None,
+                "accept_rate": row.get("accept_rate"),
+            }
+        )
+    return comparisons
+
+
+def print_comparison(rows: list[dict[str, Any]]) -> None:
+    if not rows:
+        print("No overlapping categories found for comparison.")
+        return
+    columns = [
+        ("category", "category", ""),
+        ("base_avg_pred_t/s", "base_avg_pred_t_s", "speed"),
+        ("spec_avg_pred_t/s", "spec_avg_pred_t_s", "speed"),
+        ("decode_speedup", "decode_speedup", "speedup"),
+        ("base_avg_latency", "base_avg_latency", "seconds"),
+        ("spec_avg_latency", "spec_avg_latency", "seconds"),
+        ("latency_speedup", "latency_speedup", "speedup"),
+        ("accept_rate", "accept_rate", "rate"),
+    ]
+    print_rows(rows, columns)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Compare two SPEED-Bench runs (baseline vs speculative).")
+    parser.add_argument("--baseline", required=True, help="Baseline results JSON produced by speed_bench.py --output")
+    parser.add_argument("--speculative", required=True, help="Speculative decoding results JSON produced by speed_bench.py --output")
+    args = parser.parse_args(argv)
+
+    try:
+        baseline = load_summary(args.baseline)
+        speculative = load_summary(args.speculative)
+    except Exception as exc:
+        print(f"speed_bench_compare: failed to load inputs: {exc}", file=sys.stderr)
+        return 2
+
+    comparisons = compare_rows(baseline, speculative)
+    print(f"Comparison: baseline={args.baseline} speculative={args.speculative}")
+    print_comparison(comparisons)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())