chore: import upstream snapshot with attribution

python/ray/llm/_internal/serve/benchmark/metrics.py

@@ -0,0 +1,81 @@
+"""Metrics computation and serialization for the benchmark."""
+
+from __future__ import annotations
+
+from statistics import mean
+
+import numpy as np
+
+from ray.llm._internal.serve.benchmark.models import TurnMetric
+
+
+def percentile(values: list[float], p: float) -> float:
+    """Compute the p-th percentile (0-100)."""
+    if not values:
+        return 0.0
+    return float(np.percentile(values, p))
+
+
+def summarize_metrics(metrics: list[TurnMetric], elapsed_s: float) -> dict:
+    """Compute aggregate statistics from a list of TurnMetrics.
+
+    ITL (inter-token latency) statistics are computed from raw per-token values
+    flattened across all requests, capturing the full distribution including variance.
+    """
+    if not metrics:
+        return {"requests": 0, "elapsed_s": round(elapsed_s, 2)}
+
+    ttft = [m.ttft_ms for m in metrics]
+    fc = [m.fc_ms for m in metrics]
+    # Flatten per-token ITL values across all requests for accurate distribution stats
+    itl_all = [v for m in metrics for v in m.itl_ms_list]
+    latency = [m.e2e_latency_ms for m in metrics]
+    out_tok = [m.output_tokens for m in metrics]
+    in_tok = [m.input_tokens for m in metrics]
+    total_output_tokens = sum(out_tok)
+
+    return {
+        "requests": len(metrics),
+        "elapsed_s": round(elapsed_s, 2),
+        "request_rate": round(len(metrics) / elapsed_s, 2) if elapsed_s > 0 else 0.0,
+        "throughput_tok_s": round(total_output_tokens / elapsed_s, 1)
+        if elapsed_s > 0
+        else 0.0,
+        "avg_input_tokens": round(mean(in_tok), 1),
+        "avg_output_tokens": round(mean(out_tok), 1),
+        "avg_ttft_ms": round(mean(ttft), 2),
+        "p50_ttft_ms": round(percentile(ttft, 50), 2),
+        "p90_ttft_ms": round(percentile(ttft, 90), 2),
+        "p99_ttft_ms": round(percentile(ttft, 99), 2),
+        "avg_fc_ms": round(mean(fc), 2),
+        "p50_fc_ms": round(percentile(fc, 50), 2),
+        "p90_fc_ms": round(percentile(fc, 90), 2),
+        "p99_fc_ms": round(percentile(fc, 99), 2),
+        "avg_itl_ms": round(float(np.mean(itl_all)), 2) if itl_all else 0.0,
+        "std_itl_ms": round(float(np.std(itl_all)), 2) if itl_all else 0.0,
+        "p50_itl_ms": round(percentile(itl_all, 50), 2) if itl_all else 0.0,
+        "p90_itl_ms": round(percentile(itl_all, 90), 2) if itl_all else 0.0,
+        "p99_itl_ms": round(percentile(itl_all, 99), 2) if itl_all else 0.0,
+        "avg_e2e_latency_ms": round(mean(latency), 2),
+        "p50_e2e_latency_ms": round(percentile(latency, 50), 2),
+        "p90_e2e_latency_ms": round(percentile(latency, 90), 2),
+        "p99_e2e_latency_ms": round(percentile(latency, 99), 2),
+    }
+
+
+def serialize_raw_metrics(metrics: list[TurnMetric]) -> list[dict]:
+    """Serialize TurnMetrics to dicts suitable for JSON output."""
+    return [
+        {
+            "session_id": m.session_id,
+            "turn": m.turn,
+            "ttft_ms": round(m.ttft_ms, 2),
+            "fc_ms": round(m.fc_ms, 2),
+            "itl_ms": round(m.itl_ms, 2),
+            "e2e_latency_ms": round(m.e2e_latency_ms, 2),
+            "input_tokens": m.input_tokens,
+            "output_tokens": m.output_tokens,
+            "start_time_ms": round(m.start_time_ms, 2),
+        }
+        for m in metrics
+    ]