wehub-resource-sync
82 lines
3.2 KiB
Python
82 lines
3.2 KiB
Python
"""Metrics computation and serialization for the benchmark."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from statistics import mean
|
|
|
|
import numpy as np
|
|
|
|
from ray.llm._internal.serve.benchmark.models import TurnMetric
|
|
|
|
|
|
def percentile(values: list[float], p: float) -> float:
|
|
"""Compute the p-th percentile (0-100)."""
|
|
if not values:
|
|
return 0.0
|
|
return float(np.percentile(values, p))
|
|
|
|
|
|
def summarize_metrics(metrics: list[TurnMetric], elapsed_s: float) -> dict:
|
|
"""Compute aggregate statistics from a list of TurnMetrics.
|
|
|
|
ITL (inter-token latency) statistics are computed from raw per-token values
|
|
flattened across all requests, capturing the full distribution including variance.
|
|
"""
|
|
if not metrics:
|
|
return {"requests": 0, "elapsed_s": round(elapsed_s, 2)}
|
|
|
|
ttft = [m.ttft_ms for m in metrics]
|
|
fc = [m.fc_ms for m in metrics]
|
|
# Flatten per-token ITL values across all requests for accurate distribution stats
|
|
itl_all = [v for m in metrics for v in m.itl_ms_list]
|
|
latency = [m.e2e_latency_ms for m in metrics]
|
|
out_tok = [m.output_tokens for m in metrics]
|
|
in_tok = [m.input_tokens for m in metrics]
|
|
total_output_tokens = sum(out_tok)
|
|
|
|
return {
|
|
"requests": len(metrics),
|
|
"elapsed_s": round(elapsed_s, 2),
|
|
"request_rate": round(len(metrics) / elapsed_s, 2) if elapsed_s > 0 else 0.0,
|
|
"throughput_tok_s": round(total_output_tokens / elapsed_s, 1)
|
|
if elapsed_s > 0
|
|
else 0.0,
|
|
"avg_input_tokens": round(mean(in_tok), 1),
|
|
"avg_output_tokens": round(mean(out_tok), 1),
|
|
"avg_ttft_ms": round(mean(ttft), 2),
|
|
"p50_ttft_ms": round(percentile(ttft, 50), 2),
|
|
"p90_ttft_ms": round(percentile(ttft, 90), 2),
|
|
"p99_ttft_ms": round(percentile(ttft, 99), 2),
|
|
"avg_fc_ms": round(mean(fc), 2),
|
|
"p50_fc_ms": round(percentile(fc, 50), 2),
|
|
"p90_fc_ms": round(percentile(fc, 90), 2),
|
|
"p99_fc_ms": round(percentile(fc, 99), 2),
|
|
"avg_itl_ms": round(float(np.mean(itl_all)), 2) if itl_all else 0.0,
|
|
"std_itl_ms": round(float(np.std(itl_all)), 2) if itl_all else 0.0,
|
|
"p50_itl_ms": round(percentile(itl_all, 50), 2) if itl_all else 0.0,
|
|
"p90_itl_ms": round(percentile(itl_all, 90), 2) if itl_all else 0.0,
|
|
"p99_itl_ms": round(percentile(itl_all, 99), 2) if itl_all else 0.0,
|
|
"avg_e2e_latency_ms": round(mean(latency), 2),
|
|
"p50_e2e_latency_ms": round(percentile(latency, 50), 2),
|
|
"p90_e2e_latency_ms": round(percentile(latency, 90), 2),
|
|
"p99_e2e_latency_ms": round(percentile(latency, 99), 2),
|
|
}
|
|
|
|
|
|
def serialize_raw_metrics(metrics: list[TurnMetric]) -> list[dict]:
|
|
"""Serialize TurnMetrics to dicts suitable for JSON output."""
|
|
return [
|
|
{
|
|
"session_id": m.session_id,
|
|
"turn": m.turn,
|
|
"ttft_ms": round(m.ttft_ms, 2),
|
|
"fc_ms": round(m.fc_ms, 2),
|
|
"itl_ms": round(m.itl_ms, 2),
|
|
"e2e_latency_ms": round(m.e2e_latency_ms, 2),
|
|
"input_tokens": m.input_tokens,
|
|
"output_tokens": m.output_tokens,
|
|
"start_time_ms": round(m.start_time_ms, 2),
|
|
}
|
|
for m in metrics
|
|
]
|