wehub-resource-sync
155 lines
5.6 KiB
Python
155 lines
5.6 KiB
Python
"""Reporting and result persistence for the benchmark."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from pathlib import Path
|
|
from statistics import mean
|
|
from typing import Optional
|
|
|
|
from ray.llm._internal.serve.benchmark.metrics import (
|
|
percentile,
|
|
serialize_raw_metrics,
|
|
summarize_metrics,
|
|
)
|
|
from ray.llm._internal.serve.benchmark.models import TurnMetric, WorkloadSpec
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def report_results(
|
|
metrics: list[TurnMetric],
|
|
spec: WorkloadSpec,
|
|
bench_elapsed_s: float,
|
|
first_chunk_threshold: int = 16,
|
|
save_path: Optional[str] = None,
|
|
warmup_s: float = 0.0,
|
|
discarded_warmup_requests: int = 0,
|
|
) -> None:
|
|
"""Print and optionally save benchmark results."""
|
|
if not metrics:
|
|
print("No metrics collected.")
|
|
return
|
|
|
|
all_ttft = [m.ttft_ms for m in metrics]
|
|
all_fc = [m.fc_ms for m in metrics]
|
|
all_itl = [v for m in metrics for v in m.itl_ms_list]
|
|
all_latency = [m.e2e_latency_ms for m in metrics]
|
|
all_input = [m.input_tokens for m in metrics]
|
|
all_output = [m.output_tokens for m in metrics]
|
|
|
|
total_output_tokens = sum(all_output)
|
|
throughput = total_output_tokens / bench_elapsed_s if bench_elapsed_s > 0 else 0
|
|
|
|
print()
|
|
print("=" * 70)
|
|
print("BENCHMARK RESULTS")
|
|
print("=" * 70)
|
|
print(f" Total requests: {len(metrics)}")
|
|
print(f" Unique sessions: {len({m.session_id for m in metrics})}")
|
|
print(f" Duration: {bench_elapsed_s:.1f}s")
|
|
if warmup_s > 0:
|
|
print(f" Warm-up excluded: {warmup_s:.1f}s")
|
|
if discarded_warmup_requests > 0:
|
|
print(f" Warm-up requests: {discarded_warmup_requests} (discarded)")
|
|
print(f" Throughput: {throughput:.1f} output tok/s")
|
|
print(f" Request rate: {len(metrics) / bench_elapsed_s:.1f} req/s")
|
|
print(
|
|
f" Avg input tokens: {mean(all_input):.0f} "
|
|
f"(target ISL: {spec.effective_isl:.0f})"
|
|
)
|
|
print(f" Avg output tokens: {mean(all_output):.0f} (target OSL: {spec.osl})")
|
|
print()
|
|
|
|
fc_label = f"FC({first_chunk_threshold})"
|
|
print(" Latency Statistics:")
|
|
for name, values in [
|
|
("TTFT", all_ttft),
|
|
(fc_label, all_fc),
|
|
("ITL", all_itl),
|
|
("Latency", all_latency),
|
|
]:
|
|
if not values:
|
|
continue
|
|
print(
|
|
f" {name:>8}: avg={mean(values):>8.1f}ms "
|
|
f"P50={percentile(values, 50):>8.1f}ms "
|
|
f"P90={percentile(values, 90):>8.1f}ms "
|
|
f"P99={percentile(values, 99):>8.1f}ms"
|
|
)
|
|
print()
|
|
|
|
print(" Per-Turn Breakdown:")
|
|
print(
|
|
f" {'Turn':<6} {'Count':<7} {'Avg ISL':<9} {'Avg TTFT':<10} "
|
|
f"{'Avg FC':<10} {'Avg ITL':<10} {'Avg Lat':<10}"
|
|
)
|
|
for t in range(spec.num_turns):
|
|
turn_metrics = [m for m in metrics if m.turn == t]
|
|
if not turn_metrics:
|
|
continue
|
|
t_ttft = mean([m.ttft_ms for m in turn_metrics])
|
|
t_fc = mean([m.fc_ms for m in turn_metrics])
|
|
t_itl_all = [v for m in turn_metrics for v in m.itl_ms_list]
|
|
t_itl = mean(t_itl_all) if t_itl_all else 0.0
|
|
t_lat = mean([m.e2e_latency_ms for m in turn_metrics])
|
|
t_isl = mean([m.input_tokens for m in turn_metrics])
|
|
print(
|
|
f" {t + 1:<6} {len(turn_metrics):<7} {t_isl:<9.0f} "
|
|
f"{t_ttft:<10.1f} {t_fc:<10.1f} {t_itl:<10.1f} {t_lat:<10.1f}"
|
|
)
|
|
print("=" * 70)
|
|
|
|
if save_path:
|
|
stats = summarize_metrics(metrics, bench_elapsed_s)
|
|
result = {
|
|
"config": {
|
|
"concurrency": spec.concurrency,
|
|
"request_rate": spec.request_rate,
|
|
},
|
|
"spec": spec.summary(),
|
|
"first_chunk_threshold": first_chunk_threshold,
|
|
"benchmark": {
|
|
"total_requests": len(metrics),
|
|
"duration_s": round(bench_elapsed_s, 2),
|
|
"warmup_s": round(warmup_s, 2),
|
|
"discarded_warmup_requests": discarded_warmup_requests,
|
|
},
|
|
"stats": {
|
|
("measured_request_rate" if k == "request_rate" else k): v
|
|
for k, v in stats.items()
|
|
if k not in ("requests", "elapsed_s")
|
|
},
|
|
"per_turn": [],
|
|
"raw_metrics": serialize_raw_metrics(metrics),
|
|
}
|
|
|
|
for t in range(spec.num_turns):
|
|
turn_metrics = [m for m in metrics if m.turn == t]
|
|
if not turn_metrics:
|
|
continue
|
|
t_ttft = [m.ttft_ms for m in turn_metrics]
|
|
t_fc = [m.fc_ms for m in turn_metrics]
|
|
t_itl = [v for m in turn_metrics for v in m.itl_ms_list]
|
|
t_isl = [m.input_tokens for m in turn_metrics]
|
|
result["per_turn"].append(
|
|
{
|
|
"turn": t + 1,
|
|
"count": len(turn_metrics),
|
|
"avg_isl": round(mean(t_isl), 1),
|
|
"avg_ttft_ms": round(mean(t_ttft), 2),
|
|
"avg_fc_ms": round(mean(t_fc), 2),
|
|
"avg_itl_ms": round(mean(t_itl), 2) if t_itl else 0,
|
|
"p50_fc_ms": round(percentile(t_fc, 50), 2),
|
|
"p99_ttft_ms": round(percentile(t_ttft, 99), 2),
|
|
"p99_fc_ms": round(percentile(t_fc, 99), 2),
|
|
"p99_itl_ms": (round(percentile(t_itl, 99), 2) if t_itl else 0),
|
|
}
|
|
)
|
|
|
|
Path(save_path).parent.mkdir(parents=True, exist_ok=True)
|
|
with open(save_path, "w") as f:
|
|
json.dump(result, f, indent=2)
|
|
logger.info("Results saved to %s", save_path)
|