chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,154 @@
|
||||
"""Reporting and result persistence for the benchmark."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from statistics import mean
|
||||
from typing import Optional
|
||||
|
||||
from ray.llm._internal.serve.benchmark.metrics import (
|
||||
percentile,
|
||||
serialize_raw_metrics,
|
||||
summarize_metrics,
|
||||
)
|
||||
from ray.llm._internal.serve.benchmark.models import TurnMetric, WorkloadSpec
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def report_results(
|
||||
metrics: list[TurnMetric],
|
||||
spec: WorkloadSpec,
|
||||
bench_elapsed_s: float,
|
||||
first_chunk_threshold: int = 16,
|
||||
save_path: Optional[str] = None,
|
||||
warmup_s: float = 0.0,
|
||||
discarded_warmup_requests: int = 0,
|
||||
) -> None:
|
||||
"""Print and optionally save benchmark results."""
|
||||
if not metrics:
|
||||
print("No metrics collected.")
|
||||
return
|
||||
|
||||
all_ttft = [m.ttft_ms for m in metrics]
|
||||
all_fc = [m.fc_ms for m in metrics]
|
||||
all_itl = [v for m in metrics for v in m.itl_ms_list]
|
||||
all_latency = [m.e2e_latency_ms for m in metrics]
|
||||
all_input = [m.input_tokens for m in metrics]
|
||||
all_output = [m.output_tokens for m in metrics]
|
||||
|
||||
total_output_tokens = sum(all_output)
|
||||
throughput = total_output_tokens / bench_elapsed_s if bench_elapsed_s > 0 else 0
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("BENCHMARK RESULTS")
|
||||
print("=" * 70)
|
||||
print(f" Total requests: {len(metrics)}")
|
||||
print(f" Unique sessions: {len({m.session_id for m in metrics})}")
|
||||
print(f" Duration: {bench_elapsed_s:.1f}s")
|
||||
if warmup_s > 0:
|
||||
print(f" Warm-up excluded: {warmup_s:.1f}s")
|
||||
if discarded_warmup_requests > 0:
|
||||
print(f" Warm-up requests: {discarded_warmup_requests} (discarded)")
|
||||
print(f" Throughput: {throughput:.1f} output tok/s")
|
||||
print(f" Request rate: {len(metrics) / bench_elapsed_s:.1f} req/s")
|
||||
print(
|
||||
f" Avg input tokens: {mean(all_input):.0f} "
|
||||
f"(target ISL: {spec.effective_isl:.0f})"
|
||||
)
|
||||
print(f" Avg output tokens: {mean(all_output):.0f} (target OSL: {spec.osl})")
|
||||
print()
|
||||
|
||||
fc_label = f"FC({first_chunk_threshold})"
|
||||
print(" Latency Statistics:")
|
||||
for name, values in [
|
||||
("TTFT", all_ttft),
|
||||
(fc_label, all_fc),
|
||||
("ITL", all_itl),
|
||||
("Latency", all_latency),
|
||||
]:
|
||||
if not values:
|
||||
continue
|
||||
print(
|
||||
f" {name:>8}: avg={mean(values):>8.1f}ms "
|
||||
f"P50={percentile(values, 50):>8.1f}ms "
|
||||
f"P90={percentile(values, 90):>8.1f}ms "
|
||||
f"P99={percentile(values, 99):>8.1f}ms"
|
||||
)
|
||||
print()
|
||||
|
||||
print(" Per-Turn Breakdown:")
|
||||
print(
|
||||
f" {'Turn':<6} {'Count':<7} {'Avg ISL':<9} {'Avg TTFT':<10} "
|
||||
f"{'Avg FC':<10} {'Avg ITL':<10} {'Avg Lat':<10}"
|
||||
)
|
||||
for t in range(spec.num_turns):
|
||||
turn_metrics = [m for m in metrics if m.turn == t]
|
||||
if not turn_metrics:
|
||||
continue
|
||||
t_ttft = mean([m.ttft_ms for m in turn_metrics])
|
||||
t_fc = mean([m.fc_ms for m in turn_metrics])
|
||||
t_itl_all = [v for m in turn_metrics for v in m.itl_ms_list]
|
||||
t_itl = mean(t_itl_all) if t_itl_all else 0.0
|
||||
t_lat = mean([m.e2e_latency_ms for m in turn_metrics])
|
||||
t_isl = mean([m.input_tokens for m in turn_metrics])
|
||||
print(
|
||||
f" {t + 1:<6} {len(turn_metrics):<7} {t_isl:<9.0f} "
|
||||
f"{t_ttft:<10.1f} {t_fc:<10.1f} {t_itl:<10.1f} {t_lat:<10.1f}"
|
||||
)
|
||||
print("=" * 70)
|
||||
|
||||
if save_path:
|
||||
stats = summarize_metrics(metrics, bench_elapsed_s)
|
||||
result = {
|
||||
"config": {
|
||||
"concurrency": spec.concurrency,
|
||||
"request_rate": spec.request_rate,
|
||||
},
|
||||
"spec": spec.summary(),
|
||||
"first_chunk_threshold": first_chunk_threshold,
|
||||
"benchmark": {
|
||||
"total_requests": len(metrics),
|
||||
"duration_s": round(bench_elapsed_s, 2),
|
||||
"warmup_s": round(warmup_s, 2),
|
||||
"discarded_warmup_requests": discarded_warmup_requests,
|
||||
},
|
||||
"stats": {
|
||||
("measured_request_rate" if k == "request_rate" else k): v
|
||||
for k, v in stats.items()
|
||||
if k not in ("requests", "elapsed_s")
|
||||
},
|
||||
"per_turn": [],
|
||||
"raw_metrics": serialize_raw_metrics(metrics),
|
||||
}
|
||||
|
||||
for t in range(spec.num_turns):
|
||||
turn_metrics = [m for m in metrics if m.turn == t]
|
||||
if not turn_metrics:
|
||||
continue
|
||||
t_ttft = [m.ttft_ms for m in turn_metrics]
|
||||
t_fc = [m.fc_ms for m in turn_metrics]
|
||||
t_itl = [v for m in turn_metrics for v in m.itl_ms_list]
|
||||
t_isl = [m.input_tokens for m in turn_metrics]
|
||||
result["per_turn"].append(
|
||||
{
|
||||
"turn": t + 1,
|
||||
"count": len(turn_metrics),
|
||||
"avg_isl": round(mean(t_isl), 1),
|
||||
"avg_ttft_ms": round(mean(t_ttft), 2),
|
||||
"avg_fc_ms": round(mean(t_fc), 2),
|
||||
"avg_itl_ms": round(mean(t_itl), 2) if t_itl else 0,
|
||||
"p50_fc_ms": round(percentile(t_fc, 50), 2),
|
||||
"p99_ttft_ms": round(percentile(t_ttft, 99), 2),
|
||||
"p99_fc_ms": round(percentile(t_fc, 99), 2),
|
||||
"p99_itl_ms": (round(percentile(t_itl, 99), 2) if t_itl else 0),
|
||||
}
|
||||
)
|
||||
|
||||
Path(save_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(save_path, "w") as f:
|
||||
json.dump(result, f, indent=2)
|
||||
logger.info("Results saved to %s", save_path)
|
||||
Reference in New Issue
Block a user