"""Print a markdown summary table from processed benchmark results. Usage: python3 summarize.py Reads all agg_*.json files recursively from and prints a markdown table to stdout (redirect to $GITHUB_STEP_SUMMARY to publish). """ import json import sys from pathlib import Path from tabulate import tabulate HEADERS = [ "Model", "Served Model", "Hardware", "Framework", "Precision", "ISL", "OSL", "Prefill TP", "Prefill EP", "Prefill DP Attn", "Prefill Workers", "Prefill GPUs", "Decode TP", "Decode EP", "Decode DP Attn", "Decode Workers", "Decode GPUs", "Conc", "TTFT (ms)", "TPOT (ms)", "Interactivity (tok/s/user)", "E2EL (s)", "TPUT per GPU", "Output TPUT per GPU", "Input TPUT per GPU", ] def load_json(path): try: with open(path) as f: return json.load(f) except Exception: return None def main(): if len(sys.argv) < 2: print("Usage: python3 summarize.py ") sys.exit(1) results_dir = Path(sys.argv[1]) results = [ r for path in results_dir.rglob("agg_*.json") if (r := load_json(path)) and "is_multinode" in r ] if not results: print("No processed result files found.") return results.sort( key=lambda r: ( r["infmax_model_prefix"], r["hw"], r["framework"], r["precision"], r["isl"], r["osl"], r["prefill_tp"], r["prefill_ep"], r["decode_tp"], r["decode_ep"], r["conc"], ) ) rows = [ [ r["infmax_model_prefix"], r["model"], r["hw"].upper(), r["framework"].upper(), r["precision"].upper(), r["isl"], r["osl"], r["prefill_tp"], r["prefill_ep"], r["prefill_dp_attention"], r["prefill_num_workers"], r["num_prefill_gpu"], r["decode_tp"], r["decode_ep"], r["decode_dp_attention"], r["decode_num_workers"], r["num_decode_gpu"], r["conc"], f"{r['median_ttft'] * 1000:.4f}", f"{r['median_tpot'] * 1000:.4f}", f"{r['median_intvty']:.4f}", f"{r['median_e2el']:.4f}", f"{r['tput_per_gpu']:.4f}", f"{r['output_tput_per_gpu']:.4f}", f"{r['input_tput_per_gpu']:.4f}", ] for r in results ] hw_label = "/".join(sorted({r["hw"].upper() for r in results})) if hw_label == "GB200": # NVIDIA GB200 nightly: keep the original hardcoded title untouched. print("## GB200 Nightly Benchmark Results\n") else: # AMD (e.g. MI355X) nightly: derive the title from the result hardware. print(f"## {hw_label} Nightly Benchmark Results\n") print(tabulate(rows, headers=HEADERS, tablefmt="github")) print() if __name__ == "__main__": main()