wehub-resource-sync
258 lines
8.6 KiB
Python
258 lines
8.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Plot controller benchmark results from microbenchmarks.py --run-controller.
|
|
|
|
Takes one or more JSON files (output from microbenchmarks.py --run-controller)
|
|
and produces seaborn FacetGrid plots for each metric, with file names as labels
|
|
and std for confidence intervals.
|
|
|
|
When 2+ files are provided, runs a t-test (first vs last file) to assess
|
|
statistical significance of changes.
|
|
|
|
Dependencies: matplotlib, pandas, seaborn, scipy
|
|
|
|
Example usage with the master.json and pydantic.json files:
|
|
python release/serve_tests/workloads/plot_controller_benchmark.py \
|
|
release/serve_tests/workloads/master.json \
|
|
release/serve_tests/workloads/pydantic.json \
|
|
-o /tmp/controller_plot.png
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import matplotlib.pyplot as plt
|
|
import pandas as pd
|
|
import seaborn as sns
|
|
from scipy import stats
|
|
|
|
|
|
def load_benchmark_json(path: str) -> list[dict]:
|
|
"""Load perf_metrics from a controller benchmark JSON file.
|
|
|
|
Supports both formats:
|
|
- {"perf_metrics": [...]} (from save_test_results)
|
|
- [...] (raw list of perf metric dicts)
|
|
"""
|
|
with open(path) as f:
|
|
data = json.load(f)
|
|
if isinstance(data, list):
|
|
perf_metrics = data
|
|
else:
|
|
perf_metrics = data.get("perf_metrics", [])
|
|
if not perf_metrics:
|
|
raise ValueError(f"No perf_metrics found in {path}")
|
|
return perf_metrics
|
|
|
|
|
|
def _parse_replicas_from_metric_name(name: str) -> int | None:
|
|
"""Extract replica count from metric name, e.g. controller_foo_10_replicas -> 10."""
|
|
m = re.search(r"_(\d+)_replicas$", name)
|
|
return int(m.group(1)) if m else None
|
|
|
|
|
|
def build_plot_dataframe(json_paths: list[str]) -> pd.DataFrame:
|
|
"""Build a DataFrame for plotting from multiple JSON files."""
|
|
rows = []
|
|
for path in json_paths:
|
|
label = Path(path).stem
|
|
metrics = load_benchmark_json(path)
|
|
for m in metrics:
|
|
name = m.get("perf_metric_name")
|
|
value = m.get("perf_metric_value")
|
|
std = m.get("perf_metric_std", 0.0)
|
|
n = m.get("perf_metric_sample_size", 1)
|
|
if name is not None and value is not None:
|
|
replicas = _parse_replicas_from_metric_name(name)
|
|
rows.append(
|
|
{
|
|
"metric": name,
|
|
"base_metric": name.rsplit("_", 2)[0]
|
|
if replicas is not None
|
|
else name,
|
|
"replicas": replicas if replicas is not None else -1,
|
|
"file": label,
|
|
"mean": float(value),
|
|
"std": float(std),
|
|
"n": int(n),
|
|
}
|
|
)
|
|
return pd.DataFrame(rows)
|
|
|
|
|
|
def run_ttest(df: pd.DataFrame, first_file: str, last_file: str) -> dict[str, dict]:
|
|
"""
|
|
Run independent t-test for each metric between first and last file.
|
|
Returns dict of metric_name -> {statistic, pvalue, significant}.
|
|
"""
|
|
results = {}
|
|
for metric in df["metric"].unique():
|
|
m1 = df[(df["metric"] == metric) & (df["file"] == first_file)]
|
|
m2 = df[(df["metric"] == metric) & (df["file"] == last_file)]
|
|
if m1.empty or m2.empty:
|
|
continue
|
|
row1 = m1.iloc[0]
|
|
row2 = m2.iloc[0]
|
|
try:
|
|
stat, pval = stats.ttest_ind_from_stats(
|
|
mean1=row1["mean"],
|
|
std1=row1["std"],
|
|
nobs1=row1["n"],
|
|
mean2=row2["mean"],
|
|
std2=row2["std"],
|
|
nobs2=row2["n"],
|
|
)
|
|
results[metric] = {
|
|
"statistic": float(stat),
|
|
"pvalue": float(pval),
|
|
"significant": pval < 0.05,
|
|
}
|
|
except Exception:
|
|
results[metric] = {"statistic": None, "pvalue": None, "significant": None}
|
|
return results
|
|
|
|
|
|
def plot_facet_grid(df: pd.DataFrame, output_path: str) -> None:
|
|
"""Create a seaborn FacetGrid bar plot for each metric with std as CI.
|
|
|
|
X-axis: replica count. Hue: file (when multiple files).
|
|
"""
|
|
# Use replica count on x-axis; filter out metrics without replica suffix
|
|
plot_df = df[df["replicas"] >= 0].copy()
|
|
if plot_df.empty:
|
|
# Fallback: no replica pattern found, use file on x-axis
|
|
plot_df = df.copy()
|
|
plot_df["replicas"] = plot_df["file"]
|
|
x_var, facet_var = "file", "metric"
|
|
else:
|
|
plot_df["replicas"] = plot_df["replicas"].astype(int)
|
|
x_var, facet_var = "replicas", "base_metric"
|
|
|
|
base_metrics = sorted(plot_df[facet_var].unique())
|
|
n_metrics = len(base_metrics)
|
|
col_wrap = min(3, n_metrics)
|
|
|
|
replica_order = (
|
|
sorted(plot_df["replicas"].unique()) if x_var == "replicas" else None
|
|
)
|
|
|
|
g = sns.FacetGrid(
|
|
plot_df,
|
|
col=facet_var,
|
|
col_wrap=col_wrap,
|
|
col_order=base_metrics,
|
|
sharey=False,
|
|
height=4,
|
|
aspect=1.2,
|
|
)
|
|
has_multiple_files = plot_df["file"].nunique() > 1
|
|
g.map_dataframe(
|
|
sns.barplot,
|
|
x=x_var,
|
|
y="mean",
|
|
hue="file" if has_multiple_files else x_var,
|
|
palette="husl",
|
|
legend=has_multiple_files,
|
|
order=replica_order,
|
|
hue_order=(sorted(plot_df["file"].unique()) if has_multiple_files else None),
|
|
)
|
|
|
|
# Add error bars (std as CI) to each facet
|
|
for ax, base_metric in zip(g.axes.flat, base_metrics):
|
|
subset = plot_df[plot_df[facet_var] == base_metric].sort_values(
|
|
["file", x_var] # seaborn bar order: hue then x
|
|
)
|
|
# Match bars to data: seaborn containers = one per hue, bars per x
|
|
bars = [bar for c in ax.containers for bar in c]
|
|
for (_, row), bar in zip(subset.iterrows(), bars):
|
|
ax.errorbar(
|
|
bar.get_x() + bar.get_width() / 2,
|
|
row["mean"],
|
|
yerr=row["std"],
|
|
fmt="none",
|
|
color="black",
|
|
capsize=4,
|
|
capthick=1,
|
|
)
|
|
ax.set_xlabel("Replicas" if x_var == "replicas" else "")
|
|
ax.tick_params(axis="x", rotation=0)
|
|
|
|
g.set_titles(col_template="{col_name}")
|
|
if plot_df["file"].nunique() > 1:
|
|
g.add_legend(title="File")
|
|
plt.tight_layout()
|
|
plt.savefig(output_path, dpi=150, bbox_inches="tight")
|
|
plt.close()
|
|
print(f"Saved plot to {output_path}")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
description="Plot controller benchmark JSON outputs and optionally run t-tests."
|
|
)
|
|
parser.add_argument(
|
|
"json_files",
|
|
nargs="+",
|
|
help="One or more JSON files from microbenchmarks.py --run-controller",
|
|
)
|
|
parser.add_argument(
|
|
"-o",
|
|
"--output",
|
|
default="controller_benchmark_plot.png",
|
|
help="Output plot path (default: controller_benchmark_plot.png)",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
for p in args.json_files:
|
|
if not os.path.isfile(p):
|
|
raise FileNotFoundError(f"File not found: {p}")
|
|
|
|
df = build_plot_dataframe(args.json_files)
|
|
if df.empty:
|
|
raise ValueError("No valid metrics found in the provided JSON files.")
|
|
|
|
# Plot
|
|
plot_facet_grid(df, args.output)
|
|
|
|
# T-test when 2+ files
|
|
if len(args.json_files) >= 2:
|
|
first_label = Path(args.json_files[0]).stem
|
|
last_label = Path(args.json_files[-1]).stem
|
|
ttest_results = run_ttest(df, first_label, last_label)
|
|
|
|
# Build table: rows=base_metric, cols=replicas
|
|
table_rows = []
|
|
for full_metric, r in ttest_results.items():
|
|
replicas = _parse_replicas_from_metric_name(full_metric)
|
|
base = (
|
|
full_metric.rsplit("_", 2)[0] if replicas is not None else full_metric
|
|
)
|
|
if replicas is None:
|
|
replicas = -1
|
|
if r["pvalue"] is not None:
|
|
cell = "✓" if r["significant"] else "✗"
|
|
else:
|
|
cell = "—"
|
|
table_rows.append({"base_metric": base, "replicas": replicas, "cell": cell})
|
|
|
|
if table_rows:
|
|
ttest_df = pd.DataFrame(table_rows)
|
|
pivot = ttest_df.pivot(
|
|
index="base_metric", columns="replicas", values="cell"
|
|
)
|
|
# Sort columns by replica count
|
|
pivot = pivot.reindex(columns=sorted([c for c in pivot.columns if c >= 0]))
|
|
pivot = pivot.sort_index()
|
|
|
|
print("\n--- Statistical significance (t-test: first vs last file) ---")
|
|
print(f"Comparing: {first_label} vs {last_label}")
|
|
print("✓ = significant (p<0.05), ✗ = not significant\n")
|
|
print(pivot.to_string())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|