chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Plot controller benchmark results from microbenchmarks.py --run-controller.
|
||||
|
||||
Takes one or more JSON files (output from microbenchmarks.py --run-controller)
|
||||
and produces seaborn FacetGrid plots for each metric, with file names as labels
|
||||
and std for confidence intervals.
|
||||
|
||||
When 2+ files are provided, runs a t-test (first vs last file) to assess
|
||||
statistical significance of changes.
|
||||
|
||||
Dependencies: matplotlib, pandas, seaborn, scipy
|
||||
|
||||
Example usage with the master.json and pydantic.json files:
|
||||
python release/serve_tests/workloads/plot_controller_benchmark.py \
|
||||
release/serve_tests/workloads/master.json \
|
||||
release/serve_tests/workloads/pydantic.json \
|
||||
-o /tmp/controller_plot.png
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import pandas as pd
|
||||
import seaborn as sns
|
||||
from scipy import stats
|
||||
|
||||
|
||||
def load_benchmark_json(path: str) -> list[dict]:
|
||||
"""Load perf_metrics from a controller benchmark JSON file.
|
||||
|
||||
Supports both formats:
|
||||
- {"perf_metrics": [...]} (from save_test_results)
|
||||
- [...] (raw list of perf metric dicts)
|
||||
"""
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, list):
|
||||
perf_metrics = data
|
||||
else:
|
||||
perf_metrics = data.get("perf_metrics", [])
|
||||
if not perf_metrics:
|
||||
raise ValueError(f"No perf_metrics found in {path}")
|
||||
return perf_metrics
|
||||
|
||||
|
||||
def _parse_replicas_from_metric_name(name: str) -> int | None:
|
||||
"""Extract replica count from metric name, e.g. controller_foo_10_replicas -> 10."""
|
||||
m = re.search(r"_(\d+)_replicas$", name)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def build_plot_dataframe(json_paths: list[str]) -> pd.DataFrame:
|
||||
"""Build a DataFrame for plotting from multiple JSON files."""
|
||||
rows = []
|
||||
for path in json_paths:
|
||||
label = Path(path).stem
|
||||
metrics = load_benchmark_json(path)
|
||||
for m in metrics:
|
||||
name = m.get("perf_metric_name")
|
||||
value = m.get("perf_metric_value")
|
||||
std = m.get("perf_metric_std", 0.0)
|
||||
n = m.get("perf_metric_sample_size", 1)
|
||||
if name is not None and value is not None:
|
||||
replicas = _parse_replicas_from_metric_name(name)
|
||||
rows.append(
|
||||
{
|
||||
"metric": name,
|
||||
"base_metric": name.rsplit("_", 2)[0]
|
||||
if replicas is not None
|
||||
else name,
|
||||
"replicas": replicas if replicas is not None else -1,
|
||||
"file": label,
|
||||
"mean": float(value),
|
||||
"std": float(std),
|
||||
"n": int(n),
|
||||
}
|
||||
)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def run_ttest(df: pd.DataFrame, first_file: str, last_file: str) -> dict[str, dict]:
|
||||
"""
|
||||
Run independent t-test for each metric between first and last file.
|
||||
Returns dict of metric_name -> {statistic, pvalue, significant}.
|
||||
"""
|
||||
results = {}
|
||||
for metric in df["metric"].unique():
|
||||
m1 = df[(df["metric"] == metric) & (df["file"] == first_file)]
|
||||
m2 = df[(df["metric"] == metric) & (df["file"] == last_file)]
|
||||
if m1.empty or m2.empty:
|
||||
continue
|
||||
row1 = m1.iloc[0]
|
||||
row2 = m2.iloc[0]
|
||||
try:
|
||||
stat, pval = stats.ttest_ind_from_stats(
|
||||
mean1=row1["mean"],
|
||||
std1=row1["std"],
|
||||
nobs1=row1["n"],
|
||||
mean2=row2["mean"],
|
||||
std2=row2["std"],
|
||||
nobs2=row2["n"],
|
||||
)
|
||||
results[metric] = {
|
||||
"statistic": float(stat),
|
||||
"pvalue": float(pval),
|
||||
"significant": pval < 0.05,
|
||||
}
|
||||
except Exception:
|
||||
results[metric] = {"statistic": None, "pvalue": None, "significant": None}
|
||||
return results
|
||||
|
||||
|
||||
def plot_facet_grid(df: pd.DataFrame, output_path: str) -> None:
|
||||
"""Create a seaborn FacetGrid bar plot for each metric with std as CI.
|
||||
|
||||
X-axis: replica count. Hue: file (when multiple files).
|
||||
"""
|
||||
# Use replica count on x-axis; filter out metrics without replica suffix
|
||||
plot_df = df[df["replicas"] >= 0].copy()
|
||||
if plot_df.empty:
|
||||
# Fallback: no replica pattern found, use file on x-axis
|
||||
plot_df = df.copy()
|
||||
plot_df["replicas"] = plot_df["file"]
|
||||
x_var, facet_var = "file", "metric"
|
||||
else:
|
||||
plot_df["replicas"] = plot_df["replicas"].astype(int)
|
||||
x_var, facet_var = "replicas", "base_metric"
|
||||
|
||||
base_metrics = sorted(plot_df[facet_var].unique())
|
||||
n_metrics = len(base_metrics)
|
||||
col_wrap = min(3, n_metrics)
|
||||
|
||||
replica_order = (
|
||||
sorted(plot_df["replicas"].unique()) if x_var == "replicas" else None
|
||||
)
|
||||
|
||||
g = sns.FacetGrid(
|
||||
plot_df,
|
||||
col=facet_var,
|
||||
col_wrap=col_wrap,
|
||||
col_order=base_metrics,
|
||||
sharey=False,
|
||||
height=4,
|
||||
aspect=1.2,
|
||||
)
|
||||
has_multiple_files = plot_df["file"].nunique() > 1
|
||||
g.map_dataframe(
|
||||
sns.barplot,
|
||||
x=x_var,
|
||||
y="mean",
|
||||
hue="file" if has_multiple_files else x_var,
|
||||
palette="husl",
|
||||
legend=has_multiple_files,
|
||||
order=replica_order,
|
||||
hue_order=(sorted(plot_df["file"].unique()) if has_multiple_files else None),
|
||||
)
|
||||
|
||||
# Add error bars (std as CI) to each facet
|
||||
for ax, base_metric in zip(g.axes.flat, base_metrics):
|
||||
subset = plot_df[plot_df[facet_var] == base_metric].sort_values(
|
||||
["file", x_var] # seaborn bar order: hue then x
|
||||
)
|
||||
# Match bars to data: seaborn containers = one per hue, bars per x
|
||||
bars = [bar for c in ax.containers for bar in c]
|
||||
for (_, row), bar in zip(subset.iterrows(), bars):
|
||||
ax.errorbar(
|
||||
bar.get_x() + bar.get_width() / 2,
|
||||
row["mean"],
|
||||
yerr=row["std"],
|
||||
fmt="none",
|
||||
color="black",
|
||||
capsize=4,
|
||||
capthick=1,
|
||||
)
|
||||
ax.set_xlabel("Replicas" if x_var == "replicas" else "")
|
||||
ax.tick_params(axis="x", rotation=0)
|
||||
|
||||
g.set_titles(col_template="{col_name}")
|
||||
if plot_df["file"].nunique() > 1:
|
||||
g.add_legend(title="File")
|
||||
plt.tight_layout()
|
||||
plt.savefig(output_path, dpi=150, bbox_inches="tight")
|
||||
plt.close()
|
||||
print(f"Saved plot to {output_path}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Plot controller benchmark JSON outputs and optionally run t-tests."
|
||||
)
|
||||
parser.add_argument(
|
||||
"json_files",
|
||||
nargs="+",
|
||||
help="One or more JSON files from microbenchmarks.py --run-controller",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
default="controller_benchmark_plot.png",
|
||||
help="Output plot path (default: controller_benchmark_plot.png)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
for p in args.json_files:
|
||||
if not os.path.isfile(p):
|
||||
raise FileNotFoundError(f"File not found: {p}")
|
||||
|
||||
df = build_plot_dataframe(args.json_files)
|
||||
if df.empty:
|
||||
raise ValueError("No valid metrics found in the provided JSON files.")
|
||||
|
||||
# Plot
|
||||
plot_facet_grid(df, args.output)
|
||||
|
||||
# T-test when 2+ files
|
||||
if len(args.json_files) >= 2:
|
||||
first_label = Path(args.json_files[0]).stem
|
||||
last_label = Path(args.json_files[-1]).stem
|
||||
ttest_results = run_ttest(df, first_label, last_label)
|
||||
|
||||
# Build table: rows=base_metric, cols=replicas
|
||||
table_rows = []
|
||||
for full_metric, r in ttest_results.items():
|
||||
replicas = _parse_replicas_from_metric_name(full_metric)
|
||||
base = (
|
||||
full_metric.rsplit("_", 2)[0] if replicas is not None else full_metric
|
||||
)
|
||||
if replicas is None:
|
||||
replicas = -1
|
||||
if r["pvalue"] is not None:
|
||||
cell = "✓" if r["significant"] else "✗"
|
||||
else:
|
||||
cell = "—"
|
||||
table_rows.append({"base_metric": base, "replicas": replicas, "cell": cell})
|
||||
|
||||
if table_rows:
|
||||
ttest_df = pd.DataFrame(table_rows)
|
||||
pivot = ttest_df.pivot(
|
||||
index="base_metric", columns="replicas", values="cell"
|
||||
)
|
||||
# Sort columns by replica count
|
||||
pivot = pivot.reindex(columns=sorted([c for c in pivot.columns if c >= 0]))
|
||||
pivot = pivot.sort_index()
|
||||
|
||||
print("\n--- Statistical significance (t-test: first vs last file) ---")
|
||||
print(f"Comparing: {first_label} vs {last_label}")
|
||||
print("✓ = significant (p<0.05), ✗ = not significant\n")
|
||||
print(pivot.to_string())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user