wehub-resource-sync
151 lines
4.4 KiB
Python
151 lines
4.4 KiB
Python
"""
|
|
Generate a boxplot showing the distribution of token compression per
|
|
skill, compared against a plain "Answer concisely." control.
|
|
|
|
Reads evals/snapshots/results.json and writes:
|
|
- evals/snapshots/results.html (interactive plotly)
|
|
- evals/snapshots/results.png (static export for README/PR embed)
|
|
|
|
Run: uv run --with tiktoken --with plotly --with kaleido python evals/plot.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import statistics
|
|
from pathlib import Path
|
|
|
|
import plotly.graph_objects as go
|
|
import tiktoken
|
|
|
|
ENCODING = tiktoken.get_encoding("o200k_base")
|
|
SNAPSHOT = Path(__file__).parent / "snapshots" / "results.json"
|
|
HTML_OUT = Path(__file__).parent / "snapshots" / "results.html"
|
|
PNG_OUT = Path(__file__).parent / "snapshots" / "results.png"
|
|
|
|
|
|
def count(text: str) -> int:
|
|
return len(ENCODING.encode(text))
|
|
|
|
|
|
def main() -> None:
|
|
data = json.loads(SNAPSHOT.read_text())
|
|
arms = data["arms"]
|
|
meta = data.get("metadata", {})
|
|
|
|
terse_tokens = [count(o) for o in arms["__terse__"]]
|
|
|
|
rows = []
|
|
for skill, outputs in arms.items():
|
|
if skill in ("__baseline__", "__terse__"):
|
|
continue
|
|
skill_tokens = [count(o) for o in outputs]
|
|
savings = [
|
|
(1 - (s / t)) * 100 if t else 0.0
|
|
for s, t in zip(skill_tokens, terse_tokens)
|
|
]
|
|
rows.append(
|
|
{"skill": skill, "savings": savings, "median": statistics.median(savings)}
|
|
)
|
|
|
|
rows.sort(key=lambda r: -r["median"]) # best first
|
|
|
|
fig = go.Figure()
|
|
|
|
for row in rows:
|
|
fig.add_trace(
|
|
go.Box(
|
|
y=row["savings"],
|
|
name=row["skill"],
|
|
boxpoints="all",
|
|
jitter=0.4,
|
|
pointpos=0,
|
|
marker=dict(color="#2ca02c", size=7, opacity=0.7),
|
|
line=dict(color="#2c3e50", width=2),
|
|
fillcolor="rgba(76, 120, 168, 0.25)",
|
|
boxmean=True,
|
|
hovertemplate="<b>%{x}</b><br>%{y:.1f}%<extra></extra>",
|
|
)
|
|
)
|
|
|
|
# zero line — "no effect"
|
|
fig.add_hline(
|
|
y=0,
|
|
line=dict(color="black", width=1.5, dash="dash"),
|
|
annotation_text="no effect (= same length as control)",
|
|
annotation_position="top right",
|
|
annotation_font=dict(size=11, color="black"),
|
|
)
|
|
|
|
# median labels above each box
|
|
for row in rows:
|
|
fig.add_annotation(
|
|
x=row["skill"],
|
|
y=max(row["savings"]),
|
|
text=f"<b>{row['median']:+.0f}%</b>",
|
|
showarrow=False,
|
|
yshift=22,
|
|
font=dict(size=16, color="#2c3e50"),
|
|
)
|
|
|
|
fig.update_layout(
|
|
title=dict(
|
|
text=f"<b>How much shorter does each skill make Claude's answers?</b><br>"
|
|
f"<sub>Distribution of per-prompt savings vs system prompt = "
|
|
f"<i>'Answer concisely.'</i><br>"
|
|
f"{meta.get('model', '?')} · n={meta.get('n_prompts', '?')} prompts · "
|
|
f"single run per arm</sub>",
|
|
x=0.5,
|
|
xanchor="center",
|
|
),
|
|
xaxis=dict(title="", automargin=True),
|
|
yaxis=dict(
|
|
title="↑ shorter · vs control · longer ↓",
|
|
ticksuffix="%",
|
|
zeroline=False,
|
|
gridcolor="rgba(0,0,0,0.08)",
|
|
range=[-30, 115],
|
|
),
|
|
plot_bgcolor="white",
|
|
height=560,
|
|
width=980,
|
|
margin=dict(l=140, r=80, t=120, b=120),
|
|
showlegend=False,
|
|
annotations=[
|
|
dict(
|
|
x=0.5,
|
|
y=-0.22,
|
|
xref="paper",
|
|
yref="paper",
|
|
showarrow=False,
|
|
font=dict(size=11, color="#555"),
|
|
text=(
|
|
"<b>box</b> = IQR (middle 50%) · "
|
|
"<b>line in box</b> = median · "
|
|
"<b>dashed line</b> = mean · "
|
|
"<b>green dots</b> = individual prompts"
|
|
),
|
|
)
|
|
],
|
|
)
|
|
|
|
# re-add labels after update_layout (which would otherwise wipe them)
|
|
for row in rows:
|
|
fig.add_annotation(
|
|
x=row["skill"],
|
|
y=max(row["savings"]),
|
|
text=f"<b>{row['median']:+.0f}%</b>",
|
|
showarrow=False,
|
|
yshift=22,
|
|
font=dict(size=16, color="#2c3e50"),
|
|
)
|
|
|
|
fig.write_html(HTML_OUT)
|
|
print(f"Wrote {HTML_OUT}")
|
|
fig.write_image(PNG_OUT, scale=2)
|
|
print(f"Wrote {PNG_OUT}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|