wehub-resource-sync
108 lines
3.5 KiB
Python
108 lines
3.5 KiB
Python
"""
|
||
Read evals/snapshots/results.json (produced by llm_run.py) and report
|
||
real token compression per skill against the *terse control arm* — i.e.
|
||
how much the skill adds on top of a plain "Answer concisely." instruction.
|
||
|
||
Reports median, min, max and stdev across prompts, not just the mean,
|
||
so the reader can see whether a number is solid or noisy.
|
||
|
||
Tokenizer note: tiktoken o200k_base is OpenAI's tokenizer and is only an
|
||
approximation of Claude's BPE. The ratios are still meaningful for
|
||
comparing skills against each other, but the absolute numbers should be
|
||
read as "approximate output-length reduction", not "exact Claude tokens".
|
||
|
||
Run: uv run --with tiktoken python evals/measure.py
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import statistics
|
||
from pathlib import Path
|
||
|
||
import tiktoken
|
||
|
||
ENCODING = tiktoken.get_encoding("o200k_base")
|
||
SNAPSHOT = Path(__file__).parent / "snapshots" / "results.json"
|
||
|
||
|
||
def count(text: str) -> int:
|
||
return len(ENCODING.encode(text))
|
||
|
||
|
||
def stats(savings: list[float]) -> tuple[float, float, float, float, float]:
|
||
return (
|
||
statistics.median(savings),
|
||
statistics.mean(savings),
|
||
min(savings),
|
||
max(savings),
|
||
statistics.stdev(savings) if len(savings) > 1 else 0.0,
|
||
)
|
||
|
||
|
||
def fmt_pct(x: float) -> str:
|
||
sign = "−" if x < 0 else "+"
|
||
return f"{sign}{abs(x) * 100:.0f}%"
|
||
|
||
|
||
def main() -> None:
|
||
if not SNAPSHOT.exists():
|
||
print(f"No snapshot at {SNAPSHOT}. Run `python evals/llm_run.py` first.")
|
||
return
|
||
|
||
data = json.loads(SNAPSHOT.read_text())
|
||
arms = data["arms"]
|
||
meta = data.get("metadata", {})
|
||
|
||
baseline_tokens = [count(o) for o in arms["__baseline__"]]
|
||
terse_tokens = [count(o) for o in arms["__terse__"]]
|
||
|
||
print(f"_Generated: {meta.get('generated_at', '?')}_")
|
||
print(
|
||
f"_Model: {meta.get('model', '?')} · CLI: {meta.get('claude_cli_version', '?')}_"
|
||
)
|
||
print(f"_Tokenizer: tiktoken o200k_base (approximation of Claude's BPE)_")
|
||
print(
|
||
f"_n = {meta.get('n_prompts', len(baseline_tokens))} prompts, single run per arm_"
|
||
)
|
||
print()
|
||
print(f"**Reference arms (no skill):**")
|
||
print(f"- baseline (no system prompt): {sum(baseline_tokens)} tokens total")
|
||
print(
|
||
f"- terse control (`Answer concisely.`): {sum(terse_tokens)} tokens total "
|
||
f"({fmt_pct(1 - sum(terse_tokens) / sum(baseline_tokens))} vs baseline)"
|
||
)
|
||
print()
|
||
print("**Skills, measured as additional reduction on top of the terse control:**")
|
||
print()
|
||
print("| Skill | Median | Mean | Min | Max | Stdev | Tokens (skill / terse) |")
|
||
print("|-------|--------|------|-----|-----|-------|-------------------------|")
|
||
|
||
rows = []
|
||
for skill, outputs in arms.items():
|
||
if skill in ("__baseline__", "__terse__"):
|
||
continue
|
||
skill_tokens = [count(o) for o in outputs]
|
||
savings = [
|
||
1 - (s / t) if t else 0.0 for s, t in zip(skill_tokens, terse_tokens)
|
||
]
|
||
med, mean, lo, hi, sd = stats(savings)
|
||
rows.append(
|
||
(skill, med, mean, lo, hi, sd, sum(skill_tokens), sum(terse_tokens))
|
||
)
|
||
|
||
for row in sorted(rows, key=lambda r: -r[1]):
|
||
skill, med, mean, lo, hi, sd, st, tt = row
|
||
print(
|
||
f"| **{skill}** | {fmt_pct(med)} | {fmt_pct(mean)} | "
|
||
f"{fmt_pct(lo)} | {fmt_pct(hi)} | {sd * 100:.0f}% | {st} / {tt} |"
|
||
)
|
||
|
||
print()
|
||
print("_Savings = `1 - skill_tokens / terse_tokens` per prompt._")
|
||
print(f"_Source: {SNAPSHOT.name}. Refresh with `python evals/llm_run.py`._")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|