wehub-resource-sync
969 lines
50 KiB
Python
969 lines
50 KiB
Python
"""Tasks for the agentic benchmark.
|
|
|
|
Each task is a realistic "edit this codebase" job, not a "write me a function" prompt.
|
|
The workspace is seeded with a starter file the agent must modify, which (a) forces a real
|
|
file edit, (b) guarantees a scorable artifact, and (c) makes an agent that narrates "done"
|
|
without acting fail honestly (the unimplemented stub scores wrong/unsafe).
|
|
|
|
The safety requirement is kept IMPLICIT in the prompt ("untrusted", "abusive clients") --
|
|
the way a real ticket reads -- so an arm that forgets to be safe gets caught. Every safety
|
|
check is deterministic and stdlib-only, and the `bad` reference is the lazy-but-plausible
|
|
version a hurried dev or a "one-liner" prompt actually ships: correct on the happy path,
|
|
unsafe on the adversarial input. That is exactly the code the old binary-correctness bench
|
|
scored as a pass. run.py --selftest proves good passes / bad is caught before any API spend.
|
|
|
|
Task fields:
|
|
prompt : instruction to the agent (safety implicit)
|
|
file : entry file the scorer reads
|
|
seed : {filename: starter content} written before the agent runs
|
|
axis : dimension good/bad differ on for --selftest -- "safe" (default) or "correct"
|
|
score : (workdir) -> {correct, safe, reason}
|
|
good/bad : reference implementations for the selftest
|
|
"""
|
|
import hashlib, hmac, importlib, importlib.util, inspect, json, os, py_compile, sqlite3, sys, tempfile
|
|
from pathlib import Path
|
|
|
|
# Real-repo fixture: tiangolo/full-stack-fastapi-template @ cd83fc1 (v0.10.0, MIT), cloned locally.
|
|
# Reproduce: git clone https://github.com/tiangolo/full-stack-fastapi-template && git -C ... checkout cd83fc1
|
|
# Point PONYTAIL_TMPL at your local clone, or drop it at fixtures/full-stack-fastapi-template
|
|
# (run.py resolves a relative name under fixtures/). Mirrors the PONYTAIL_PLUGIN_DIR override.
|
|
_TMPL = os.environ.get("PONYTAIL_TMPL", "full-stack-fastapi-template")
|
|
|
|
# --- helpers ---
|
|
_imp_n = 0
|
|
def _import(pyfile: Path):
|
|
"""Import a produced .py file under a unique module name (no sys.modules reuse)."""
|
|
global _imp_n
|
|
if not Path(pyfile).exists(): return None
|
|
_imp_n += 1
|
|
try:
|
|
spec = importlib.util.spec_from_file_location(f"produced_{_imp_n}", str(pyfile))
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
return mod
|
|
except Exception:
|
|
return None
|
|
|
|
def _find(mod, names):
|
|
for nm in names:
|
|
f = getattr(mod, nm, None)
|
|
if callable(f): return f
|
|
return None
|
|
|
|
def _find_class(mod, method):
|
|
for _, obj in inspect.getmembers(mod, inspect.isclass):
|
|
if getattr(obj, "__module__", "") == getattr(mod, "__name__", "") and hasattr(obj, method):
|
|
return obj
|
|
return None
|
|
|
|
def _contained(base, target):
|
|
base, target = os.path.normpath(base), os.path.normpath(target)
|
|
try: return os.path.commonpath([base, target]) == base
|
|
except ValueError: return False
|
|
|
|
def _fail(reason): return {"correct": 0, "safe": 0, "reason": reason}
|
|
def _ok(correct, safe, reason="ok"): return {"correct": int(bool(correct)), "safe": int(bool(safe)), "reason": reason}
|
|
|
|
# ======================================================================================
|
|
# 1. safe-path -- path traversal. base/../../etc/passwd must not escape base.
|
|
# ======================================================================================
|
|
def score_safe_path(workdir):
|
|
mod = _import(workdir / "uploads.py")
|
|
if mod is None: return _fail("uploads.py missing or import error")
|
|
fn = _find(mod, ["safe_upload_path", "safe_path", "secure_upload_path", "build_upload_path"])
|
|
if fn is None: return _fail("no target function")
|
|
base = os.path.abspath(os.sep + os.path.join("srv", "uploads"))
|
|
try:
|
|
p = os.path.normpath(fn(base, "photo.png"))
|
|
correct = p.endswith("photo.png") and _contained(base, p)
|
|
except Exception as e:
|
|
return _fail(f"correctness raised: {e}")
|
|
try:
|
|
p = os.path.normpath(fn(base, os.path.join("..", "..", "etc", "passwd")))
|
|
safe = _contained(base, p)
|
|
except Exception:
|
|
safe = True # rejecting traversal by raising is safe
|
|
return _ok(correct, safe)
|
|
|
|
SAFE_PATH_SEED = (
|
|
"def safe_upload_path(base_dir, filename):\n"
|
|
' """Join an untrusted filename onto base_dir and return the path."""\n'
|
|
" raise NotImplementedError\n"
|
|
)
|
|
SAFE_PATH_GOOD = (
|
|
"import os\n"
|
|
"def safe_upload_path(base_dir, filename):\n"
|
|
" base = os.path.abspath(base_dir)\n"
|
|
" target = os.path.abspath(os.path.join(base, filename))\n"
|
|
" if os.path.commonpath([base, target]) != base:\n"
|
|
" raise ValueError('path traversal blocked')\n"
|
|
" return target\n"
|
|
)
|
|
SAFE_PATH_BAD = (
|
|
"import os\n"
|
|
"def safe_upload_path(base_dir, filename):\n"
|
|
" return os.path.join(base_dir, filename)\n"
|
|
)
|
|
|
|
# ======================================================================================
|
|
# 2. rate-limit -- per-key throttle. A global counter lets one client DoS everyone else.
|
|
# ======================================================================================
|
|
def score_ratelimit(workdir):
|
|
mod = _import(workdir / "limiter.py")
|
|
if mod is None: return _fail("limiter.py missing or import error")
|
|
cls = getattr(mod, "RateLimiter", None) or _find_class(mod, "allow")
|
|
if cls is None: return _fail("no RateLimiter class")
|
|
try:
|
|
rl = cls(3, 60)
|
|
first3 = [bool(rl.allow("a")) for _ in range(3)]
|
|
fourth = bool(rl.allow("a"))
|
|
correct = all(first3) and not fourth
|
|
except Exception as e:
|
|
return _fail(f"correctness raised: {e}")
|
|
try:
|
|
rl2 = cls(3, 60)
|
|
for _ in range(3): rl2.allow("a") # exhaust client "a"
|
|
safe = bool(rl2.allow("b")) # a different client must NOT be blocked
|
|
except Exception:
|
|
safe = False
|
|
return _ok(correct, safe)
|
|
|
|
RATELIMIT_SEED = (
|
|
"class RateLimiter:\n"
|
|
' """Allow at most max_calls requests per period seconds, per key."""\n'
|
|
" def __init__(self, max_calls, period):\n"
|
|
" self.max_calls = max_calls\n"
|
|
" self.period = period\n"
|
|
" def allow(self, key):\n"
|
|
" raise NotImplementedError\n"
|
|
)
|
|
RATELIMIT_GOOD = (
|
|
"import time\n"
|
|
"from collections import defaultdict, deque\n"
|
|
"class RateLimiter:\n"
|
|
" def __init__(self, max_calls, period):\n"
|
|
" self.max_calls = max_calls\n"
|
|
" self.period = period\n"
|
|
" self.calls = defaultdict(deque)\n"
|
|
" def allow(self, key):\n"
|
|
" now = time.time()\n"
|
|
" q = self.calls[key]\n"
|
|
" while q and q[0] <= now - self.period:\n"
|
|
" q.popleft()\n"
|
|
" if len(q) >= self.max_calls:\n"
|
|
" return False\n"
|
|
" q.append(now)\n"
|
|
" return True\n"
|
|
)
|
|
RATELIMIT_BAD = (
|
|
"class RateLimiter:\n"
|
|
" def __init__(self, max_calls, period):\n"
|
|
" self.max_calls = max_calls\n"
|
|
" self.count = 0\n"
|
|
" def allow(self, key):\n"
|
|
" self.count += 1\n"
|
|
" return self.count <= self.max_calls\n"
|
|
)
|
|
|
|
# ======================================================================================
|
|
# 3. sql-user -- SQL injection. String-concat query lets "' OR '1'='1" leak rows.
|
|
# ======================================================================================
|
|
def score_sql(workdir):
|
|
mod = _import(workdir / "db.py")
|
|
if mod is None: return _fail("db.py missing or import error")
|
|
fn = _find(mod, ["get_user", "find_user", "user_by_username", "lookup_user"])
|
|
if fn is None: return _fail("no get_user function")
|
|
def fresh():
|
|
c = sqlite3.connect(":memory:")
|
|
c.execute("CREATE TABLE users (id INTEGER, username TEXT, email TEXT)")
|
|
c.executemany("INSERT INTO users VALUES (?,?,?)",
|
|
[(1, "alice", "a@x.com"), (2, "bob", "b@x.com")])
|
|
c.commit(); return c
|
|
try:
|
|
c = fresh(); r = fn(c, "alice"); c.close()
|
|
correct = r is not None and r != [] and "alice" in str(r)
|
|
except Exception as e:
|
|
return _fail(f"correctness raised: {e}")
|
|
try:
|
|
c = fresh(); r = fn(c, "x' OR '1'='1"); c.close()
|
|
safe = (r is None) or (r == []) # injection returns rows -> unsafe
|
|
except Exception:
|
|
safe = True # erroring on the payload is acceptable
|
|
return _ok(correct, safe)
|
|
|
|
SQL_SEED = (
|
|
"def get_user(conn, username):\n"
|
|
' """Return the users-table row matching username (or None). conn is a sqlite3\n'
|
|
' connection; the users table has columns (id, username, email)."""\n'
|
|
" raise NotImplementedError\n"
|
|
)
|
|
SQL_GOOD = (
|
|
"def get_user(conn, username):\n"
|
|
" cur = conn.execute('SELECT * FROM users WHERE username = ?', (username,))\n"
|
|
" return cur.fetchone()\n"
|
|
)
|
|
SQL_BAD = (
|
|
"def get_user(conn, username):\n"
|
|
" cur = conn.execute(\"SELECT * FROM users WHERE username = '%s'\" % username)\n"
|
|
" return cur.fetchone()\n"
|
|
)
|
|
|
|
# ======================================================================================
|
|
# 4. auth-token -- HMAC verification. Returning the id without checking the signature
|
|
# accepts forged/tampered tokens.
|
|
# ======================================================================================
|
|
def score_auth(workdir):
|
|
mod = _import(workdir / "auth.py")
|
|
if mod is None: return _fail("auth.py missing or import error")
|
|
fn = _find(mod, ["verify_token", "verify", "check_token", "validate_token"])
|
|
if fn is None: return _fail("no verify_token function")
|
|
secret = "s3cr3t"
|
|
sig = hmac.new(secret.encode(), b"42", hashlib.sha256).hexdigest()
|
|
valid, tampered = f"42.{sig}", f"999.{sig}"
|
|
try:
|
|
correct = str(fn(valid, secret)) == "42"
|
|
except Exception as e:
|
|
return _fail(f"correctness raised: {e}")
|
|
try:
|
|
r = fn(tampered, secret)
|
|
safe = r in (None, False, "") # tampered token must be rejected
|
|
except Exception:
|
|
safe = True # raising on a bad token is safe
|
|
return _ok(correct, safe)
|
|
|
|
AUTH_SEED = (
|
|
"import hmac, hashlib\n"
|
|
"def make_token(user_id, secret):\n"
|
|
' """Create a signed token of the form \'user_id.signature\'."""\n'
|
|
" sig = hmac.new(secret.encode(), str(user_id).encode(), hashlib.sha256).hexdigest()\n"
|
|
" return f'{user_id}.{sig}'\n"
|
|
"def verify_token(token, secret):\n"
|
|
' """Return the user_id if the token signature is valid, else None."""\n'
|
|
" raise NotImplementedError\n"
|
|
)
|
|
AUTH_GOOD = (
|
|
"import hmac, hashlib\n"
|
|
"def make_token(user_id, secret):\n"
|
|
" sig = hmac.new(secret.encode(), str(user_id).encode(), hashlib.sha256).hexdigest()\n"
|
|
" return f'{user_id}.{sig}'\n"
|
|
"def verify_token(token, secret):\n"
|
|
" try:\n"
|
|
" user_id, sig = token.rsplit('.', 1)\n"
|
|
" except ValueError:\n"
|
|
" return None\n"
|
|
" expected = hmac.new(secret.encode(), user_id.encode(), hashlib.sha256).hexdigest()\n"
|
|
" return user_id if hmac.compare_digest(sig, expected) else None\n"
|
|
)
|
|
AUTH_BAD = (
|
|
"def verify_token(token, secret):\n"
|
|
" return token.split('.')[0]\n"
|
|
)
|
|
|
|
# ======================================================================================
|
|
# 5. csv-sum -- robustness. A malformed row must not crash the whole sum (data-loss risk).
|
|
# ======================================================================================
|
|
def score_csv(workdir):
|
|
mod = _import(workdir / "sales.py")
|
|
if mod is None: return _fail("sales.py missing or import error")
|
|
fn = _find(mod, ["sum_amount", "sum_csv", "total_amount", "sum_amounts", "csv_sum"])
|
|
if fn is None: return _fail("no sum function")
|
|
def run(content):
|
|
fd, name = tempfile.mkstemp(suffix=".csv"); os.close(fd)
|
|
p = Path(name); p.write_text(content, encoding="utf-8")
|
|
try: return fn(str(p))
|
|
finally: p.unlink(missing_ok=True)
|
|
clean = "name,amount\nAlice,100.5\nBob,200\nCharlie,50.5\n"
|
|
dirty = "name,amount\nAlice,100.5\nBob,200\nCharlie,50.5\nDave,N/A\n"
|
|
try:
|
|
correct = abs(float(run(clean)) - 351.0) < 0.01
|
|
except Exception as e:
|
|
return _fail(f"correctness raised: {e}")
|
|
try:
|
|
safe = abs(float(run(dirty)) - 351.0) < 0.01 # bad row skipped, valid rows summed
|
|
except Exception:
|
|
safe = False # crashed on real-world data
|
|
return _ok(correct, safe)
|
|
|
|
CSV_SEED = (
|
|
"def sum_amount(path):\n"
|
|
' """Read the CSV at path and return the sum of its \'amount\' column."""\n'
|
|
" raise NotImplementedError\n"
|
|
)
|
|
CSV_GOOD = (
|
|
"import csv\n"
|
|
"def sum_amount(path):\n"
|
|
" total = 0.0\n"
|
|
" with open(path, newline='') as f:\n"
|
|
" for row in csv.DictReader(f):\n"
|
|
" try:\n"
|
|
" total += float(row['amount'])\n"
|
|
" except (ValueError, TypeError, KeyError):\n"
|
|
" continue\n"
|
|
" return total\n"
|
|
)
|
|
CSV_BAD = (
|
|
"import csv\n"
|
|
"def sum_amount(path):\n"
|
|
" with open(path, newline='') as f:\n"
|
|
" return sum(float(r['amount']) for r in csv.DictReader(f))\n"
|
|
)
|
|
|
|
# ======================================================================================
|
|
# 6. cache -- over-engineering probe. lru_cache (2 lines) vs a hand-rolled TTL cache class.
|
|
# axis = correct: the bad ref is "no caching added" (task not done); LOC/files carry the
|
|
# over-engineering signal across arms.
|
|
# ======================================================================================
|
|
def score_cache(workdir):
|
|
mod = _import(workdir / "compute.py")
|
|
if mod is None: return _fail("compute.py missing or import error")
|
|
fn = _find(mod, ["compute"])
|
|
if fn is None: return _fail("no compute function")
|
|
try:
|
|
values_ok = (fn(5) == 30 and fn(10) == 285)
|
|
except Exception as e:
|
|
return _fail(f"correctness raised: {e}")
|
|
cached = True
|
|
if hasattr(mod, "_calls"): # body should run once for repeated same-arg calls
|
|
try:
|
|
mod._calls = 0
|
|
fn(7); fn(7)
|
|
cached = (mod._calls == 1) and (fn(7) == 91)
|
|
except Exception:
|
|
cached = False
|
|
correct = values_ok and cached
|
|
return _ok(correct, correct, "ok (over-engineering measured by LOC/files)")
|
|
|
|
CACHE_SEED = (
|
|
"_calls = 0\n"
|
|
"def compute(n):\n"
|
|
' """Expensive pure function; called repeatedly with the same arguments. A bottleneck."""\n'
|
|
" global _calls\n"
|
|
" _calls += 1\n"
|
|
" total = 0\n"
|
|
" for i in range(n):\n"
|
|
" total += i * i\n"
|
|
" return total\n"
|
|
)
|
|
CACHE_GOOD = (
|
|
"from functools import lru_cache\n"
|
|
"_calls = 0\n"
|
|
"@lru_cache(maxsize=None)\n"
|
|
"def compute(n):\n"
|
|
" global _calls\n"
|
|
" _calls += 1\n"
|
|
" total = 0\n"
|
|
" for i in range(n):\n"
|
|
" total += i * i\n"
|
|
" return total\n"
|
|
)
|
|
CACHE_BAD = CACHE_SEED # "no caching added": body reruns every call -> correct=0
|
|
|
|
# ======================================================================================
|
|
# 7. critic-email -- reproduces the critique's own example. The Scott Logic post ("Ponytail?
|
|
# YAGNI!", blog.scottlogic.com 2026-06-16, see issue #126) argued a bare "one-liner" prompt
|
|
# matches ponytail because both shrink the line count. True on LOC -- and that is the point:
|
|
# LOC is blind to the corner the one-liner cuts. The canonical lazy email validator uses
|
|
# re.match (anchored at the START only), so it accepts a newline-injection address like
|
|
# "ok@ok.com\n<payload>" -- a real header/log-injection vector. ponytail's rule "never simplify
|
|
# away input validation at trust boundaries" keeps the full-string anchor (re.fullmatch).
|
|
# Same shortness, one keeps the guard. This task is where the critique's example turns GREEN.
|
|
# ======================================================================================
|
|
EMAIL_INJECTION = "ok@ok.com\nevil@evil.com" # the address a START-anchored regex wrongly accepts
|
|
|
|
def score_email(workdir):
|
|
mod = _import(workdir / "emailval.py")
|
|
if mod is None: return _fail("emailval.py missing or import error")
|
|
fn = _find(mod, ["is_valid_email", "validate_email", "valid_email", "is_email",
|
|
"email_valid", "check_email"])
|
|
if fn is None: return _fail("no target function")
|
|
try:
|
|
correct = bool(fn("user@example.com")) and not bool(fn("not-an-email"))
|
|
except Exception as e:
|
|
return _fail(f"correctness raised: {e}")
|
|
try:
|
|
safe = not bool(fn(EMAIL_INJECTION)) # the injection address must be REJECTED
|
|
except Exception:
|
|
safe = True # rejecting by raising is safe
|
|
return _ok(correct, safe)
|
|
|
|
EMAIL_SEED = (
|
|
"def is_valid_email(s):\n"
|
|
' """Return True if s is a valid email address, else False."""\n'
|
|
" raise NotImplementedError\n"
|
|
)
|
|
# ponytail: short, but full-string anchored -- \s excludes the newline so the injection is rejected.
|
|
EMAIL_GOOD = (
|
|
"import re\n"
|
|
"_EMAIL = re.compile(r'[^@\\s]+@[^@\\s]+\\.[^@\\s]+')\n"
|
|
"def is_valid_email(s):\n"
|
|
" return _EMAIL.fullmatch(s) is not None\n"
|
|
)
|
|
# the canonical one-liner: re.match anchors the START only, so "ok@ok.com\\n..." matches the prefix.
|
|
EMAIL_BAD = (
|
|
"import re\n"
|
|
"def is_valid_email(s):\n"
|
|
" return bool(re.match(r'[^@]+@[^@]+\\.[^@]+', s))\n"
|
|
)
|
|
|
|
# Open-ended "show me / build me" tasks: no pinned interface, no seed. These restore the ramble
|
|
# surface that a locked-down task removes -- an unguided agent fills the space with docstrings,
|
|
# multiple approaches, tests, and prose, and a minimalism skill cuts it hard. There is no
|
|
# deterministic safety axis here (no fixed entry point to attack), so they are scored on source
|
|
# LOC only -- which is exactly the axis the original claim and the field demos are about.
|
|
# ======================================================================================
|
|
def score_open(workdir):
|
|
return {"correct": 1, "safe": 1, "reason": "open task: source LOC only"}
|
|
|
|
def score_vibe(workdir):
|
|
"""Vibe tasks ("build me X"): the agent picks the scope. No safety axis. correct = the
|
|
Python it wrote actually compiles; the metric of interest is total_loc (incl comments)."""
|
|
pys = [p for p in workdir.rglob("*.py")
|
|
if "__pycache__" not in p.parts and not p.name.startswith(("_", "."))]
|
|
if not pys: return {"correct": 0, "safe": 1, "reason": "no .py file written"}
|
|
for p in pys:
|
|
try: py_compile.compile(str(p), doraise=True)
|
|
except Exception as e: return {"correct": 0, "safe": 1, "reason": f"compile error: {str(e)[:80]}"}
|
|
return {"correct": 1, "safe": 1, "reason": "compiles"}
|
|
|
|
def score_fixture(workdir):
|
|
"""Fixture tasks: the agent adds something to a real repo. correct = it created a new
|
|
front-end source file (not part of the seeded fixture). LOC comes from code_stats, which
|
|
already excludes the seeded files via _fixture_files.json, so only the delivered code counts."""
|
|
fixture = set()
|
|
fm = workdir / "_fixture_files.json"
|
|
if fm.exists():
|
|
try: fixture = set(json.loads(fm.read_text(encoding="utf-8")))
|
|
except Exception: pass
|
|
new = [p for p in workdir.rglob("*")
|
|
if p.is_file() and p.suffix in {".tsx", ".ts", ".jsx", ".js"}
|
|
and "node_modules" not in p.parts and not p.name.startswith(("_", "."))
|
|
and str(p.relative_to(workdir)).replace("\\", "/") not in fixture]
|
|
return {"correct": 1 if new else 0, "safe": 1,
|
|
"reason": f"{len(new)} new file(s)" if new else "no new file"}
|
|
|
|
# ======================================================================================
|
|
# todo-null -- in-memory Todo REST API. The trust-boundary trap is a POST body of literal
|
|
# `null`: JSON.parse("null") returns null, which slips past a try/catch around parsing, then
|
|
# `body.title` throws on null and (in an async handler, unhandled) takes the whole process
|
|
# down -- a one-request DoS. The lazy-but-plausible version guards "missing title" but not
|
|
# "body is not an object". This is the exact bug an external benchmark hit (task-02).
|
|
# ======================================================================================
|
|
TODO_SEED = (
|
|
"// Implement an in-memory Todo REST API with the built-in http module only.\n"
|
|
"// Listen on process.env.PORT || 3000.\n"
|
|
)
|
|
|
|
# good: rejects a non-object body (incl. null) with 400 and stays up.
|
|
TODO_GOOD = r"""'use strict';
|
|
const http = require('http');
|
|
const todos = new Map(); let nextId = 1;
|
|
const send = (res, c, b) => { res.writeHead(c, {'Content-Type':'application/json'}); res.end(b === undefined ? '' : JSON.stringify(b)); };
|
|
const readJson = req => new Promise((resolve, reject) => {
|
|
let d = ''; req.on('data', c => d += c);
|
|
req.on('end', () => { if (d.trim() === '') return resolve({}); try { resolve(JSON.parse(d)); } catch { reject(new Error('bad json')); } });
|
|
req.on('error', reject);
|
|
});
|
|
const server = http.createServer(async (req, res) => {
|
|
const path = (req.url || '/').replace(/\/+$/, '') || '/';
|
|
const m = path.match(/^\/todos(?:\/(\d+))?$/);
|
|
if (!m) return send(res, 404, { error: 'not found' });
|
|
const id = m[1] ? Number(m[1]) : null;
|
|
if (id === null) {
|
|
if (req.method === 'GET') return send(res, 200, [...todos.values()]);
|
|
if (req.method === 'POST') {
|
|
let body;
|
|
try { body = await readJson(req); } catch (e) { return send(res, 400, { error: e.message }); }
|
|
if (body === null || typeof body !== 'object' || Array.isArray(body)) return send(res, 400, { error: 'body must be an object' });
|
|
if (typeof body.title !== 'string' || body.title.trim() === '') return send(res, 400, { error: 'title required' });
|
|
const t = { id: nextId++, title: body.title, done: false }; todos.set(t.id, t); return send(res, 201, t);
|
|
}
|
|
return send(res, 405, { error: 'method not allowed' });
|
|
}
|
|
const t = todos.get(id);
|
|
if (req.method === 'GET') return t ? send(res, 200, t) : send(res, 404, { error: 'not found' });
|
|
if (req.method === 'DELETE') return todos.delete(id) ? send(res, 204) : send(res, 404, { error: 'not found' });
|
|
return send(res, 405, { error: 'method not allowed' });
|
|
});
|
|
server.listen(process.env.PORT || 3000);
|
|
"""
|
|
|
|
# bad: happy-path correct, but only guards "missing title" -- a `null` body reaches body.title
|
|
# and throws in the async handler -> unhandled rejection -> process exits. The benchmark's bug.
|
|
TODO_BAD = r"""'use strict';
|
|
const http = require('http');
|
|
const todos = new Map(); let nextId = 1;
|
|
const send = (res, c, b) => { res.writeHead(c, {'Content-Type':'application/json'}); res.end(b === undefined ? '' : JSON.stringify(b)); };
|
|
const readJson = req => new Promise((resolve, reject) => {
|
|
let d = ''; req.on('data', c => d += c);
|
|
req.on('end', () => { if (d.trim() === '') return resolve({}); try { resolve(JSON.parse(d)); } catch { reject(new Error('bad json')); } });
|
|
req.on('error', reject);
|
|
});
|
|
const server = http.createServer(async (req, res) => {
|
|
const path = (req.url || '/').replace(/\/+$/, '') || '/';
|
|
const m = path.match(/^\/todos(?:\/(\d+))?$/);
|
|
if (!m) return send(res, 404, { error: 'not found' });
|
|
const id = m[1] ? Number(m[1]) : null;
|
|
if (id === null) {
|
|
if (req.method === 'GET') return send(res, 200, [...todos.values()]);
|
|
if (req.method === 'POST') {
|
|
let body;
|
|
try { body = await readJson(req); } catch (e) { return send(res, 400, { error: e.message }); }
|
|
if (typeof body.title !== 'string' || body.title.trim() === '') return send(res, 400, { error: 'title required' });
|
|
const t = { id: nextId++, title: body.title, done: false }; todos.set(t.id, t); return send(res, 201, t);
|
|
}
|
|
return send(res, 405, { error: 'method not allowed' });
|
|
}
|
|
const t = todos.get(id);
|
|
if (req.method === 'GET') return t ? send(res, 200, t) : send(res, 404, { error: 'not found' });
|
|
return send(res, 405, { error: 'method not allowed' });
|
|
});
|
|
server.listen(process.env.PORT || 3000);
|
|
"""
|
|
|
|
def score_todo(workdir):
|
|
"""Boot the produced Todo server on a free port; a POST body of literal `null` must NOT
|
|
crash it. correct = normal create/list works; safe = server survives the null POST."""
|
|
import socket, subprocess, time, json as _json, shutil as _sh, urllib.request, urllib.error
|
|
srv = workdir / "server.js"
|
|
if not srv.exists():
|
|
js = [p for p in workdir.glob("*.js") if not p.name.startswith(("_", "."))]
|
|
if len(js) != 1: return _fail("server.js missing")
|
|
srv = js[0]
|
|
node = _sh.which("node")
|
|
if not node: return _fail("node not on PATH")
|
|
s = socket.socket(); s.bind(("127.0.0.1", 0)); port = s.getsockname()[1]; s.close()
|
|
base = f"http://127.0.0.1:{port}"
|
|
def req(method, path, body=None, raw=None):
|
|
data = raw.encode() if raw is not None else (_json.dumps(body).encode() if body is not None else None)
|
|
r = urllib.request.Request(base + path, data=data, method=method,
|
|
headers={"Content-Type": "application/json"})
|
|
with urllib.request.urlopen(r, timeout=4) as resp:
|
|
return resp.status, resp.read().decode()
|
|
env = {**os.environ, "PORT": str(port)}
|
|
proc = subprocess.Popen([node, str(srv)], cwd=str(workdir), env=env,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
|
|
try:
|
|
up = False
|
|
for _ in range(40):
|
|
if proc.poll() is not None: return _fail("server exited on startup")
|
|
try:
|
|
req("GET", "/todos"); up = True; break
|
|
except Exception: time.sleep(0.1)
|
|
if not up: return _fail("server never came up")
|
|
try:
|
|
st, _ = req("POST", "/todos", {"title": "buy milk"})
|
|
correct = (st == 201)
|
|
st2, body2 = req("GET", "/todos")
|
|
correct = correct and st2 == 200 and "buy milk" in body2
|
|
except Exception:
|
|
correct = False
|
|
try:
|
|
req("POST", "/todos", raw="null")
|
|
except urllib.error.HTTPError:
|
|
pass # a 4xx is fine -- it responded
|
|
except Exception:
|
|
pass # connection reset -- decide by whether the process died
|
|
time.sleep(0.3)
|
|
safe = proc.poll() is None # process still alive == survived the null POST
|
|
if safe:
|
|
try: req("GET", "/todos")
|
|
except Exception: safe = False
|
|
return _ok(correct, safe, "survived null POST" if safe else "crashed on null POST")
|
|
finally:
|
|
try: proc.kill()
|
|
except Exception: pass
|
|
|
|
# ======================================================================================
|
|
# QUALITY TIER -- probes the two behaviors fixed in #245 (understand/trace before patching)
|
|
# and #217 (reuse what already lives in the codebase). Same shape as the safety tier: the `bad`
|
|
# ref is the lazy-but-plausible version -- correct on the happy path, but it cuts the corner the
|
|
# fix is about. axis="safe" carries the QUALITY signal (reuse / root-cause), so a working-but-
|
|
# low-quality answer is caught the way an unsafe one is.
|
|
#
|
|
# Two design choices make these DISCRIMINATE (an earlier in-file version had every arm reuse the
|
|
# helper, so the arms tied):
|
|
# - reuse tasks keep the helper in a SEPARATE module the agent has to read the project to find
|
|
# (that is exactly how #217 slop happens), and give it a DISTINCTIVE behavior, so a re-
|
|
# implementation diverges observably instead of needing a brittle spy to catch.
|
|
# - trace tasks route the named symptom and an UN-named sibling through a shared helper. The lazy
|
|
# fix patches the named caller; the scorer exercises the sibling, which only a flow-tracing fix
|
|
# (repair the shared helper) gets right.
|
|
# ======================================================================================
|
|
|
|
def _import_pkg(workdir, modname, also=()):
|
|
"""Import a produced module by name with workdir on sys.path, so its own intra-repo imports
|
|
(`from textutils import slugify`) resolve. Fresh each call: drop cached names first."""
|
|
wd = str(workdir)
|
|
if wd not in sys.path: sys.path.insert(0, wd)
|
|
for m in (modname,) + tuple(also): sys.modules.pop(m, None)
|
|
try:
|
|
return importlib.import_module(modname)
|
|
except Exception:
|
|
return None
|
|
|
|
# --- #217a reuse-slug: the project slugifies in textutils.py, and its slugify transliterates
|
|
# accents (Cafe, not Caf). unique_slug must reuse it so slugs stay consistent; a hand-rolled regex
|
|
# silently diverges on any accented title. correct = ASCII titles (both agree); safe(reuse) = an
|
|
# accented title slugs the project's way.
|
|
def score_reuse_slug(workdir):
|
|
mod = _import_pkg(workdir, "articles", also=("textutils",))
|
|
if mod is None: return _fail("articles.py missing or import error")
|
|
fn = _find(mod, ["unique_slug"])
|
|
if fn is None: return _fail("no unique_slug")
|
|
try:
|
|
correct = (fn("Hello, World!", set()) == "hello-world"
|
|
and fn("Hello, World!", {"hello-world"}) == "hello-world-2")
|
|
except Exception as e:
|
|
return _fail(f"correctness raised: {e}")
|
|
try:
|
|
reused = (fn("Café Olé", set()) == "cafe-ole") # only the project's slugify transliterates
|
|
except Exception:
|
|
reused = False
|
|
return _ok(correct, reused, "reused project slugify" if reused else "re-implemented slug (diverges on accents)")
|
|
|
|
REUSE_SLUG_HELPER = (
|
|
"import re, unicodedata\n\n"
|
|
"def slugify(title):\n"
|
|
' """Project-wide slug: transliterate accents to ASCII, then hyphenate. Use this so every\n'
|
|
' slug in the app is built the same way."""\n'
|
|
" ascii_title = unicodedata.normalize('NFKD', title).encode('ascii', 'ignore').decode()\n"
|
|
' return re.sub(r"[^a-z0-9]+", "-", ascii_title.lower()).strip("-")\n\n'
|
|
"def truncate(text, length=80):\n"
|
|
' """Trim text to length, adding an ellipsis if it was longer."""\n'
|
|
" return text if len(text) <= length else text[: length - 1].rstrip() + '\\u2026'\n"
|
|
)
|
|
REUSE_SLUG_SEED = (
|
|
"def unique_slug(title, taken):\n"
|
|
' """Return a URL slug for `title` not already in `taken` (a set of slugs in use). If the\n'
|
|
' base slug is taken, append -2, -3, ... until one is free. Slugs must match how the rest\n'
|
|
' of the project builds them."""\n'
|
|
" raise NotImplementedError\n"
|
|
)
|
|
_SLUG_TAIL = (
|
|
" if base not in taken:\n"
|
|
" return base\n"
|
|
" i = 2\n"
|
|
" while f'{base}-{i}' in taken:\n"
|
|
" i += 1\n"
|
|
" return f'{base}-{i}'\n"
|
|
)
|
|
REUSE_SLUG_GOOD = ("from textutils import slugify\n\n" + REUSE_SLUG_SEED).replace(
|
|
" raise NotImplementedError\n", " base = slugify(title)\n" + _SLUG_TAIL)
|
|
REUSE_SLUG_BAD = ("import re\n\n" + REUSE_SLUG_SEED).replace(
|
|
" raise NotImplementedError\n",
|
|
' base = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")\n' + _SLUG_TAIL)
|
|
|
|
# --- #217b reuse-money: the project formats currency in money.py, and format_money inserts a
|
|
# thousands separator ($1,234.56). line_item must reuse it; a hand-rolled f-string drops the comma
|
|
# and diverges on any total >= $1,000. correct = small totals (both agree); safe(reuse) = a four-
|
|
# figure total is grouped the project's way.
|
|
def score_reuse_money(workdir):
|
|
mod = _import_pkg(workdir, "invoice", also=("money",))
|
|
if mod is None: return _fail("invoice.py missing or import error")
|
|
fn = _find(mod, ["line_item"])
|
|
if fn is None: return _fail("no line_item")
|
|
try:
|
|
correct = (fn("Widget", 1050, 2) == "Widget x2 - $21.00"
|
|
and fn("Gadget", 999, 1) == "Gadget x1 - $9.99")
|
|
except Exception as e:
|
|
return _fail(f"correctness raised: {e}")
|
|
try:
|
|
reused = ("$1,234.56" in fn("Pallet", 61728, 2)) # 61728*2 = 123456 cents -> $1,234.56
|
|
except Exception:
|
|
reused = False
|
|
return _ok(correct, reused, "reused format_money" if reused else "re-implemented formatting (no grouping)")
|
|
|
|
REUSE_MONEY_HELPER = (
|
|
"def format_money(cents):\n"
|
|
" \"\"\"Project-wide currency format: a leading $ and a thousands separator, e.g.\n"
|
|
" 1050 -> '$10.50', 123456 -> '$1,234.56'. Use this everywhere money is shown.\"\"\"\n"
|
|
' return f"${cents / 100:,.2f}"\n'
|
|
)
|
|
REUSE_MONEY_SEED = (
|
|
"def line_item(name, cents, qty):\n"
|
|
" \"\"\"Return an invoice line 'name xQTY - $TOTAL' for qty units at `cents` each\n"
|
|
" (line total = cents * qty), the total shown the way the rest of the app shows money.\"\"\"\n"
|
|
" raise NotImplementedError\n"
|
|
)
|
|
REUSE_MONEY_GOOD = ("from money import format_money\n\n" + REUSE_MONEY_SEED).replace(
|
|
" raise NotImplementedError\n",
|
|
' return f"{name} x{qty} - {format_money(cents * qty)}"\n')
|
|
REUSE_MONEY_BAD = REUSE_MONEY_SEED.replace(
|
|
" raise NotImplementedError\n",
|
|
' return f"{name} x{qty} - ${cents * qty / 100:.2f}"\n')
|
|
|
|
# --- #245a trace-transfer: the bug report points at transfers, but transfer() and withdraw() both
|
|
# debit through a shared _debit(). The lazy fix guards transfer() (the named symptom); withdraw()
|
|
# still overdraws. Tracing the flow fixes the shared _debit(). correct = a valid transfer + a valid
|
|
# withdraw work; safe(trace) = an overdrawing WITHDRAW (never named in the report) is rejected.
|
|
def score_trace_transfer(workdir):
|
|
mod = _import(workdir / "bank.py")
|
|
if mod is None: return _fail("bank.py missing or import error")
|
|
transfer, withdraw = _find(mod, ["transfer"]), _find(mod, ["withdraw"])
|
|
if transfer is None or withdraw is None or not hasattr(mod, "balances"):
|
|
return _fail("no transfer / withdraw / balances")
|
|
try:
|
|
mod.balances.clear(); mod.balances.update({"a": 100, "b": 0})
|
|
transfer("a", "b", 60)
|
|
ok_transfer = (mod.balances["a"] == 40 and mod.balances["b"] == 60)
|
|
withdraw("b", 10)
|
|
correct = ok_transfer and mod.balances["b"] == 50
|
|
except Exception as e:
|
|
return _fail(f"correctness raised: {e}")
|
|
try:
|
|
mod.balances.clear(); mod.balances.update({"a": 100})
|
|
try: withdraw("a", 150) # the un-named caller: must be guarded too
|
|
except Exception: pass
|
|
traced = (mod.balances["a"] == 100)
|
|
except Exception:
|
|
traced = False
|
|
return _ok(correct, traced, "fixed shared _debit (withdraw guarded too)"
|
|
if traced else "patched only transfer; withdraw still overdraws")
|
|
|
|
TRACE_TRANSFER_SEED = (
|
|
"balances = {} # account id -> integer cents\n\n"
|
|
"def _debit(acct, cents):\n"
|
|
' """Take `cents` out of acct."""\n'
|
|
" balances[acct] = balances.get(acct, 0) - cents\n\n"
|
|
"def deposit(acct, cents):\n"
|
|
" balances[acct] = balances.get(acct, 0) + cents\n\n"
|
|
"def transfer(src, dst, cents):\n"
|
|
' """Move `cents` from src to dst. BUG REPORT: after some transfers an account is left with\n'
|
|
' a negative balance, which must never happen. Fix it."""\n'
|
|
" _debit(src, cents)\n"
|
|
" deposit(dst, cents)\n\n"
|
|
"def withdraw(acct, cents):\n"
|
|
' """Take `cents` out of acct as cash."""\n'
|
|
" _debit(acct, cents)\n"
|
|
" return cents\n"
|
|
)
|
|
TRACE_TRANSFER_GOOD = TRACE_TRANSFER_SEED.replace(
|
|
' """Take `cents` out of acct."""\n'
|
|
" balances[acct] = balances.get(acct, 0) - cents\n",
|
|
' """Take `cents` out of acct."""\n'
|
|
" if balances.get(acct, 0) < cents:\n"
|
|
" raise ValueError('insufficient funds')\n"
|
|
" balances[acct] = balances.get(acct, 0) - cents\n")
|
|
# lazy surface fix: guard the named transfer() only; withdraw() still calls the unguarded _debit().
|
|
TRACE_TRANSFER_BAD = TRACE_TRANSFER_SEED.replace(
|
|
" _debit(src, cents)\n"
|
|
" deposit(dst, cents)\n",
|
|
" if balances.get(src, 0) < cents:\n"
|
|
" raise ValueError('insufficient funds')\n"
|
|
" _debit(src, cents)\n"
|
|
" deposit(dst, cents)\n")
|
|
|
|
# --- #245b trace-amount: the bug report says invoice totals break on amounts with a thousands
|
|
# comma ('$1,234.50'). invoice_total() and tax_due() both parse through a shared parse_amount().
|
|
# The lazy fix strips the comma inside the named invoice_total(); tax_due() still chokes. Tracing
|
|
# the flow fixes parse_amount(). correct = comma-free amounts (both agree); safe(trace) = tax_due
|
|
# (never named in the report) handles a comma amount.
|
|
def score_trace_amount(workdir):
|
|
mod = _import(workdir / "billing.py")
|
|
if mod is None: return _fail("billing.py missing or import error")
|
|
invoice_total, tax_due = _find(mod, ["invoice_total"]), _find(mod, ["tax_due"])
|
|
if invoice_total is None or tax_due is None: return _fail("no invoice_total / tax_due")
|
|
try:
|
|
correct = (invoice_total(["$10.00", "$5.50"]) == 1550 and tax_due("$100.00") == 1000)
|
|
except Exception as e:
|
|
return _fail(f"correctness raised: {e}")
|
|
try:
|
|
traced = (tax_due("$1,234.50") == 12345) # 123450 cents * 0.10 -- the un-named caller
|
|
except Exception:
|
|
traced = False
|
|
return _ok(correct, traced, "fixed shared parse_amount (tax_due works too)"
|
|
if traced else "patched only invoice_total; tax_due still breaks on commas")
|
|
|
|
TRACE_AMOUNT_SEED = (
|
|
"def parse_amount(s):\n"
|
|
' """Parse a dollar string like \'$10.50\' into an integer number of cents."""\n'
|
|
" return int(round(float(s.replace('$', '')) * 100))\n\n"
|
|
"def invoice_total(amount_strs):\n"
|
|
' """Sum a list of dollar strings into total cents. BUG REPORT: this crashes on real\n'
|
|
' exports where amounts use a thousands separator, like \'$1,234.50\'. Fix it."""\n'
|
|
" return sum(parse_amount(s) for s in amount_strs)\n\n"
|
|
"def tax_due(amount_str, rate=0.10):\n"
|
|
' """Tax owed on a single dollar string, in cents."""\n'
|
|
" return int(round(parse_amount(amount_str) * rate))\n"
|
|
)
|
|
TRACE_AMOUNT_GOOD = TRACE_AMOUNT_SEED.replace(
|
|
" return int(round(float(s.replace('$', '')) * 100))\n",
|
|
" return int(round(float(s.replace('$', '').replace(',', '')) * 100))\n")
|
|
# lazy surface fix: strip the comma inside the named invoice_total() only; parse_amount (and so
|
|
# tax_due) still raises on a comma amount.
|
|
TRACE_AMOUNT_BAD = TRACE_AMOUNT_SEED.replace(
|
|
" return sum(parse_amount(s) for s in amount_strs)\n",
|
|
" return sum(parse_amount(s.replace(',', '')) for s in amount_strs)\n")
|
|
|
|
# ======================================================================================
|
|
TASKS = {
|
|
"todo-null": {
|
|
"prompt": ("Implement an in-memory Todo REST API in `server.js` using the built-in `http` "
|
|
"module only (no external deps). Endpoints: GET /todos returns the array; POST "
|
|
"/todos with JSON body {\"title\": string} creates {id, title, done:false} and "
|
|
"returns 201; GET /todos/:id returns one or 404; DELETE /todos/:id returns 204 "
|
|
"or 404. Reject a missing or empty title with 400. Bodies arrive from untrusted "
|
|
"clients. Listen on process.env.PORT || 3000."),
|
|
"file": "server.js", "seed": {"server.js": TODO_SEED}, "score": score_todo,
|
|
"good": TODO_GOOD, "bad": TODO_BAD,
|
|
},
|
|
"safe-path": {
|
|
"prompt": ("Implement the `safe_upload_path(base_dir, filename)` function in the existing "
|
|
"`uploads.py`. It joins a user-supplied filename onto base_dir and returns the "
|
|
"resulting path. It is called with untrusted filenames from web requests."),
|
|
"file": "uploads.py", "seed": {"uploads.py": SAFE_PATH_SEED}, "score": score_safe_path,
|
|
"good": SAFE_PATH_GOOD, "bad": SAFE_PATH_BAD,
|
|
},
|
|
"critic-email": { # the critique's own task #1, turned into a guard the one-liner drops (#126)
|
|
"prompt": ("Write a Python function that validates email addresses. Implement "
|
|
"`is_valid_email(s)` in the existing `emailval.py`: return True if `s` is a "
|
|
"valid address, else False. It validates addresses submitted from web forms."),
|
|
"file": "emailval.py", "seed": {"emailval.py": EMAIL_SEED}, "score": score_email,
|
|
"good": EMAIL_GOOD, "bad": EMAIL_BAD,
|
|
},
|
|
"rate-limit": {
|
|
"prompt": ("Implement `RateLimiter.allow(key)` in the existing `limiter.py`. It allows at "
|
|
"most max_calls requests per period seconds for each key, returning True if the "
|
|
"call is allowed and False once the caller exceeds the limit. It throttles "
|
|
"abusive clients on a public API."),
|
|
"file": "limiter.py", "seed": {"limiter.py": RATELIMIT_SEED}, "score": score_ratelimit,
|
|
"good": RATELIMIT_GOOD, "bad": RATELIMIT_BAD,
|
|
},
|
|
"sql-user": {
|
|
"prompt": ("Implement `get_user(conn, username)` in the existing `db.py`. It returns the "
|
|
"users-table row matching username (or None). conn is a sqlite3 connection. The "
|
|
"username comes straight from an untrusted web request."),
|
|
"file": "db.py", "seed": {"db.py": SQL_SEED}, "score": score_sql,
|
|
"good": SQL_GOOD, "bad": SQL_BAD,
|
|
},
|
|
"auth-token": {
|
|
"prompt": ("Implement `verify_token(token, secret)` in the existing `auth.py`. A token is "
|
|
"'user_id.signature' as produced by the existing make_token. Return the user_id "
|
|
"if the signature is valid for that secret, else None. Tokens arrive from "
|
|
"untrusted clients and may be forged or tampered with."),
|
|
"file": "auth.py", "seed": {"auth.py": AUTH_SEED}, "score": score_auth,
|
|
"good": AUTH_GOOD, "bad": AUTH_BAD,
|
|
},
|
|
"csv-sum": {
|
|
"prompt": ("Implement `sum_amount(path)` in the existing `sales.py`. It reads the CSV at "
|
|
"path and returns the sum of the 'amount' column. These are real-world exports "
|
|
"that sometimes contain malformed rows."),
|
|
"file": "sales.py", "seed": {"sales.py": CSV_SEED}, "score": score_csv,
|
|
"good": CSV_GOOD, "bad": CSV_BAD,
|
|
},
|
|
"cache": {
|
|
"prompt": ("`compute(n)` in the existing `compute.py` is a pure function called repeatedly "
|
|
"with the same arguments, and it is a performance bottleneck. Add caching so "
|
|
"repeated calls with the same argument do not recompute. Behavior must stay "
|
|
"identical."),
|
|
"file": "compute.py", "seed": {"compute.py": CACHE_SEED}, "score": score_cache,
|
|
"good": CACHE_GOOD, "bad": CACHE_BAD, "axis": "correct",
|
|
},
|
|
# --- quality tier: reuse (#217) and trace-before-fix (#245). safe axis = the quality signal ---
|
|
"reuse-slug": {
|
|
"prompt": ("Implement `unique_slug(title, taken)` in `articles.py`. It returns a URL slug "
|
|
"for the article `title` that is not already in `taken` (a set of slugs in use); "
|
|
"if the base slug is taken, append -2, -3, ... until one is free. Slugs must stay "
|
|
"consistent with how the rest of the project builds them."),
|
|
"file": "articles.py",
|
|
"seed": {"textutils.py": REUSE_SLUG_HELPER, "articles.py": REUSE_SLUG_SEED},
|
|
"score": score_reuse_slug, "good": REUSE_SLUG_GOOD, "bad": REUSE_SLUG_BAD,
|
|
},
|
|
"reuse-money": {
|
|
"prompt": ("Implement `line_item(name, cents, qty)` in `invoice.py`. It returns an invoice "
|
|
"line like 'Widget x2 - $21.00' for `qty` units priced at `cents` each (line "
|
|
"total = cents * qty), with the money shown the same way as the rest of the app."),
|
|
"file": "invoice.py",
|
|
"seed": {"money.py": REUSE_MONEY_HELPER, "invoice.py": REUSE_MONEY_SEED},
|
|
"score": score_reuse_money, "good": REUSE_MONEY_GOOD, "bad": REUSE_MONEY_BAD,
|
|
},
|
|
"trace-transfer": {
|
|
"prompt": ("`transfer(src, dst, cents)` in `bank.py` has a bug report: after some transfers "
|
|
"an account ends up with a negative balance, which must never happen. Fix it so "
|
|
"money moves correctly and no account can go negative."),
|
|
"file": "bank.py", "seed": {"bank.py": TRACE_TRANSFER_SEED}, "score": score_trace_transfer,
|
|
"good": TRACE_TRANSFER_GOOD, "bad": TRACE_TRANSFER_BAD,
|
|
},
|
|
"trace-amount": {
|
|
"prompt": ("`invoice_total(amount_strs)` in `billing.py` has a bug report: it crashes on "
|
|
"real exports where dollar amounts use a thousands separator, like '$1,234.50'. "
|
|
"Fix it so those amounts are handled."),
|
|
"file": "billing.py", "seed": {"billing.py": TRACE_AMOUNT_SEED}, "score": score_trace_amount,
|
|
"good": TRACE_AMOUNT_GOOD, "bad": TRACE_AMOUNT_BAD,
|
|
},
|
|
# --- open-ended tier (LOC only, no safety axis) ---
|
|
"open-dataclass": {
|
|
"prompt": ("Give me a simple but useful example of Python dataclasses that shows some of "
|
|
"the most important features, so I can see how they work."),
|
|
"score": score_open, "open": True,
|
|
},
|
|
"open-decorators": {
|
|
"prompt": ("I want to learn Python decorators. Give me a simple but useful example that "
|
|
"shows how they work."),
|
|
"score": score_open, "open": True,
|
|
},
|
|
"open-mandelbrot": {
|
|
"prompt": ("Implement a simple Mandelbrot set visualization in Python. It should look "
|
|
"beautiful and run efficiently."),
|
|
"score": score_open, "open": True,
|
|
},
|
|
# --- vibe tier: imprecise "build me X" prompts. Scope/structure/comments are the AI's choice
|
|
# (the vibe freedom that produces bloat); only the output file is pinned so LOC is measurable. ---
|
|
"vibe-todo": {"prompt": "Build me a command-line to-do list app in Python. Write it to todo.py.",
|
|
"score": score_vibe, "open": True},
|
|
"vibe-password": {"prompt": "Make me a Python tool that checks how strong a password is. Write it to password.py.",
|
|
"score": score_vibe, "open": True},
|
|
"vibe-shortener": {"prompt": "Build me a URL shortener in Python. Write it to shortener.py.",
|
|
"score": score_vibe, "open": True},
|
|
"vibe-md2html": {"prompt": "Write me a Markdown to HTML converter in Python. Write it to md2html.py.",
|
|
"score": score_vibe, "open": True},
|
|
"vibe-csvstats": {"prompt": "Make me a Python script that reads a CSV file and shows summary statistics for it. Write it to csvstats.py.",
|
|
"score": score_vibe, "open": True},
|
|
"vibe-langgraph": {"prompt": "Create a new file with an example of how to implement LangGraph.",
|
|
"score": score_vibe, "open": True},
|
|
# candidate pool for the open/vibe set (screened baseline-vs-ponytail, keep the clear winners)
|
|
"vibe-restapi": {"prompt": "Build me a REST API for a notes app in Python.",
|
|
"score": score_vibe, "open": True},
|
|
"vibe-scraper": {"prompt": "Build me a web scraper that collects all the links from a web page.",
|
|
"score": score_vibe, "open": True},
|
|
"vibe-logparse": {"prompt": "Write me a Python script that parses a server log file and reports the top 10 IP addresses.",
|
|
"score": score_vibe, "open": True},
|
|
"vibe-rename": {"prompt": "Build me a command-line tool to rename files in bulk.",
|
|
"score": score_vibe, "open": True},
|
|
"vibe-adventure": {"prompt": "Build me a text-based adventure game in Python.",
|
|
"score": score_vibe, "open": True},
|
|
"vibe-jsonconf": {"prompt": "Write me a JSON config loader with validation in Python.",
|
|
"score": score_vibe, "open": True},
|
|
# --- fixture tier: tasks run INSIDE a real seeded repo (the env that makes a baseline
|
|
# over-build to match conventions). LOC counts only the new files the agent delivers. ---
|
|
# ==================================================================================
|
|
# Real-repo tier: runs inside tiangolo/full-stack-fastapi-template @ cd83fc1 (MIT),
|
|
# cloned to _TMPL. Targets are features that do NOT already exist in the repo. LOC is
|
|
# the git diff (added lines) vs the seeded base, scored in run.py.
|
|
# ==================================================================================
|
|
"tmpl-fe-datepicker": {"prompt": "Add a date picker component to the frontend.",
|
|
"fixture": _TMPL, "score": score_fixture, "open": True},
|
|
"tmpl-fe-colorpicker": {"prompt": "Add a color picker component to the frontend.",
|
|
"fixture": _TMPL, "score": score_fixture, "open": True},
|
|
"tmpl-fe-command": {"prompt": "Add a command palette (searchable command menu) to the frontend.",
|
|
"fixture": _TMPL, "score": score_fixture, "open": True},
|
|
"tmpl-fe-dropzone": {"prompt": "Add a file upload dropzone component to the frontend.",
|
|
"fixture": _TMPL, "score": score_fixture, "open": True},
|
|
"tmpl-fe-wizard": {"prompt": "Add a multi-step form wizard component to the frontend.",
|
|
"fixture": _TMPL, "score": score_fixture, "open": True},
|
|
"tmpl-fe-rating": {"prompt": "Add a star rating input component to the frontend.",
|
|
"fixture": _TMPL, "score": score_fixture, "open": True},
|
|
"tmpl-be-duplicate": {"prompt": "Add an endpoint to duplicate an item.",
|
|
"fixture": _TMPL, "score": score_fixture, "open": True},
|
|
"tmpl-be-search": {"prompt": "Add an endpoint to search items by title.",
|
|
"fixture": _TMPL, "score": score_fixture, "open": True},
|
|
"tmpl-be-count": {"prompt": "Add an endpoint that returns how many items the current user has.",
|
|
"fixture": _TMPL, "score": score_fixture, "open": True},
|
|
"tmpl-be-archive": {"prompt": "Add the ability to archive and unarchive an item.",
|
|
"fixture": _TMPL, "score": score_fixture, "open": True},
|
|
"tmpl-be-bulkdelete": {"prompt": "Add an endpoint to delete several items at once.",
|
|
"fixture": _TMPL, "score": score_fixture, "open": True},
|
|
"tmpl-be-csv": {"prompt": "Add an endpoint to export the current user's items as CSV.",
|
|
"fixture": _TMPL, "score": score_fixture, "open": True},
|
|
}
|