wehub-resource-sync
278 lines
9.1 KiB
Python
278 lines
9.1 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
"""LiveCodeBench benchmark.
|
|
|
|
Tests code generation ability using competitive programming problems.
|
|
Generates code, executes it in a sandboxed subprocess, and checks output.
|
|
Dataset bundled from livecodebench/code_generation_lite on HuggingFace.
|
|
|
|
SECURITY NOTE: This benchmark executes model-generated code on the local
|
|
machine. Mitigations: subprocess with timeout, memory limits via resource
|
|
module, temp file cleanup. Users are warned in the UI before running.
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import resource
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Optional
|
|
|
|
from .base import BaseBenchmark, BenchmarkResult, QuestionResult
|
|
from .datasets import deterministic_sample, load_jsonl
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DATA_DIR = Path(__file__).parent / "data"
|
|
|
|
# Execution limits
|
|
EXEC_TIMEOUT_SECONDS = 30
|
|
EXEC_MEMORY_LIMIT_BYTES = 256 * 1024 * 1024 # 256 MB
|
|
|
|
|
|
def _extract_code(response: str) -> str:
|
|
"""Extract Python code from model response.
|
|
|
|
Looks for ```python...``` blocks first, then ```...``` blocks,
|
|
then falls back to the entire response.
|
|
"""
|
|
match = re.search(r"```python\s*\n(.*?)```", response, re.DOTALL)
|
|
if match:
|
|
return match.group(1).strip()
|
|
|
|
match = re.search(r"```\s*\n(.*?)```", response, re.DOTALL)
|
|
if match:
|
|
return match.group(1).strip()
|
|
|
|
lines = response.strip().split("\n")
|
|
code_lines = []
|
|
in_code = False
|
|
for line in lines:
|
|
if not in_code and (
|
|
line.startswith("def ")
|
|
or line.startswith("class ")
|
|
or line.startswith("import ")
|
|
or line.startswith("from ")
|
|
or line.startswith("#")
|
|
):
|
|
in_code = True
|
|
if in_code:
|
|
code_lines.append(line)
|
|
|
|
return "\n".join(code_lines) if code_lines else response.strip()
|
|
|
|
|
|
def _set_resource_limits():
|
|
"""Set resource limits for subprocess. Called via preexec_fn."""
|
|
try:
|
|
resource.setrlimit(resource.RLIMIT_AS, (EXEC_MEMORY_LIMIT_BYTES, EXEC_MEMORY_LIMIT_BYTES))
|
|
except (ValueError, resource.error):
|
|
pass
|
|
try:
|
|
resource.setrlimit(resource.RLIMIT_CPU, (EXEC_TIMEOUT_SECONDS + 5, EXEC_TIMEOUT_SECONDS + 5))
|
|
except (ValueError, resource.error):
|
|
pass
|
|
|
|
|
|
def _execute_code(code: str, stdin_input: str = "") -> tuple[str, bool, str]:
|
|
"""Execute Python code in a subprocess with safety limits.
|
|
|
|
Returns:
|
|
(stdout, success, error_message)
|
|
"""
|
|
with tempfile.NamedTemporaryFile(
|
|
mode="w", suffix=".py", delete=False
|
|
) as f:
|
|
f.write(code)
|
|
tmp_path = f.name
|
|
|
|
try:
|
|
result = subprocess.run(
|
|
["python3", tmp_path],
|
|
input=stdin_input,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=EXEC_TIMEOUT_SECONDS,
|
|
preexec_fn=_set_resource_limits,
|
|
env={
|
|
"PATH": os.environ.get("PATH", "/usr/bin:/usr/local/bin"),
|
|
"HOME": os.environ.get("HOME", "/tmp"),
|
|
"LANG": "en_US.UTF-8",
|
|
},
|
|
)
|
|
if result.returncode == 0:
|
|
return result.stdout, True, ""
|
|
else:
|
|
return result.stdout, False, result.stderr[:500]
|
|
except subprocess.TimeoutExpired:
|
|
return "", False, "Execution timed out"
|
|
except Exception as e:
|
|
return "", False, str(e)[:500]
|
|
finally:
|
|
try:
|
|
os.unlink(tmp_path)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
class LiveCodeBenchBenchmark(BaseBenchmark):
|
|
"""LiveCodeBench: code generation with sandboxed execution."""
|
|
|
|
name = "livecodebench"
|
|
quick_size = 100
|
|
|
|
async def load_dataset(self, sample_size: int = 0) -> list[dict]:
|
|
"""Load LiveCodeBench from bundled data."""
|
|
items = load_jsonl(DATA_DIR / "livecodebench.jsonl")
|
|
|
|
normalized = []
|
|
for i, item in enumerate(items):
|
|
test_cases_str = item.get("public_test_cases", "[]")
|
|
if isinstance(test_cases_str, str):
|
|
try:
|
|
test_cases = json.loads(test_cases_str)
|
|
except (json.JSONDecodeError, TypeError):
|
|
test_cases = []
|
|
else:
|
|
test_cases = test_cases_str
|
|
|
|
if not isinstance(test_cases, list) or not test_cases:
|
|
continue
|
|
|
|
inputs = [tc.get("input", "") for tc in test_cases]
|
|
outputs = [tc.get("output", "") for tc in test_cases]
|
|
|
|
if not inputs or not outputs:
|
|
continue
|
|
|
|
normalized.append({
|
|
"id": item.get("question_id", str(i)),
|
|
"title": item.get("question_title", f"Problem {i}"),
|
|
"description": item.get("question_content", ""),
|
|
"inputs": inputs,
|
|
"outputs": outputs,
|
|
"difficulty": item.get("difficulty", ""),
|
|
"starter_code": item.get("starter_code", ""),
|
|
})
|
|
|
|
logger.info(f"LiveCodeBench: loaded {len(normalized)} problems")
|
|
|
|
if sample_size == 0:
|
|
return normalized
|
|
|
|
return deterministic_sample(normalized, sample_size)
|
|
|
|
def get_max_tokens(self) -> int:
|
|
return 16384
|
|
|
|
def format_prompt(self, item: dict) -> list[dict[str, str]]:
|
|
"""Format as a coding problem prompt."""
|
|
description = item["description"]
|
|
prompt = (
|
|
"Solve the following programming problem in Python. "
|
|
"Read input from stdin and print the output to stdout. "
|
|
"Provide only the complete Python code, no explanations.\n\n"
|
|
f"Problem:\n{description}\n\n"
|
|
"Solution:"
|
|
)
|
|
return [{"role": "user", "content": prompt}]
|
|
|
|
def extract_answer(self, response: str, item: dict) -> str:
|
|
"""Extract code from the response (last code block to skip drafts)."""
|
|
return self._extract_last_code_block(response)
|
|
|
|
def check_answer(self, predicted: str, item: dict) -> bool:
|
|
"""Execute code and check against test cases.
|
|
|
|
Runs the first 3 test cases to keep execution time reasonable.
|
|
"""
|
|
if not predicted.strip():
|
|
return False
|
|
|
|
inputs = item["inputs"][:3]
|
|
outputs = item["outputs"][:3]
|
|
|
|
for inp, expected_out in zip(inputs, outputs):
|
|
stdin_input = inp if isinstance(inp, str) else str(inp)
|
|
expected = expected_out.strip() if isinstance(expected_out, str) else str(expected_out).strip()
|
|
|
|
stdout, success, error = _execute_code(predicted, stdin_input)
|
|
if not success:
|
|
return False
|
|
|
|
actual = stdout.strip()
|
|
if actual != expected:
|
|
return False
|
|
|
|
return True
|
|
|
|
async def run(
|
|
self,
|
|
engine: Any,
|
|
items: list[dict],
|
|
on_progress: Optional[Callable[[int, int], Any]] = None,
|
|
batch_size: int = 1,
|
|
sampling_kwargs: Optional[dict] = None,
|
|
enable_thinking: bool = False,
|
|
) -> BenchmarkResult:
|
|
"""Override run: generation is batched, code execution is sequential."""
|
|
results: list[QuestionResult] = []
|
|
correct = 0
|
|
start_time = time.time()
|
|
completed = 0
|
|
|
|
for batch_start in range(0, len(items), batch_size):
|
|
batch_end = min(batch_start + batch_size, len(items))
|
|
batch = items[batch_start:batch_end]
|
|
batch_time = time.time()
|
|
|
|
# Batch the generation phase
|
|
gen_tasks = [
|
|
self._eval_single(engine, item, batch_start + j, sampling_kwargs, enable_thinking)
|
|
for j, item in enumerate(batch)
|
|
]
|
|
gen_results = await asyncio.gather(*gen_tasks)
|
|
gen_elapsed = time.time() - batch_time
|
|
|
|
# Code execution is sequential (subprocess safety)
|
|
for idx, item, response_text, prompt_text, _raw in sorted(gen_results, key=lambda x: x[0]):
|
|
code = self.extract_answer(response_text, item)
|
|
is_correct = self.check_answer(code, item)
|
|
|
|
if is_correct:
|
|
correct += 1
|
|
|
|
results.append(
|
|
QuestionResult(
|
|
question_id=str(item.get("id", idx)),
|
|
correct=is_correct,
|
|
expected="(test cases)",
|
|
predicted=code[:200] + "..." if len(code) > 200 else code,
|
|
time_seconds=gen_elapsed / len(batch),
|
|
question_text=prompt_text,
|
|
raw_response=response_text,
|
|
category=self.get_category(item),
|
|
)
|
|
)
|
|
|
|
completed += len(batch)
|
|
if on_progress:
|
|
await on_progress(completed, len(items))
|
|
|
|
total_time = time.time() - start_time
|
|
total = len(items)
|
|
|
|
return BenchmarkResult(
|
|
benchmark_name=self.name,
|
|
accuracy=correct / total if total > 0 else 0.0,
|
|
total_questions=total,
|
|
correct_count=correct,
|
|
time_seconds=total_time,
|
|
question_results=results,
|
|
thinking_used=enable_thinking,
|
|
)
|