promptfoo--promptfoo/examples/integration-e2b/validate_and_run_code_e2b.py

# validate_and_run_code_e2b.py
import json
import logging
import re
import time

from e2b_code_interpreter import Sandbox
from metrics import write_metrics

# Robust fenced code regex
FENCE_RE = re.compile(r"```(?:\s*python)?\s*\r?\n(.*?)```", re.DOTALL | re.IGNORECASE)

# Unsafe patterns: pre-exec static scanner
UNSAFE_PATTERNS = [
    r"\bimport\s+socket\b",
    r"\bimport\s+requests\b",
    r"\bimport\s+urllib\b",
    r"\bos\.system\b",
    r"\bsubprocess\b",
    r"\beval\s*\(",
    r"\bexec\s*\(",
    r"open\(\s*['\"]\/etc",
    r"open\(\s*['\"]\/proc",
    r"__import__\(",
]

logger = logging.getLogger(__name__)


def is_unsafe(code: str) -> bool:
    """Return True when generated code contains blocked imports or APIs."""
    for p in UNSAFE_PATTERNS:
        if re.search(p, code):
            return True
    return False


def _extract_function(output: str, fn_name: str) -> str | None:
    """Extract the generated function from fenced code or plain text output."""
    m = FENCE_RE.search(output)
    if m:
        return m.group(1).strip()

    by_name = re.search(
        rf"(def\s+{re.escape(fn_name)}\s*\(.*?\)\s*:[\s\S]*?)(?=\n\s*\n|^```|^class\s+|^def\s+)",
        output,
        re.IGNORECASE | re.MULTILINE,
    )
    if by_name:
        return by_name.group(1).strip()

    any_def = re.search(r"(def\s+\w+\s*\(.*?\)\s*:[\s\S]*)", output)
    if any_def:
        return any_def.group(1).strip()
    return None


def _try_parse_logs_obj(logs_obj):
    """Try multiple ways to extract stdout/stderr from logs object or string."""
    if isinstance(logs_obj, (dict, list)):
        if isinstance(logs_obj, dict) and "stdout" in logs_obj:
            out = logs_obj.get("stdout")
            err = logs_obj.get("stderr", "")
            if isinstance(out, list):
                out = "".join(map(str, out))
            if isinstance(err, list):
                err = "".join(map(str, err))
            return str(out).strip(), str(err).strip()
        if isinstance(logs_obj, list):
            outs, errs = [], []
            for it in logs_obj:
                if isinstance(it, dict):
                    if it.get("stdout"):
                        outs.append(it.get("stdout"))
                    if it.get("stderr"):
                        errs.append(it.get("stderr"))
            if outs or errs:
                outs = "".join(map(str, outs))
                errs = "".join(map(str, errs))
                return str(outs).strip(), str(errs).strip()

    if hasattr(logs_obj, "to_json"):
        try:
            j = logs_obj.to_json()
            return _try_parse_logs_obj(j)
        except Exception:
            logger.debug("Failed to parse sandbox logs via to_json()", exc_info=True)

    if hasattr(logs_obj, "stdout") or hasattr(logs_obj, "stderr"):
        try:
            out = getattr(logs_obj, "stdout", "")
            err = getattr(logs_obj, "stderr", "")
            if isinstance(out, list):
                out = "".join(map(str, out))
            if isinstance(err, list):
                err = "".join(map(str, err))
            return str(out).strip(), str(err).strip()
        except Exception:
            logger.debug("Failed to parse sandbox log stream fields", exc_info=True)

    if isinstance(logs_obj, str):
        try:
            parsed = json.loads(logs_obj)
            return _try_parse_logs_obj(parsed)
        except Exception:
            logger.debug("Failed to decode sandbox logs as JSON", exc_info=True)
            m = re.search(r"stdout:\s*\[([^\]]*)\]", logs_obj)
            if m:
                inner = m.group(1).strip()
                items = re.findall(r"'(.*?)'|\"(.*?)\"", inner)
                strs = []
                for a, b in items:
                    if a:
                        strs.append(a)
                    elif b:
                        strs.append(b)
                out = "".join(strs)
                return out.strip(), ""
    return "", ""


def _stdout_from_result(res) -> tuple[str, str]:
    """Return stdout and stderr strings from an E2B execution result."""
    if hasattr(res, "results") and res.results:
        try:
            first = res.results[0]
            if isinstance(first, dict):
                out = (
                    first.get("stdout")
                    or first.get("output")
                    or first.get("text")
                    or ""
                )
                err = first.get("stderr") or ""
                if isinstance(out, list):
                    out = "".join(map(str, out))
                if isinstance(err, list):
                    err = "".join(map(str, err))
                if out or err:
                    return str(out).strip(), str(err).strip()
        except Exception:
            logger.debug("Failed to parse sandbox result entries", exc_info=True)

    logs_field = getattr(res, "logs", None)
    if logs_field is not None:
        out, err = _try_parse_logs_obj(logs_field)
        if out or err:
            return out, err

    if getattr(res, "text", None):
        return str(res.text).strip(), ""

    if hasattr(res, "to_json"):
        try:
            j = res.to_json()
            return _try_parse_logs_obj(j)
        except Exception:
            logger.debug(
                "Failed to serialize sandbox result via to_json()", exc_info=True
            )

    return "", ""


def _run_code_in_sandbox(sbx, code: str):
    """Try a few run_code signatures safely (SDKs vary)."""
    # Preferred: named args (limits, disable network)
    try:
        return sbx.run_code(
            code=code,
            language="python",
            limits={"cputime": 1, "wall_time": 5, "memory": 128},
            allow_network=False,
        )
    except TypeError:
        pass
    except Exception:
        logger.debug("Preferred sandbox run_code signature failed", exc_info=True)
    # Try alternate signatures
    try:
        return sbx.run_code(
            code, "python", {"cputime": 1, "wall_time": 5, "memory": 128}
        )
    except Exception:
        logger.debug("Positional sandbox run_code signature failed", exc_info=True)
    # Last resort
    return sbx.run_code(code)


def get_assert(output, context):
    """Execute the generated function in E2B and compare stdout with expected output."""
    task_id = context.get("id", str(time.time()))
    provider = context.get("provider", "unknown")
    model = context.get("model", "unknown")

    fn_name = context["vars"]["function_name"]
    test_input = context["vars"]["test_input"]
    expected = str(context["vars"]["expected_output"])

    function_code = _extract_function(output, fn_name)
    if not function_code:
        snippet = output.strip()[:300].replace("\n", " ")
        write_metrics(
            task_id, provider, model, False, 0.0, extra={"reason": "no_code_found"}
        )
        return {
            "pass": False,
            "score": 0,
            "reason": f"No Python code block found (first 300 chars: {snippet})",
        }

    # Pre-exec safety
    if is_unsafe(function_code):
        write_metrics(
            task_id, provider, model, False, 0.0, extra={"reason": "unsafe_pattern"}
        )
        return {
            "pass": False,
            "score": 0,
            "reason": "Unsafe pattern detected in generated code",
        }

    test_program = f"""{function_code}

print({fn_name}({test_input}))
"""

    start = time.time()
    with Sandbox.create() as sbx:
        try:
            res = _run_code_in_sandbox(sbx, test_program)
        except Exception as e:
            duration = time.time() - start
            write_metrics(
                task_id, provider, model, False, duration, extra={"error": str(e)}
            )
            return {
                "pass": False,
                "score": 0,
                "reason": f"Sandbox execution error: {e}",
            }

    duration = time.time() - start
    stdout, stderr = _stdout_from_result(res)
    err_obj = getattr(res, "error", None)

    if err_obj or stderr:
        dbg = ""
        try:
            dbg = (
                getattr(res, "to_json")()
                if hasattr(res, "to_json")
                else str(getattr(res, "logs", ""))[:800]
            )
        except Exception:
            dbg = str(getattr(res, "logs", ""))[:800]
        write_metrics(
            task_id,
            provider,
            model,
            False,
            duration,
            extra={"error": err_obj or stderr, "debug": dbg},
        )
        return {
            "pass": False,
            "score": 0,
            "reason": f"Execution error: {err_obj or stderr} | debug: {dbg}",
        }

    success = stdout == expected
    write_metrics(
        task_id, provider, model, success, duration, extra={"stdout": stdout[:500]}
    )
    if success:
        return {"pass": True, "score": 1, "reason": f"Correct output: {stdout}"}
    logs_preview = getattr(res, "logs", None)
    if isinstance(logs_preview, str) and len(logs_preview) > 400:
        logs_preview = logs_preview[:400] + "...(truncated)"
    return {
        "pass": False,
        "score": 0,
        "reason": f"Expected {expected}, got {stdout or '(empty)'} | logs: {logs_preview}",
    }