567-labs--instructor/scripts/benchmark_import.py

"""Benchmark instructor import costs.

Run with:

    python scripts/benchmark_import.py
"""

from __future__ import annotations

import subprocess
import sys
import textwrap


def run_import_benchmark(code: str, label: str) -> dict[str, str]:
    """Run a Python snippet in a subprocess and capture import stats."""
    script = textwrap.dedent(
        f"""
        import sys
        import time
        import tracemalloc

        tracemalloc.start()
        start = time.time()

        {code}

        elapsed = time.time() - start
        current, peak = tracemalloc.get_traced_memory()
        tracemalloc.stop()

        print(f"{{elapsed:.3f}}|{{current}}|{{peak}}|{{len(sys.modules)}}")
        """
    )

    result = subprocess.run(
        [sys.executable, "-c", script],
        capture_output=True,
        text=True,
        timeout=60,
        check=False,
    )
    if result.returncode != 0:
        return {
            "label": label,
            "time": "ERROR",
            "memory_mb": "ERROR",
            "peak_mb": "ERROR",
            "modules": "ERROR",
            "error": result.stderr.strip(),
        }

    elapsed, current, peak, mod_count = result.stdout.strip().split("|")
    return {
        "label": label,
        "time": f"{float(elapsed):.3f}s",
        "memory_mb": f"{int(current) / 1024 / 1024:.1f} MB",
        "peak_mb": f"{int(peak) / 1024 / 1024:.1f} MB",
        "modules": mod_count,
    }


def main() -> None:
    print("=" * 70)
    print(" INSTRUCTOR IMPORT BENCHMARK")
    print("=" * 70)
    print(f" Python: {sys.version.split()[0]}")
    print(f" Executable: {sys.executable}")
    print("=" * 70)
    print()

    benchmarks = [
        ("import sys", "baseline (sys only)"),
        ("import instructor", "import instructor"),
        ("import instructor; _ = instructor.from_provider", "access from_provider"),
        ("import instructor; _ = instructor.from_openai", "access from_openai"),
        ("import instructor; _ = instructor.from_anthropic", "access from_anthropic"),
        ("import instructor; _ = instructor.from_genai", "access from_genai"),
        ("import instructor; _ = instructor.Mode", "access Mode"),
        ("import instructor; _ = instructor.Instructor", "access Instructor class"),
        ("import instructor; _ = instructor.Partial", "access Partial"),
    ]

    results = [run_import_benchmark(code, label) for code, label in benchmarks]

    header = f"{'Benchmark':<30} {'Time':>8} {'Memory':>10} {'Peak':>10} {'Modules':>8}"
    print(header)
    print("-" * len(header))

    for result in results:
        if result.get("error"):
            print(f"{result['label']:<30} {'ERROR':>8} (missing dependency?)")
        else:
            print(
                f"{result['label']:<30} {result['time']:>8} "
                f"{result['memory_mb']:>10} {result['peak_mb']:>10} "
                f"{result['modules']:>8}"
            )

    print()
    print("Notes:")
    print("  - 'import instructor' should stay small because of lazy loading.")
    print("  - Provider access loads each SDK on demand.")


if __name__ == "__main__":
    main()