chore: import upstream snapshot with attribution

src/python/persistent_wrapper.py

@@ -0,0 +1,477 @@
+#!/usr/bin/env python3
+"""
+Persistent Python wrapper for Promptfoo.
+
+This wrapper loads a user script once and handles multiple requests
+via a simple control protocol over stdin/stdout.
+
+Protocol:
+  - Node sends: "CALL|<function_name>|<request_file>|<response_file>\n"
+  - Worker executes function, writes response to file
+  - Worker sends: "DONE|<response_file>\n"
+  - Node sends: "SHUTDOWN\n" to exit
+
+Data transfer uses files (proven UTF-8 handling), control uses stdin/stdout.
+Note: Using pipe (|) delimiter to avoid conflicts with Windows drive letters (C:).
+"""
+
+import asyncio
+import importlib.util
+import inspect
+import json
+import os
+import sys
+import traceback
+
+# ============================================================================
+# OpenTelemetry Tracing Support (Optional)
+# ============================================================================
+# When PROMPTFOO_ENABLE_OTEL=true, automatically instruments Python provider
+# calls with OpenTelemetry spans that link to the parent trace from Promptfoo.
+#
+# Requirements (optional):
+#   pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http
+# ============================================================================
+
+_tracer = None
+_tracing_enabled = False
+
+
+def _init_tracing():
+    """Initialize OpenTelemetry tracing if enabled and packages are available."""
+    global _tracer, _tracing_enabled
+
+    if os.getenv("PROMPTFOO_ENABLE_OTEL", "").lower() not in ("true", "1", "yes"):
+        return
+
+    try:
+        from opentelemetry import trace
+        from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+            OTLPSpanExporter,
+        )
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+
+        # Get endpoint from environment or use default Promptfoo OTLP receiver
+        base_endpoint = os.getenv(
+            "OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318"
+        )
+        endpoint = f"{base_endpoint.rstrip('/')}/v1/traces"
+
+        provider = TracerProvider()
+        exporter = OTLPSpanExporter(endpoint=endpoint)
+        provider.add_span_processor(SimpleSpanProcessor(exporter))
+        trace.set_tracer_provider(provider)
+
+        _tracer = trace.get_tracer("promptfoo.python.provider")
+        _tracing_enabled = True
+
+    except ImportError:
+        print(
+            "[PythonProvider] OpenTelemetry packages not installed, tracing disabled. "
+            "Install with: pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http",
+            file=sys.stderr,
+            flush=True,
+        )
+    except Exception as e:
+        print(
+            f"[PythonProvider] Failed to initialize tracing: {e}",
+            file=sys.stderr,
+            flush=True,
+        )
+
+
+# Initialize tracing at module load
+_init_tracing()
+
+
+def load_user_module(script_path):
+    """Load and return the user's Python module."""
+    script_dir = os.path.dirname(os.path.abspath(script_path))
+    module_name = os.path.basename(script_path).rsplit(".", 1)[0]
+
+    if script_dir not in sys.path:
+        sys.path.insert(0, script_dir)
+
+    spec = importlib.util.spec_from_file_location(module_name, script_path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Cannot load module from {script_path}")
+
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    return module
+
+
+def get_callable(module, method_name):
+    """Get the callable method from module, supporting 'Class.method' syntax."""
+    try:
+        if "." in method_name:
+            class_name, classmethod_name = method_name.split(".", 1)
+            cls = getattr(module, class_name)
+            return getattr(cls, classmethod_name)
+        else:
+            return getattr(module, method_name)
+    except AttributeError as e:
+        # Provide helpful error message when function not found
+        available_funcs = [
+            name
+            for name in dir(module)
+            if callable(getattr(module, name, None)) and not name.startswith("_")
+        ]
+
+        error_lines = [
+            f"Function '{method_name}' not found in module '{module.__name__}'",
+            "",
+            f"Available functions in your module: {', '.join(available_funcs) if available_funcs else '(none)'}",
+            "",
+            "Expected function names for promptfoo:",
+            "  • call_api(prompt, options, context) - for chat/completions",
+            "  • call_embedding_api(prompt, options) - for embeddings",
+            "  • call_classification_api(prompt, options) - for classification",
+            "",
+        ]
+
+        # Fuzzy match suggestion
+        if available_funcs:
+            # Check for common mistakes
+            method_lower = method_name.lower()
+            for func in available_funcs:
+                func_lower = func.lower()
+                # Check if user used 'get_' instead of 'call_'
+                if method_lower.replace("call_", "") == func_lower.replace("get_", ""):
+                    error_lines.append(
+                        f"💡 Did you mean to rename '{func}' to '{method_name}'?"
+                    )
+                    break
+                # Check if function name is similar (missing 'call_' prefix)
+                elif method_lower.replace("call_", "") == func_lower:
+                    error_lines.append(
+                        f"💡 Did you mean to rename '{func}' to '{method_name}'?"
+                    )
+                    break
+                # Check if it's just a typo (Levenshtein-like)
+                elif (
+                    len(set(method_lower) & set(func_lower)) > len(method_lower) * 0.6
+                    and abs(len(method_lower) - len(func_lower)) <= 3
+                ):
+                    error_lines.append(f"💡 Did you mean '{func}'?")
+                    break
+
+        error_lines.append(
+            "\nSee https://www.promptfoo.dev/docs/providers/python/ for details."
+        )
+
+        raise AttributeError("\n".join(error_lines)) from e
+
+
+def call_method(method_callable, args):
+    """Call the method, handling both sync and async functions."""
+    if inspect.iscoroutinefunction(method_callable):
+        return asyncio.run(method_callable(*args))
+    else:
+        return method_callable(*args)
+
+
+def _truncate_body(text, max_length=4096):
+    """Truncate text to max_length, adding indicator if truncated."""
+    if not isinstance(text, str):
+        text = str(text)
+    if len(text) <= max_length:
+        return text
+    return text[: max_length - 13] + "... [truncated]"
+
+
+def _traced_call(method_callable, args, function_name):
+    """
+    Call method with OpenTelemetry tracing if enabled.
+
+    Extracts traceparent from context (3rd arg) and creates a child span
+    that links to the parent trace from Promptfoo.
+    """
+    global _tracer, _tracing_enabled
+
+    # Fast path: if tracing not enabled, just call the method
+    if not _tracing_enabled or _tracer is None:
+        return call_method(method_callable, args)
+
+    # Extract traceparent from context (3rd argument for call_api)
+    traceparent = None
+    context_arg = None
+    if len(args) >= 3:
+        context_arg = args[2]
+        if isinstance(context_arg, dict):
+            traceparent = context_arg.get("traceparent")
+
+    # If no traceparent, fall back to untraced call
+    if not traceparent:
+        return call_method(method_callable, args)
+
+    try:
+        from opentelemetry.propagate import extract
+        from opentelemetry.trace import SpanKind, Status, StatusCode
+
+        # Extract parent context from W3C traceparent header
+        parent_ctx = extract({"traceparent": traceparent})
+
+        # Determine span name following GenAI conventions
+        span_name = f"python {function_name}"
+
+        with _tracer.start_as_current_span(
+            span_name, context=parent_ctx, kind=SpanKind.CLIENT
+        ) as span:
+            # Set GenAI semantic convention attributes
+            span.set_attribute("gen_ai.system", "python")
+            span.set_attribute("gen_ai.operation.name", function_name)
+
+            # Set request attributes from prompt (1st arg)
+            if len(args) >= 1:
+                prompt = args[0]
+                if isinstance(prompt, str):
+                    span.set_attribute("promptfoo.request.body", _truncate_body(prompt))
+
+            # Set model from config if available (2nd arg)
+            if len(args) >= 2:
+                options = args[1]
+                if isinstance(options, dict):
+                    config = options.get("config", {})
+                    if config.get("model"):
+                        span.set_attribute("gen_ai.request.model", config["model"])
+                    # Also check for provider id
+                    if options.get("id"):
+                        span.set_attribute("promptfoo.provider.id", options["id"])
+
+            # Set evaluation metadata from context
+            if context_arg:
+                if context_arg.get("evaluationId"):
+                    span.set_attribute("promptfoo.eval.id", context_arg["evaluationId"])
+                if context_arg.get("testCaseId"):
+                    span.set_attribute("promptfoo.test.id", context_arg["testCaseId"])
+
+            try:
+                # Execute the user's function
+                result = call_method(method_callable, args)
+
+                # Set response attributes
+                if isinstance(result, dict):
+                    # Response body
+                    if "output" in result:
+                        output = result["output"]
+                        if isinstance(output, str):
+                            span.set_attribute(
+                                "promptfoo.response.body", _truncate_body(output)
+                            )
+                        elif output is not None:
+                            span.set_attribute(
+                                "promptfoo.response.body",
+                                _truncate_body(json.dumps(output)),
+                            )
+
+                    # Token usage following GenAI conventions
+                    if "tokenUsage" in result and isinstance(
+                        result["tokenUsage"], dict
+                    ):
+                        usage = result["tokenUsage"]
+                        if "total" in usage:
+                            span.set_attribute(
+                                "gen_ai.usage.total_tokens", usage["total"]
+                            )
+                        if "prompt" in usage:
+                            span.set_attribute(
+                                "gen_ai.usage.input_tokens", usage["prompt"]
+                            )
+                        if "completion" in usage:
+                            span.set_attribute(
+                                "gen_ai.usage.output_tokens", usage["completion"]
+                            )
+
+                    # Cache hit
+                    if "cached" in result:
+                        span.set_attribute(
+                            "promptfoo.cache_hit", bool(result["cached"])
+                        )
+
+                    # Handle error in result
+                    if result.get("error"):
+                        span.set_status(Status(StatusCode.ERROR, str(result["error"])))
+                    else:
+                        span.set_status(Status(StatusCode.OK))
+                else:
+                    span.set_status(Status(StatusCode.OK))
+
+                return result
+
+            except Exception as e:
+                # Record exception and set error status
+                span.record_exception(e)
+                span.set_status(Status(StatusCode.ERROR, str(e)))
+                raise
+
+    except Exception as tracing_error:
+        # If tracing itself fails, log and fall back to untraced call
+        print(
+            f"[PythonProvider] Tracing error (continuing without trace): {tracing_error}",
+            file=sys.stderr,
+            flush=True,
+        )
+        return call_method(method_callable, args)
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(
+            "Usage: persistent_wrapper.py <script_path> <function_name>",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    script_path = sys.argv[1]
+    function_name = sys.argv[2]
+
+    # Load user module once
+    try:
+        user_module = load_user_module(script_path)
+        # Note: We don't validate the default function exists at initialization.
+        # With the persistent worker protocol supporting dynamic function calls per request,
+        # users may only define specific functions (e.g., call_embedding_api for embeddings-only).
+        # Functions are validated when actually called in handle_call(), providing clear
+        # error messages with available functions and suggestions.
+    except Exception as e:
+        print(f"ERROR: Failed to load module: {e}", file=sys.stderr, flush=True)
+        print(traceback.format_exc(), file=sys.stderr, flush=True)
+        sys.exit(1)
+
+    # Signal ready
+    print("READY", flush=True)
+
+    # Main loop - wait for commands
+    while True:
+        try:
+            line = sys.stdin.readline()
+            if not line:
+                # stdin closed, exit gracefully
+                break
+
+            line = line.strip()
+
+            if line.startswith("SHUTDOWN"):
+                break
+            elif line.startswith("CALL|"):
+                handle_call(line, user_module, function_name)
+            else:
+                print(f"ERROR: Unknown command: {line}", file=sys.stderr, flush=True)
+
+        except KeyboardInterrupt:
+            break
+        except Exception as e:
+            print(f"ERROR in main loop: {e}", file=sys.stderr, flush=True)
+            print(traceback.format_exc(), file=sys.stderr, flush=True)
+
+
+def handle_call(command_line, user_module, default_function_name):
+    """Handle a CALL command."""
+    response_file = None
+    try:
+        # Parse command: "CALL|<function_name>|<request_file>|<response_file>"
+        # or legacy: "CALL|<request_file>|<response_file>"
+        # Note: Using pipe (|) delimiter to avoid conflicts with Windows drive letters (C:)
+        parts = command_line.split("|", 3)
+
+        # Extract response_file first (always the last part) so it's available even if validation fails
+        # This ensures we can write an error response file if command format is invalid
+        if len(parts) >= 3:
+            response_file = parts[-1]
+
+        if len(parts) == 4:
+            # New format: CALL|<function_name>|<request_file>|<response_file>
+            _, function_name, request_file, _ = parts
+        elif len(parts) == 3:
+            # Legacy format: CALL|<request_file>|<response_file>
+            _, request_file, _ = parts
+            function_name = default_function_name
+        else:
+            raise ValueError(f"Invalid CALL command format: {command_line}")
+
+        # Resolve the callable for this call
+        method_callable = get_callable(user_module, function_name)
+
+        # Read request
+        with open(request_file, "r", encoding="utf-8") as f:
+            args = json.load(f)
+
+        # Execute user function (with automatic tracing if enabled)
+        try:
+            result = _traced_call(method_callable, args, function_name)
+            response = {"type": "result", "data": result}
+        except Exception as e:
+            response = {
+                "type": "error",
+                "error": str(e),
+                "traceback": traceback.format_exc(),
+            }
+
+        # Write response
+        with open(response_file, "w", encoding="utf-8") as f:
+            json.dump(response, f, ensure_ascii=False)
+            f.flush()  # Flush Python buffer
+            os.fsync(f.fileno())  # Force OS to write to disk (critical for Windows)
+
+        # Verify file is readable before signaling done (prevents race conditions)
+        # Retry up to 3 times with small delays if file isn't immediately readable
+        for verify_attempt in range(3):
+            try:
+                with open(response_file, "r", encoding="utf-8") as f:
+                    _ = f.read()
+                break  # Successfully read, exit retry loop
+            except Exception as e:
+                if verify_attempt < 2:
+                    import time
+
+                    time.sleep(0.1)  # 100ms delay before retry
+                    continue
+                # Final attempt failed
+                print(
+                    f"ERROR: Failed to verify response file after 3 attempts: {e}",
+                    file=sys.stderr,
+                    flush=True,
+                )
+                # Still send DONE to avoid hanging Node, but Node will handle missing file
+
+        # Signal completion with the response path so provider stdout cannot
+        # accidentally satisfy another request's control message.
+        print(f"DONE|{response_file}", flush=True)
+
+    except Exception as e:
+        print(f"ERROR handling call: {e}", file=sys.stderr, flush=True)
+        print(traceback.format_exc(), file=sys.stderr, flush=True)
+
+        # Write error response if we have response_file
+        # This ensures Node.js always has a file to read, preventing ENOENT errors
+        # when errors occur before the normal response file write (e.g., invalid command,
+        # non-existent function, file I/O errors)
+        if response_file:
+            try:
+                error_response = {
+                    "type": "error",
+                    "error": str(e),
+                    "traceback": traceback.format_exc(),
+                }
+                with open(response_file, "w", encoding="utf-8") as f:
+                    json.dump(error_response, f, ensure_ascii=False)
+                    f.flush()
+                    os.fsync(f.fileno())
+            except Exception as write_error:
+                print(
+                    f"ERROR: Failed to write error response: {write_error}",
+                    file=sys.stderr,
+                    flush=True,
+                )
+
+            # Signal done so Node can read the error response.
+            print(f"DONE|{response_file}", flush=True)
+        # No DONE| when response_file is unknown: Node's path match would fail
+        # and the call would resolve via timeout anyway; the stderr is the signal.
+
+
+if __name__ == "__main__":
+    main()

src/python/persistent_wrapper_test.py

@@ -0,0 +1,634 @@
+import asyncio
+import io
+import json
+import os
+import sys
+import tempfile
+import unittest
+from types import ModuleType
+from unittest.mock import MagicMock, patch
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from python import persistent_wrapper
+
+
+class TestCallMethod(unittest.TestCase):
+    """Tests for call_method function."""
+
+    def test_sync_function(self) -> None:
+        """Tests call_method with a synchronous function."""
+
+        def sync_func(a, b):
+            return a + b
+
+        result = persistent_wrapper.call_method(sync_func, [2, 3])
+        self.assertEqual(result, 5)
+
+    def test_async_function(self) -> None:
+        """Tests call_method with an asynchronous function."""
+
+        async def async_func(a, b):
+            await asyncio.sleep(0.001)
+            return a * b
+
+        result = persistent_wrapper.call_method(async_func, [3, 4])
+        self.assertEqual(result, 12)
+
+    def test_detects_async_correctly(self) -> None:
+        """Tests that call_method uses inspect.iscoroutinefunction (not deprecated asyncio version)."""
+
+        async def real_async_func():
+            return "async"
+
+        def real_sync_func():
+            return "sync"
+
+        # Test async function is detected and run with asyncio.run
+        def close_coroutine(coro):
+            coro.close()
+            return "mocked_async"
+
+        with patch("asyncio.run", side_effect=close_coroutine) as mock_run:
+            result = persistent_wrapper.call_method(real_async_func, [])
+            mock_run.assert_called_once()
+            self.assertEqual(result, "mocked_async")
+
+        # Test sync function bypasses asyncio.run
+        with patch("asyncio.run") as mock_run:
+            result = persistent_wrapper.call_method(real_sync_func, [])
+            mock_run.assert_not_called()
+            self.assertEqual(result, "sync")
+
+
+class TestLoadUserModule(unittest.TestCase):
+    """Tests for load_user_module function."""
+
+    def test_loads_valid_module(self) -> None:
+        """Tests loading a user module from a file."""
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
+            f.write("def test_func(x):\n    return x * 2\n")
+            f.flush()
+            script_path = f.name
+
+        try:
+            module = persistent_wrapper.load_user_module(script_path)
+            self.assertTrue(hasattr(module, "test_func"))
+            self.assertEqual(module.test_func(5), 10)
+        finally:
+            os.unlink(script_path)
+
+    def test_file_not_found(self) -> None:
+        """Tests loading a non-existent module raises FileNotFoundError."""
+        with self.assertRaises(FileNotFoundError):
+            persistent_wrapper.load_user_module("/nonexistent/path.py")
+
+    def test_invalid_spec_raises_import_error(self) -> None:
+        """Tests that ImportError is raised when spec is None."""
+        with patch("importlib.util.spec_from_file_location", return_value=None):
+            with self.assertRaises(ImportError) as context:
+                persistent_wrapper.load_user_module("/some/path.py")
+            self.assertIn("Cannot load module", str(context.exception))
+
+    def test_adds_script_dir_to_path(self) -> None:
+        """Tests that script directory is added to sys.path."""
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
+            f.write("x = 1\n")
+            f.flush()
+            script_path = f.name
+            script_dir = os.path.dirname(os.path.abspath(script_path))
+
+        try:
+            # Remove from path if present
+            if script_dir in sys.path:
+                sys.path.remove(script_dir)
+
+            persistent_wrapper.load_user_module(script_path)
+            self.assertIn(script_dir, sys.path)
+        finally:
+            os.unlink(script_path)
+            if script_dir in sys.path:
+                sys.path.remove(script_dir)
+
+
+class TestGetCallable(unittest.TestCase):
+    """Tests for get_callable function."""
+
+    def test_gets_top_level_function(self) -> None:
+        """Tests getting a top-level function from a module."""
+        mock_module = MagicMock()
+        mock_module.my_func = lambda x: x
+
+        result = persistent_wrapper.get_callable(mock_module, "my_func")
+        self.assertEqual(result, mock_module.my_func)
+
+    def test_gets_class_method(self) -> None:
+        """Tests getting a class method using 'Class.method' syntax."""
+        mock_module = MagicMock()
+        mock_module.MyClass.my_method = lambda x: x
+
+        result = persistent_wrapper.get_callable(mock_module, "MyClass.my_method")
+        self.assertEqual(result, mock_module.MyClass.my_method)
+
+    def test_not_found_shows_available_functions(self) -> None:
+        """Tests that AttributeError shows available functions."""
+        mock_module = MagicMock(spec=["existing_func"])
+        mock_module.__name__ = "test_module"
+        mock_module.existing_func = lambda: None
+
+        with self.assertRaises(AttributeError) as context:
+            persistent_wrapper.get_callable(mock_module, "nonexistent_func")
+
+        error_message = str(context.exception)
+        self.assertIn("nonexistent_func", error_message)
+        self.assertIn("not found", error_message)
+
+    def test_suggests_get_to_call_rename(self) -> None:
+        """Tests suggestion when user has get_api instead of call_api."""
+        # Create a real module-like object
+        mock_module = type("MockModule", (), {"__name__": "test_module"})()
+        mock_module.get_api = lambda: None
+
+        with self.assertRaises(AttributeError) as context:
+            persistent_wrapper.get_callable(mock_module, "call_api")
+
+        error_message = str(context.exception)
+        self.assertIn("Did you mean to rename", error_message)
+        self.assertIn("get_api", error_message)
+
+    def test_suggests_missing_call_prefix(self) -> None:
+        """Tests suggestion when user has 'api' instead of 'call_api'."""
+        mock_module = type("MockModule", (), {"__name__": "test_module"})()
+        mock_module.api = lambda: None
+
+        with self.assertRaises(AttributeError) as context:
+            persistent_wrapper.get_callable(mock_module, "call_api")
+
+        error_message = str(context.exception)
+        self.assertIn("Did you mean to rename", error_message)
+        self.assertIn("api", error_message)
+
+    def test_suggests_typo_correction(self) -> None:
+        """Tests suggestion for typo-like mistakes."""
+        mock_module = type("MockModule", (), {"__name__": "test_module"})()
+        mock_module.call_apii = lambda: None  # typo: extra 'i'
+
+        with self.assertRaises(AttributeError) as context:
+            persistent_wrapper.get_callable(mock_module, "call_api")
+
+        error_message = str(context.exception)
+        self.assertIn("Did you mean", error_message)
+        self.assertIn("call_apii", error_message)
+
+    def test_no_suggestion_when_no_similar_function(self) -> None:
+        """Tests no suggestion when no similar function exists."""
+        mock_module = type("MockModule", (), {"__name__": "test_module"})()
+        mock_module.completely_different = lambda: None
+
+        with self.assertRaises(AttributeError) as context:
+            persistent_wrapper.get_callable(mock_module, "call_api")
+
+        error_message = str(context.exception)
+        self.assertIn("call_api", error_message)
+        self.assertNotIn("Did you mean", error_message)
+
+
+class TestInitTracing(unittest.TestCase):
+    """Tests for OpenTelemetry initialization."""
+
+    def setUp(self) -> None:
+        self.original_tracer = persistent_wrapper._tracer
+        self.original_tracing_enabled = persistent_wrapper._tracing_enabled
+        persistent_wrapper._tracer = None
+        persistent_wrapper._tracing_enabled = False
+
+    def tearDown(self) -> None:
+        persistent_wrapper._tracer = self.original_tracer
+        persistent_wrapper._tracing_enabled = self.original_tracing_enabled
+
+    def test_successful_init_does_not_write_to_stderr(self) -> None:
+        """Tests that successful tracing initialization stays quiet."""
+        fake_trace = MagicMock()
+        fake_tracer = object()
+        fake_trace.get_tracer.return_value = fake_tracer
+
+        opentelemetry_module = ModuleType("opentelemetry")
+        opentelemetry_exporter_module = ModuleType("opentelemetry.exporter")
+        opentelemetry_exporter_otlp_module = ModuleType("opentelemetry.exporter.otlp")
+        opentelemetry_exporter_otlp_proto_module = ModuleType(
+            "opentelemetry.exporter.otlp.proto"
+        )
+        opentelemetry_exporter_otlp_proto_http_module = ModuleType(
+            "opentelemetry.exporter.otlp.proto.http"
+        )
+        trace_exporter_module = ModuleType(
+            "opentelemetry.exporter.otlp.proto.http.trace_exporter"
+        )
+        sdk_module = ModuleType("opentelemetry.sdk")
+        sdk_trace_module = ModuleType("opentelemetry.sdk.trace")
+        sdk_trace_export_module = ModuleType("opentelemetry.sdk.trace.export")
+
+        class FakeOTLPSpanExporter:
+            def __init__(self, endpoint):
+                self.endpoint = endpoint
+
+        class FakeSimpleSpanProcessor:
+            def __init__(self, exporter):
+                self.exporter = exporter
+
+        class FakeTracerProvider:
+            def __init__(self):
+                self.processors = []
+
+            def add_span_processor(self, processor):
+                self.processors.append(processor)
+
+        trace_exporter_module.OTLPSpanExporter = FakeOTLPSpanExporter
+        sdk_trace_module.TracerProvider = FakeTracerProvider
+        sdk_trace_export_module.SimpleSpanProcessor = FakeSimpleSpanProcessor
+
+        opentelemetry_module.trace = fake_trace
+        opentelemetry_module.exporter = opentelemetry_exporter_module
+        opentelemetry_module.sdk = sdk_module
+        opentelemetry_exporter_module.otlp = opentelemetry_exporter_otlp_module
+        opentelemetry_exporter_otlp_module.proto = (
+            opentelemetry_exporter_otlp_proto_module
+        )
+        opentelemetry_exporter_otlp_proto_module.http = (
+            opentelemetry_exporter_otlp_proto_http_module
+        )
+        opentelemetry_exporter_otlp_proto_http_module.trace_exporter = (
+            trace_exporter_module
+        )
+        sdk_module.trace = sdk_trace_module
+        sdk_trace_module.export = sdk_trace_export_module
+
+        def getenv(name, default=None):
+            if name == "PROMPTFOO_ENABLE_OTEL":
+                return "true"
+            return default
+
+        fake_modules = {
+            "opentelemetry": opentelemetry_module,
+            "opentelemetry.exporter": opentelemetry_exporter_module,
+            "opentelemetry.exporter.otlp": opentelemetry_exporter_otlp_module,
+            "opentelemetry.exporter.otlp.proto": opentelemetry_exporter_otlp_proto_module,
+            "opentelemetry.exporter.otlp.proto.http": (
+                opentelemetry_exporter_otlp_proto_http_module
+            ),
+            "opentelemetry.exporter.otlp.proto.http.trace_exporter": (
+                trace_exporter_module
+            ),
+            "opentelemetry.sdk": sdk_module,
+            "opentelemetry.sdk.trace": sdk_trace_module,
+            "opentelemetry.sdk.trace.export": sdk_trace_export_module,
+        }
+
+        with patch.dict(sys.modules, fake_modules, clear=False):
+            with patch("python.persistent_wrapper.os.getenv", side_effect=getenv):
+                with patch("sys.stderr", new_callable=io.StringIO) as stderr_capture:
+                    persistent_wrapper._init_tracing()
+
+        self.assertTrue(persistent_wrapper._tracing_enabled)
+        self.assertIs(persistent_wrapper._tracer, fake_tracer)
+        self.assertEqual(stderr_capture.getvalue(), "")
+
+
+class TestMain(unittest.TestCase):
+    """Tests for main function."""
+
+    def test_exits_with_too_few_arguments(self) -> None:
+        """Tests that main exits with error when not enough arguments."""
+        with patch.object(sys, "argv", ["persistent_wrapper.py"]):
+            with patch("sys.exit", side_effect=SystemExit(1)) as mock_exit:
+                with patch("sys.stderr", new_callable=io.StringIO):
+                    with self.assertRaises(SystemExit):
+                        persistent_wrapper.main()
+                    mock_exit.assert_called_with(1)
+
+    def test_exits_on_module_load_failure(self) -> None:
+        """Tests that main exits when module fails to load."""
+        with patch.object(
+            sys, "argv", ["persistent_wrapper.py", "/nonexistent.py", "func"]
+        ):
+            with patch("sys.exit", side_effect=SystemExit(1)) as mock_exit:
+                with patch("sys.stderr", new_callable=io.StringIO):
+                    with self.assertRaises(SystemExit):
+                        persistent_wrapper.main()
+                    mock_exit.assert_called_with(1)
+
+    def test_signals_ready_and_handles_shutdown(self) -> None:
+        """Tests READY signal and SHUTDOWN command."""
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
+            f.write("def test_func(): pass\n")
+            f.flush()
+            script_path = f.name
+
+        try:
+            stdout_capture = io.StringIO()
+            stdin_mock = io.StringIO("SHUTDOWN\n")
+
+            with patch.object(
+                sys, "argv", ["persistent_wrapper.py", script_path, "test_func"]
+            ):
+                with patch("sys.stdin", stdin_mock):
+                    with patch("sys.stdout", stdout_capture):
+                        persistent_wrapper.main()
+
+            output = stdout_capture.getvalue()
+            self.assertIn("READY", output)
+        finally:
+            os.unlink(script_path)
+
+    def test_handles_stdin_close(self) -> None:
+        """Tests graceful exit when stdin is closed."""
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
+            f.write("def test_func(): pass\n")
+            f.flush()
+            script_path = f.name
+
+        try:
+            stdout_capture = io.StringIO()
+            stdin_mock = io.StringIO("")  # Empty = EOF
+
+            with patch.object(
+                sys, "argv", ["persistent_wrapper.py", script_path, "test_func"]
+            ):
+                with patch("sys.stdin", stdin_mock):
+                    with patch("sys.stdout", stdout_capture):
+                        persistent_wrapper.main()
+
+            output = stdout_capture.getvalue()
+            self.assertIn("READY", output)
+        finally:
+            os.unlink(script_path)
+
+    def test_handles_unknown_command(self) -> None:
+        """Tests error message for unknown commands."""
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
+            f.write("def test_func(): pass\n")
+            f.flush()
+            script_path = f.name
+
+        try:
+            stderr_capture = io.StringIO()
+            stdin_mock = io.StringIO("UNKNOWN_CMD\nSHUTDOWN\n")
+
+            with patch.object(
+                sys, "argv", ["persistent_wrapper.py", script_path, "test_func"]
+            ):
+                with patch("sys.stdin", stdin_mock):
+                    with patch("sys.stderr", stderr_capture):
+                        with patch("sys.stdout", io.StringIO()):
+                            persistent_wrapper.main()
+
+            stderr_output = stderr_capture.getvalue()
+            self.assertIn("Unknown command", stderr_output)
+        finally:
+            os.unlink(script_path)
+
+    def test_routes_call_command(self) -> None:
+        """Tests that CALL commands are routed to handle_call."""
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
+            f.write("def test_func(): return 'ok'\n")
+            f.flush()
+            script_path = f.name
+
+        try:
+            with patch("python.persistent_wrapper.handle_call") as mock_handle_call:
+                stdin_mock = io.StringIO("CALL|func|req|resp\nSHUTDOWN\n")
+
+                with patch.object(
+                    sys, "argv", ["persistent_wrapper.py", script_path, "test_func"]
+                ):
+                    with patch("sys.stdin", stdin_mock):
+                        with patch("sys.stdout", io.StringIO()):
+                            persistent_wrapper.main()
+
+                mock_handle_call.assert_called_once()
+        finally:
+            os.unlink(script_path)
+
+
+class TestHandleCall(unittest.TestCase):
+    """Tests for handle_call function."""
+
+    def setUp(self):
+        """Set up test fixtures."""
+        self.temp_dir = tempfile.mkdtemp()
+        self.request_file = os.path.join(self.temp_dir, "request.json")
+        self.response_file = os.path.join(self.temp_dir, "response.json")
+
+        # Create a mock module
+        self.mock_module = type("MockModule", (), {"__name__": "test_module"})()
+        self.mock_module.test_func = lambda a, b: a + b
+        self.mock_module.error_func = self._raise_error
+
+    def _raise_error(self):
+        raise ValueError("Test error")
+
+    def tearDown(self):
+        """Clean up temp files."""
+        import shutil
+
+        shutil.rmtree(self.temp_dir, ignore_errors=True)
+
+    def test_new_format_success(self) -> None:
+        """Tests successful CALL with new 4-part format."""
+        with open(self.request_file, "w") as f:
+            json.dump([2, 3], f)
+
+        stdout_capture = io.StringIO()
+        with patch("sys.stdout", stdout_capture):
+            persistent_wrapper.handle_call(
+                f"CALL|test_func|{self.request_file}|{self.response_file}",
+                self.mock_module,
+                "default_func",
+            )
+
+        self.assertIn("DONE", stdout_capture.getvalue())
+
+        with open(self.response_file) as f:
+            response = json.load(f)
+        self.assertEqual(response["type"], "result")
+        self.assertEqual(response["data"], 5)
+
+    def test_legacy_format_success(self) -> None:
+        """Tests successful CALL with legacy 3-part format."""
+        with open(self.request_file, "w") as f:
+            json.dump([10, 20], f)
+
+        stdout_capture = io.StringIO()
+        with patch("sys.stdout", stdout_capture):
+            persistent_wrapper.handle_call(
+                f"CALL|{self.request_file}|{self.response_file}",
+                self.mock_module,
+                "test_func",  # Uses default function
+            )
+
+        self.assertIn("DONE", stdout_capture.getvalue())
+
+        with open(self.response_file) as f:
+            response = json.load(f)
+        self.assertEqual(response["type"], "result")
+        self.assertEqual(response["data"], 30)
+
+    def test_invalid_format_writes_error(self) -> None:
+        """Tests that invalid command format reports the error without emitting a bogus DONE marker."""
+        stdout_capture = io.StringIO()
+        stderr_capture = io.StringIO()
+
+        with patch("sys.stdout", stdout_capture):
+            with patch("sys.stderr", stderr_capture):
+                persistent_wrapper.handle_call(
+                    "CALL|only_two_parts",
+                    self.mock_module,
+                    "test_func",
+                )
+
+        # No response_file means no useful DONE| can be emitted; the stderr is
+        # the real signal and Node will resolve the call via timeout.
+        self.assertNotIn("DONE", stdout_capture.getvalue())
+        self.assertIn("Invalid CALL command format", stderr_capture.getvalue())
+
+    def test_function_execution_error(self) -> None:
+        """Tests that function errors are captured in response."""
+        with open(self.request_file, "w") as f:
+            json.dump([], f)
+
+        stdout_capture = io.StringIO()
+        with patch("sys.stdout", stdout_capture):
+            persistent_wrapper.handle_call(
+                f"CALL|error_func|{self.request_file}|{self.response_file}",
+                self.mock_module,
+                "default_func",
+            )
+
+        self.assertIn("DONE", stdout_capture.getvalue())
+
+        with open(self.response_file) as f:
+            response = json.load(f)
+        self.assertEqual(response["type"], "error")
+        self.assertIn("Test error", response["error"])
+
+    def test_nonexistent_function_writes_error(self) -> None:
+        """Tests that missing function writes error response."""
+        with open(self.request_file, "w") as f:
+            json.dump([], f)
+
+        stdout_capture = io.StringIO()
+        stderr_capture = io.StringIO()
+
+        with patch("sys.stdout", stdout_capture):
+            with patch("sys.stderr", stderr_capture):
+                persistent_wrapper.handle_call(
+                    f"CALL|nonexistent|{self.request_file}|{self.response_file}",
+                    self.mock_module,
+                    "default_func",
+                )
+
+        self.assertIn("DONE", stdout_capture.getvalue())
+
+        with open(self.response_file) as f:
+            response = json.load(f)
+        self.assertEqual(response["type"], "error")
+
+    def test_handles_unicode_in_response(self) -> None:
+        """Tests that Unicode characters are preserved in response."""
+        self.mock_module.unicode_func = lambda: {"emoji": "🎉", "chinese": "中文"}
+
+        with open(self.request_file, "w") as f:
+            json.dump([], f)
+
+        with patch("sys.stdout", io.StringIO()):
+            persistent_wrapper.handle_call(
+                f"CALL|unicode_func|{self.request_file}|{self.response_file}",
+                self.mock_module,
+                "default_func",
+            )
+
+        with open(self.response_file, encoding="utf-8") as f:
+            response = json.load(f)
+
+        self.assertEqual(response["data"]["emoji"], "🎉")
+        self.assertEqual(response["data"]["chinese"], "中文")
+
+    def test_file_verification_retry(self) -> None:
+        """Tests that file verification retries on failure."""
+        with open(self.request_file, "w") as f:
+            json.dump([1, 2], f)
+
+        # Mock open to fail first two reads, succeed on third
+        original_open = open
+        verification_read_count = 0
+
+        def mock_open_func(path, *args, **kwargs):
+            nonlocal verification_read_count
+            mode = args[0] if args else kwargs.get("mode", "r")
+            if path == self.response_file and "r" in mode:
+                verification_read_count += 1
+                if verification_read_count <= 2:
+                    raise IOError("Simulated read failure")
+            return original_open(path, *args, **kwargs)
+
+        stdout_capture = io.StringIO()
+        with patch("sys.stdout", stdout_capture):
+            with patch("time.sleep") as mock_sleep:
+                with patch("builtins.open", side_effect=mock_open_func):
+                    persistent_wrapper.handle_call(
+                        f"CALL|test_func|{self.request_file}|{self.response_file}",
+                        self.mock_module,
+                        "default_func",
+                    )
+
+        self.assertIn("DONE", stdout_capture.getvalue())
+        self.assertEqual(verification_read_count, 3)
+        self.assertEqual(mock_sleep.call_count, 2)
+
+        with open(self.response_file) as f:
+            response = json.load(f)
+        self.assertEqual(response["type"], "result")
+        self.assertEqual(response["data"], 3)
+
+
+class TestAsyncFunctionHandling(unittest.TestCase):
+    """Tests for async function handling in handle_call."""
+
+    def setUp(self):
+        self.temp_dir = tempfile.mkdtemp()
+        self.request_file = os.path.join(self.temp_dir, "request.json")
+        self.response_file = os.path.join(self.temp_dir, "response.json")
+
+        self.mock_module = type("MockModule", (), {"__name__": "test_module"})()
+
+        async def async_add(a, b):
+            await asyncio.sleep(0.001)
+            return a + b
+
+        self.mock_module.async_add = async_add
+
+    def tearDown(self):
+        import shutil
+
+        shutil.rmtree(self.temp_dir, ignore_errors=True)
+
+    def test_async_function_in_handle_call(self) -> None:
+        """Tests that async functions work correctly through handle_call."""
+        with open(self.request_file, "w") as f:
+            json.dump([5, 7], f)
+
+        with patch("sys.stdout", io.StringIO()):
+            persistent_wrapper.handle_call(
+                f"CALL|async_add|{self.request_file}|{self.response_file}",
+                self.mock_module,
+                "default_func",
+            )
+
+        with open(self.response_file) as f:
+            response = json.load(f)
+
+        self.assertEqual(response["type"], "result")
+        self.assertEqual(response["data"], 12)
+
+
+if __name__ == "__main__":
+    unittest.main()

src/python/pythonUtils.ts

@@ -0,0 +1,395 @@
+import { execFile } from 'child_process';
+import fs from 'fs/promises';
+import path from 'path';
+import { promisify } from 'util';
+
+import { PythonShell } from 'python-shell';
+import { getEnvBool, getEnvString } from '../envars';
+import { getWrapperDir } from '../esm';
+import logger from '../logger';
+import { safeJsonStringify } from '../util/json';
+import {
+  createSecureTempDirectory,
+  removeSecureTempDirectory,
+  writeSecureTempFile,
+} from '../util/secureTempFiles';
+import { PythonStderrLogger } from './stderr';
+
+const execFileAsync = promisify(execFile);
+
+import type { Options as PythonShellOptions } from 'python-shell';
+
+/**
+ * Gets an integer value from an environment variable.
+ * @param key - The environment variable name
+ * @returns The parsed integer value, or undefined if not set or not a valid integer
+ */
+export function getEnvInt(key: string): number | undefined {
+  const value = process.env[key];
+  if (value === undefined) {
+    return undefined;
+  }
+  const parsed = parseInt(value, 10);
+  return isNaN(parsed) ? undefined : parsed;
+}
+
+/**
+ * Resolves the Python executable path from explicit config and environment.
+ * This centralizes the fallback logic: configPath > PROMPTFOO_PYTHON env var.
+ *
+ * Note: Does NOT apply the final 'python' default - that's handled by
+ * validatePythonPath. This preserves the distinction between "explicitly
+ * configured" (should fail if invalid) and "using system default" (should
+ * try fallback detection).
+ *
+ * @param configPath - Explicitly configured Python path from provider config
+ * @returns The configured path, or undefined if neither config nor env var is set
+ */
+export function getConfiguredPythonPath(configPath?: string): string | undefined {
+  if (configPath) {
+    return configPath;
+  }
+  const envPath = getEnvString('PROMPTFOO_PYTHON');
+  return envPath || undefined;
+}
+
+export const state: {
+  cachedPythonPath: string | null;
+  validationPromise: Promise<string> | null;
+} = {
+  cachedPythonPath: null,
+  validationPromise: null,
+};
+
+/**
+ * Try to find Python using Windows 'where' command, filtering out Microsoft Store stubs.
+ */
+async function tryWindowsWhere(): Promise<string | null> {
+  try {
+    const result = await execFileAsync('where', ['python']);
+    const output = result.stdout.trim();
+
+    // Handle empty output
+    if (!output) {
+      logger.debug("Windows 'where python' returned empty output");
+      return null;
+    }
+
+    const paths = output.split('\n').filter((path) => path.trim());
+
+    for (const pythonPath of paths) {
+      const trimmedPath = pythonPath.trim();
+
+      // Skip Microsoft Store stubs and non-executables
+      if (trimmedPath.includes('WindowsApps') || !trimmedPath.endsWith('.exe')) {
+        continue;
+      }
+
+      const validated = await tryPath(trimmedPath);
+      if (validated) {
+        return validated;
+      }
+    }
+  } catch (error) {
+    const errorMsg = error instanceof Error ? error.message : String(error);
+    logger.debug(`Windows 'where python' failed: ${errorMsg}`);
+
+    // Log permission/access errors differently
+    if (errorMsg.includes('Access is denied') || errorMsg.includes('EACCES')) {
+      logger.warn(`Permission denied when searching for Python: ${errorMsg}`);
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Try Python commands to get sys.executable path.
+ */
+async function tryPythonCommands(commands: string[]): Promise<string | null> {
+  for (const cmd of commands) {
+    try {
+      const result = await execFileAsync(cmd, ['-c', 'import sys; print(sys.executable)']);
+      const executablePath = result.stdout.trim();
+      if (executablePath && executablePath !== 'None') {
+        // On Windows, ensure .exe suffix if missing (but only for Windows-style paths)
+        if (process.platform === 'win32' && !executablePath.toLowerCase().endsWith('.exe')) {
+          // Only add .exe for Windows-style paths (drive letter or UNC paths)
+          if (executablePath.includes('\\') || /^[A-Za-z]:/.test(executablePath)) {
+            return executablePath + '.exe';
+          }
+        }
+        return executablePath;
+      }
+    } catch (error) {
+      const errorMsg = error instanceof Error ? error.message : String(error);
+      logger.debug(`Python command "${cmd}" failed: ${errorMsg}`);
+
+      // Log permission/access errors differently
+      if (
+        errorMsg.includes('Access is denied') ||
+        errorMsg.includes('EACCES') ||
+        errorMsg.includes('EPERM')
+      ) {
+        logger.warn(`Permission denied when trying Python command "${cmd}": ${errorMsg}`);
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ * Try direct command validation as final fallback.
+ */
+async function tryDirectCommands(commands: string[]): Promise<string | null> {
+  for (const cmd of commands) {
+    try {
+      const validated = await tryPath(cmd);
+      if (validated) {
+        return validated;
+      }
+    } catch (error) {
+      const errorMsg = error instanceof Error ? error.message : String(error);
+      logger.debug(`Direct command "${cmd}" failed: ${errorMsg}`);
+
+      // Log permission/access errors differently
+      if (
+        errorMsg.includes('Access is denied') ||
+        errorMsg.includes('EACCES') ||
+        errorMsg.includes('EPERM')
+      ) {
+        logger.warn(`Permission denied when trying Python command "${cmd}": ${errorMsg}`);
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ * Attempts to get the Python executable path using platform-appropriate strategies.
+ * @returns The Python executable path if successful, or null if failed.
+ */
+export async function getSysExecutable(): Promise<string | null> {
+  if (process.platform === 'win32') {
+    // Windows: Try 'where python' first to avoid Microsoft Store stubs
+    const whereResult = await tryWindowsWhere();
+    if (whereResult) {
+      return whereResult;
+    }
+
+    // Then try py launcher commands (removing python3 as it's uncommon on Windows)
+    const sysResult = await tryPythonCommands(['py', 'py -3']);
+    if (sysResult) {
+      return sysResult;
+    }
+
+    // Final fallback to direct python command
+    return await tryDirectCommands(['python']);
+  } else {
+    // Unix: Standard python3/python detection
+    return await tryPythonCommands(['python3', 'python']);
+  }
+}
+
+/**
+ * Attempts to validate a Python executable path.
+ * @param path - The path to the Python executable to test.
+ * @returns The validated path if successful, or null if invalid.
+ */
+export async function tryPath(path: string): Promise<string | null> {
+  let timeoutId: NodeJS.Timeout | undefined;
+
+  try {
+    const timeoutPromise = new Promise<never>((_, reject) => {
+      timeoutId = setTimeout(() => reject(new Error('Command timed out')), 2500);
+    });
+
+    const result = await Promise.race([execFileAsync(path, ['--version']), timeoutPromise]);
+
+    if (timeoutId) {
+      clearTimeout(timeoutId);
+    }
+
+    const versionOutput = (result as { stdout: string }).stdout.trim();
+    if (versionOutput.startsWith('Python')) {
+      return path;
+    }
+    return null;
+  } catch {
+    if (timeoutId) {
+      clearTimeout(timeoutId);
+    }
+    return null;
+  }
+}
+
+/**
+ * Validates and caches the Python executable path.
+ *
+ * @param pythonPath - Path to the Python executable.
+ * @param isExplicit - If true, only tries the provided path.
+ * @returns Validated Python executable path.
+ * @throws {Error} If no valid Python executable is found.
+ */
+export async function validatePythonPath(pythonPath: string, isExplicit: boolean): Promise<string> {
+  // Return cached result if available
+  if (state.cachedPythonPath) {
+    return state.cachedPythonPath;
+  }
+
+  // Create validation promise atomically if it doesn't exist
+  // This prevents race conditions where multiple calls create separate validations
+  if (!state.validationPromise) {
+    state.validationPromise = (async () => {
+      try {
+        const primaryPath = await tryPath(pythonPath);
+        if (primaryPath) {
+          state.cachedPythonPath = primaryPath;
+          state.validationPromise = null;
+          return primaryPath;
+        }
+
+        if (isExplicit) {
+          const error = new Error(
+            `Python 3 not found. Tried "${pythonPath}" ` +
+              `Please ensure Python 3 is installed and set the PROMPTFOO_PYTHON environment variable ` +
+              `to your Python 3 executable path (e.g., '${process.platform === 'win32' ? 'C:\\Python39\\python.exe' : '/usr/bin/python3'}').`,
+          );
+          // Clear promise on error to allow retry
+          state.validationPromise = null;
+          throw error;
+        }
+
+        // Try to get Python executable using comprehensive detection
+        const detectedPath = await getSysExecutable();
+        if (detectedPath) {
+          state.cachedPythonPath = detectedPath;
+          state.validationPromise = null;
+          return detectedPath;
+        }
+
+        const error = new Error(
+          `Python 3 not found. Tried "${pythonPath}", sys.executable detection, and fallback commands. ` +
+            `Please ensure Python 3 is installed and set the PROMPTFOO_PYTHON environment variable ` +
+            `to your Python 3 executable path (e.g., '${process.platform === 'win32' ? 'C:\\Python39\\python.exe' : '/usr/bin/python3'}').`,
+        );
+        // Clear promise on error to allow retry
+        state.validationPromise = null;
+        throw error;
+      } catch (error) {
+        // Ensure promise is cleared on any error
+        state.validationPromise = null;
+        throw error;
+      }
+    })();
+  }
+
+  // Return the existing or newly-created promise
+  return state.validationPromise;
+}
+
+/**
+ * Runs a Python script with the specified method and arguments.
+ *
+ * @param scriptPath - The path to the Python script to run.
+ * @param method - The name of the method to call in the Python script.
+ * @param args - An array of arguments to pass to the Python script.
+ * @param options - Optional settings for running the Python script.
+ * @param options.pythonExecutable - Optional path to the Python executable.
+ * @returns A promise that resolves to the output of the Python script.
+ * @throws An error if there's an issue running the Python script or parsing its output.
+ */
+export async function runPython<T = unknown>(
+  scriptPath: string,
+  method: string,
+  args: (string | number | object | undefined)[],
+  options: { pythonExecutable?: string } = {},
+): Promise<T> {
+  const absPath = path.resolve(scriptPath);
+  const customPath = getConfiguredPythonPath(options.pythonExecutable);
+  let pythonPath = customPath || 'python';
+  let tempDirectory: string | undefined;
+
+  pythonPath = await validatePythonPath(pythonPath, typeof customPath === 'string');
+
+  try {
+    tempDirectory = await createSecureTempDirectory('promptfoo-python-');
+    const tempJsonPath = await writeSecureTempFile(
+      tempDirectory,
+      'input.json',
+      safeJsonStringify(args) as string,
+    );
+    const outputPath = await writeSecureTempFile(tempDirectory, 'output.json', '');
+    const pythonOptions: PythonShellOptions = {
+      args: [absPath, method, tempJsonPath, outputPath],
+      env: process.env,
+      mode: 'binary',
+      pythonPath,
+      scriptPath: getWrapperDir('python'),
+      // When `inherit` is used, `import pdb; pdb.set_trace()` will work.
+      ...(getEnvBool('PROMPTFOO_PYTHON_DEBUG_ENABLED') && { stdio: 'inherit' }),
+    };
+
+    logger.debug('[Python] Running script', { scriptPath: absPath, method });
+
+    await new Promise<void>((resolve, reject) => {
+      try {
+        const pyshell = new PythonShell('wrapper.py', pythonOptions);
+        const stderrLogger = new PythonStderrLogger();
+
+        pyshell.stdout?.on('data', (chunk: Buffer) => {
+          logger.debug(chunk.toString('utf-8').trim());
+        });
+
+        pyshell.stderr?.on('data', (chunk: Buffer) => {
+          stderrLogger.handleData(chunk);
+        });
+
+        pyshell.end((err) => {
+          stderrLogger.flush();
+          if (err) {
+            reject(err);
+          } else {
+            resolve();
+          }
+        });
+      } catch (error) {
+        reject(error);
+      }
+    });
+
+    const output = await fs.readFile(outputPath, 'utf-8');
+    logger.debug('[Python] Script returned a result', { scriptPath: absPath });
+
+    let result: { type: 'final_result'; data: T } | undefined;
+    try {
+      result = JSON.parse(output);
+    } catch (error) {
+      throw new Error(
+        `Invalid JSON returned by Python script: ${(error as Error).message}\nStack Trace: ${
+          (error as Error).stack
+        }`,
+      );
+    }
+    if (result?.type !== 'final_result') {
+      throw new Error('The Python script `call_api` function must return a dict with an `output`');
+    }
+
+    return result.data;
+  } catch (error) {
+    const message = `Error running Python script: ${(error as Error).message}\nStack Trace: ${
+      (error as Error).stack?.replace('--- Python Traceback ---', 'Python Traceback: ') ||
+      'No Python traceback available'
+    }`;
+    logger.error(message);
+    throw new Error(message);
+  } finally {
+    if (tempDirectory) {
+      try {
+        await removeSecureTempDirectory(tempDirectory);
+      } catch (error) {
+        logger.error(`Error removing temporary Python directory: ${error}`);
+      }
+    }
+  }
+}

src/python/stderr.ts

@@ -0,0 +1,115 @@
+import { StringDecoder } from 'string_decoder';
+
+import logger from '../logger';
+
+export const MAX_STDERR_BUFFER_LENGTH = 16_384;
+
+type StderrLevel = 'debug' | 'info' | 'warn' | 'error';
+
+export class PythonStderrLogger {
+  private buffer = '';
+  private decoder = new StringDecoder('utf8');
+  private inTraceback = false;
+
+  constructor(private readonly prefix = '') {}
+
+  handleData(data: Buffer | string): void {
+    const text = typeof data === 'string' ? data : this.decoder.write(data);
+    this.buffer += text;
+
+    const carry = this.buffer.endsWith('\r') ? '\r' : '';
+    const normalized = (carry ? this.buffer.slice(0, -1) : this.buffer).replace(/\r\n?/g, '\n');
+    const lines = normalized.split('\n');
+    this.buffer = (lines.pop() ?? '') + carry;
+
+    for (const line of lines) {
+      this.logLine(line);
+    }
+
+    if (this.buffer.length >= MAX_STDERR_BUFFER_LENGTH) {
+      this.logLine(this.buffer);
+      this.buffer = '';
+    }
+  }
+
+  flush(): void {
+    const remaining = this.buffer + this.decoder.end();
+    this.buffer = '';
+
+    if (remaining) {
+      for (const line of remaining.split(/\r\n|[\r\n]/)) {
+        this.logLine(line);
+      }
+    }
+
+    this.inTraceback = false;
+    this.decoder = new StringDecoder('utf8');
+  }
+
+  private logLine(line: string): void {
+    if (!line.trim()) {
+      this.inTraceback = false;
+      return;
+    }
+
+    if (this.inTraceback && /^\s/.test(line)) {
+      this.writeLog('error', line);
+      return;
+    }
+
+    const trimmedStart = line.trimStart();
+    if (/^Traceback \(most recent call last\):/i.test(trimmedStart)) {
+      this.inTraceback = true;
+      this.writeLog('error', line);
+      return;
+    }
+
+    const prefixMatch = /^(DEBUG|INFO|WARN|WARNING|ERROR|CRITICAL|FATAL)\b[: ]?/i.exec(
+      trimmedStart,
+    );
+    if (prefixMatch) {
+      this.inTraceback = false;
+      this.writeLog(this.normalizeLevel(prefixMatch[1]), line);
+      return;
+    }
+
+    if (this.inTraceback) {
+      this.inTraceback = false;
+      this.writeLog('error', line);
+      return;
+    }
+
+    if (
+      /^(During handling of the above exception|The above exception was the direct cause)/i.test(
+        trimmedStart,
+      )
+    ) {
+      this.writeLog('error', line);
+      return;
+    }
+
+    this.writeLog('warn', line);
+  }
+
+  private normalizeLevel(level: string): StderrLevel {
+    switch (level.toUpperCase()) {
+      case 'DEBUG':
+        return 'debug';
+      case 'INFO':
+        return 'info';
+      case 'WARN':
+      case 'WARNING':
+        return 'warn';
+      case 'ERROR':
+      case 'CRITICAL':
+      case 'FATAL':
+        return 'error';
+      default:
+        return 'warn';
+    }
+  }
+
+  private writeLog(level: StderrLevel, line: string): void {
+    logger[level](`${this.prefix}${line}`);
+  }
+}

src/python/worker.ts

@@ -0,0 +1,310 @@
+import fs from 'fs/promises';
+import path from 'path';
+
+import { PythonShell } from 'python-shell';
+import { getWrapperDir } from '../esm';
+import logger from '../logger';
+import { getRequestTimeoutMs } from '../providers/shared';
+import { safeJsonStringify } from '../util/json';
+import {
+  createSecureTempDirectory,
+  removeSecureTempDirectory,
+  writeSecureTempFile,
+} from '../util/secureTempFiles';
+import { validatePythonPath } from './pythonUtils';
+import { PythonStderrLogger } from './stderr';
+
+export { MAX_STDERR_BUFFER_LENGTH } from './stderr';
+
+export class PythonWorker {
+  private process: PythonShell | null = null;
+  private ready: boolean = false;
+  private busy: boolean = false;
+  private shuttingDown: boolean = false;
+  private crashCount: number = 0;
+  private stderrLogger = new PythonStderrLogger('Python worker stderr: ');
+  private readonly maxCrashes: number = 3;
+  private pendingRequest: {
+    responseFile: string;
+    resolve: (result: unknown) => void;
+    reject: (error: Error) => void;
+  } | null = null;
+  private requestTimeout: NodeJS.Timeout | null = null;
+
+  constructor(
+    private scriptPath: string,
+    private functionName: string,
+    private pythonPath?: string,
+    private timeout: number = getRequestTimeoutMs(),
+    private onReady?: () => void,
+  ) {}
+
+  async initialize(): Promise<void> {
+    return this.startWorker();
+  }
+
+  private async startWorker(): Promise<void> {
+    const wrapperPath = path.join(getWrapperDir('python'), 'persistent_wrapper.py');
+
+    // Validate and resolve Python path using smart detection (tries python3, then python)
+    const resolvedPythonPath = await validatePythonPath(
+      this.pythonPath || 'python',
+      typeof this.pythonPath === 'string',
+    );
+
+    this.process = new PythonShell(wrapperPath, {
+      mode: 'text',
+      pythonPath: resolvedPythonPath,
+      args: [this.scriptPath, this.functionName],
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+
+    // Listen for READY signal
+    return new Promise((resolve, reject) => {
+      const readyTimeout = setTimeout(() => {
+        // Kill the process to prevent orphaned Python processes
+        // and avoid triggering handleCrash() which would retry
+        this.shuttingDown = true;
+        if (this.process) {
+          this.process.kill('SIGTERM');
+          this.process = null;
+        }
+        reject(new Error('Worker failed to become ready within timeout'));
+      }, 30000);
+
+      this.process!.on('message', (message: string) => {
+        if (message.trim() === 'READY') {
+          clearTimeout(readyTimeout);
+          this.ready = true;
+          logger.debug(`Python worker ready for ${this.scriptPath}`);
+          // Notify pool that worker is ready (triggers queue processing)
+          if (this.onReady) {
+            this.onReady();
+          }
+          resolve();
+        } else if (message.startsWith('DONE|')) {
+          this.handleDone(message.slice('DONE|'.length));
+        }
+      });
+
+      this.process!.on('error', (err) => {
+        clearTimeout(readyTimeout);
+        reject(err);
+      });
+
+      this.process!.on('close', () => {
+        this.flushStderr();
+        if (!this.shuttingDown) {
+          this.handleCrash();
+        }
+      });
+
+      this.process!.stderr?.on('data', (data) => {
+        this.handleStderr(data);
+      });
+    });
+  }
+
+  private handleStderr(data: Buffer | string): void {
+    this.stderrLogger.handleData(data);
+  }
+
+  private flushStderr(): void {
+    this.stderrLogger.flush();
+  }
+
+  async call(functionName: string, args: unknown[]): Promise<unknown> {
+    if (!this.ready) {
+      throw new Error('Worker not ready');
+    }
+
+    if (this.busy) {
+      throw new Error('Worker is busy');
+    }
+
+    this.busy = true;
+
+    try {
+      return await Promise.race([this.executeCall(functionName, args), this.createTimeout()]);
+    } finally {
+      this.busy = false;
+      if (this.requestTimeout) {
+        clearTimeout(this.requestTimeout);
+        this.requestTimeout = null;
+      }
+    }
+  }
+
+  private async executeCall(functionName: string, args: unknown[]): Promise<unknown> {
+    let tempDirectory: string | undefined;
+
+    try {
+      tempDirectory = await createSecureTempDirectory('promptfoo-worker-');
+      const requestFile = await writeSecureTempFile(
+        tempDirectory,
+        'request.json',
+        safeJsonStringify(args) as string,
+      );
+      const responseFile = await writeSecureTempFile(tempDirectory, 'response.json', '');
+
+      // Send CALL command with function name
+      // Note: PythonShell.send() adds newline automatically in 'text' mode
+      // Using pipe (|) delimiter to avoid conflicts with Windows drive letters (C:)
+      const command = `CALL|${functionName}|${requestFile}|${responseFile}`;
+      this.process!.send(command);
+
+      // Wait for DONE
+      await new Promise<unknown>((resolve, reject) => {
+        this.pendingRequest = { responseFile, resolve, reject };
+      });
+
+      // Read response with exponential backoff retry.
+      // Python verifies file readability before sending DONE, but OS-level delays may still occur.
+      let responseData: string | undefined;
+      let lastError: unknown;
+
+      // Exponential backoff: 1ms, 2ms, 4ms, 8ms, 16ms, 32ms, 64ms, 128ms, 256ms, 512ms, 1024ms, 2048ms, 4096ms, 5000ms (capped)...
+      // Total max wait: ~18 seconds (handles severe filesystem delays)
+      for (let attempt = 0, delay = 1; attempt < 16; attempt++, delay = Math.min(delay * 2, 5000)) {
+        try {
+          responseData = await fs.readFile(responseFile, 'utf-8');
+          if (attempt > 0) {
+            logger.debug(`Response file read succeeded on attempt ${attempt + 1}`);
+          }
+          break;
+        } catch (error: unknown) {
+          lastError = error;
+          if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') {
+            // File doesn't exist yet, wait and retry with exponential backoff.
+            await new Promise((resolve) => setTimeout(resolve, delay));
+            continue;
+          }
+          // Non-ENOENT error, don't retry.
+          throw error;
+        }
+      }
+
+      // If we exhausted all retries, throw with debugging info
+      if (!responseData) {
+        try {
+          const files = await fs.readdir(tempDirectory);
+          logger.error(
+            `Failed to read response file after 16 attempts (~18s). Expected: ${path.basename(responseFile)}, Found in temporary directory: ${files.join(', ')}`,
+          );
+        } catch {
+          logger.error(
+            `Failed to read Python worker response file: ${path.basename(responseFile)}`,
+          );
+        }
+        throw lastError instanceof Error
+          ? lastError
+          : new Error('Python worker response file was empty after completion signal');
+      }
+
+      const response = JSON.parse(responseData);
+
+      if (response.type === 'error') {
+        throw new Error(`Python error: ${response.error}\n${response.traceback || ''}`);
+      }
+
+      return response.data;
+    } finally {
+      if (tempDirectory) {
+        try {
+          await removeSecureTempDirectory(tempDirectory);
+        } catch (error) {
+          logger.error(`Error removing temporary Python worker directory: ${error}`);
+        }
+      }
+    }
+  }
+
+  private createTimeout(): Promise<never> {
+    return new Promise((_, reject) => {
+      this.requestTimeout = setTimeout(() => {
+        reject(new Error(`Python worker timed out after ${this.timeout}ms`));
+      }, this.timeout);
+      // Prevent timeout from keeping Node.js event loop alive
+      this.requestTimeout.unref();
+    });
+  }
+
+  private handleDone(responseFile: string): void {
+    const normalizedResponseFile = responseFile.replace(/[\r\n]+$/, '');
+    if (this.pendingRequest?.responseFile === normalizedResponseFile) {
+      this.pendingRequest.resolve(undefined);
+      this.pendingRequest = null;
+      return;
+    }
+    // Either no request is in flight, or the path does not match the one we
+    // dispatched. Do not record either path: the received marker is
+    // provider-controlled and the pending path belongs to a private temp dir.
+    logger.debug('Python worker ignored DONE marker that did not match the in-flight request', {
+      hasPendingRequest: this.pendingRequest !== null,
+    });
+  }
+
+  private handleCrash(): void {
+    this.ready = false;
+    this.crashCount++;
+
+    if (this.pendingRequest) {
+      this.pendingRequest.reject(new Error('Worker crashed'));
+      this.pendingRequest = null;
+    }
+
+    if (this.crashCount < this.maxCrashes) {
+      logger.warn(`Python worker crashed (${this.crashCount}/${this.maxCrashes}), restarting...`);
+      this.startWorker().catch((err) => {
+        logger.error(`Failed to restart worker: ${err}`);
+      });
+    } else {
+      logger.error(`Python worker crashed ${this.maxCrashes} times, marking as dead`);
+    }
+  }
+
+  isReady(): boolean {
+    return this.ready;
+  }
+
+  isBusy(): boolean {
+    return this.busy;
+  }
+
+  async shutdown(): Promise<void> {
+    if (!this.process) {
+      return;
+    }
+
+    try {
+      this.shuttingDown = true;
+
+      // Reject any in-flight request promptly
+      if (this.pendingRequest) {
+        this.pendingRequest.reject(new Error('Worker shutting down'));
+        this.pendingRequest = null;
+      }
+
+      // Note: PythonShell.send() adds newline automatically in 'text' mode
+      this.process.send('SHUTDOWN');
+
+      // Wait for exit (5s timeout)
+      await Promise.race([
+        new Promise<void>((resolve) => {
+          this.process!.on('close', () => resolve());
+        }),
+        new Promise<void>((resolve) => setTimeout(resolve, 5000).unref()),
+      ]);
+    } catch (error) {
+      logger.error(`Error during worker shutdown: ${error}`);
+    } finally {
+      if (this.process) {
+        this.process.kill('SIGTERM');
+        this.process = null;
+      }
+      this.ready = false;
+      this.busy = false;
+      this.shuttingDown = false;
+    }
+  }
+}

src/python/workerPool.ts

@@ -0,0 +1,144 @@
+import logger from '../logger';
+import { PythonWorker } from './worker';
+
+interface QueuedRequest {
+  functionName: string;
+  args: unknown[];
+  resolve: (result: unknown) => void;
+  reject: (error: Error) => void;
+}
+
+export class PythonWorkerPool {
+  private workers: PythonWorker[] = [];
+  private queue: QueuedRequest[] = [];
+  private isInitialized: boolean = false;
+
+  constructor(
+    private scriptPath: string,
+    private functionName: string,
+    private workerCount: number = 1,
+    private pythonPath?: string,
+    private timeout?: number,
+  ) {}
+
+  async initialize(): Promise<void> {
+    if (this.isInitialized) {
+      return;
+    }
+
+    // Validate worker count
+    if (this.workerCount < 1) {
+      throw new Error(`Invalid worker count: ${this.workerCount}. Must be at least 1.`);
+    }
+
+    // Warn on excessive workers
+    if (this.workerCount > 8) {
+      logger.warn(
+        `Spawning ${this.workerCount} Python workers for ${this.scriptPath}. ` +
+          `This may use significant memory if your script has heavy imports.`,
+      );
+    }
+
+    logger.debug(
+      `Initializing Python worker pool with ${this.workerCount} workers for ${this.scriptPath}`,
+    );
+
+    // Start all workers in parallel
+    const initPromises = [];
+    for (let i = 0; i < this.workerCount; i++) {
+      const worker = new PythonWorker(
+        this.scriptPath,
+        this.functionName,
+        this.pythonPath,
+        this.timeout,
+        () => this.processQueue(), // Resume queue processing when worker becomes ready
+      );
+      initPromises.push(worker.initialize());
+      this.workers.push(worker);
+    }
+
+    await Promise.all(initPromises);
+    this.isInitialized = true;
+    logger.debug(`Python worker pool initialized with ${this.workerCount} workers`);
+  }
+
+  // biome-ignore lint/suspicious/noExplicitAny: FIXME
+  async execute(functionName: string, args: unknown[]): Promise<any> {
+    if (!this.isInitialized) {
+      throw new Error('Worker pool not initialized');
+    }
+
+    // Try to get available worker
+    const worker = this.getAvailableWorker();
+
+    if (worker) {
+      // Worker available, execute immediately and trigger queue processing when done
+      return worker.call(functionName, args).finally(() => this.processQueue());
+    } else {
+      // All workers busy, queue the request
+      return new Promise<unknown>((resolve, reject) => {
+        this.queue.push({ functionName, args, resolve, reject });
+        logger.debug(`Request queued (queue size: ${this.queue.length})`);
+      });
+    }
+  }
+
+  private getAvailableWorker(): PythonWorker | null {
+    for (const worker of this.workers) {
+      if (worker.isReady() && !worker.isBusy()) {
+        return worker;
+      }
+    }
+    return null;
+  }
+
+  private processQueue(): void {
+    // Drain the entire queue - process all waiting requests with available workers
+    while (this.queue.length > 0) {
+      const worker = this.getAvailableWorker();
+      if (!worker) {
+        return; // No workers available right now
+      }
+
+      const request = this.queue.shift();
+      if (!request) {
+        return;
+      }
+
+      logger.debug(`Processing queued request (${this.queue.length} remaining)`);
+
+      // Execute and attach queue processing to continue draining when done
+      worker
+        .call(request.functionName, request.args)
+        .then(request.resolve)
+        .catch(request.reject)
+        .finally(() => this.processQueue());
+    }
+  }
+
+  getWorkerCount(): number {
+    return this.workers.length;
+  }
+
+  async shutdown(): Promise<void> {
+    logger.debug(`Shutting down Python worker pool (${this.workers.length} workers)`);
+
+    // Reject any queued requests
+    for (const req of this.queue) {
+      try {
+        req.reject(new Error('Worker pool shutting down'));
+      } catch {
+        // Ignore errors from rejecting
+      }
+    }
+
+    // Shutdown all workers in parallel
+    await Promise.all(this.workers.map((w) => w.shutdown()));
+
+    this.workers = [];
+    this.queue = [];
+    this.isInitialized = false;
+
+    logger.debug('Python worker pool shutdown complete');
+  }
+}

src/python/wrapper.py

@@ -0,0 +1,43 @@
+import asyncio
+import importlib.util
+import inspect
+import json
+import os
+import sys
+
+
+def call_method(script_path, method_name, *args):
+    script_dir = os.path.dirname(os.path.abspath(script_path))
+    module_name = os.path.basename(script_path).rsplit(".", 1)[0]
+    if script_dir not in sys.path:
+        sys.path.insert(0, script_dir)
+
+    print(f"Importing module {module_name} from {script_dir} ...")
+
+    spec = importlib.util.spec_from_file_location(module_name, script_path)
+    script_module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(script_module)
+
+    if "." in method_name:
+        class_name, classmethod_name = method_name.split(".")
+        method_to_call = getattr(getattr(script_module, class_name), classmethod_name)
+    else:
+        method_to_call = getattr(script_module, method_name)
+    if inspect.iscoroutinefunction(method_to_call):
+        return asyncio.run(method_to_call(*args))
+    else:
+        return method_to_call(*args)
+
+
+if __name__ == "__main__":
+    script_path = sys.argv[1]
+    method_name = sys.argv[2]
+    json_path = sys.argv[3]
+    output_path = sys.argv[4]
+    with open(json_path, "r", encoding="utf-8") as fp:
+        data = json.load(fp)
+
+    result = call_method(script_path, method_name, *data)
+    with open(output_path, "w", encoding="utf-8") as fp:
+        # Ensure Unicode is preserved by using ensure_ascii=False
+        json.dump({"type": "final_result", "data": result}, fp, ensure_ascii=False)

src/python/wrapper.ts

@@ -0,0 +1,40 @@
+import logger from '../logger';
+import {
+  createSecureTempDirectory,
+  removeSecureTempDirectory,
+  writeSecureTempFile,
+} from '../util/secureTempFiles';
+import { runPython } from './pythonUtils';
+
+/**
+ * Executes Python code by writing it to a temporary file
+ * @param {string} code - The Python code to execute.
+ * @param {string} method - The method name to call in the Python script.
+ * @param {(string | object | undefined)[]} args - The list of arguments to pass to the Python method.
+ * @returns {Promise<T>} - The result from executing the Python code.
+ */
+export async function runPythonCode<T = unknown>(
+  code: string,
+  method: string,
+  args: (string | object | undefined)[],
+): Promise<T> {
+  let tempDirectory: string | undefined;
+  try {
+    tempDirectory = await createSecureTempDirectory('promptfoo-python-code-');
+    const tempFilePath = await writeSecureTempFile(tempDirectory, 'script.py', code);
+    // Necessary to await so temp file doesn't get deleted.
+    const result = await runPython<T>(tempFilePath, method, args);
+    return result;
+  } catch (error) {
+    logger.error(`Error executing Python code: ${error}`);
+    throw error;
+  } finally {
+    if (tempDirectory) {
+      try {
+        await removeSecureTempDirectory(tempDirectory);
+      } catch (error) {
+        logger.error(`Error removing temporary Python code directory: ${error}`);
+      }
+    }
+  }
+}

src/python/wrapper_test.py

@@ -0,0 +1,110 @@
+import os
+import sys
+import unittest
+from unittest.mock import MagicMock, mock_open, patch
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from python import wrapper
+
+
+class TestWrapper(unittest.TestCase):
+    def create_temp_script(self, content: str) -> str:
+        """Creates a mock temporary script with the given content.
+
+        Args:
+            content: The content of the mock script.
+
+        Returns:
+            A string representing the path to the mock script.
+        """
+        with patch("builtins.open", mock_open(read_data=content)):
+            return "/path/to/mock_script.py"
+
+    def test_call_method_sync(self) -> None:
+        """Tests the call_method function with a synchronous function.
+
+        This test creates a mock script with a synchronous function,
+        patches the necessary modules, and verifies that call_method
+        correctly executes the function and returns the expected result.
+        """
+        script_content = """
+def test_function(arg1, arg2):
+    return arg1 + arg2
+"""
+        script_path = self.create_temp_script(script_content)
+        with patch("importlib.util.spec_from_file_location") as mock_spec:
+            mock_module = MagicMock()
+            mock_module.test_function.return_value = 3
+            mock_spec.return_value.loader.exec_module.side_effect = lambda m: setattr(
+                m, "test_function", mock_module.test_function
+            )
+            result = wrapper.call_method(script_path, "test_function", 1, 2)
+            self.assertEqual(result, 3)
+
+    def test_call_method_async(self) -> None:
+        """Tests the call_method function with an asynchronous function.
+
+        This test creates a mock script with an asynchronous function,
+        patches the necessary modules and asyncio functions, and verifies
+        that call_method correctly executes the async function and returns
+        the expected result.
+        """
+        script_content = """
+import asyncio
+
+async def test_async_function(arg1, arg2):
+    await asyncio.sleep(0.1)
+    return arg1 * arg2
+"""
+        script_path = self.create_temp_script(script_content)
+
+        # Create a real async function to avoid MagicMock + inspect recursion issues
+        async def fake_async(*args):
+            return 12
+
+        with patch("importlib.util.spec_from_file_location") as mock_spec:
+            mock_spec.return_value.loader.exec_module.side_effect = lambda m: setattr(
+                m, "test_async_function", fake_async
+            )
+            with patch("asyncio.run", return_value=12):
+                result = wrapper.call_method(script_path, "test_async_function", 3, 4)
+                self.assertEqual(result, 12)
+
+    def test_call_method_nonexistent_script(self) -> None:
+        """Tests the call_method function with a nonexistent script.
+
+        This test verifies that call_method raises a FileNotFoundError
+        when trying to execute a method from a nonexistent script.
+        """
+        with self.assertRaises(FileNotFoundError):
+            wrapper.call_method("/path/to/nonexistent/script.py", "some_method")
+
+    def test_call_method_with_classmethod(self) -> None:
+        """Tests the call_method function with methods that are Python's classmethods or staticmethods.
+
+        This test creates a mock script with a classmethod,
+        patches the necessary modules, and verifies that call_method
+        correctly executes the classmethod and returns the expected result.
+        """
+        script_content = """
+class TestClass:
+    @classmethod
+    def test_classmethod(cls, arg1, arg2):
+        return arg1 - arg2
+"""
+
+        script_path = self.create_temp_script(script_content)
+        with patch("importlib.util.spec_from_file_location") as mock_spec:
+            mock_module = MagicMock()
+            mock_module.TestClass.test_classmethod.return_value = 3
+            mock_spec.return_value.loader.exec_module.side_effect = lambda m: setattr(
+                m, "TestClass", mock_module.TestClass
+            )
+            result = wrapper.call_method(
+                script_path, "TestClass.test_classmethod", 4, 1
+            )
+            self.assertEqual(result, 3)
+
+
+if __name__ == "__main__":
+    unittest.main()