chore: import upstream snapshot with attribution

libs/python/agent/cua_agent/adapters/__init__.py

@@ -0,0 +1,19 @@
+"""
+Adapters package for agent - Custom LLM adapters for LiteLLM
+"""
+
+from .azure_ml_adapter import AzureMLAdapter
+from .cua_adapter import CUAAdapter
+from .huggingfacelocal_adapter import HuggingFaceLocalAdapter
+from .human_adapter import HumanAdapter
+from .mlxvlm_adapter import MLXVLMAdapter
+from .yutori_adapter import YutoriAdapter
+
+__all__ = [
+    "AzureMLAdapter",
+    "HuggingFaceLocalAdapter",
+    "HumanAdapter",
+    "MLXVLMAdapter",
+    "CUAAdapter",
+    "YutoriAdapter",
+]

libs/python/agent/cua_agent/adapters/azure_ml_adapter.py

@@ -0,0 +1,283 @@
+"""
+Azure ML Custom Provider Adapter for LiteLLM.
+
+This adapter provides direct OpenAI-compatible API access to Azure ML endpoints
+without message transformation, specifically for models like Fara-7B that require
+exact OpenAI message formatting.
+"""
+
+import json
+from typing import Any, AsyncIterator, Dict, Iterator, List, Optional
+
+import httpx
+from litellm import acompletion, completion
+from litellm.llms.custom_llm import CustomLLM
+from litellm.types.utils import GenericStreamingChunk, ModelResponse
+
+
+class AzureMLAdapter(CustomLLM):
+    """
+    Azure ML Adapter for OpenAI-compatible endpoints.
+
+    Makes direct HTTP calls to Azure ML foundry inference endpoints
+    using the OpenAI-compatible API format without transforming messages.
+
+    Usage:
+        model = "azure_ml/Fara-7B"
+        api_base = "https://foundry-inference-xxx.centralus.inference.ml.azure.com"
+        api_key = "your-api-key"
+
+        response = litellm.completion(
+            model=model,
+            messages=[...],
+            api_base=api_base,
+            api_key=api_key
+        )
+    """
+
+    def __init__(self, **kwargs):
+        """Initialize the adapter."""
+        super().__init__()
+        self._client: Optional[httpx.Client] = None
+        self._async_client: Optional[httpx.AsyncClient] = None
+
+    def _get_client(self) -> httpx.Client:
+        """Get or create sync HTTP client."""
+        if self._client is None:
+            self._client = httpx.Client(timeout=600.0)
+        return self._client
+
+    def _get_async_client(self) -> httpx.AsyncClient:
+        """Get or create async HTTP client."""
+        if self._async_client is None:
+            self._async_client = httpx.AsyncClient(timeout=600.0)
+        return self._async_client
+
+    def _prepare_request(self, **kwargs) -> tuple[str, dict, dict]:
+        """
+        Prepare the HTTP request without transforming messages.
+
+        Applies Azure ML workaround: double-encodes function arguments to work around
+        Azure ML's bug where it parses arguments before validation.
+
+        Returns:
+            Tuple of (url, headers, json_data)
+        """
+        # Extract required params
+        api_base = kwargs.get("api_base")
+        api_key = kwargs.get("api_key")
+        model = kwargs.get("model", "").replace("azure_ml/", "")
+        messages = kwargs.get("messages", [])
+
+        if not api_base:
+            raise ValueError("api_base is required for azure_ml provider")
+        if not api_key:
+            raise ValueError("api_key is required for azure_ml provider")
+
+        # Build OpenAI-compatible endpoint URL
+        base_url = api_base.rstrip("/")
+        url = f"{base_url}/chat/completions"
+
+        # Prepare headers
+        headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+
+        # WORKAROUND for Azure ML bug:
+        # Azure ML incorrectly parses the arguments field before validation,
+        # causing it to reject valid JSON strings. We double-encode arguments
+        # so that after Azure ML's parse, they remain as strings.
+        messages_copy = []
+        for message in messages:
+            msg_copy = message.copy()
+
+            # Check if message has tool_calls that need double-encoding
+            if "tool_calls" in msg_copy:
+                tool_calls_copy = []
+                for tool_call in msg_copy["tool_calls"]:
+                    tc_copy = tool_call.copy()
+
+                    if "function" in tc_copy and "arguments" in tc_copy["function"]:
+                        func_copy = tc_copy["function"].copy()
+                        arguments = func_copy["arguments"]
+
+                        # If arguments is already a string, double-encode it
+                        if isinstance(arguments, str):
+                            func_copy["arguments"] = json.dumps(arguments)
+
+                        tc_copy["function"] = func_copy
+
+                    tool_calls_copy.append(tc_copy)
+
+                msg_copy["tool_calls"] = tool_calls_copy
+
+            messages_copy.append(msg_copy)
+
+        # Prepare request body with double-encoded messages
+        json_data = {"model": model, "messages": messages_copy}
+
+        # Add optional parameters if provided
+        optional_params = [
+            "temperature",
+            "top_p",
+            "n",
+            "stream",
+            "stop",
+            "max_tokens",
+            "presence_penalty",
+            "frequency_penalty",
+            "logit_bias",
+            "user",
+            "response_format",
+            "seed",
+            "tools",
+            "tool_choice",
+        ]
+
+        for param in optional_params:
+            if param in kwargs and kwargs[param] is not None:
+                json_data[param] = kwargs[param]
+
+        return url, headers, json_data
+
+    def completion(self, *args, **kwargs) -> ModelResponse:
+        """
+        Synchronous completion method.
+
+        Makes a direct HTTP POST to Azure ML's OpenAI-compatible endpoint.
+        """
+        url, headers, json_data = self._prepare_request(**kwargs)
+
+        client = self._get_client()
+        response = client.post(url, headers=headers, json=json_data)
+        response.raise_for_status()
+
+        # Parse response
+        response_json = response.json()
+
+        # Return using litellm's completion with the actual response
+        return completion(
+            model=f"azure_ml/{kwargs.get('model', '')}",
+            mock_response=response_json["choices"][0]["message"]["content"],
+            messages=kwargs.get("messages", []),
+        )
+
+    async def acompletion(self, *args, **kwargs) -> ModelResponse:
+        """
+        Asynchronous completion method.
+
+        Makes a direct async HTTP POST to Azure ML's OpenAI-compatible endpoint.
+        """
+        url, headers, json_data = self._prepare_request(**kwargs)
+
+        client = self._get_async_client()
+        response = await client.post(url, headers=headers, json=json_data)
+        response.raise_for_status()
+
+        # Parse response
+        response_json = response.json()
+
+        # Return using litellm's acompletion with the actual response
+        return await acompletion(
+            model=f"azure_ml/{kwargs.get('model', '')}",
+            mock_response=response_json["choices"][0]["message"]["content"],
+            messages=kwargs.get("messages", []),
+        )
+
+    def streaming(self, *args, **kwargs) -> Iterator[GenericStreamingChunk]:
+        """
+        Synchronous streaming method.
+
+        Makes a streaming HTTP POST to Azure ML's OpenAI-compatible endpoint.
+        """
+        url, headers, json_data = self._prepare_request(**kwargs)
+        json_data["stream"] = True
+
+        client = self._get_client()
+
+        with client.stream("POST", url, headers=headers, json=json_data) as response:
+            response.raise_for_status()
+
+            for line in response.iter_lines():
+                if line.startswith("data: "):
+                    data = line[6:]  # Remove "data: " prefix
+                    if data == "[DONE]":
+                        break
+
+                    try:
+                        chunk_json = json.loads(data)
+                        delta = chunk_json["choices"][0].get("delta", {})
+                        content = delta.get("content", "")
+                        finish_reason = chunk_json["choices"][0].get("finish_reason")
+
+                        generic_streaming_chunk: GenericStreamingChunk = {
+                            "finish_reason": finish_reason,
+                            "index": 0,
+                            "is_finished": finish_reason is not None,
+                            "text": content,
+                            "tool_use": None,
+                            "usage": chunk_json.get(
+                                "usage",
+                                {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0},
+                            ),
+                        }
+
+                        yield generic_streaming_chunk
+                    except json.JSONDecodeError:
+                        continue
+
+    async def astreaming(self, *args, **kwargs) -> AsyncIterator[GenericStreamingChunk]:
+        """
+        Asynchronous streaming method.
+
+        Makes an async streaming HTTP POST to Azure ML's OpenAI-compatible endpoint.
+        """
+        url, headers, json_data = self._prepare_request(**kwargs)
+        json_data["stream"] = True
+
+        client = self._get_async_client()
+
+        async with client.stream("POST", url, headers=headers, json=json_data) as response:
+            response.raise_for_status()
+
+            async for line in response.aiter_lines():
+                if line.startswith("data: "):
+                    data = line[6:]  # Remove "data: " prefix
+                    if data == "[DONE]":
+                        break
+
+                    try:
+                        chunk_json = json.loads(data)
+                        delta = chunk_json["choices"][0].get("delta", {})
+                        content = delta.get("content", "")
+                        finish_reason = chunk_json["choices"][0].get("finish_reason")
+
+                        generic_streaming_chunk: GenericStreamingChunk = {
+                            "finish_reason": finish_reason,
+                            "index": 0,
+                            "is_finished": finish_reason is not None,
+                            "text": content,
+                            "tool_use": None,
+                            "usage": chunk_json.get(
+                                "usage",
+                                {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0},
+                            ),
+                        }
+
+                        yield generic_streaming_chunk
+                    except json.JSONDecodeError:
+                        continue
+
+    def __del__(self):
+        """Cleanup HTTP clients."""
+        if self._client is not None:
+            self._client.close()
+        if self._async_client is not None:
+            import asyncio
+
+            try:
+                loop = asyncio.get_event_loop()
+                if loop.is_running():
+                    loop.create_task(self._async_client.aclose())
+                else:
+                    loop.run_until_complete(self._async_client.aclose())
+            except Exception:
+                pass

libs/python/agent/cua_agent/adapters/cua_adapter.py

@@ -0,0 +1,250 @@
+import os
+from typing import Any, AsyncIterator, Iterator
+
+from cua_core.http import cua_version_headers
+from litellm import acompletion, completion
+from litellm.llms.custom_llm import CustomLLM
+from litellm.types.utils import GenericStreamingChunk, ModelResponse
+
+
+class CUAAdapter(CustomLLM):
+    def __init__(self, base_url: str | None = None, api_key: str | None = None, **_: Any):
+        super().__init__()
+        self.base_url = base_url or os.environ.get("CUA_BASE_URL") or "https://inference.cua.ai/v1"
+        self.api_key = (
+            api_key or os.environ.get("CUA_INFERENCE_API_KEY") or os.environ.get("CUA_API_KEY")
+        )
+
+    def _normalize_model(self, model: str) -> str:
+        """Strip known prefixes to get the base model name."""
+        known_prefixes = ("cua/", "anthropic/", "gemini/", "google/", "openai/")
+        result = model
+        for prefix in known_prefixes:
+            if result.startswith(prefix):
+                result = result[len(prefix) :]
+        return result
+
+    def _resolve_route(self, model: str, api_base: str) -> tuple[str, str]:
+        """Return (prefixed_model, api_base) for the CUA inference API."""
+        if "anthropic/" in model:
+            return f"anthropic/{self._normalize_model(model)}", api_base.removesuffix("/v1")
+        elif "gemini/" in model or "google/" in model:
+            return f"gemini/{self._normalize_model(model)}", api_base + "/gemini"
+        else:
+            return f"openai/{self._normalize_model(model)}", api_base
+
+    def _resolve_api_key(self, kwargs: dict | None = None) -> str:
+        """Resolve the CUA API key, raising a clear error if missing.
+
+        Checks kwargs (from ComputerAgent api_key param) then falls back
+        to self.api_key (from CUA_API_KEY / CUA_INFERENCE_API_KEY env vars).
+
+        This validation must run before the inner litellm call because that
+        call uses an anthropic/ or openai/ model prefix, which would cause
+        litellm to fall back to ANTHROPIC_API_KEY from env — sending the
+        wrong key to the CUA inference endpoint.
+        """
+        resolved = (kwargs.get("api_key") if kwargs else None) or self.api_key
+        if not resolved:
+            raise ValueError(
+                "No CUA API key provided for cua/ model inference. "
+                "Please either set the CUA_API_KEY environment variable "
+                "or pass api_key to ComputerAgent()."
+            )
+        return resolved
+
+    def completion(self, *args, **kwargs) -> ModelResponse:
+        model, api_base = self._resolve_route(
+            kwargs.get("model", ""), kwargs.get("api_base") or self.base_url
+        )
+
+        api_key = self._resolve_api_key(kwargs)
+
+        # Ensure the CUA inference API always receives Bearer auth;
+        # merge caller headers first, then force Authorization so it cannot be overridden.
+        extra_headers = {}
+        if "extra_headers" in kwargs:
+            extra_headers.update(kwargs.pop("extra_headers"))
+        extra_headers["Authorization"] = f"Bearer {api_key}"
+
+        params = {
+            "model": model,
+            "messages": kwargs.get("messages", []),
+            "api_base": api_base,
+            "api_key": api_key,
+            "extra_headers": extra_headers,
+            "stream": False,
+        }
+
+        # Forward tools if provided
+        if "tools" in kwargs:
+            params["tools"] = kwargs["tools"]
+
+        if "optional_params" in kwargs:
+            protected_keys = {"api_key", "extra_headers", "model", "api_base", "stream"}
+            filtered = {
+                k: v for k, v in kwargs["optional_params"].items() if k not in protected_keys
+            }
+            params.update(filtered)
+            del kwargs["optional_params"]
+
+        if "headers" in kwargs:
+            params["headers"] = kwargs["headers"]
+            del kwargs["headers"]
+
+        # Always include CUA version headers
+        version_hdrs = cua_version_headers()
+        if version_hdrs:
+            params["headers"] = {**version_hdrs, **params.get("headers", {})}
+
+        # Print dropped parameters
+        original_keys = set(kwargs.keys())
+        used_keys = set(params.keys())  # Only these are extracted from kwargs
+        ignored_keys = {
+            "litellm_params",
+            "client",
+            "print_verbose",
+            "acompletion",
+            "timeout",
+            "logging_obj",
+            "encoding",
+            "custom_prompt_dict",
+            "model_response",
+            "logger_fn",
+        }
+        dropped_keys = original_keys - used_keys - ignored_keys
+        if dropped_keys:
+            dropped_keyvals = {k: kwargs[k] for k in dropped_keys}
+            # print(f"CUAAdapter.completion: Dropped parameters: {dropped_keyvals}")
+
+        return completion(**params)  # type: ignore
+
+    async def acompletion(self, *args, **kwargs) -> ModelResponse:
+        model, api_base = self._resolve_route(
+            kwargs.get("model", ""), kwargs.get("api_base") or self.base_url
+        )
+
+        api_key = self._resolve_api_key(kwargs)
+
+        # Ensure the CUA inference API always receives Bearer auth;
+        # merge caller headers first, then force Authorization so it cannot be overridden.
+        extra_headers = {}
+        if "extra_headers" in kwargs:
+            extra_headers.update(kwargs.pop("extra_headers"))
+        extra_headers["Authorization"] = f"Bearer {api_key}"
+
+        params = {
+            "model": model,
+            "messages": kwargs.get("messages", []),
+            "api_base": api_base,
+            "api_key": api_key,
+            "extra_headers": extra_headers,
+            "stream": False,
+        }
+
+        # Forward tools if provided
+        if "tools" in kwargs:
+            params["tools"] = kwargs["tools"]
+
+        if "optional_params" in kwargs:
+            protected_keys = {"api_key", "extra_headers", "model", "api_base", "stream"}
+            filtered = {
+                k: v for k, v in kwargs["optional_params"].items() if k not in protected_keys
+            }
+            params.update(filtered)
+            del kwargs["optional_params"]
+
+        if "headers" in kwargs:
+            params["headers"] = kwargs["headers"]
+            del kwargs["headers"]
+
+        # Always include CUA version headers
+        version_hdrs = cua_version_headers()
+        if version_hdrs:
+            params["headers"] = {**version_hdrs, **params.get("headers", {})}
+
+        # Print dropped parameters
+        original_keys = set(kwargs.keys())
+        used_keys = set(params.keys())  # Only these are extracted from kwargs
+        ignored_keys = {
+            "litellm_params",
+            "client",
+            "print_verbose",
+            "acompletion",
+            "timeout",
+            "logging_obj",
+            "encoding",
+            "custom_prompt_dict",
+            "model_response",
+            "logger_fn",
+        }
+        dropped_keys = original_keys - used_keys - ignored_keys
+        if dropped_keys:
+            dropped_keyvals = {k: kwargs[k] for k in dropped_keys}
+            # print(f"CUAAdapter.acompletion: Dropped parameters: {dropped_keyvals}")
+
+        response = await acompletion(**params)  # type: ignore
+
+        return response
+
+    def streaming(self, *args, **kwargs) -> Iterator[GenericStreamingChunk]:
+        params = dict(kwargs)
+        model, api_base = self._resolve_route(
+            params.get("model", ""), params.get("api_base") or self.base_url
+        )
+        api_key = self._resolve_api_key(kwargs)
+
+        # Ensure the CUA inference API always receives Bearer auth;
+        # merge caller headers first, then force Authorization so it cannot be overridden.
+        extra_headers = {}
+        if "extra_headers" in params:
+            extra_headers.update(params.pop("extra_headers"))
+        extra_headers["Authorization"] = f"Bearer {api_key}"
+
+        params.update(
+            {
+                "model": model,
+                "api_base": api_base,
+                "api_key": api_key,
+                "extra_headers": extra_headers,
+                "stream": True,
+            }
+        )
+        # Always include CUA version headers
+        version_hdrs = cua_version_headers()
+        if version_hdrs:
+            params["headers"] = {**version_hdrs, **params.get("headers", {})}
+        # Yield chunks directly from LiteLLM's streaming generator
+        for chunk in completion(**params):  # type: ignore
+            yield chunk  # type: ignore
+
+    async def astreaming(self, *args, **kwargs) -> AsyncIterator[GenericStreamingChunk]:
+        params = dict(kwargs)
+        model, api_base = self._resolve_route(
+            params.get("model", ""), params.get("api_base") or self.base_url
+        )
+        api_key = self._resolve_api_key(kwargs)
+
+        # Ensure the CUA inference API always receives Bearer auth;
+        # merge caller headers first, then force Authorization so it cannot be overridden.
+        extra_headers = {}
+        if "extra_headers" in params:
+            extra_headers.update(params.pop("extra_headers"))
+        extra_headers["Authorization"] = f"Bearer {api_key}"
+
+        params.update(
+            {
+                "model": model,
+                "api_base": api_base,
+                "api_key": api_key,
+                "extra_headers": extra_headers,
+                "stream": True,
+            }
+        )
+        # Always include CUA version headers
+        version_hdrs = cua_version_headers()
+        if version_hdrs:
+            params["headers"] = {**version_hdrs, **params.get("headers", {})}
+        stream = await acompletion(**params)  # type: ignore
+        async for chunk in stream:  # type: ignore
+            yield chunk  # type: ignore

libs/python/agent/cua_agent/adapters/huggingfacelocal_adapter.py

@@ -0,0 +1,186 @@
+import asyncio
+import functools
+import warnings
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, AsyncIterator, Dict, Iterator, List, Optional
+
+from litellm import acompletion, completion
+from litellm.llms.custom_llm import CustomLLM
+from litellm.types.utils import GenericStreamingChunk, ModelResponse
+
+# Try to import HuggingFace dependencies
+try:
+    import torch
+    from transformers import AutoModelForImageTextToText, AutoProcessor
+
+    HF_AVAILABLE = True
+except ImportError:
+    HF_AVAILABLE = False
+
+from .models import load_model as load_model_handler
+
+
+class HuggingFaceLocalAdapter(CustomLLM):
+    """HuggingFace Local Adapter for running vision-language models locally."""
+
+    def __init__(self, device: str = "auto", trust_remote_code: bool = False, **kwargs):
+        """Initialize the adapter.
+
+        Args:
+            device: Device to load model on ("auto", "cuda", "cpu", etc.)
+            trust_remote_code: Whether to trust remote code
+            **kwargs: Additional arguments
+        """
+        super().__init__()
+        self.device = device
+        self.trust_remote_code = trust_remote_code
+        # Cache for model handlers keyed by model_name
+        self._handlers: Dict[str, Any] = {}
+        self._executor = ThreadPoolExecutor(max_workers=1)  # Single thread pool
+
+    def _get_handler(self, model_name: str):
+        """Get or create a model handler for the given model name."""
+        if model_name not in self._handlers:
+            self._handlers[model_name] = load_model_handler(
+                model_name=model_name, device=self.device, trust_remote_code=self.trust_remote_code
+            )
+        return self._handlers[model_name]
+
+    def _convert_messages(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Convert OpenAI format messages to HuggingFace format.
+
+        Args:
+            messages: Messages in OpenAI format
+
+        Returns:
+            Messages in HuggingFace format
+        """
+        converted_messages = []
+
+        for message in messages:
+            converted_message = {"role": message["role"], "content": []}
+
+            content = message.get("content", [])
+            if isinstance(content, str):
+                # Simple text content
+                converted_message["content"].append({"type": "text", "text": content})
+            elif isinstance(content, list):
+                # Multi-modal content
+                for item in content:
+                    if item.get("type") == "text":
+                        converted_message["content"].append(
+                            {"type": "text", "text": item.get("text", "")}
+                        )
+                    elif item.get("type") == "image_url":
+                        # Convert image_url format to image format
+                        image_url = item.get("image_url", {}).get("url", "")
+                        converted_message["content"].append({"type": "image", "image": image_url})
+
+            converted_messages.append(converted_message)
+
+        return converted_messages
+
+    def _generate(self, **kwargs) -> str:
+        """Generate response using the local HuggingFace model.
+
+        Args:
+            **kwargs: Keyword arguments containing messages and model info
+
+        Returns:
+            Generated text response
+        """
+        if not HF_AVAILABLE:
+            raise ImportError(
+                "HuggingFace transformers dependencies not found. "
+                'Please install with: pip install "cua-agent[uitars-hf]"'
+            )
+
+        # Extract messages and model from kwargs
+        messages = kwargs.get("messages", [])
+        model_name = kwargs.get("model", "ByteDance-Seed/UI-TARS-1.5-7B")
+        max_new_tokens = kwargs.get("max_tokens", 128)
+
+        # Warn about ignored kwargs
+        ignored_kwargs = set(kwargs.keys()) - {"messages", "model", "max_tokens"}
+        if ignored_kwargs:
+            warnings.warn(f"Ignoring unsupported kwargs: {ignored_kwargs}")
+
+        # Convert messages to HuggingFace format
+        hf_messages = self._convert_messages(messages)
+
+        # Delegate to model handler
+        handler = self._get_handler(model_name)
+        generated_text = handler.generate(hf_messages, max_new_tokens=max_new_tokens)
+        return generated_text
+
+    def completion(self, *args, **kwargs) -> ModelResponse:
+        """Synchronous completion method.
+
+        Returns:
+            ModelResponse with generated text
+        """
+        generated_text = self._generate(**kwargs)
+
+        return completion(
+            model=f"huggingface-local/{kwargs['model']}",
+            mock_response=generated_text,
+        )
+
+    async def acompletion(self, *args, **kwargs) -> ModelResponse:
+        """Asynchronous completion method.
+
+        Returns:
+            ModelResponse with generated text
+        """
+        # Run _generate in thread pool to avoid blocking
+        loop = asyncio.get_event_loop()
+        generated_text = await loop.run_in_executor(
+            self._executor, functools.partial(self._generate, **kwargs)
+        )
+
+        return await acompletion(
+            model=f"huggingface-local/{kwargs['model']}",
+            mock_response=generated_text,
+        )
+
+    def streaming(self, *args, **kwargs) -> Iterator[GenericStreamingChunk]:
+        """Synchronous streaming method.
+
+        Returns:
+            Iterator of GenericStreamingChunk
+        """
+        generated_text = self._generate(**kwargs)
+
+        generic_streaming_chunk: GenericStreamingChunk = {
+            "finish_reason": "stop",
+            "index": 0,
+            "is_finished": True,
+            "text": generated_text,
+            "tool_use": None,
+            "usage": {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0},
+        }
+
+        yield generic_streaming_chunk
+
+    async def astreaming(self, *args, **kwargs) -> AsyncIterator[GenericStreamingChunk]:
+        """Asynchronous streaming method.
+
+        Returns:
+            AsyncIterator of GenericStreamingChunk
+        """
+        # Run _generate in thread pool to avoid blocking
+        loop = asyncio.get_event_loop()
+        generated_text = await loop.run_in_executor(
+            self._executor, functools.partial(self._generate, **kwargs)
+        )
+
+        generic_streaming_chunk: GenericStreamingChunk = {
+            "finish_reason": "stop",
+            "index": 0,
+            "is_finished": True,
+            "text": generated_text,
+            "tool_use": None,
+            "usage": {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0},
+        }
+
+        yield generic_streaming_chunk

libs/python/agent/cua_agent/adapters/human_adapter.py

@@ -0,0 +1,350 @@
+import asyncio
+import os
+from typing import Any, AsyncIterator, Dict, Iterator, List
+
+import requests
+from litellm import acompletion, completion
+from litellm.llms.custom_llm import CustomLLM
+from litellm.types.utils import GenericStreamingChunk, ModelResponse
+
+
+class HumanAdapter(CustomLLM):
+    """Human Adapter for human-in-the-loop completions.
+
+    This adapter sends completion requests to a human completion server
+    where humans can review and respond to AI requests.
+    """
+
+    def __init__(self, base_url: str | None = None, timeout: float = 300.0, **kwargs):
+        """Initialize the human adapter.
+
+        Args:
+            base_url: Base URL for the human completion server.
+                     Defaults to HUMAN_BASE_URL environment variable or http://localhost:8002
+            timeout: Timeout in seconds for waiting for human response
+            **kwargs: Additional arguments
+        """
+        super().__init__()
+        self.base_url = base_url or os.getenv("HUMAN_BASE_URL", "http://localhost:8002")
+        self.timeout = timeout
+
+        # Ensure base_url doesn't end with slash
+        self.base_url = self.base_url.rstrip("/")
+
+    def _queue_completion(self, messages: List[Dict[str, Any]], model: str) -> str:
+        """Queue a completion request and return the call ID.
+
+        Args:
+            messages: Messages in OpenAI format
+            model: Model name
+
+        Returns:
+            Call ID for tracking the request
+
+        Raises:
+            Exception: If queueing fails
+        """
+        try:
+            response = requests.post(
+                f"{self.base_url}/queue", json={"messages": messages, "model": model}, timeout=10
+            )
+            response.raise_for_status()
+            return response.json()["id"]
+        except requests.RequestException as e:
+            raise Exception(f"Failed to queue completion request: {e}")
+
+    def _wait_for_completion(self, call_id: str) -> Dict[str, Any]:
+        """Wait for human to complete the call.
+
+        Args:
+            call_id: ID of the queued completion call
+
+        Returns:
+            Dict containing response and/or tool_calls
+
+        Raises:
+            TimeoutError: If timeout is exceeded
+            Exception: If completion fails
+        """
+        import time
+
+        start_time = time.time()
+
+        while True:
+            try:
+                # Check status
+                status_response = requests.get(f"{self.base_url}/status/{call_id}")
+                status_response.raise_for_status()
+                status_data = status_response.json()
+
+                if status_data["status"] == "completed":
+                    result = {}
+                    if "response" in status_data and status_data["response"]:
+                        result["response"] = status_data["response"]
+                    if "tool_calls" in status_data and status_data["tool_calls"]:
+                        result["tool_calls"] = status_data["tool_calls"]
+                    return result
+                elif status_data["status"] == "failed":
+                    error_msg = status_data.get("error", "Unknown error")
+                    raise Exception(f"Completion failed: {error_msg}")
+
+                # Check timeout
+                if time.time() - start_time > self.timeout:
+                    raise TimeoutError(
+                        f"Timeout waiting for human response after {self.timeout} seconds"
+                    )
+
+                # Wait before checking again
+                time.sleep(1.0)
+
+            except requests.RequestException as e:
+                if time.time() - start_time > self.timeout:
+                    raise TimeoutError(f"Timeout waiting for human response: {e}")
+                # Continue trying if we haven't timed out
+                time.sleep(1.0)
+
+    async def _async_wait_for_completion(self, call_id: str) -> Dict[str, Any]:
+        """Async version of wait_for_completion.
+
+        Args:
+            call_id: ID of the queued completion call
+
+        Returns:
+            Dict containing response and/or tool_calls
+
+        Raises:
+            TimeoutError: If timeout is exceeded
+            Exception: If completion fails
+        """
+        import time
+
+        import aiohttp
+
+        start_time = time.time()
+
+        async with aiohttp.ClientSession() as session:
+            while True:
+                try:
+                    # Check status
+                    async with session.get(f"{self.base_url}/status/{call_id}") as response:
+                        response.raise_for_status()
+                        status_data = await response.json()
+
+                    if status_data["status"] == "completed":
+                        result = {}
+                        if "response" in status_data and status_data["response"]:
+                            result["response"] = status_data["response"]
+                        if "tool_calls" in status_data and status_data["tool_calls"]:
+                            result["tool_calls"] = status_data["tool_calls"]
+                        return result
+                    elif status_data["status"] == "failed":
+                        error_msg = status_data.get("error", "Unknown error")
+                        raise Exception(f"Completion failed: {error_msg}")
+
+                    # Check timeout
+                    if time.time() - start_time > self.timeout:
+                        raise TimeoutError(
+                            f"Timeout waiting for human response after {self.timeout} seconds"
+                        )
+
+                    # Wait before checking again
+                    await asyncio.sleep(1.0)
+
+                except Exception as e:
+                    if time.time() - start_time > self.timeout:
+                        raise TimeoutError(f"Timeout waiting for human response: {e}")
+                    # Continue trying if we haven't timed out
+                    await asyncio.sleep(1.0)
+
+    def _generate_response(self, messages: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
+        """Generate a human response for the given messages.
+
+        Args:
+            messages: Messages in OpenAI format
+            model: Model name
+
+        Returns:
+            Dict containing response and/or tool_calls
+        """
+        # Queue the completion request
+        call_id = self._queue_completion(messages, model)
+
+        # Wait for human response
+        response = self._wait_for_completion(call_id)
+
+        return response
+
+    async def _async_generate_response(
+        self, messages: List[Dict[str, Any]], model: str
+    ) -> Dict[str, Any]:
+        """Async version of _generate_response.
+
+        Args:
+            messages: Messages in OpenAI format
+            model: Model name
+
+        Returns:
+            Dict containing response and/or tool_calls
+        """
+        # Queue the completion request (sync operation)
+        call_id = self._queue_completion(messages, model)
+
+        # Wait for human response (async)
+        response = await self._async_wait_for_completion(call_id)
+
+        return response
+
+    def completion(self, *args, **kwargs) -> ModelResponse:
+        """Synchronous completion method.
+
+        Returns:
+            ModelResponse with human-generated text or tool calls
+        """
+        messages = kwargs.get("messages", [])
+        model = kwargs.get("model", "human")
+
+        # Generate human response
+        human_response_data = self._generate_response(messages, model)
+
+        # Create ModelResponse with proper structure
+        import time
+        import uuid
+
+        from litellm.types.utils import Choices, Message, ModelResponse
+
+        # Create message content based on response type
+        if "tool_calls" in human_response_data and human_response_data["tool_calls"]:
+            # Tool calls response
+            message = Message(
+                role="assistant",
+                content=human_response_data.get("response", ""),
+                tool_calls=human_response_data["tool_calls"],
+            )
+        else:
+            # Text response
+            message = Message(role="assistant", content=human_response_data.get("response", ""))
+
+        choice = Choices(finish_reason="stop", index=0, message=message)
+
+        result = ModelResponse(
+            id=f"human-{uuid.uuid4()}",
+            choices=[choice],
+            created=int(time.time()),
+            model=f"human/{model}",
+            object="chat.completion",
+        )
+
+        return result
+
+    async def acompletion(self, *args, **kwargs) -> ModelResponse:
+        """Asynchronous completion method.
+
+        Returns:
+            ModelResponse with human-generated text or tool calls
+        """
+        messages = kwargs.get("messages", [])
+        model = kwargs.get("model", "human")
+
+        # Generate human response
+        human_response_data = await self._async_generate_response(messages, model)
+
+        # Create ModelResponse with proper structure
+        import time
+        import uuid
+
+        from litellm.types.utils import Choices, Message, ModelResponse
+
+        # Create message content based on response type
+        if "tool_calls" in human_response_data and human_response_data["tool_calls"]:
+            # Tool calls response
+            message = Message(
+                role="assistant",
+                content=human_response_data.get("response", ""),
+                tool_calls=human_response_data["tool_calls"],
+            )
+        else:
+            # Text response
+            message = Message(role="assistant", content=human_response_data.get("response", ""))
+
+        choice = Choices(finish_reason="stop", index=0, message=message)
+
+        result = ModelResponse(
+            id=f"human-{uuid.uuid4()}",
+            choices=[choice],
+            created=int(time.time()),
+            model=f"human/{model}",
+            object="chat.completion",
+        )
+
+        return result
+
+    def streaming(self, *args, **kwargs) -> Iterator[GenericStreamingChunk]:
+        """Synchronous streaming method.
+
+        Yields:
+            Streaming chunks with human-generated text or tool calls
+        """
+        messages = kwargs.get("messages", [])
+        model = kwargs.get("model", "human")
+
+        # Generate human response
+        human_response_data = self._generate_response(messages, model)
+
+        import time
+
+        # Handle tool calls vs text response
+        if "tool_calls" in human_response_data and human_response_data["tool_calls"]:
+            # Stream tool calls as a single chunk
+            generic_chunk: GenericStreamingChunk = {
+                "finish_reason": "tool_calls",
+                "index": 0,
+                "is_finished": True,
+                "text": human_response_data.get("response", ""),
+                "tool_use": human_response_data["tool_calls"],
+                "usage": {"completion_tokens": 1, "prompt_tokens": 0, "total_tokens": 1},
+            }
+            yield generic_chunk
+        else:
+            # Stream text response
+            response_text = human_response_data.get("response", "")
+            generic_chunk: GenericStreamingChunk = {
+                "finish_reason": "stop",
+                "index": 0,
+                "is_finished": True,
+                "text": response_text,
+                "tool_use": None,
+                "usage": {
+                    "completion_tokens": len(response_text.split()),
+                    "prompt_tokens": 0,
+                    "total_tokens": len(response_text.split()),
+                },
+            }
+            yield generic_chunk
+
+    async def astreaming(self, *args, **kwargs) -> AsyncIterator[GenericStreamingChunk]:
+        """Asynchronous streaming method.
+
+        Yields:
+            Streaming chunks with human-generated text or tool calls
+        """
+        messages = kwargs.get("messages", [])
+        model = kwargs.get("model", "human")
+
+        # Generate human response
+        human_response = await self._async_generate_response(messages, model)
+
+        # Return as single streaming chunk
+        generic_streaming_chunk: GenericStreamingChunk = {
+            "finish_reason": "stop",
+            "index": 0,
+            "is_finished": True,
+            "text": human_response,
+            "tool_use": None,
+            "usage": {
+                "completion_tokens": len(human_response.split()),
+                "prompt_tokens": 0,
+                "total_tokens": len(human_response.split()),
+            },
+        }
+
+        yield generic_streaming_chunk

libs/python/agent/cua_agent/adapters/mlxvlm_adapter.py

@@ -0,0 +1,370 @@
+import asyncio
+import base64
+import functools
+import io
+import math
+import re
+import warnings
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Tuple, cast
+
+from litellm import acompletion, completion
+from litellm.llms.custom_llm import CustomLLM
+from litellm.types.utils import GenericStreamingChunk, ModelResponse
+from PIL import Image
+
+# Try to import MLX dependencies
+try:
+    import mlx.core as mx
+    from mlx_vlm import generate, load
+    from mlx_vlm.prompt_utils import apply_chat_template
+    from mlx_vlm.utils import load_config
+    from transformers.tokenization_utils import PreTrainedTokenizer
+
+    MLX_AVAILABLE = True
+except ImportError:
+    MLX_AVAILABLE = False
+
+# Constants for smart_resize
+IMAGE_FACTOR = 28
+MIN_PIXELS = 100 * 28 * 28
+MAX_PIXELS = 16384 * 28 * 28
+MAX_RATIO = 200
+
+
+def round_by_factor(number: float, factor: int) -> int:
+    """Returns the closest integer to 'number' that is divisible by 'factor'."""
+    return round(number / factor) * factor
+
+
+def ceil_by_factor(number: float, factor: int) -> int:
+    """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
+    return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: float, factor: int) -> int:
+    """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
+    return math.floor(number / factor) * factor
+
+
+def smart_resize(
+    height: int,
+    width: int,
+    factor: int = IMAGE_FACTOR,
+    min_pixels: int = MIN_PIXELS,
+    max_pixels: int = MAX_PIXELS,
+) -> tuple[int, int]:
+    """
+    Rescales the image so that the following conditions are met:
+
+    1. Both dimensions (height and width) are divisible by 'factor'.
+    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+    3. The aspect ratio of the image is maintained as closely as possible.
+    """
+    if max(height, width) / min(height, width) > MAX_RATIO:
+        raise ValueError(
+            f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
+        )
+    h_bar = max(factor, round_by_factor(height, factor))
+    w_bar = max(factor, round_by_factor(width, factor))
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = floor_by_factor(height / beta, factor)
+        w_bar = floor_by_factor(width / beta, factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = ceil_by_factor(height * beta, factor)
+        w_bar = ceil_by_factor(width * beta, factor)
+    return h_bar, w_bar
+
+
+class MLXVLMAdapter(CustomLLM):
+    """MLX VLM Adapter for running vision-language models locally using MLX."""
+
+    def __init__(self, **kwargs):
+        """Initialize the adapter.
+
+        Args:
+            **kwargs: Additional arguments
+        """
+        super().__init__()
+
+        self.models = {}  # Cache for loaded models
+        self.processors = {}  # Cache for loaded processors
+        self.configs = {}  # Cache for loaded configs
+        self._executor = ThreadPoolExecutor(max_workers=1)  # Single thread pool
+
+    def _load_model_and_processor(self, model_name: str):
+        """Load model and processor if not already cached.
+
+        Args:
+            model_name: Name of the model to load
+
+        Returns:
+            Tuple of (model, processor, config)
+        """
+        if not MLX_AVAILABLE:
+            raise ImportError("MLX VLM dependencies not available. Please install mlx-vlm.")
+
+        if model_name not in self.models:
+            # Load model and processor
+            model_obj, processor = load(
+                model_name, processor_kwargs={"min_pixels": MIN_PIXELS, "max_pixels": MAX_PIXELS}
+            )
+            config = load_config(model_name)
+
+            # Cache them
+            self.models[model_name] = model_obj
+            self.processors[model_name] = processor
+            self.configs[model_name] = config
+
+        return self.models[model_name], self.processors[model_name], self.configs[model_name]
+
+    def _process_coordinates(
+        self, text: str, original_size: Tuple[int, int], model_size: Tuple[int, int]
+    ) -> str:
+        """Process coordinates in box tokens based on image resizing using smart_resize approach.
+
+        Args:
+            text: Text containing box tokens
+            original_size: Original image size (width, height)
+            model_size: Model processed image size (width, height)
+
+        Returns:
+            Text with processed coordinates
+        """
+        # Find all box tokens
+        box_pattern = r"<\|box_start\|>\((\d+),\s*(\d+)\)<\|box_end\|>"
+
+        def process_coords(match):
+            model_x, model_y = int(match.group(1)), int(match.group(2))
+            # Scale coordinates from model space to original image space
+            # Both original_size and model_size are in (width, height) format
+            new_x = int(model_x * original_size[0] / model_size[0])  # Width
+            new_y = int(model_y * original_size[1] / model_size[1])  # Height
+            return f"<|box_start|>({new_x},{new_y})<|box_end|>"
+
+        return re.sub(box_pattern, process_coords, text)
+
+    def _convert_messages(self, messages: List[Dict[str, Any]]) -> Tuple[
+        List[Dict[str, Any]],
+        List[Image.Image],
+        Dict[int, Tuple[int, int]],
+        Dict[int, Tuple[int, int]],
+    ]:
+        """Convert OpenAI format messages to MLX VLM format and extract images.
+
+        Args:
+            messages: Messages in OpenAI format
+
+        Returns:
+            Tuple of (processed_messages, images, original_sizes, model_sizes)
+        """
+        processed_messages = []
+        images = []
+        original_sizes = {}  # Track original sizes of images for coordinate mapping
+        model_sizes = {}  # Track model processed sizes
+        image_index = 0
+
+        for message in messages:
+            processed_message = {"role": message["role"], "content": []}
+
+            content = message.get("content", [])
+            if isinstance(content, str):
+                # Simple text content
+                processed_message["content"] = content
+            elif isinstance(content, list):
+                # Multi-modal content
+                processed_content = []
+                for item in content:
+                    if item.get("type") == "text":
+                        processed_content.append({"type": "text", "text": item.get("text", "")})
+                    elif item.get("type") == "image_url":
+                        image_url = item.get("image_url", {}).get("url", "")
+                        pil_image = None
+
+                        if image_url.startswith("data:image/"):
+                            # Extract base64 data
+                            base64_data = image_url.split(",")[1]
+                            # Convert base64 to PIL Image
+                            image_data = base64.b64decode(base64_data)
+                            pil_image = Image.open(io.BytesIO(image_data))
+                        else:
+                            # Handle file path or URL
+                            pil_image = Image.open(image_url)
+
+                        # Store original image size for coordinate mapping
+                        original_size = pil_image.size
+                        original_sizes[image_index] = original_size
+
+                        # Use smart_resize to determine model size
+                        # Note: smart_resize expects (height, width) but PIL gives (width, height)
+                        height, width = original_size[1], original_size[0]
+                        new_height, new_width = smart_resize(height, width)
+                        # Store model size in (width, height) format for consistent coordinate processing
+                        model_sizes[image_index] = (new_width, new_height)
+
+                        # Resize the image using the calculated dimensions from smart_resize
+                        resized_image = pil_image.resize((new_width, new_height))
+                        images.append(resized_image)
+
+                        # Add image placeholder to content
+                        processed_content.append({"type": "image"})
+
+                        image_index += 1
+
+                processed_message["content"] = processed_content
+
+            processed_messages.append(processed_message)
+
+        return processed_messages, images, original_sizes, model_sizes
+
+    def _generate(self, **kwargs) -> str:
+        """Generate response using the local MLX VLM model.
+
+        Args:
+            **kwargs: Keyword arguments containing messages and model info
+
+        Returns:
+            Generated text response
+        """
+        messages = kwargs.get("messages", [])
+        model_name = kwargs.get("model", "mlx-community/UI-TARS-1.5-7B-4bit")
+        max_tokens = kwargs.get("max_tokens", 128)
+
+        # Warn about ignored kwargs
+        ignored_kwargs = set(kwargs.keys()) - {"messages", "model", "max_tokens"}
+        if ignored_kwargs:
+            warnings.warn(f"Ignoring unsupported kwargs: {ignored_kwargs}")
+
+        # Load model and processor
+        model, processor, config = self._load_model_and_processor(model_name)
+
+        # Convert messages and extract images
+        processed_messages, images, original_sizes, model_sizes = self._convert_messages(messages)
+
+        # Process user text input with box coordinates after image processing
+        # Swap original_size and model_size arguments for inverse transformation
+        for msg_idx, msg in enumerate(processed_messages):
+            if msg.get("role") == "user" and isinstance(msg.get("content"), str):
+                content = msg.get("content", "")
+                if (
+                    "<|box_start|>" in content
+                    and original_sizes
+                    and model_sizes
+                    and 0 in original_sizes
+                    and 0 in model_sizes
+                ):
+                    orig_size = original_sizes[0]
+                    model_size = model_sizes[0]
+                    # Swap arguments to perform inverse transformation for user input
+                    processed_messages[msg_idx]["content"] = self._process_coordinates(
+                        content, model_size, orig_size
+                    )
+
+        try:
+            # Format prompt according to model requirements using the processor directly
+            prompt = processor.apply_chat_template(
+                processed_messages, tokenize=False, add_generation_prompt=True, return_tensors="pt"
+            )
+            tokenizer = cast(PreTrainedTokenizer, processor)
+
+            # Generate response
+            text_content, usage = generate(
+                model,
+                tokenizer,
+                str(prompt),
+                images,  # type: ignore
+                verbose=False,
+                max_tokens=max_tokens,
+            )
+
+        except Exception as e:
+            raise RuntimeError(f"Error generating response: {str(e)}") from e
+
+        # Process coordinates in the response back to original image space
+        if original_sizes and model_sizes and 0 in original_sizes and 0 in model_sizes:
+            # Get original image size and model size (using the first image)
+            orig_size = original_sizes[0]
+            model_size = model_sizes[0]
+
+            # Check if output contains box tokens that need processing
+            if "<|box_start|>" in text_content:
+                # Process coordinates from model space back to original image space
+                text_content = self._process_coordinates(text_content, orig_size, model_size)
+
+        return text_content
+
+    def completion(self, *args, **kwargs) -> ModelResponse:
+        """Synchronous completion method.
+
+        Returns:
+            ModelResponse with generated text
+        """
+        generated_text = self._generate(**kwargs)
+
+        result = completion(
+            model=f"mlx/{kwargs.get('model', 'mlx-community/UI-TARS-1.5-7B-4bit')}",
+            mock_response=generated_text,
+        )
+        return cast(ModelResponse, result)
+
+    async def acompletion(self, *args, **kwargs) -> ModelResponse:
+        """Asynchronous completion method.
+
+        Returns:
+            ModelResponse with generated text
+        """
+        # Run _generate in thread pool to avoid blocking
+        loop = asyncio.get_event_loop()
+        generated_text = await loop.run_in_executor(
+            self._executor, functools.partial(self._generate, **kwargs)
+        )
+
+        result = await acompletion(
+            model=f"mlx/{kwargs.get('model', 'mlx-community/UI-TARS-1.5-7B-4bit')}",
+            mock_response=generated_text,
+        )
+        return cast(ModelResponse, result)
+
+    def streaming(self, *args, **kwargs) -> Iterator[GenericStreamingChunk]:
+        """Synchronous streaming method.
+
+        Returns:
+            Iterator of GenericStreamingChunk
+        """
+        generated_text = self._generate(**kwargs)
+
+        generic_streaming_chunk: GenericStreamingChunk = {
+            "finish_reason": "stop",
+            "index": 0,
+            "is_finished": True,
+            "text": generated_text,
+            "tool_use": None,
+            "usage": {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0},
+        }
+
+        yield generic_streaming_chunk
+
+    async def astreaming(self, *args, **kwargs) -> AsyncIterator[GenericStreamingChunk]:
+        """Asynchronous streaming method.
+
+        Returns:
+            AsyncIterator of GenericStreamingChunk
+        """
+        # Run _generate in thread pool to avoid blocking
+        loop = asyncio.get_event_loop()
+        generated_text = await loop.run_in_executor(
+            self._executor, functools.partial(self._generate, **kwargs)
+        )
+
+        generic_streaming_chunk: GenericStreamingChunk = {
+            "finish_reason": "stop",
+            "index": 0,
+            "is_finished": True,
+            "text": generated_text,
+            "tool_use": None,
+            "usage": {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0},
+        }
+
+        yield generic_streaming_chunk

libs/python/agent/cua_agent/adapters/models/__init__.py

@@ -0,0 +1,41 @@
+from typing import Optional
+
+try:
+    from transformers import AutoConfig
+
+    HF_AVAILABLE = True
+except ImportError:
+    HF_AVAILABLE = False
+
+from .generic import GenericHFModel
+from .internvl import InternVLModel
+from .opencua import OpenCUAModel
+from .qwen2_5_vl import Qwen2_5_VLModel
+
+
+def load_model(model_name: str, device: str = "auto", trust_remote_code: bool = False):
+    """Factory function to load and return the right model handler instance.
+
+    - If the underlying transformers config class matches OpenCUA, return OpenCUAModel
+    - Otherwise, return GenericHFModel
+    """
+    if not HF_AVAILABLE:
+        raise ImportError(
+            'HuggingFace transformers dependencies not found. Install with: pip install "cua-agent[uitars-hf]"'
+        )
+    cfg = AutoConfig.from_pretrained(model_name, trust_remote_code=trust_remote_code)
+    cls = cfg.__class__.__name__
+    print(f"cls: {cls}")
+    if "OpenCUA" in cls:
+        return OpenCUAModel(
+            model_name=model_name, device=device, trust_remote_code=trust_remote_code
+        )
+    elif "Qwen2_5_VL" in cls:
+        return Qwen2_5_VLModel(
+            model_name=model_name, device=device, trust_remote_code=trust_remote_code
+        )
+    elif "InternVL" in cls:
+        return InternVLModel(
+            model_name=model_name, device=device, trust_remote_code=trust_remote_code
+        )
+    return GenericHFModel(model_name=model_name, device=device, trust_remote_code=trust_remote_code)

libs/python/agent/cua_agent/adapters/models/generic.py

@@ -0,0 +1,78 @@
+from typing import Any, Dict, List, Optional
+
+# Hugging Face imports are local to avoid hard dependency at module import
+try:
+    import torch  # type: ignore
+    from transformers import AutoModel, AutoProcessor  # type: ignore
+
+    HF_AVAILABLE = True
+except Exception:
+    HF_AVAILABLE = False
+
+
+class GenericHFModel:
+    """Generic Hugging Face vision-language model handler.
+    Loads an AutoModelForImageTextToText and AutoProcessor and generates text.
+    """
+
+    def __init__(
+        self, model_name: str, device: str = "auto", trust_remote_code: bool = False
+    ) -> None:
+        if not HF_AVAILABLE:
+            raise ImportError(
+                'HuggingFace transformers dependencies not found. Install with: pip install "cua-agent[uitars-hf]"'
+            )
+        self.model_name = model_name
+        self.device = device
+        self.model = None
+        self.processor = None
+        self.trust_remote_code = trust_remote_code
+        self._load()
+
+    def _load(self) -> None:
+        # Load model
+        self.model = AutoModel.from_pretrained(
+            self.model_name,
+            torch_dtype=torch.float16,
+            device_map=self.device,
+            attn_implementation="sdpa",
+            trust_remote_code=self.trust_remote_code,
+        )
+        # Load processor
+        self.processor = AutoProcessor.from_pretrained(
+            self.model_name,
+            min_pixels=3136,
+            max_pixels=4096 * 2160,
+            device_map=self.device,
+            trust_remote_code=self.trust_remote_code,
+        )
+
+    def generate(self, messages: List[Dict[str, Any]], max_new_tokens: int = 128) -> str:
+        """Generate text for the given HF-format messages.
+        messages: [{ role, content: [{type:'text'|'image', text|image}] }]
+        """
+        assert self.model is not None and self.processor is not None
+        # Apply chat template and tokenize
+        inputs = self.processor.apply_chat_template(
+            messages,
+            add_generation_prompt=True,
+            tokenize=True,
+            return_dict=True,
+            return_tensors="pt",
+        )
+        # Move inputs to the same device as model
+        inputs = inputs.to(self.model.device)
+        # Generate
+        with torch.no_grad():
+            generated_ids = self.model.generate(**inputs, max_new_tokens=max_new_tokens)
+        # Trim prompt tokens from output
+        generated_ids_trimmed = [
+            out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+        ]
+        # Decode
+        output_text = self.processor.batch_decode(
+            generated_ids_trimmed,
+            skip_special_tokens=True,
+            clean_up_tokenization_spaces=False,
+        )
+        return output_text[0] if output_text else ""

libs/python/agent/cua_agent/adapters/models/internvl.py

@@ -0,0 +1,290 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+# Hugging Face imports are local to avoid hard dependency at module import
+try:
+    import base64  # type: ignore
+    from io import BytesIO  # type: ignore
+
+    # Attempt to import InternVL's model dependencies
+    import einops as _  # type: ignore
+    import requests  # type: ignore
+    import timm as _  # type: ignore
+    import torch  # type: ignore
+    import torchvision.transforms as T  # type: ignore
+    from PIL import Image  # type: ignore
+    from torchvision.transforms.functional import InterpolationMode  # type: ignore
+    from transformers import AutoModel, AutoTokenizer  # type: ignore
+
+    HF_AVAILABLE = True
+except Exception:
+    HF_AVAILABLE = False
+
+
+class InternVLModel:
+    """Generic Hugging Face vision-language model handler.
+    Uses InternVL's native `model.chat()` interface with `AutoTokenizer`.
+    Provides preprocessing to support multi-turn conversations with multiple images.
+    """
+
+    def __init__(
+        self, model_name: str, device: str = "auto", trust_remote_code: bool = False
+    ) -> None:
+        if not HF_AVAILABLE:
+            raise ImportError(
+                'InternVL dependencies not found. Install with: pip install "cua-agent[internvl-hf]"'
+            )
+        self.model_name = model_name
+        self.device = device
+        self.model = None
+        self.tokenizer = None
+        self.trust_remote_code = trust_remote_code
+        self._load()
+
+    def _load(self) -> None:
+        # Load model
+        self.model = AutoModel.from_pretrained(
+            self.model_name,
+            torch_dtype=torch.bfloat16,
+            low_cpu_mem_usage=True,
+            use_flash_attn=True,
+            device_map=self.device,
+            trust_remote_code=self.trust_remote_code,
+        ).eval()
+        # Load tokenizer (InternVL requires trust_remote_code=True and often use_fast=False)
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            self.model_name,
+            trust_remote_code=self.trust_remote_code,
+            use_fast=False,
+        )
+
+    # ---- Image preprocessing utilities adapted from InternVL docs ----
+    IMAGENET_MEAN = (0.485, 0.456, 0.406)
+    IMAGENET_STD = (0.229, 0.224, 0.225)
+
+    def _build_transform(self, input_size: int) -> T.Compose:
+        MEAN, STD = self.IMAGENET_MEAN, self.IMAGENET_STD
+        transform = T.Compose(
+            [
+                T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img),
+                T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
+                T.ToTensor(),
+                T.Normalize(mean=MEAN, std=STD),
+            ]
+        )
+        return transform
+
+    def _find_closest_aspect_ratio(
+        self,
+        aspect_ratio: float,
+        target_ratios: List[tuple],
+        width: int,
+        height: int,
+        image_size: int,
+    ):
+        best_ratio_diff = float("inf")
+        best_ratio = (1, 1)
+        area = width * height
+        for ratio in target_ratios:
+            target_aspect_ratio = ratio[0] / ratio[1]
+            ratio_diff = abs(aspect_ratio - target_aspect_ratio)
+            if ratio_diff < best_ratio_diff:
+                best_ratio_diff = ratio_diff
+                best_ratio = ratio
+            elif ratio_diff == best_ratio_diff:
+                if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
+                    best_ratio = ratio
+        return best_ratio
+
+    def _dynamic_preprocess(
+        self,
+        image: Image.Image,
+        min_num: int = 1,
+        max_num: int = 12,
+        image_size: int = 448,
+        use_thumbnail: bool = True,
+    ) -> List[Image.Image]:
+        orig_width, orig_height = image.size
+        aspect_ratio = orig_width / orig_height
+
+        target_ratios = set(
+            (i, j)
+            for n in range(min_num, max_num + 1)
+            for i in range(1, n + 1)
+            for j in range(1, n + 1)
+            if i * j <= max_num and i * j >= min_num
+        )
+        target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
+
+        target_aspect_ratio = self._find_closest_aspect_ratio(
+            aspect_ratio, target_ratios, orig_width, orig_height, image_size
+        )
+
+        target_width = image_size * target_aspect_ratio[0]
+        target_height = image_size * target_aspect_ratio[1]
+        blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
+
+        resized_img = image.resize((target_width, target_height))
+        processed_images: List[Image.Image] = []
+        for i in range(blocks):
+            box = (
+                (i % (target_width // image_size)) * image_size,
+                (i // (target_width // image_size)) * image_size,
+                ((i % (target_width // image_size)) + 1) * image_size,
+                ((i // (target_width // image_size)) + 1) * image_size,
+            )
+            split_img = resized_img.crop(box)
+            processed_images.append(split_img)
+        assert len(processed_images) == blocks
+        if use_thumbnail and len(processed_images) != 1:
+            thumbnail_img = image.resize((image_size, image_size))
+            processed_images.append(thumbnail_img)
+        return processed_images
+
+    def _load_image_from_source(self, src: str) -> Image.Image:
+        """Load PIL image from various sources: data URL, http(s), or local path."""
+        if src.startswith("data:image/"):
+            # data URL base64
+            header, b64data = src.split(",", 1)
+            img_bytes = base64.b64decode(b64data)
+            return Image.open(BytesIO(img_bytes)).convert("RGB")
+        if src.startswith("http://") or src.startswith("https://"):
+            resp = requests.get(src, timeout=10)
+            resp.raise_for_status()
+            return Image.open(BytesIO(resp.content)).convert("RGB")
+        # Assume local file path
+        return Image.open(src).convert("RGB")
+
+    def _images_to_pixel_values(
+        self, images: List[Image.Image], input_size: int = 448, max_num: int = 12
+    ):
+        transform = self._build_transform(input_size=input_size)
+        pixel_values_list = []
+        num_patches_list: List[int] = []
+        for img in images:
+            tiles = self._dynamic_preprocess(
+                img, image_size=input_size, use_thumbnail=True, max_num=max_num
+            )
+            pv = [transform(tile) for tile in tiles]
+            pv = torch.stack(pv)
+            num_patches_list.append(pv.shape[0])
+            pixel_values_list.append(pv)
+        if not pixel_values_list:
+            return None, []
+        pixel_values = torch.cat(pixel_values_list)
+        return pixel_values, num_patches_list
+
+    def generate(self, messages: List[Dict[str, Any]], max_new_tokens: int = 128) -> str:
+        """Generate text for the given HF-format messages.
+        messages: [{ role, content: [{type:'text'|'image', text|image}] }]
+
+        This implementation constructs InternVL-compatible inputs and uses
+        `model.chat(tokenizer, pixel_values, question, history=...)` to avoid
+        relying on AutoProcessor (which fails for some tokenizers).
+        """
+        assert self.model is not None and self.tokenizer is not None
+
+        # Build textual context and collect images and the final question
+        context_lines: List[str] = []
+        all_images: List[Image.Image] = []
+        last_user_text_parts: List[str] = []
+
+        for msg in messages:
+            role = msg.get("role", "user")
+            content = msg.get("content", [])
+            if isinstance(content, str):
+                content_items = [{"type": "text", "text": content}]
+            else:
+                content_items = content
+
+            if role == "user":
+                # Collect text and images
+                parts_text: List[str] = []
+                for item in content_items:
+                    if item.get("type") == "text":
+                        t = item.get("text", "")
+                        if t:
+                            parts_text.append(t)
+                    elif item.get("type") == "image":
+                        url = item.get("image", "")
+                        if url:
+                            try:
+                                all_images.append(self._load_image_from_source(url))
+                            except Exception:
+                                # Ignore failed image loads but keep going
+                                pass
+                text = "\n".join(parts_text).strip()
+                if text:
+                    context_lines.append(f"User: {text}")
+                # Track last user text separately for question
+                last_user_text_parts = parts_text or last_user_text_parts
+            elif role == "assistant":
+                # Only keep text content for history
+                parts_text = [
+                    item.get("text", "") for item in content_items if item.get("type") == "text"
+                ]
+                text = "\n".join(parts_text).strip()
+                if text:
+                    context_lines.append(f"Assistant: {text}")
+
+        # Prepare pixel values for all collected images (across turns)
+        pixel_values = None
+        num_patches_list: List[int] = []
+        if all_images:
+            pixel_values, num_patches_list = self._images_to_pixel_values(
+                all_images, input_size=448, max_num=12
+            )
+            if pixel_values is not None:
+                # Convert dtype/device as in docs
+                pixel_values = pixel_values.to(torch.bfloat16)
+                # Chat API expects tensors on CUDA when model is on CUDA
+                try:
+                    pixel_values = pixel_values.to(self.model.device)
+                except Exception:
+                    pass
+
+        # Build question with any prior context and numbered image placeholders
+        if all_images:
+            # Separate images layout: Image-1: <image> ... then question text
+            prefix_lines = [f"Image-{i+1}: <image>" for i in range(len(all_images))]
+            prefix = "\n".join(prefix_lines) + "\n"
+        else:
+            prefix = ""
+
+        last_user_text = "\n".join(last_user_text_parts).strip()
+        # Combine prior text-only turns as context to emulate multi-turn
+        context_text = "\n".join(context_lines[:-1]) if len(context_lines) > 1 else ""
+        base_question = last_user_text if last_user_text else "Describe the image(s) in detail."
+        if context_text:
+            question = (context_text + "\n" + prefix + base_question).strip()
+        else:
+            question = (prefix + base_question).strip()
+
+        # Generation config
+        generation_config = dict(max_new_tokens=max_new_tokens, do_sample=False)
+
+        # Call InternVL chat
+        try:
+            if pixel_values is None:
+                # Pure-text conversation (embed prior turns in question)
+                response = self.model.chat(self.tokenizer, None, question, generation_config)
+            else:
+                # Multi-image: pass num_patches_list if >1 image
+                if len(num_patches_list) > 1:
+                    response = self.model.chat(
+                        self.tokenizer,
+                        pixel_values,
+                        question,
+                        generation_config,
+                        num_patches_list=num_patches_list,
+                    )
+                else:
+                    response = self.model.chat(
+                        self.tokenizer, pixel_values, question, generation_config
+                    )
+        except Exception as e:
+            # Fallback: return empty string to avoid crashing the adapter
+            return ""
+
+        return response or ""

libs/python/agent/cua_agent/adapters/models/opencua.py

@@ -0,0 +1,115 @@
+import base64
+import re
+from io import BytesIO
+from typing import Any, Dict, List
+
+try:
+    import blobfile as _  # assert blobfile is installed
+    import torch  # type: ignore
+    from PIL import Image  # type: ignore
+    from transformers import (  # type: ignore
+        AutoImageProcessor,
+        AutoModel,
+        AutoTokenizer,
+    )
+
+    OPENCUA_AVAILABLE = True
+except Exception:
+    OPENCUA_AVAILABLE = False
+
+
+class OpenCUAModel:
+    """OpenCUA model handler using AutoTokenizer, AutoModel and AutoImageProcessor."""
+
+    def __init__(
+        self, model_name: str, device: str = "auto", trust_remote_code: bool = False
+    ) -> None:
+        if not OPENCUA_AVAILABLE:
+            raise ImportError(
+                'OpenCUA requirements not found. Install with: pip install "cua-agent[opencua-hf]"'
+            )
+        self.model_name = model_name
+        self.device = device
+        self.model = None
+        self.tokenizer = None
+        self.image_processor = None
+        self.trust_remote_code = trust_remote_code
+        self._load()
+
+    def _load(self) -> None:
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            self.model_name, trust_remote_code=self.trust_remote_code
+        )
+        self.model = AutoModel.from_pretrained(
+            self.model_name,
+            torch_dtype="auto",
+            device_map=self.device,
+            trust_remote_code=self.trust_remote_code,
+            attn_implementation="sdpa",
+        )
+        self.image_processor = AutoImageProcessor.from_pretrained(
+            self.model_name, trust_remote_code=self.trust_remote_code
+        )
+
+    @staticmethod
+    def _extract_last_image_b64(messages: List[Dict[str, Any]]) -> str:
+        # Expect HF-format messages with content items type: "image" with data URL
+        for msg in reversed(messages):
+            for item in reversed(msg.get("content", [])):
+                if isinstance(item, dict) and item.get("type") == "image":
+                    url = item.get("image", "")
+                    if isinstance(url, str) and url.startswith("data:image/"):
+                        return url.split(",", 1)[1]
+        return ""
+
+    def generate(self, messages: List[Dict[str, Any]], max_new_tokens: int = 512) -> str:
+        assert (
+            self.model is not None
+            and self.tokenizer is not None
+            and self.image_processor is not None
+        )
+
+        # Tokenize text side using chat template
+        input_ids = self.tokenizer.apply_chat_template(
+            messages, tokenize=True, add_generation_prompt=True
+        )
+        input_ids = torch.tensor([input_ids]).to(self.model.device)
+
+        # Prepare image inputs from last data URL image
+        image_b64 = self._extract_last_image_b64(messages)
+        pixel_values = None
+        grid_thws = None
+        if image_b64:
+            image = Image.open(BytesIO(base64.b64decode(image_b64))).convert("RGB")
+            image_info = self.image_processor.preprocess(images=[image])
+            pixel_values = torch.tensor(image_info["pixel_values"]).to(
+                dtype=torch.bfloat16, device=self.model.device
+            )
+            grid_thws = (
+                torch.tensor(image_info["image_grid_thw"])
+                if "image_grid_thw" in image_info
+                else None
+            )
+
+        gen_kwargs: Dict[str, Any] = {
+            "max_new_tokens": max_new_tokens,
+            "temperature": 0,
+        }
+        if pixel_values is not None:
+            gen_kwargs["pixel_values"] = pixel_values
+        if grid_thws is not None:
+            gen_kwargs["grid_thws"] = grid_thws
+
+        with torch.no_grad():
+            generated_ids = self.model.generate(
+                input_ids,
+                **gen_kwargs,
+            )
+
+        # Remove prompt tokens
+        prompt_len = input_ids.shape[1]
+        generated_ids = generated_ids[:, prompt_len:]
+        output_text = self.tokenizer.batch_decode(
+            generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )[0]
+        return output_text

libs/python/agent/cua_agent/adapters/models/qwen2_5_vl.py

@@ -0,0 +1,78 @@
+from typing import Any, Dict, List, Optional
+
+# Hugging Face imports are local to avoid hard dependency at module import
+try:
+    import torch  # type: ignore
+    from transformers import AutoModelForImageTextToText, AutoProcessor  # type: ignore
+
+    HF_AVAILABLE = True
+except Exception:
+    HF_AVAILABLE = False
+
+
+class Qwen2_5_VLModel:
+    """Qwen2.5-VL Hugging Face vision-language model handler.
+    Loads an AutoModelForImageTextToText and AutoProcessor and generates text.
+    """
+
+    def __init__(
+        self, model_name: str, device: str = "auto", trust_remote_code: bool = False
+    ) -> None:
+        if not HF_AVAILABLE:
+            raise ImportError(
+                'HuggingFace transformers dependencies not found. Install with: pip install "cua-agent[uitars-hf]"'
+            )
+        self.model_name = model_name
+        self.device = device
+        self.model = None
+        self.processor = None
+        self.trust_remote_code = trust_remote_code
+        self._load()
+
+    def _load(self) -> None:
+        # Load model
+        self.model = AutoModelForImageTextToText.from_pretrained(
+            self.model_name,
+            torch_dtype=torch.bfloat16,
+            device_map=self.device,
+            attn_implementation="sdpa",
+            trust_remote_code=self.trust_remote_code,
+        )
+        # Load processor
+        self.processor = AutoProcessor.from_pretrained(
+            self.model_name,
+            min_pixels=3136,
+            max_pixels=4096 * 2160,
+            device_map=self.device,
+            trust_remote_code=self.trust_remote_code,
+        )
+
+    def generate(self, messages: List[Dict[str, Any]], max_new_tokens: int = 128) -> str:
+        """Generate text for the given HF-format messages.
+        messages: [{ role, content: [{type:'text'|'image', text|image}] }]
+        """
+        assert self.model is not None and self.processor is not None
+        # Apply chat template and tokenize
+        inputs = self.processor.apply_chat_template(
+            messages,
+            add_generation_prompt=True,
+            tokenize=True,
+            return_dict=True,
+            return_tensors="pt",
+        )
+        # Move inputs to the same device as model
+        inputs = inputs.to(self.model.device)
+        # Generate
+        with torch.no_grad():
+            generated_ids = self.model.generate(**inputs, max_new_tokens=max_new_tokens)
+        # Trim prompt tokens from output
+        generated_ids_trimmed = [
+            out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+        ]
+        # Decode
+        output_text = self.processor.batch_decode(
+            generated_ids_trimmed,
+            skip_special_tokens=True,
+            clean_up_tokenization_spaces=False,
+        )
+        return output_text[0] if output_text else ""

libs/python/agent/cua_agent/adapters/yutori_adapter.py

@@ -0,0 +1,99 @@
+"""
+Yutori adapter for litellm - routes yutori/ prefixed models to the Yutori API.
+"""
+
+import os
+from typing import Any, AsyncIterator, Iterator
+
+from litellm import acompletion, completion
+from litellm.llms.custom_llm import CustomLLM
+from litellm.types.utils import GenericStreamingChunk, ModelResponse
+
+YUTORI_API_BASE = "https://api.yutori.com/v1"
+
+
+class YutoriAdapter(CustomLLM):
+    def __init__(self, base_url: str | None = None, api_key: str | None = None, **_: Any):
+        super().__init__()
+        self.base_url = base_url or os.environ.get("YUTORI_API_BASE") or YUTORI_API_BASE
+        self.api_key = api_key or os.environ.get("YUTORI_API_KEY")
+
+    def _normalize_model(self, model: str) -> str:
+        """Strip the yutori/ prefix to get the bare model name."""
+        if model.startswith("yutori/"):
+            return model[len("yutori/") :]
+        return model
+
+    def _resolve_api_key(self, kwargs: dict | None = None) -> str:
+        """Resolve the Yutori API key, raising a clear error if missing."""
+        resolved = (kwargs.get("api_key") if kwargs else None) or self.api_key
+        if not resolved:
+            raise ValueError(
+                "No Yutori API key provided. "
+                "Please either set the YUTORI_API_KEY environment variable "
+                "or pass api_key to ComputerAgent()."
+            )
+        return resolved
+
+    def _build_params(self, kwargs: dict) -> dict:
+        """Build parameters for the inner litellm call."""
+        model = self._normalize_model(kwargs.get("model", ""))
+        api_key = self._resolve_api_key(kwargs)
+
+        extra_headers = {}
+        if "extra_headers" in kwargs:
+            extra_headers.update(kwargs.pop("extra_headers"))
+        extra_headers["Authorization"] = f"Bearer {api_key}"
+
+        params = {
+            "model": f"openai/{model}",
+            "messages": kwargs.get("messages", []),
+            "api_base": self.base_url,
+            "api_key": api_key,
+            "extra_headers": extra_headers,
+            "stream": False,
+        }
+
+        # Forward tools if provided
+        if "tools" in kwargs:
+            params["tools"] = kwargs["tools"]
+        if "tool_choice" in kwargs:
+            params["tool_choice"] = kwargs["tool_choice"]
+
+        # Forward optional generation params
+        for key in (
+            "temperature",
+            "top_p",
+            "max_completion_tokens",
+            "max_tokens",
+            "response_format",
+        ):
+            if key in kwargs:
+                params[key] = kwargs[key]
+
+        if "optional_params" in kwargs:
+            protected_keys = {"api_key", "extra_headers", "model", "api_base", "stream"}
+            filtered = {
+                k: v for k, v in kwargs["optional_params"].items() if k not in protected_keys
+            }
+            params.update(filtered)
+
+        if "headers" in kwargs:
+            params["headers"] = kwargs["headers"]
+
+        return params
+
+    def completion(self, *args, **kwargs) -> ModelResponse:
+        params = self._build_params(kwargs)
+        return completion(**params)  # type: ignore
+
+    async def acompletion(self, *args, **kwargs) -> ModelResponse:
+        params = self._build_params(kwargs)
+        response = await acompletion(**params)  # type: ignore
+        return response
+
+    def streaming(self, *args, **kwargs) -> Iterator[GenericStreamingChunk]:
+        raise NotImplementedError("Yutori n1 does not support streaming.")
+
+    async def astreaming(self, *args, **kwargs) -> AsyncIterator[GenericStreamingChunk]:
+        raise NotImplementedError("Yutori n1 does not support streaming.")