chore: import upstream snapshot with attribution

libs/python/agent/cua_agent/loops/__init__.py

@@ -0,0 +1,50 @@
+"""
+Agent loops for agent
+"""
+
+# Import the loops to register them
+from . import (
+    anthropic,
+    composed_grounded,
+    fara,
+    gelato,
+    gemini,
+    generic_vlm,
+    glm45v,
+    gta1,
+    holo,
+    internvl,
+    moondream3,
+    omniparser,
+    openai,
+    opencua,
+    qwen3vl,
+    qwen35,
+    uiins,
+    uitars,
+    uitars2,
+    yutori,
+)
+
+__all__ = [
+    "anthropic",
+    "composed_grounded",
+    "gelato",
+    "gemini",
+    "generic_vlm",
+    "fara",
+    "glm45v",
+    "gta1",
+    "holo",
+    "internvl",
+    "moondream3",
+    "omniparser",
+    "openai",
+    "opencua",
+    "qwen3vl",
+    "qwen35",
+    "uiins",
+    "uitars",
+    "uitars2",
+    "yutori",
+]

libs/python/agent/cua_agent/loops/base.py

@@ -0,0 +1,80 @@
+"""
+Base protocol for async agent configurations
+"""
+
+from abc import abstractmethod
+from typing import Any, Dict, List, Optional, Protocol, Tuple, Union
+
+from ..types import AgentCapability
+
+
+class AsyncAgentConfig(Protocol):
+    """Protocol defining the interface for async agent configurations."""
+
+    @abstractmethod
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **generation_config,
+    ) -> Dict[str, Any]:
+        """
+        Predict the next step based on input items.
+
+        Args:
+            messages: Input items following Responses format (message, function_call, computer_call)
+            model: Model name to use
+            tools: Optional list of tool schemas
+            max_retries: Maximum number of retries for failed API calls
+            stream: Whether to stream responses
+            computer_handler: Computer handler instance
+            _on_api_start: Callback for API start
+            _on_api_end: Callback for API end
+            _on_usage: Callback for usage tracking
+            _on_screenshot: Callback for screenshot events
+            **generation_config: Additional arguments to pass to the model provider
+                - api_key: Optional API key for the provider
+                - api_base: Optional API base URL for the provider
+
+        Returns:
+            Dictionary with "output" (output items) and "usage" array
+        """
+        ...
+
+    @abstractmethod
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **generation_config
+    ) -> Optional[Tuple[int, int]]:
+        """
+        Predict click coordinates based on image and instruction.
+
+        Args:
+            model: Model name to use
+            image_b64: Base64 encoded image
+            instruction: Instruction for where to click
+            **generation_config: Additional arguments to pass to the model provider
+                - api_key: Optional API key for the provider
+                - api_base: Optional API base URL for the provider
+
+        Returns:
+            None or tuple with (x, y) coordinates
+        """
+        ...
+
+    @abstractmethod
+    def get_capabilities(self) -> List[AgentCapability]:
+        """
+        Get list of capabilities supported by this agent config.
+
+        Returns:
+            List of capability strings (e.g., ["step", "click"])
+        """
+        ...

libs/python/agent/cua_agent/loops/composed_grounded.py

@@ -0,0 +1,316 @@
+"""
+Composed-grounded agent loop implementation that combines grounding and thinking models.
+Uses a two-stage approach: grounding model for element detection, thinking model for reasoning.
+"""
+
+import asyncio
+import base64
+import json
+import uuid
+from io import BytesIO
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from PIL import Image
+
+from ..agent import find_agent_config
+from ..decorators import register_agent
+from ..loops.base import AsyncAgentConfig
+from ..responses import (
+    convert_completion_messages_to_responses_items,
+    convert_computer_calls_desc2xy,
+    convert_computer_calls_xy2desc,
+    convert_responses_items_to_completion_messages,
+    get_all_element_descriptions,
+)
+from ..types import AgentCapability, AgentResponse, Messages, Tools
+
+GROUNDED_COMPUTER_TOOL_SCHEMA = {
+    "type": "function",
+    "function": {
+        "name": "computer",
+        "description": "Control a computer by taking screenshots and interacting with UI elements. This tool uses element descriptions to locate and interact with UI elements on the screen (e.g., 'red submit button', 'search text field', 'hamburger menu icon', 'close button in top right corner').",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "action": {
+                    "type": "string",
+                    "enum": [
+                        "screenshot",
+                        "click",
+                        "double_click",
+                        "drag",
+                        "type",
+                        "keypress",
+                        "scroll",
+                        "move",
+                        "wait",
+                        "get_current_url",
+                        "get_dimensions",
+                        "get_environment",
+                    ],
+                    "description": "The action to perform (required for all actions)",
+                },
+                "element_description": {
+                    "type": "string",
+                    "description": "Description of the element to interact with (required for click, double_click, move, scroll actions)",
+                },
+                "start_element_description": {
+                    "type": "string",
+                    "description": "Description of the element to start dragging from (required for drag action)",
+                },
+                "end_element_description": {
+                    "type": "string",
+                    "description": "Description of the element to drag to (required for drag action)",
+                },
+                "text": {
+                    "type": "string",
+                    "description": "The text to type (required for type action)",
+                },
+                "keys": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Key(s) to press (required for keypress action)",
+                },
+                "button": {
+                    "type": "string",
+                    "enum": ["left", "right", "wheel", "back", "forward"],
+                    "description": "The mouse button to use for click action (required for click and double_click action)",
+                },
+                "scroll_x": {
+                    "type": "integer",
+                    "description": "Horizontal scroll amount for scroll action (required for scroll action)",
+                },
+                "scroll_y": {
+                    "type": "integer",
+                    "description": "Vertical scroll amount for scroll action (required for scroll action)",
+                },
+            },
+            "required": ["action"],
+        },
+    },
+}
+
+
+def _prepare_tools_for_grounded(tool_schemas: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Prepare tools for grounded API format"""
+    grounded_tools = []
+
+    for schema in tool_schemas:
+        if schema["type"] == "computer":
+            grounded_tools.append(GROUNDED_COMPUTER_TOOL_SCHEMA)
+        else:
+            grounded_tools.append(schema)
+
+    return grounded_tools
+
+
+def get_last_computer_call_image(messages: List[Dict[str, Any]]) -> Optional[str]:
+    """Get the last computer call output image from messages."""
+    for message in reversed(messages):
+        if (
+            isinstance(message, dict)
+            and message.get("type") == "computer_call_output"
+            and isinstance(message.get("output"), dict)
+            and message["output"].get("type") == "input_image"
+        ):
+            image_url = message["output"].get("image_url", "")
+            if image_url.startswith("data:image/png;base64,"):
+                return image_url.split(",", 1)[1]
+    return None
+
+
+@register_agent(r".*\+.*", priority=1)
+class ComposedGroundedConfig(AsyncAgentConfig):
+    """
+    Composed-grounded agent configuration that uses both grounding and thinking models.
+
+    The model parameter should be in format: "grounding_model+thinking_model"
+    e.g., "huggingface-local/HelloKKMe/GTA1-7B+gemini/gemini-1.5-pro"
+    """
+
+    def __init__(self):
+        self.desc2xy: Dict[str, Tuple[float, float]] = {}
+
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        use_prompt_caching: Optional[bool] = False,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        """
+        Composed-grounded predict step implementation.
+
+        Process:
+        0. Store last computer call image, if none then take a screenshot
+        1. Convert computer calls from xy to descriptions
+        2. Convert responses items to completion messages
+        3. Call thinking model with litellm.acompletion
+        4. Convert completion messages to responses items
+        5. Get all element descriptions and populate desc2xy mapping
+        6. Convert computer calls from descriptions back to xy coordinates
+        7. Return output and usage
+        """
+        # Parse the composed model
+        if "+" not in model:
+            raise ValueError(
+                f"Composed model must be in format 'grounding_model+thinking_model', got: {model}"
+            )
+        grounding_model, thinking_model = model.split("+", 1)
+
+        pre_output_items = []
+
+        # Step 0: Store last computer call image, if none then take a screenshot
+        last_image_b64 = get_last_computer_call_image(messages)
+        if last_image_b64 is None:
+            # Take a screenshot
+            screenshot_b64 = await computer_handler.screenshot()  # type: ignore
+            if screenshot_b64:
+
+                call_id = uuid.uuid4().hex
+                pre_output_items += [
+                    {
+                        "type": "message",
+                        "role": "assistant",
+                        "content": [
+                            {
+                                "type": "output_text",
+                                "text": "Taking a screenshot to see the current computer screen.",
+                            }
+                        ],
+                    },
+                    {
+                        "action": {"type": "screenshot"},
+                        "call_id": call_id,
+                        "status": "completed",
+                        "type": "computer_call",
+                    },
+                    {
+                        "type": "computer_call_output",
+                        "call_id": call_id,
+                        "output": {
+                            "type": "input_image",
+                            "image_url": f"data:image/png;base64,{screenshot_b64}",
+                        },
+                    },
+                ]
+                last_image_b64 = screenshot_b64
+
+                # Call screenshot callback if provided
+                if _on_screenshot:
+                    await _on_screenshot(screenshot_b64)
+
+        tool_schemas = _prepare_tools_for_grounded(tools)  # type: ignore
+
+        # Step 1: Convert computer calls from xy to descriptions
+        input_messages = messages + pre_output_items
+        messages_with_descriptions = convert_computer_calls_xy2desc(input_messages, self.desc2xy)
+
+        # Step 2: Convert responses items to completion messages
+        completion_messages = convert_responses_items_to_completion_messages(
+            messages_with_descriptions, allow_images_in_tool_results=False
+        )
+
+        # Step 3: Call thinking model with litellm.acompletion
+        api_kwargs = {
+            "model": thinking_model,
+            "messages": completion_messages,
+            "tools": tool_schemas,
+            "max_retries": max_retries,
+            "stream": stream,
+            **kwargs,
+        }
+
+        if use_prompt_caching:
+            api_kwargs["use_prompt_caching"] = use_prompt_caching
+
+        # Call API start hook
+        if _on_api_start:
+            await _on_api_start(api_kwargs)
+
+        # Make the completion call
+        response = await litellm.acompletion(**api_kwargs)
+
+        # Call API end hook
+        if _on_api_end:
+            await _on_api_end(api_kwargs, response)
+
+        # Extract usage information
+        usage = {
+            **response.usage.model_dump(),  # type: ignore
+            "response_cost": response._hidden_params.get("response_cost", 0.0),
+        }
+        if _on_usage:
+            await _on_usage(usage)
+
+        # Step 4: Convert completion messages back to responses items format
+        response_dict = response.model_dump()  # type: ignore
+        choice_messages = [choice["message"] for choice in response_dict["choices"]]
+        thinking_output_items = []
+
+        for choice_message in choice_messages:
+            thinking_output_items.extend(
+                convert_completion_messages_to_responses_items([choice_message])
+            )
+
+        # Step 5: Get all element descriptions and populate desc2xy mapping
+        element_descriptions = get_all_element_descriptions(thinking_output_items)
+
+        if element_descriptions and last_image_b64:
+            # Use grounding model to predict coordinates for each description
+            grounding_agent_conf = find_agent_config(grounding_model)
+            if grounding_agent_conf:
+                grounding_agent = grounding_agent_conf.agent_class()
+
+                for desc in element_descriptions:
+                    for _ in range(3):  # try 3 times
+                        coords = await grounding_agent.predict_click(
+                            model=grounding_model, image_b64=last_image_b64, instruction=desc
+                        )
+                        if coords:
+                            self.desc2xy[desc] = coords
+                            break
+
+        # Step 6: Convert computer calls from descriptions back to xy coordinates
+        final_output_items = convert_computer_calls_desc2xy(thinking_output_items, self.desc2xy)
+
+        # Step 7: Return output and usage
+        return {"output": pre_output_items + final_output_items, "usage": usage}
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[int, int]]:
+        """
+        Predict click coordinates using the grounding model.
+
+        For composed models, uses only the grounding model part for click prediction.
+        """
+        # Parse the composed model to get grounding model
+        if "+" not in model:
+            raise ValueError(
+                f"Composed model must be in format 'grounding_model+thinking_model', got: {model}"
+            )
+        grounding_model, thinking_model = model.split("+", 1)
+
+        # Find and use the grounding agent
+        grounding_agent_conf = find_agent_config(grounding_model)
+        if grounding_agent_conf:
+            grounding_agent = grounding_agent_conf.agent_class()
+            return await grounding_agent.predict_click(
+                model=grounding_model, image_b64=image_b64, instruction=instruction, **kwargs
+            )
+
+        return None
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        """Return the capabilities supported by this agent."""
+        return ["click", "step"]

libs/python/agent/cua_agent/loops/fara/__init__.py

@@ -0,0 +1,8 @@
+"""
+FARA-7B agent loop implementation.
+Original implementation from Microsoft: https://github.com/microsoft/Fara
+"""
+
+from .config import FaraVlmConfig
+
+__all__ = ("FaraVlmConfig",)

libs/python/agent/cua_agent/loops/fara/config.py

@@ -0,0 +1,661 @@
+"""FARA VLM agent configuration."""
+
+from __future__ import annotations
+
+import ast
+import json
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from litellm.responses.litellm_completion_transformation.transformation import (
+    LiteLLMCompletionResponsesConfig,
+)
+
+from ...decorators import register_agent
+from ...loops.base import AsyncAgentConfig
+from ...responses import (
+    make_click_item,
+    make_double_click_item,
+    make_drag_item,
+    make_keypress_item,
+    make_move_item,
+    make_output_text_item,
+    make_reasoning_item,
+    make_screenshot_item,
+    make_scroll_item,
+    make_type_item,
+    make_wait_item,
+)
+from ...types import AgentCapability
+from .helpers import (
+    _convert_responses_items_to_fara_messages,
+    build_nous_system,
+    parse_tool_call_from_text,
+)
+
+
+def _scale_fara_coordinates(
+    args: Dict[str, Any],
+    original_dims: Tuple[int, int],
+    resized_dims: Tuple[int, int],
+) -> Dict[str, Any]:
+    """
+    Scale FARA coordinates from resized image space to original viewport space.
+
+    FARA outputs pixel coordinates on the resized image (after smart_resize).
+    This scales them back to the original browser viewport, matching FARA's
+    convert_resized_coords_to_original() in fara_agent.py:
+        scale_x = og_w / rsz_w
+        return [coords[0] * scale_x, coords[1] * scale_y]
+
+    Args:
+        args: Action arguments containing "coordinate" key
+        original_dims: (width, height) of original browser viewport
+        resized_dims: (width, height) after smart_resize
+    """
+    coord = args.get("coordinate")
+    if not coord or not isinstance(coord, (list, tuple)) or len(coord) < 2:
+        return args
+
+    x, y = float(coord[0]), float(coord[1])
+    original_w, original_h = float(original_dims[0]), float(original_dims[1])
+    resized_w, resized_h = float(resized_dims[0]), float(resized_dims[1])
+
+    # Scale from resized to original: x_final = x * (original / resized)
+    scale_x = original_w / resized_w
+    scale_y = original_h / resized_h
+
+    x_scaled = max(0.0, min(original_w, x * scale_x))
+    y_scaled = max(0.0, min(original_h, y * scale_y))
+
+    return {**args, "coordinate": [round(x_scaled), round(y_scaled)]}
+
+
+def _fara_args_to_sdk_item(args: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """
+    Convert FARA model output args to SDK item using make_*_item helpers.
+
+    FARA format: {"action": "left_click", "coordinate": [100, 200]}
+    SDK format:  ResponseComputerToolCallParam with action={"type": "click", "x": 100, "y": 200}
+    """
+    action = args.get("action", "")
+    coordinate = args.get("coordinate", [0, 0])
+    x = coordinate[0] if len(coordinate) > 0 else 0
+    y = coordinate[1] if len(coordinate) > 1 else 0
+
+    # Click actions
+    if action in ("left_click", "click"):
+        return make_click_item(x=x, y=y, button="left")
+    if action == "right_click":
+        return make_click_item(x=x, y=y, button="right")
+    if action == "middle_click":
+        return make_click_item(x=x, y=y, button="wheel")
+    if action == "double_click":
+        return make_double_click_item(x=x, y=y)
+
+    # Type action
+    if action == "type":
+        return make_type_item(text=args.get("text", ""))
+
+    # Key action
+    if action in ("key", "keypress"):
+        keys = args.get("keys", [])
+        if isinstance(keys, str):
+            keys = keys.split("+")
+        return make_keypress_item(keys=keys)
+
+    # Move action
+    if action in ("mouse_move", "move"):
+        return make_move_item(x=x, y=y)
+
+    # Scroll action
+    if action == "scroll":
+        pixels = args.get("pixels") or 0  # Handle None explicitly
+        # FARA: positive = up, negative = down
+        scroll_y = -pixels  # SDK: positive = down
+        return make_scroll_item(x=x, y=y, scroll_x=0, scroll_y=scroll_y)
+
+    if action == "hscroll":
+        pixels = args.get("pixels") or 0  # Handle None explicitly
+        return make_scroll_item(x=x, y=y, scroll_x=pixels, scroll_y=0)
+
+    # Drag action
+    if action == "left_click_drag":
+        start_coord = args.get("start_coordinate", [0, 0])
+        end_coord = args.get("end_coordinate", [0, 0])
+        return make_drag_item(
+            path=[
+                {"x": start_coord[0], "y": start_coord[1]},
+                {"x": end_coord[0], "y": end_coord[1]},
+            ]
+        )
+
+    # Screenshot
+    if action == "screenshot":
+        return make_screenshot_item()
+
+    # Wait
+    if action == "wait":
+        return make_wait_item()
+
+    # Terminate - return None so no action is executed
+    # The caller checks for terminate action and adds an assistant message to stop the loop
+    if action == "terminate":
+        return None
+
+    # FARA browser-specific actions - create computer_call items directly
+    # agent.py uses getattr(computer, action_type) to call these methods
+    if action == "visit_url":
+        return {
+            "type": "computer_call",
+            "call_id": f"call_{id(args)}",
+            "action": {"type": "visit_url", "url": args.get("url", "")},
+            "pending_safety_checks": [],
+            "status": "completed",
+        }
+
+    if action == "web_search":
+        return {
+            "type": "computer_call",
+            "call_id": f"call_{id(args)}",
+            "action": {"type": "web_search", "query": args.get("query", "")},
+            "pending_safety_checks": [],
+            "status": "completed",
+        }
+
+    if action == "history_back":
+        return {
+            "type": "computer_call",
+            "call_id": f"call_{id(args)}",
+            "action": {"type": "history_back"},
+            "pending_safety_checks": [],
+            "status": "completed",
+        }
+
+    return None
+
+
+@register_agent(models=r"(?i).*fara-7b.*", tool_type="browser")
+class FaraVlmConfig(AsyncAgentConfig):
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        use_prompt_caching: Optional[bool] = False,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        # Check if the last message is a terminate function_call_output
+        # If so, return a final assistant message to stop the loop
+        if messages:
+            last_msg = messages[-1]
+            if last_msg.get("type") in ("function_call_output", "computer_call_output"):
+                output_data = last_msg.get("output")
+
+                # Parse string if needed (could be JSON or Python dict literal)
+                if isinstance(output_data, str):
+                    try:
+                        output_data = json.loads(output_data)
+                    except:
+                        try:
+                            output_data = ast.literal_eval(output_data)
+                        except:
+                            pass
+
+                # Check if it's a terminate action output (contains "terminated": True)
+                if isinstance(output_data, dict) and output_data.get("terminated") is True:
+                    return {
+                        "output": [
+                            {
+                                "type": "message",
+                                "role": "assistant",
+                                "content": [{"type": "output_text", "text": "Task completed."}],
+                            }
+                        ],
+                        "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0},
+                    }
+
+        # Build messages using FARA's dedicated conversion layer
+        # This converts SDK format to FARA's native format (action + coordinate)
+        converted_msgs = _convert_responses_items_to_fara_messages(
+            messages, allow_images_in_tool_results=False
+        )
+
+        # Build function schemas from tools array
+        function_schemas = []
+        if tools:
+            from ...computers import is_agent_computer
+
+            for tool in tools:
+                tool_type = tool.get("type")
+
+                if tool_type == "computer":
+                    # For computer tools, use FARA_COMPUTER_TOOL schema
+                    computer = tool.get("computer")
+                    if computer and is_agent_computer(computer):
+                        function_schemas.append(FARA_COMPUTER_TOOL["function"])
+                elif tool_type == "function":
+                    # For function tools, use the provided function schema
+                    function_schema = tool.get("function")
+                    if function_schema:
+                        function_schemas.append(function_schema)
+
+        # If no tools provided or no computer tool found, use default FARA_COMPUTER_TOOL
+        if not function_schemas:
+            function_schemas = [FARA_COMPUTER_TOOL["function"]]
+
+        # Prepend Nous-generated system if available
+        nous_system = build_nous_system(function_schemas)
+        completion_messages = ([nous_system] if nous_system else []) + converted_msgs
+
+        # If there is no screenshot in the conversation, take one now and inject it.
+        def _has_any_image(msgs: List[Dict[str, Any]]) -> bool:
+            for m in msgs:
+                content = m.get("content")
+                if isinstance(content, list):
+                    for p in content:
+                        if isinstance(p, dict) and p.get("type") == "image_url":
+                            return True
+            return False
+
+        pre_output_items: List[Dict[str, Any]] = []
+        if not _has_any_image(completion_messages):
+            if computer_handler is None or not hasattr(computer_handler, "screenshot"):
+                raise RuntimeError(
+                    "No screenshots present and computer_handler.screenshot is not available."
+                )
+            screenshot_b64 = await computer_handler.screenshot()
+            if not screenshot_b64:
+                raise RuntimeError("Failed to capture screenshot from computer_handler.")
+
+            await _on_screenshot(screenshot_b64, "screenshot_before")
+
+            # Check if computer_handler has get_current_url method
+            screenshot_text = "Here is the next screenshot. Think about what to do next."
+            if hasattr(computer_handler, "get_current_url"):
+                try:
+                    current_url = await computer_handler.get_current_url()
+                    screenshot_text = f"Current URL: {current_url[:100]}\nHere is the next screenshot. Think about what to do next."
+                except Exception:
+                    # If get_current_url fails, fall back to default text
+                    pass
+
+            # Inject a user message with the screenshot so the model can see current context
+            screenshot_msg = {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:image/png;base64,{screenshot_b64}"},
+                    },
+                    {"type": "text", "text": screenshot_text},
+                ],
+            }
+            completion_messages.append(screenshot_msg)
+
+        # Smart-resize all screenshots and attach min/max pixel hints. Fail fast if deps missing.
+        # Track both original and resized dimensions for coordinate scaling.
+        last_original_w: Optional[int] = None
+        last_original_h: Optional[int] = None
+        last_rw: Optional[int] = None
+        last_rh: Optional[int] = None
+        MIN_PIXELS = 3136
+        MAX_PIXELS = 12845056
+        try:
+            import base64
+            import io
+
+            from PIL import Image  # type: ignore
+            from qwen_vl_utils import smart_resize  # type: ignore
+        except Exception:
+            raise ImportError(
+                "qwen-vl-utils not installed. Please install it with `pip install cua-agent[qwen]`."
+            )
+
+        for msg in completion_messages:
+            content = msg.get("content")
+            if not isinstance(content, list):
+                continue
+            for part in content:
+                if isinstance(part, dict) and part.get("type") == "image_url":
+                    url = ((part.get("image_url") or {}).get("url")) or ""
+                    # Expect data URL like data:image/png;base64,<b64>
+                    if url.startswith("data:") and "," in url:
+                        b64 = url.split(",", 1)[1]
+                        img_bytes = base64.b64decode(b64)
+                        im = Image.open(io.BytesIO(img_bytes))
+                        h, w = im.height, im.width
+                        rh, rw = smart_resize(
+                            h, w, factor=28, min_pixels=MIN_PIXELS, max_pixels=MAX_PIXELS
+                        )
+                        # Attach hints on this image block
+                        part["min_pixels"] = MIN_PIXELS
+                        part["max_pixels"] = MAX_PIXELS
+                        # Track both original and resized dimensions
+                        last_original_w, last_original_h = w, h
+                        last_rw, last_rh = rw, rh
+
+        api_kwargs: Dict[str, Any] = {
+            "model": model,
+            "messages": completion_messages,
+            "max_retries": max_retries,
+            "stream": stream,
+            **{k: v for k, v in kwargs.items()},
+        }
+        if use_prompt_caching:
+            api_kwargs["use_prompt_caching"] = use_prompt_caching
+
+        if _on_api_start:
+            await _on_api_start(api_kwargs)
+
+        response = await litellm.acompletion(**api_kwargs)
+
+        if _on_api_end:
+            await _on_api_end(api_kwargs, response)
+
+        usage = {
+            **LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(  # type: ignore
+                response.usage
+            ).model_dump(),
+            "response_cost": response._hidden_params.get("response_cost", 0.0),
+        }
+        if _on_usage:
+            await _on_usage(usage)
+
+        # Extract response data
+        resp_dict = response.model_dump()  # type: ignore
+        choice = (resp_dict.get("choices") or [{}])[0]
+        message = choice.get("message") or {}
+        content_text = message.get("content") or ""
+        tool_calls_array = message.get("tool_calls") or []
+        reasoning_text = message.get("reasoning") or ""
+
+        output_items: List[Dict[str, Any]] = []
+        has_terminate = False
+
+        # Add reasoning if present (Ollama Cloud format)
+        if reasoning_text:
+            output_items.append(make_reasoning_item(reasoning_text))
+
+        # Extract thoughts (text before <tool_call> tag)
+        thoughts = ""
+        if "<tool_call>" in content_text:
+            thoughts = content_text.split("<tool_call>")[0].strip()
+
+        # Add thoughts as assistant message if present
+        if thoughts:
+            output_items.append(make_output_text_item(thoughts))
+
+        # Priority 1: Try to parse tool call from content text (OpenRouter format)
+        tool_call = parse_tool_call_from_text(content_text)
+
+        if tool_call and isinstance(tool_call, dict):
+            fn_name = tool_call.get("name") or "computer"
+            raw_args = tool_call.get("arguments") or {}
+
+            # Scale coordinates from resized image space to original viewport
+            if (
+                last_rw is None
+                or last_rh is None
+                or last_original_w is None
+                or last_original_h is None
+            ):
+                raise RuntimeError(
+                    "No screenshots found to derive dimensions for coordinate scaling."
+                )
+            args = _scale_fara_coordinates(
+                raw_args,
+                original_dims=(last_original_w, last_original_h),
+                resized_dims=(last_rw, last_rh),
+            )
+
+            # Convert FARA output to SDK format using make_*_item helpers
+            if fn_name in ("computer", "computer_use"):
+                item = _fara_args_to_sdk_item(args)
+                if item:
+                    output_items.append(item)
+                # Check for terminate (even if item is None)
+                if args.get("action") == "terminate":
+                    has_terminate = True
+
+        elif tool_calls_array:
+            # Priority 2: Use tool_calls field if present (Ollama Cloud format)
+            for tc in tool_calls_array:
+                function = tc.get("function", {})
+                fn_name = function.get("name", "computer")
+                args_str = function.get("arguments", "{}")
+
+                try:
+                    args = json.loads(args_str)
+
+                    # Scale coordinates from resized image space to original viewport
+                    if "coordinate" in args and last_rw is not None and last_rh is not None:
+                        if last_original_w is not None and last_original_h is not None:
+                            args = _scale_fara_coordinates(
+                                args,
+                                original_dims=(last_original_w, last_original_h),
+                                resized_dims=(last_rw, last_rh),
+                            )
+
+                    # Convert FARA output to SDK format
+                    if fn_name in ("computer", "computer_use"):
+                        item = _fara_args_to_sdk_item(args)
+                        if item:
+                            output_items.append(item)
+                        # Check for terminate (even if item is None)
+                        if args.get("action") == "terminate":
+                            has_terminate = True
+                except json.JSONDecodeError:
+                    pass
+
+        elif content_text:
+            # No tool calls found, return text response
+            output_items.append(make_output_text_item(content_text))
+
+        # If terminate detected, ensure LAST item is an assistant message to exit the loop
+        # The generic agent loop checks: while new_items[-1].get("role") != "assistant"
+        if has_terminate:
+            output_items.append(
+                {
+                    "type": "message",
+                    "role": "assistant",
+                    "content": [{"type": "output_text", "text": ""}],
+                }
+            )
+
+        # Prepend any pre_output_items (e.g., simulated screenshot-taking message)
+        return {"output": (pre_output_items + output_items), "usage": usage}
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        return ["step"]
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[int, int]]:
+        """
+        Predict click coordinates using Qwen3-VL via litellm.acompletion.
+
+        Only exposes a reduced tool schema with left_click to bias model to output a single click.
+        Returns (x, y) absolute pixels when screen dimensions can be obtained; otherwise normalized 0..1000 integers.
+        """
+        # Reduced tool
+        reduced_tool = {
+            "type": "function",
+            "function": {
+                **FARA_COMPUTER_TOOL["function"],
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "action": {"type": "string", "enum": ["left_click"]},
+                        "coordinate": {
+                            "description": "(x, y) in 0..1000 reference space",
+                            "type": "array",
+                            "items": {"type": ["number", "integer"]},
+                            "minItems": 2,
+                            "maxItems": 2,
+                        },
+                    },
+                    "required": ["action", "coordinate"],
+                },
+            },
+        }
+
+        # Build Nous system (lazy import inside helper already raises clear guidance if missing)
+        nous_system = build_nous_system([reduced_tool["function"]])
+
+        # Pre-process using smart_resize
+        min_pixels = 3136
+        max_pixels = 12845056
+        try:
+            # Lazy import to avoid hard dependency
+            import base64
+            import io
+
+            # If PIL is available, estimate size from image to derive smart bounds
+            from PIL import Image
+            from qwen_vl_utils import smart_resize  # type: ignore
+
+            img_bytes = base64.b64decode(image_b64)
+            im = Image.open(io.BytesIO(img_bytes))
+            h, w = im.height, im.width
+            rh, rw = smart_resize(h, w, factor=28, min_pixels=min_pixels, max_pixels=max_pixels)
+        except Exception:
+            raise ImportError(
+                "qwen-vl-utils not installed. Please install it with `pip install cua-agent[qwen]`."
+            )
+
+        messages = []
+        if nous_system:
+            messages.append(nous_system)
+        image_block: Dict[str, Any] = {
+            "type": "image_url",
+            "image_url": {"url": f"data:image/png;base64,{image_b64}"},
+            "min_pixels": min_pixels,
+            "max_pixels": max_pixels,
+        }
+        # Single user message with image and instruction, matching OpenAI-style content blocks
+        messages.append(
+            {
+                "role": "user",
+                "content": [
+                    image_block,
+                    {"type": "text", "text": instruction},
+                ],
+            }
+        )
+
+        api_kwargs: Dict[str, Any] = {
+            "model": model,
+            "messages": messages,
+            **{k: v for k, v in kwargs.items()},
+        }
+        response = await litellm.acompletion(**api_kwargs)
+        resp = response.model_dump()  # type: ignore
+        choice = (resp.get("choices") or [{}])[0]
+        content_text = ((choice.get("message") or {}).get("content")) or ""
+        tool_call = parse_tool_call_from_text(content_text) or {}
+        args = tool_call.get("arguments") or {}
+        # Scale from resized image space to original viewport
+        args = _scale_fara_coordinates(
+            args,
+            original_dims=(w, h),
+            resized_dims=(rw, rh),
+        )
+        coord = args.get("coordinate")
+        if isinstance(coord, (list, tuple)) and len(coord) >= 2:
+            return int(coord[0]), int(coord[1])
+        return None
+
+
+# FARA-specific ComputerUse tool schema (OpenAI function tool format)
+# This schema is tailored for FARA-7B model and includes browser-specific actions
+# NOTE: Tool name MUST be "computer_use" to match what FARA-7B was trained on
+FARA_COMPUTER_TOOL: dict[str, Any] = {
+    "type": "function",
+    "function": {
+        "name": "computer_use",
+        "description": (
+            "Use a mouse and keyboard to interact with a computer, and take screenshots.\n"
+            "* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.\n"
+            "* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try wait and taking another screenshot.\n"
+            "* The screen's resolution is 1000x1000.\n"
+            "* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.\n"
+            "* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.\n"
+            "* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges.\n"
+            "* Use terminate action when you have completed the task or cannot proceed further."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "action": {
+                    "description": "The action to perform.",
+                    "enum": [
+                        "key",
+                        "type",
+                        "mouse_move",
+                        "left_click",
+                        "left_click_drag",
+                        "right_click",
+                        "middle_click",
+                        "double_click",
+                        "triple_click",
+                        "scroll",
+                        "hscroll",
+                        "screenshot",
+                        "wait",
+                        "visit_url",
+                        "web_search",
+                        "history_back",
+                        "terminate",
+                    ],
+                    "type": "string",
+                },
+                "keys": {
+                    "description": "Required only by action=key.",
+                    "type": "array",
+                    "items": {"type": "string"},
+                },
+                "text": {
+                    "description": "Required only by action=type.",
+                    "type": "string",
+                },
+                "coordinate": {
+                    "description": "(x, y): Pixel coordinates from top-left.",
+                    "type": "array",
+                    "items": {"type": ["number", "integer"]},
+                    "minItems": 2,
+                    "maxItems": 2,
+                },
+                "pixels": {
+                    "description": "Scroll amount. Positive=up, negative=down. For scroll/hscroll.",
+                    "type": "number",
+                },
+                "time": {
+                    "description": "Seconds to wait (action=wait).",
+                    "type": "number",
+                },
+                "url": {
+                    "description": "The URL to visit. Required only by action=visit_url.",
+                    "type": "string",
+                },
+                "query": {
+                    "description": "The search query. Required only by action=web_search.",
+                    "type": "string",
+                },
+                "status": {
+                    "description": "Task completion status. Required only by action=terminate.",
+                    "type": "string",
+                    "enum": ["success", "failure"],
+                },
+            },
+            "required": ["action"],
+        },
+    },
+}

libs/python/agent/cua_agent/loops/fara/helpers.py

@@ -0,0 +1,817 @@
+# Source: https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/llm/fncall_prompts/nous_fncall_prompt.py
+
+import copy
+import json
+import os
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union
+
+from .schema import ContentItem, Message
+
+FN_CALL_TEMPLATE_QWEN = """# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{tool_descs}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{{"name": <function-name>, "arguments": <args-json-object>}}
+</tool_call>"""
+
+FN_CALL_TEMPLATE = """You are a web automation agent that performs actions on websites to fulfill user requests by calling various tools.
+* You should stop execution at Critical Points. A Critical Point would be encountered in tasks like 'Checkout', 'Book', 'Purchase', 'Call', 'Email', 'Order', etc where a binding transaction/agreement would require the user's permission/personal or sensitive information (name, email, credit card, address, payment information, resume, etc) in order to complete a transaction (purchase, reservation, sign-up etc), or to communicate in a way that a human would be expected to do (call, email, apply to a job, etc).
+* Solve the task as far as you can up until a Critical Point:
+    - For example, if the task is to "call a restaurant to make a reservation", you should not actually make the call but should navigate to the restaurant's page and find the phone number.
+    - Similarly, if the task is to "order new size 12 running shoes" you should not actually place the order but should instead search for the right shoes that meet the criteria and add them to the cart.
+    - Some tasks, like answering questions, may not encounter a Critical Point at all.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{tool_descs}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{{"name": <function-name>, "arguments": <args-json-object>}}
+</tool_call>"""
+
+
+SPECIAL_CODE_MODE = os.getenv("SPECIAL_CODE_MODE", "false").lower() == "true"
+CODE_TOOL_PATTERN = "code_interpreter"
+FN_CALL_TEMPLATE_WITH_CI = """# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{tool_descs}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{{"name": <function-name>, "arguments": <args-json-object>}}
+</tool_call>
+For code parameters, use placeholders first, and then put the code within <code></code> XML tags, such as:
+<tool_call>
+{{"name": <function-name>, "arguments": {{"code": ""}}}}
+<code>
+Here is the code.
+</code>
+</tool_call>"""
+
+
+class NousFnCallPrompt:
+    def __init__(self, template_name: str = "default"):
+        """Initialize NousFnCallPrompt with a specific template.
+
+        Args:
+            template_name: Name of the template to use. Options:
+                          "default", "qwen", "with_ci"
+        """
+        self.template_name = template_name
+        self.template_map = {
+            "default": FN_CALL_TEMPLATE,
+            "qwen": FN_CALL_TEMPLATE_QWEN,
+            "with_ci": FN_CALL_TEMPLATE_WITH_CI,
+        }
+
+        if template_name not in self.template_map:
+            raise ValueError(
+                f"Unknown template_name: {template_name}. "
+                f"Available options: {list(self.template_map.keys())}"
+            )
+
+    def preprocess_fncall_messages(
+        self,
+        messages: List[Message],
+        functions: List[dict],
+        lang: Literal["en", "zh"],
+        parallel_function_calls: bool = True,
+        function_choice: Union[Literal["auto"], str] = "auto",
+    ) -> List[Message]:
+        del lang  # ignored
+        del parallel_function_calls  # ignored
+        if function_choice != "auto":
+            raise NotImplementedError
+
+        ori_messages = messages
+
+        # Change function_call responses to plaintext responses:
+        messages = []
+        for msg in copy.deepcopy(ori_messages):
+            role, content, reasoning_content = (
+                msg.role,
+                msg.content,
+                msg.reasoning_content,
+            )
+            if role in ("system", "user"):
+                messages.append(msg)
+            elif role == "assistant":
+                content = content or []
+                fn_call = msg.function_call
+                if fn_call:
+                    if (not SPECIAL_CODE_MODE) or (CODE_TOOL_PATTERN not in fn_call.name):
+                        fc = {
+                            "name": fn_call.name,
+                            "arguments": json.loads(fn_call.arguments),
+                        }
+                        fc = json.dumps(fc, ensure_ascii=False)
+                        fc = f"<tool_call>\n{fc}\n</tool_call>"
+                    else:
+                        para = json.loads(fn_call.arguments)
+                        code = para["code"]
+                        para["code"] = ""
+                        fc = {"name": fn_call.name, "arguments": para}
+                        fc = json.dumps(fc, ensure_ascii=False)
+                        fc = f"<tool_call>\n{fc}\n<code>\n{code}\n</code>\n</tool_call>"
+
+                    content.append(ContentItem(text=fc))
+                if messages[-1].role == "assistant":
+                    messages[-1].content.append(ContentItem(text="\n"))
+                    messages[-1].content.extend(content)
+                else:
+                    # TODO: Assuming there will only be one continuous reasoning_content here
+                    messages.append(
+                        Message(
+                            role=role,
+                            content=content,
+                            reasoning_content=reasoning_content,
+                        )
+                    )
+            elif role == "function":
+                assert isinstance(content, list)
+                assert len(content) == 1
+                assert content[0].text
+                fc = f"<tool_response>\n{content[0].text}\n</tool_response>"
+                content = [ContentItem(text=fc)]
+                if messages[-1].role == "user":
+                    messages[-1].content.append(ContentItem(text="\n"))
+                    messages[-1].content.extend(content)
+                else:
+                    messages.append(Message(role="user", content=content))
+            else:
+                raise TypeError
+
+        tool_descs = [{"type": "function", "function": f} for f in functions]
+        tool_names = [
+            function.get("name_for_model", function.get("name", "")) for function in functions
+        ]
+        tool_descs = "\n".join([json.dumps(f, ensure_ascii=False) for f in tool_descs])
+
+        # Select template based on configuration
+        if SPECIAL_CODE_MODE and any([CODE_TOOL_PATTERN in x for x in tool_names]):
+            selected_template = FN_CALL_TEMPLATE_WITH_CI
+        else:
+            selected_template = self.template_map[self.template_name]
+
+        tool_system = selected_template.format(tool_descs=tool_descs)
+        if messages[0].role == "system":
+            messages[0].content.append(ContentItem(text="\n\n" + tool_system))
+        else:
+            messages = [Message(role="system", content=[ContentItem(text=tool_system)])] + messages
+        return messages
+
+
+# Mainly for removing incomplete special tokens when streaming the output
+# This assumes that '<tool_call>\n{"name": "' is the special token for the NousFnCallPrompt
+def remove_incomplete_special_tokens(text: str) -> str:
+    if text in '<tool_call>\n{"name": "':
+        text = ""
+    return text
+
+
+def extract_fn(text: str):
+    fn_name, fn_args = "", ""
+    fn_name_s = '"name": "'
+    fn_name_e = '", "'
+    fn_args_s = '"arguments": '
+    i = text.find(fn_name_s)
+    k = text.find(fn_args_s)
+    if i > 0:
+        _text = text[i + len(fn_name_s) :]
+        j = _text.find(fn_name_e)
+        if j > -1:
+            fn_name = _text[:j]
+    if k > 0:
+        fn_args = text[k + len(fn_args_s) :]
+
+    if len(fn_args) > 5:
+        fn_args = fn_args[:-5]
+    else:
+        fn_args = ""
+    return fn_name, fn_args
+
+
+def build_nous_system(functions: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+    """Use original FARA NousFnCallPrompt to generate a system message embedding tool schema."""
+    from .schema import ContentItem as NousContentItem
+    from .schema import Message as NousMessage
+
+    msgs = NousFnCallPrompt().preprocess_fncall_messages(
+        messages=[
+            NousMessage(
+                role="system", content=[NousContentItem(text="You are a helpful assistant.")]
+            )
+        ],
+        functions=functions,
+        lang="en",
+    )
+    sys = msgs[0].model_dump()
+    # Convert structured content to OpenAI-style content list
+    content = [{"type": "text", "text": c["text"]} for c in sys.get("content", [])]
+    return {"role": "system", "content": content}
+
+
+def fix_fara_tool_call_format(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """
+    Fix tool call format in conversation history for FARA compatibility.
+
+    The shared `convert_responses_items_to_completion_messages` function outputs:
+    - Tool name as "computer" (should be "computer_use")
+    - Action key as "type" (should be "action")
+
+    This function post-processes assistant messages to fix these issues.
+    """
+    import re
+
+    # Valid FARA action types
+    valid_actions = {
+        "left_click",
+        "right_click",
+        "middle_click",
+        "double_click",
+        "triple_click",
+        "click",
+        "type",
+        "key",
+        "scroll",
+        "hscroll",
+        "mouse_move",
+        "wait",
+        "visit_url",
+        "web_search",
+        "history_back",
+        "screenshot",
+        "terminate",
+    }
+
+    fixed_messages = []
+    for msg in messages:
+        if msg.get("role") != "assistant":
+            fixed_messages.append(msg)
+            continue
+
+        content = msg.get("content", "")
+        if not isinstance(content, str) or "<tool_call>" not in content:
+            fixed_messages.append(msg)
+            continue
+
+        # Find and fix all tool calls in the content
+        def fix_tool_call(match):
+            tool_call_content = match.group(1)
+            try:
+                tool_call = json.loads(tool_call_content)
+
+                # Fix tool name: "computer" -> "computer_use"
+                if tool_call.get("name") == "computer":
+                    tool_call["name"] = "computer_use"
+
+                # Fix arguments: "type" -> "action" and x/y -> coordinate
+                args = tool_call.get("arguments", {})
+                if isinstance(args, dict):
+                    # If "type" contains a valid action, rename to "action"
+                    if "type" in args and args["type"] in valid_actions:
+                        args["action"] = args.pop("type")
+
+                    # Convert internal x/y format back to FARA coordinate format
+                    if "x" in args and "y" in args and "coordinate" not in args:
+                        args["coordinate"] = [args.pop("x"), args.pop("y")]
+
+                    # Normalize action names: "click" -> "left_click"
+                    if args.get("action") == "click":
+                        args["action"] = "left_click"
+
+                    # Remove "button" field - FARA doesn't use it (action name implies button)
+                    args.pop("button", None)
+
+                    # If "action" is empty but we can infer from other keys
+                    if args.get("action") == "" and "coordinate" in args:
+                        args["action"] = "left_click"
+
+                    tool_call["arguments"] = args
+
+                return f"<tool_call>\n{json.dumps(tool_call)}\n</tool_call>"
+            except (json.JSONDecodeError, TypeError):
+                return match.group(0)  # Return original if parsing fails
+
+        # Match <tool_call>...</tool_call> or <tool_call>...</tool_call>
+        fixed_content = re.sub(
+            r"<tool_call>\s*(\{.*?\})\s*</tool_call>", fix_tool_call, content, flags=re.DOTALL
+        )
+
+        # Also handle malformed closing tags like <tool_call> used as closing
+        fixed_content = re.sub(
+            r"<tool_call>(\{.*?\})<tool_call>", fix_tool_call, fixed_content, flags=re.DOTALL
+        )
+
+        fixed_messages.append({**msg, "content": fixed_content})
+
+    return fixed_messages
+
+
+def parse_tool_call_from_text(text: str) -> Optional[Dict[str, Any]]:
+    """Extract JSON object within <tool_call>...</tool_call> from model text.
+
+    Accepts both </tool_call> and <tool_call> as closing tags for robustness.
+    Handles nested braces in JSON objects.
+    """
+    # Find the opening tag
+    start_idx = text.find("<tool_call>")
+    if start_idx == -1:
+        return None
+
+    # Find the start of JSON (first '{' after opening tag)
+    json_start = text.find("{", start_idx)
+    if json_start == -1:
+        return None
+
+    # Extract JSON by counting braces
+    brace_count = 0
+    json_end = json_start
+    for i in range(json_start, len(text)):
+        if text[i] == "{":
+            brace_count += 1
+        elif text[i] == "}":
+            brace_count -= 1
+            if brace_count == 0:
+                json_end = i + 1
+                break
+
+    if brace_count != 0:
+        return None
+
+    json_str = text[json_start:json_end]
+    try:
+        return json.loads(json_str)
+    except Exception:
+        return None
+
+
+async def unnormalize_coordinate(args: Dict[str, Any], dims: Tuple[int, int]) -> Dict[str, Any]:
+    """Coordinates appear in 0..1000 space, scale to actual screen size using dims if provided."""
+    coord = args.get("coordinate")
+    if not coord or not isinstance(coord, (list, tuple)) or len(coord) < 2:
+        return args
+    x, y = float(coord[0]), float(coord[1])
+    width, height = float(dims[0]), float(dims[1])
+    x_abs = max(0.0, min(width, (x / 1000.0) * width))
+    y_abs = max(0.0, min(height, (y / 1000.0) * height))
+    args = {**args, "coordinate": [round(x_abs), round(y_abs)]}
+    return args
+
+
+def convert_qwen_tool_args_to_computer_action(args: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """
+    Convert Qwen computer tool arguments to the Computer Calls action schema.
+
+    Qwen (example):
+        {"action": "left_click", "coordinate": [114, 68]}
+
+    Target (example):
+        {"action": "left_click", "x": 114, "y": 68}
+
+    Other mappings:
+    - right_click, middle_click, double_click (triple_click -> double_click)
+    - mouse_move -> { action: "move", x, y }
+    - key -> { action: "keypress", keys: [...] }
+    - type -> { action: "type", text }
+    - scroll/hscroll -> { action: "scroll", scroll_x, scroll_y, x, y }
+    - wait -> { action: "wait" }
+    - terminate/answer are not direct UI actions; return None for now
+    """
+    if not isinstance(args, dict):
+        return None
+
+    action = args.get("action")
+    if not isinstance(action, str):
+        return None
+
+    # Coordinates helper
+    coord = args.get("coordinate")
+    x = y = None
+    if isinstance(coord, (list, tuple)) and len(coord) >= 2:
+        try:
+            x = int(round(float(coord[0])))
+            y = int(round(float(coord[1])))
+        except Exception:
+            x = y = None
+
+    # Map actions
+    a = action.lower()
+    if a in {"left_click", "right_click", "middle_click", "double_click"}:
+        if x is None or y is None:
+            return None
+        return {"action": a, "x": x, "y": y}
+    if a == "triple_click":
+        # Approximate as double_click
+        if x is None or y is None:
+            return None
+        return {"action": "double_click", "x": x, "y": y}
+    if a == "mouse_move":
+        if x is None or y is None:
+            return None
+        return {"action": "move", "x": x, "y": y}
+    if a == "key":
+        keys = args.get("keys")
+        if isinstance(keys, list) and all(isinstance(k, str) for k in keys):
+            return {"action": "keypress", "keys": keys}
+        return None
+    if a == "type":
+        text = args.get("text")
+        if isinstance(text, str):
+            return {"action": "type", "text": text}
+        return None
+    if a in {"scroll", "hscroll"}:
+        pixels = args.get("pixels") or 0
+        try:
+            pixels_val = int(round(float(pixels)))
+        except Exception:
+            pixels_val = 0
+        scroll_x = pixels_val if a == "hscroll" else 0
+        scroll_y = pixels_val if a == "scroll" else 0
+        # Include cursor position if available (optional)
+        out: Dict[str, Any] = {"action": "scroll", "scroll_x": scroll_x, "scroll_y": scroll_y}
+        if x is not None and y is not None:
+            out.update({"x": x, "y": y})
+        return out
+    if a == "wait":
+        return {"action": "wait"}
+
+    # Non-UI or terminal actions: terminate/answer -> not mapped here
+    return None
+
+
+def convert_fara_args_to_browser_tool_format(args: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Convert FARA model output format to BrowserTool compatible format.
+
+    FARA model may output extra parameters that BrowserTool methods don't accept.
+    This function cleans up the arguments and maps them to the correct format.
+
+    Examples:
+        Input:  {"action": "click", "button": "left", "x": 378, "y": 144}
+        Output: {"action": "left_click", "coordinate": [378, 144]}
+
+        Input:  {"action": "visit_url", "url": "https://...", "text": "..."}
+        Output: {"action": "visit_url", "url": "https://..."}
+
+        Input:  {"action": "terminate", "url": "...", "text": "...", "status": "success"}
+        Output: {"action": "terminate", "status": "success"}
+    """
+    if not isinstance(args, dict):
+        return args
+
+    action = args.get("action", "")
+    if not isinstance(action, str):
+        return args
+
+    a = action.lower()
+    result: Dict[str, Any] = {"action": a}
+
+    # Handle coordinate-based actions
+    # Check for both coordinate array and separate x/y fields
+    coord = args.get("coordinate")
+    x = args.get("x")
+    y = args.get("y")
+
+    if coord and isinstance(coord, (list, tuple)) and len(coord) >= 2:
+        x, y = coord[0], coord[1]
+
+    # Click actions - normalize to left_click with coordinate
+    if a in {"click", "left_click"}:
+        if x is not None and y is not None:
+            result["action"] = "left_click"
+            result["coordinate"] = [x, y]
+        return result
+
+    if a in {"right_click", "middle_click", "double_click", "triple_click"}:
+        if x is not None and y is not None:
+            result["coordinate"] = [x, y]
+        return result
+
+    if a == "mouse_move":
+        if x is not None and y is not None:
+            result["coordinate"] = [x, y]
+        return result
+
+    if a == "left_click_drag":
+        if x is not None and y is not None:
+            result["coordinate"] = [x, y]
+        # Also handle start/end coordinates if present
+        start_coord = args.get("start_coordinate")
+        end_coord = args.get("end_coordinate")
+        if start_coord:
+            result["start_coordinate"] = start_coord
+        if end_coord:
+            result["end_coordinate"] = end_coord
+        return result
+
+    # Keyboard actions
+    if a == "key":
+        keys = args.get("keys")
+        if keys:
+            result["keys"] = keys
+        return result
+
+    if a == "type":
+        text = args.get("text")
+        if text:
+            result["text"] = text
+        # Include coordinate if typing at a specific location
+        if x is not None and y is not None:
+            result["coordinate"] = [x, y]
+        return result
+
+    # Scroll actions
+    if a in {"scroll", "hscroll"}:
+        pixels = args.get("pixels")
+        if pixels is not None:
+            result["pixels"] = pixels
+        if x is not None and y is not None:
+            result["coordinate"] = [x, y]
+        return result
+
+    # Browser-specific actions
+    if a == "visit_url":
+        url = args.get("url")
+        if url:
+            result["url"] = url
+        return result
+
+    if a == "web_search":
+        query = args.get("query")
+        if query:
+            result["query"] = query
+        return result
+
+    if a == "history_back":
+        return result
+
+    # Wait action
+    if a == "wait":
+        time_val = args.get("time")
+        if time_val is not None:
+            result["time"] = time_val
+        return result
+
+    # Screenshot action
+    if a == "screenshot":
+        return result
+
+    # Terminate action
+    if a == "terminate":
+        status = args.get("status", "success")
+        result["status"] = status
+        return result
+
+    # For any other action, return cleaned args (just action + known fields)
+    return result
+
+
+def _convert_responses_items_to_fara_messages(
+    messages: List[Dict[str, Any]],
+    allow_images_in_tool_results: bool = False,
+) -> List[Dict[str, Any]]:
+    """
+    Convert SDK responses_items format to FARA-compatible completion messages.
+
+    This is FARA's dedicated conversion layer (similar to Anthropic's pattern).
+    It handles the conversion from SDK's OpenAI-style format to FARA's native format:
+
+    SDK format:
+        {"type": "click", "x": 100, "y": 200, "button": "left"}
+
+    FARA format (in XML tool_call):
+        {"name": "computer_use", "arguments": {"action": "left_click", "coordinate": [100, 200]}}
+    """
+    completion_messages: List[Dict[str, Any]] = []
+
+    for message in messages:
+        msg_type = message.get("type")
+        role = message.get("role")
+
+        # Handle user messages
+        if role == "user" or msg_type == "user":
+            content = message.get("content", "")
+            if isinstance(content, list):
+                converted_content = []
+                for item in content:
+                    if isinstance(item, dict):
+                        item_type = item.get("type")
+                        if item_type == "input_image":
+                            image_url = item.get("image_url", "")
+                            if image_url and image_url != "[omitted]":
+                                converted_content.append(
+                                    {"type": "image_url", "image_url": {"url": image_url}}
+                                )
+                        elif item_type == "input_text":
+                            converted_content.append({"type": "text", "text": item.get("text", "")})
+                        elif item_type == "image_url":
+                            # Already in correct format
+                            converted_content.append(item)
+                        elif item_type == "text":
+                            converted_content.append(item)
+                        else:
+                            converted_content.append(item)
+                    else:
+                        converted_content.append({"type": "text", "text": str(item)})
+                completion_messages.append({"role": "user", "content": converted_content})
+            else:
+                completion_messages.append({"role": "user", "content": content})
+
+        # Handle assistant messages
+        elif role == "assistant" and msg_type == "message":
+            content = message.get("content", [])
+            if isinstance(content, str):
+                completion_messages.append({"role": "assistant", "content": content})
+            elif isinstance(content, list):
+                text_parts = []
+                for item in content:
+                    if isinstance(item, dict) and item.get("type") == "output_text":
+                        text_parts.append(item.get("text", ""))
+                completion_messages.append({"role": "assistant", "content": "\n".join(text_parts)})
+
+        # Handle reasoning
+        elif msg_type == "reasoning":
+            summary = message.get("summary", [])
+            reasoning_text = ""
+            if isinstance(summary, list) and summary:
+                for item in summary:
+                    if isinstance(item, dict) and item.get("type") == "summary_text":
+                        reasoning_text = item.get("text", "")
+                        break
+            if reasoning_text:
+                completion_messages.append({"role": "assistant", "content": reasoning_text})
+
+        # Handle computer_call - convert SDK format to FARA's XML tool_call format
+        elif msg_type == "computer_call":
+            action = message.get("action", {})
+            action_type = action.get("type")
+
+            # Convert SDK action to FARA format
+            fara_args = _sdk_action_to_fara_args(action)
+
+            # Build FARA's XML tool_call format
+            tool_call_json = json.dumps({"name": "computer_use", "arguments": fara_args})
+            tool_call_text = f"<tool_call>\n{tool_call_json}\n</tool_call>"
+
+            # Append to last assistant message or create new one
+            if completion_messages and completion_messages[-1].get("role") == "assistant":
+                prev_content = completion_messages[-1].get("content", "")
+                completion_messages[-1]["content"] = f"{prev_content}\n{tool_call_text}".strip()
+            else:
+                completion_messages.append({"role": "assistant", "content": tool_call_text})
+
+        # Handle computer_call_output - convert to FARA's tool_response format
+        elif msg_type == "computer_call_output":
+            output = message.get("output", {})
+
+            # Build response content
+            if isinstance(output, dict) and output.get("type") == "input_image":
+                image_url = output.get("image_url", "")
+                response_text = "<tool_response>\nAction executed successfully. Here is the next screenshot.\n</tool_response>"
+
+                # Add as user message with image
+                if allow_images_in_tool_results and image_url and image_url != "[omitted]":
+                    completion_messages.append(
+                        {
+                            "role": "user",
+                            "content": [
+                                {"type": "text", "text": response_text},
+                                {"type": "image_url", "image_url": {"url": image_url}},
+                            ],
+                        }
+                    )
+                else:
+                    completion_messages.append(
+                        {
+                            "role": "user",
+                            "content": [
+                                {"type": "text", "text": response_text},
+                            ],
+                        }
+                    )
+            elif isinstance(output, dict) and output.get("terminated"):
+                response_text = "<tool_response>\nTask terminated.\n</tool_response>"
+                completion_messages.append({"role": "user", "content": response_text})
+            else:
+                response_text = f"<tool_response>\n{json.dumps(output) if isinstance(output, dict) else str(output)}\n</tool_response>"
+                completion_messages.append({"role": "user", "content": response_text})
+
+        # Handle function_call (non-computer tools)
+        elif msg_type == "function_call":
+            fn_name = message.get("name", "")
+            fn_args = message.get("arguments", "{}")
+
+            tool_call_json = json.dumps(
+                {
+                    "name": fn_name,
+                    "arguments": json.loads(fn_args) if isinstance(fn_args, str) else fn_args,
+                }
+            )
+            tool_call_text = f"<tool_call>\n{tool_call_json}\n</tool_call>"
+
+            if completion_messages and completion_messages[-1].get("role") == "assistant":
+                prev_content = completion_messages[-1].get("content", "")
+                completion_messages[-1]["content"] = f"{prev_content}\n{tool_call_text}".strip()
+            else:
+                completion_messages.append({"role": "assistant", "content": tool_call_text})
+
+        # Handle function_call_output
+        elif msg_type == "function_call_output":
+            output = message.get("output", "")
+            response_text = f"<tool_response>\n{output}\n</tool_response>"
+            completion_messages.append({"role": "user", "content": response_text})
+
+    return completion_messages
+
+
+def _sdk_action_to_fara_args(action: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Convert SDK action format to FARA arguments format.
+
+    SDK format:  {"type": "click", "x": 100, "y": 200, "button": "left"}
+    FARA format: {"action": "left_click", "coordinate": [100, 200]}
+    """
+    action_type = action.get("type", "")
+
+    # Click actions
+    if action_type == "click":
+        button = action.get("button", "left")
+        action_name = {
+            "left": "left_click",
+            "right": "right_click",
+            "wheel": "middle_click",
+            "middle": "middle_click",
+        }.get(button, "left_click")
+        return {"action": action_name, "coordinate": [action.get("x", 0), action.get("y", 0)]}
+
+    if action_type == "double_click":
+        return {"action": "double_click", "coordinate": [action.get("x", 0), action.get("y", 0)]}
+
+    # Type action
+    if action_type == "type":
+        result = {"action": "type", "text": action.get("text", "")}
+        # Include coordinate if present (for click-then-type)
+        if "x" in action and "y" in action:
+            result["coordinate"] = [action.get("x", 0), action.get("y", 0)]
+        return result
+
+    # Keypress action
+    if action_type == "keypress":
+        keys = action.get("keys", [])
+        return {"action": "key", "keys": keys}
+
+    # Move action
+    if action_type in ("move", "mouse_move"):
+        return {"action": "mouse_move", "coordinate": [action.get("x", 0), action.get("y", 0)]}
+
+    # Scroll action
+    if action_type == "scroll":
+        scroll_x = action.get("scroll_x", 0)
+        scroll_y = action.get("scroll_y", 0)
+        # FARA uses pixels (positive = up/left, negative = down/right)
+        pixels = scroll_y if scroll_y != 0 else scroll_x
+        result = {"action": "scroll", "pixels": pixels}
+        if "x" in action and "y" in action:
+            result["coordinate"] = [action.get("x", 0), action.get("y", 0)]
+        return result
+
+    # Drag action
+    if action_type == "drag":
+        path = action.get("path", [])
+        if len(path) >= 2:
+            return {
+                "action": "left_click_drag",
+                "start_coordinate": [path[0].get("x", 0), path[0].get("y", 0)],
+                "end_coordinate": [path[-1].get("x", 0), path[-1].get("y", 0)],
+            }
+        return {"action": "left_click_drag"}
+
+    # Screenshot
+    if action_type == "screenshot":
+        return {"action": "screenshot"}
+
+    # Wait
+    if action_type == "wait":
+        return {"action": "wait"}
+
+    # Terminate
+    if action_type == "terminate":
+        return {"action": "terminate", "status": action.get("status", "success")}
+
+    # Fallback - return as-is with type renamed to action
+    return {"action": action_type, **{k: v for k, v in action.items() if k != "type"}}

libs/python/agent/cua_agent/loops/fara/schema.py

@@ -0,0 +1,143 @@
+# Source: https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/llm/schema.py
+
+from typing import List, Literal, Optional, Tuple, Union
+
+from pydantic import BaseModel, field_validator, model_validator
+
+
+class BaseModelCompatibleDict(BaseModel):
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def model_dump(self, **kwargs):
+        if "exclude_none" not in kwargs:
+            kwargs["exclude_none"] = True
+        return super().model_dump(**kwargs)
+
+    def model_dump_json(self, **kwargs):
+        if "exclude_none" not in kwargs:
+            kwargs["exclude_none"] = True
+        return super().model_dump_json(**kwargs)
+
+    def get(self, key, default=None):
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            return default
+
+    def __str__(self):
+        return f"{self.model_dump()}"
+
+
+class FunctionCall(BaseModelCompatibleDict):
+    name: str
+    arguments: str
+
+    def __init__(self, name: str, arguments: str):
+        super().__init__(name=name, arguments=arguments)
+
+    def __repr__(self):
+        return f"FunctionCall({self.model_dump()})"
+
+
+class ContentItem(BaseModelCompatibleDict):
+    text: Optional[str] = None
+    image: Optional[str] = None
+    file: Optional[str] = None
+    audio: Optional[Union[str, dict]] = None
+    video: Optional[Union[str, list]] = None
+
+    def __init__(
+        self,
+        text: Optional[str] = None,
+        image: Optional[str] = None,
+        file: Optional[str] = None,
+        audio: Optional[Union[str, dict]] = None,
+        video: Optional[Union[str, list]] = None,
+    ):
+        super().__init__(text=text, image=image, file=file, audio=audio, video=video)
+
+    @model_validator(mode="after")
+    def check_exclusivity(self):
+        provided_fields = 0
+        if self.text is not None:
+            provided_fields += 1
+        if self.image:
+            provided_fields += 1
+        if self.file:
+            provided_fields += 1
+        if self.audio:
+            provided_fields += 1
+        if self.video:
+            provided_fields += 1
+
+        if provided_fields != 1:
+            raise ValueError(
+                "Exactly one of 'text', 'image', 'file', 'audio', or 'video' must be provided."
+            )
+        return self
+
+    def __repr__(self):
+        return f"ContentItem({self.model_dump()})"
+
+    def get_type_and_value(
+        self,
+    ) -> Tuple[Literal["text", "image", "file", "audio", "video"], str]:
+        ((t, v),) = self.model_dump().items()
+        assert t in ("text", "image", "file", "audio", "video")
+        return t, v
+
+    @property
+    def type(self) -> Literal["text", "image", "file", "audio", "video"]:
+        t, _ = self.get_type_and_value()
+        return t
+
+    @property
+    def value(self) -> str:
+        _, v = self.get_type_and_value()
+        return v
+
+
+class Message(BaseModelCompatibleDict):
+    role: str
+    content: Union[str, List[ContentItem]]
+    reasoning_content: Optional[Union[str, List[ContentItem]]] = None
+    name: Optional[str] = None
+    function_call: Optional[FunctionCall] = None
+    extra: Optional[dict] = None
+
+    def __init__(
+        self,
+        role: str,
+        content: Union[str, List[ContentItem]],
+        reasoning_content: Optional[Union[str, List[ContentItem]]] = None,
+        name: Optional[str] = None,
+        function_call: Optional[FunctionCall] = None,
+        extra: Optional[dict] = None,
+        **kwargs,
+    ):
+        if content is None:
+            content = ""
+        if reasoning_content is None:
+            reasoning_content = ""
+        super().__init__(
+            role=role,
+            content=content,
+            reasoning_content=reasoning_content,
+            name=name,
+            function_call=function_call,
+            extra=extra,
+        )
+
+    def __repr__(self):
+        return f"Message({self.model_dump()})"
+
+    @field_validator("role")
+    def role_checker(cls, value: str) -> str:
+        values = ["system", "user", "assistant", "function"]
+        if value not in values:
+            raise ValueError(f'{value} must be one of {",".join(values)}')
+        return value

libs/python/agent/cua_agent/loops/gelato.py

@@ -0,0 +1,183 @@
+"""
+Gelato agent loop implementation for click prediction using litellm.acompletion
+Model: https://huggingface.co/mlfoundations/Gelato-30B-A3B
+Code: https://github.com/mlfoundations/Gelato/tree/main
+"""
+
+import base64
+import math
+import re
+from io import BytesIO
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from PIL import Image
+
+from ..decorators import register_agent
+from ..loops.base import AsyncAgentConfig
+from ..types import AgentCapability
+
+SYSTEM_PROMPT = """
+You are an expert UI element locator. Given a GUI image and a user's element description, provide the coordinates of the specified element as a single (x,y) point. For elements with area, return the center point.
+
+Output the coordinate pair exactly:
+(x,y)
+"""
+
+
+def extract_coordinates(raw_string):
+    """
+    Extract the coordinates from the raw string.
+    Args:
+        raw_string: str (e.g. "(100, 200)")
+    Returns:
+        x: float (e.g. 100.0)
+        y: float (e.g. 200.0)
+    """
+    try:
+        matches = re.findall(r"\((-?\d*\.?\d+),\s*(-?\d*\.?\d+)\)", raw_string)
+        return [tuple(map(int, match)) for match in matches][0]
+    except:
+        return 0, 0
+
+
+def smart_resize(
+    height: int,
+    width: int,
+    factor: int = 28,
+    min_pixels: int = 3136,
+    max_pixels: int = 8847360,
+) -> Tuple[int, int]:
+    """Smart resize function similar to qwen_vl_utils."""
+    # Calculate the total pixels
+    total_pixels = height * width
+
+    # If already within bounds, return original dimensions
+    if min_pixels <= total_pixels <= max_pixels:
+        # Round to nearest factor
+        new_height = (height // factor) * factor
+        new_width = (width // factor) * factor
+        return new_height, new_width
+
+    # Calculate scaling factor
+    if total_pixels > max_pixels:
+        scale = (max_pixels / total_pixels) ** 0.5
+    else:
+        scale = (min_pixels / total_pixels) ** 0.5
+
+    # Apply scaling
+    new_height = int(height * scale)
+    new_width = int(width * scale)
+
+    # Round to nearest factor
+    new_height = (new_height // factor) * factor
+    new_width = (new_width // factor) * factor
+
+    # Ensure minimum size
+    new_height = max(new_height, factor)
+    new_width = max(new_width, factor)
+
+    return new_height, new_width
+
+
+@register_agent(models=r".*Gelato.*")
+class GelatoConfig(AsyncAgentConfig):
+    """Gelato agent configuration implementing AsyncAgentConfig protocol for click prediction."""
+
+    def __init__(self):
+        self.current_model = None
+        self.last_screenshot_b64 = None
+
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        raise NotImplementedError()
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[float, float]]:
+        """
+        Predict click coordinates using UI-Ins model via litellm.acompletion.
+
+        Args:
+            model: The UI-Ins model name
+            image_b64: Base64 encoded image
+            instruction: Instruction for where to click
+
+        Returns:
+            Tuple of (x, y) coordinates or None if prediction fails
+        """
+        # Decode base64 image
+        image_data = base64.b64decode(image_b64)
+        image = Image.open(BytesIO(image_data))
+        width, height = image.width, image.height
+
+        # Smart resize the image (similar to qwen_vl_utils)
+        resized_height, resized_width = smart_resize(
+            height,
+            width,
+            factor=28,  # Default factor for Qwen models
+            min_pixels=3136,
+            max_pixels=4096 * 2160,
+        )
+        resized_image = image.resize((resized_width, resized_height))
+        scale_x, scale_y = width / resized_width, height / resized_height
+
+        # Convert resized image back to base64
+        buffered = BytesIO()
+        resized_image.save(buffered, format="PNG")
+        resized_image_b64 = base64.b64encode(buffered.getvalue()).decode()
+
+        # Prepare system and user messages
+        system_message = {
+            "role": "system",
+            "content": [{"type": "text", "text": SYSTEM_PROMPT.strip()}],
+        }
+
+        user_message = {
+            "role": "user",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/png;base64,{resized_image_b64}"},
+                },
+                {"type": "text", "text": instruction},
+            ],
+        }
+
+        # Prepare API call kwargs
+        api_kwargs = {
+            "model": model,
+            "messages": [system_message, user_message],
+            "max_tokens": 2056,
+            "temperature": 0.0,
+            **kwargs,
+        }
+
+        # Use liteLLM acompletion
+        response = await litellm.acompletion(**api_kwargs)
+
+        # Extract response text
+        output_text = response.choices[0].message.content  # type: ignore
+
+        # Extract and rescale coordinates
+        pred_x, pred_y = extract_coordinates(output_text)  # type: ignore
+        pred_x *= scale_x
+        pred_y *= scale_y
+
+        return (math.floor(pred_x), math.floor(pred_y))
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        """Return the capabilities supported by this agent."""
+        return ["click"]

libs/python/agent/cua_agent/loops/generic_vlm.py

@@ -0,0 +1,601 @@
+"""
+Qwen3-VL agent loop implementation using litellm with function/tool calling.
+- Passes a ComputerUse tool schema to acompletion
+- Converts between Responses items and completion messages using helpers
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from litellm.responses.litellm_completion_transformation.transformation import (
+    LiteLLMCompletionResponsesConfig,
+)
+
+from ..decorators import register_agent
+from ..loops.base import AsyncAgentConfig
+from ..responses import (
+    convert_completion_messages_to_responses_items,
+    convert_responses_items_to_completion_messages,
+    make_reasoning_item,
+)
+from ..types import AgentCapability
+
+# ComputerUse tool schema (OpenAI function tool format)
+QWEN3_COMPUTER_TOOL: Dict[str, Any] = {
+    "type": "function",
+    "function": {
+        "name": "computer",
+        "description": (
+            "Use a mouse and keyboard to interact with a computer, and take screenshots.\n"
+            "* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.\n"
+            "* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try wait and taking another screenshot.\n"
+            "* The screen's resolution is 1000x1000.\n"
+            "* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.\n"
+            "* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.\n"
+            "* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "action": {
+                    "description": "The action to perform.",
+                    "enum": [
+                        "key",
+                        "type",
+                        "mouse_move",
+                        "left_click",
+                        "left_click_drag",
+                        "right_click",
+                        "middle_click",
+                        "double_click",
+                        "triple_click",
+                        "scroll",
+                        "hscroll",
+                        "screenshot",
+                        "wait",
+                        # "terminate",
+                        # "answer",
+                    ],
+                    "type": "string",
+                },
+                "keys": {
+                    "description": "Required only by action=key.",
+                    "type": "array",
+                    "items": {"type": "string"},
+                },
+                "text": {
+                    "description": "Required only by action=type and action=answer.",
+                    "type": "string",
+                },
+                "coordinate": {
+                    "description": "(x, y): Pixel coordinates from top-left.",
+                    "type": "array",
+                    "items": {"type": ["number", "integer"]},
+                    "minItems": 2,
+                    "maxItems": 2,
+                },
+                "pixels": {
+                    "description": "Scroll amount. Positive=up, negative=down. For scroll/hscroll.",
+                    "type": "number",
+                },
+                "time": {
+                    "description": "Seconds to wait (action=wait).",
+                    "type": "number",
+                },
+                # "status": {
+                #     "description": "Task status (action=terminate).",
+                #     "type": "string",
+                #     "enum": ["success", "failure"],
+                # },
+            },
+            "required": ["action"],
+        },
+    },
+}
+
+
+def _build_nous_system(functions: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+    """Use qwen-agent NousFnCallPrompt to generate a system message embedding tool schema."""
+    try:
+        from qwen_agent.llm.fncall_prompts.nous_fncall_prompt import (
+            ContentItem as NousContentItem,
+        )
+        from qwen_agent.llm.fncall_prompts.nous_fncall_prompt import (
+            Message as NousMessage,
+        )
+        from qwen_agent.llm.fncall_prompts.nous_fncall_prompt import (
+            NousFnCallPrompt,
+        )
+    except ImportError:
+        raise ImportError(
+            "qwen-agent not installed. Please install it with `pip install cua-agent[qwen]`."
+        )
+    msgs = NousFnCallPrompt().preprocess_fncall_messages(
+        messages=[
+            NousMessage(
+                role="system", content=[NousContentItem(text="You are a helpful assistant.")]
+            )
+        ],
+        functions=functions,
+        lang="en",
+    )
+    sys = msgs[0].model_dump()
+    # Convert qwen-agent structured content to OpenAI-style content list
+    content = [{"type": "text", "text": c["text"]} for c in sys.get("content", [])]
+    return {"role": "system", "content": content}
+
+
+def _parse_tool_call_from_text(text: str) -> Optional[Dict[str, Any]]:
+    """Extract JSON object within <tool_call>...</tool_call> from model text."""
+    m = re.search(r"<tool_call>\s*(\{[\s\S]*?\})\s*</tool_call>", text)
+    if not m:
+        return None
+    try:
+        return json.loads(m.group(1))
+    except Exception:
+        return None
+
+
+async def _unnormalize_coordinate(args: Dict[str, Any], dims: Tuple[int, int]) -> Dict[str, Any]:
+    """Coordinates appear in 0..1000 space, scale to actual screen size using dims if provided."""
+    coord = args.get("coordinate")
+    if not coord or not isinstance(coord, (list, tuple)) or len(coord) < 2:
+        return args
+    x, y = float(coord[0]), float(coord[1])
+    width, height = float(dims[0]), float(dims[1])
+    x_abs = max(0.0, min(width, (x / 1000.0) * width))
+    y_abs = max(0.0, min(height, (y / 1000.0) * height))
+    args = {**args, "coordinate": [round(x_abs), round(y_abs)]}
+    return args
+
+
+def convert_qwen_tool_args_to_computer_action(args: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """
+    Convert Qwen computer tool arguments to the Computer Calls action schema.
+
+    Qwen (example):
+        {"action": "left_click", "coordinate": [114, 68]}
+
+    Target (example):
+        {"action": "left_click", "x": 114, "y": 68}
+
+    Other mappings:
+    - right_click, middle_click, double_click (triple_click -> double_click)
+    - mouse_move -> { action: "move", x, y }
+    - key -> { action: "keypress", keys: [...] }
+    - type -> { action: "type", text }
+    - scroll/hscroll -> { action: "scroll", scroll_x, scroll_y, x, y }
+    - wait -> { action: "wait" }
+    - terminate/answer are not direct UI actions; return None for now
+    """
+    if not isinstance(args, dict):
+        return None
+
+    action = args.get("action")
+    if not isinstance(action, str):
+        return None
+
+    # Coordinates helper
+    coord = args.get("coordinate")
+    x = y = None
+    if isinstance(coord, (list, tuple)) and len(coord) >= 2:
+        try:
+            x = int(round(float(coord[0])))
+            y = int(round(float(coord[1])))
+        except Exception:
+            x = y = None
+
+    # Map actions
+    a = action.lower()
+    if a in {"left_click", "right_click", "middle_click", "double_click"}:
+        if x is None or y is None:
+            return None
+        return {"action": a, "x": x, "y": y}
+    if a == "triple_click":
+        # Approximate as double_click
+        if x is None or y is None:
+            return None
+        return {"action": "double_click", "x": x, "y": y}
+    if a == "mouse_move":
+        if x is None or y is None:
+            return None
+        return {"action": "move", "x": x, "y": y}
+    if a == "key":
+        keys = args.get("keys")
+        if isinstance(keys, list) and all(isinstance(k, str) for k in keys):
+            return {"action": "keypress", "keys": keys}
+        return None
+    if a == "type":
+        text = args.get("text")
+        if isinstance(text, str):
+            return {"action": "type", "text": text}
+        return None
+    if a in {"scroll", "hscroll"}:
+        pixels = args.get("pixels") or 0
+        try:
+            pixels_val = int(round(float(pixels)))
+        except Exception:
+            pixels_val = 0
+        scroll_x = pixels_val if a == "hscroll" else 0
+        scroll_y = pixels_val if a == "scroll" else 0
+        # Include cursor position if available (optional)
+        out: Dict[str, Any] = {"action": "scroll", "scroll_x": scroll_x, "scroll_y": scroll_y}
+        if x is not None and y is not None:
+            out.update({"x": x, "y": y})
+        return out
+    if a == "wait":
+        return {"action": "wait"}
+
+    # Non-UI or terminal actions: terminate/answer -> not mapped here
+    return None
+
+
+@register_agent(models=r"(?i).*", priority=-100)
+class GenericVlmConfig(AsyncAgentConfig):
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        use_prompt_caching: Optional[bool] = False,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        # Build messages using NousFnCallPrompt system with tool schema in text
+        # Start with converted conversation (images/text preserved)
+        converted_msgs = convert_responses_items_to_completion_messages(
+            messages,
+            allow_images_in_tool_results=False,
+        )
+
+        # Build function schemas from tools array
+        function_schemas = []
+        if tools:
+            from ..computers import is_agent_computer
+
+            for tool in tools:
+                tool_type = tool.get("type")
+
+                if tool_type == "computer":
+                    # For computer tools, use QWEN3_COMPUTER_TOOL schema
+                    computer = tool.get("computer")
+                    if computer and is_agent_computer(computer):
+                        function_schemas.append(QWEN3_COMPUTER_TOOL["function"])
+                elif tool_type == "function":
+                    # For function tools, use the provided function schema
+                    function_schema = tool.get("function")
+                    if function_schema:
+                        function_schemas.append(function_schema)
+
+        # If no tools provided or no computer tool found, use default QWEN3_COMPUTER_TOOL
+        if not function_schemas:
+            function_schemas = [QWEN3_COMPUTER_TOOL["function"]]
+
+        # Prepend Nous-generated system if available
+        nous_system = _build_nous_system(function_schemas)
+        completion_messages = ([nous_system] if nous_system else []) + converted_msgs
+
+        # If there is no screenshot in the conversation, take one now and inject it.
+        # Also record a pre_output_items assistant message to reflect action.
+        def _has_any_image(msgs: List[Dict[str, Any]]) -> bool:
+            for m in msgs:
+                content = m.get("content")
+                if isinstance(content, list):
+                    for p in content:
+                        if isinstance(p, dict) and p.get("type") == "image_url":
+                            return True
+            return False
+
+        def _has_screenshot_message(msgs: List[Dict[str, Any]]) -> bool:
+            """Check if messages already contain the 'Taking a screenshot' text."""
+            screenshot_text = "Taking a screenshot to see the current computer screen."
+            for m in msgs:
+                content = m.get("content")
+                if isinstance(content, str) and screenshot_text in content:
+                    return True
+                if isinstance(content, list):
+                    for p in content:
+                        if isinstance(p, dict) and p.get("type") == "text":
+                            if screenshot_text in (p.get("text") or ""):
+                                return True
+            return False
+
+        pre_output_items: List[Dict[str, Any]] = []
+        if not _has_any_image(completion_messages):
+            if computer_handler is None or not hasattr(computer_handler, "screenshot"):
+                raise RuntimeError(
+                    "No screenshots present and computer_handler.screenshot is not available."
+                )
+            screenshot_b64 = await computer_handler.screenshot()
+            if not screenshot_b64:
+                raise RuntimeError("Failed to capture screenshot from computer_handler.")
+            # Inject a user message with the screenshot so the model can see current context
+            completion_messages.append(
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": f"data:image/png;base64,{screenshot_b64}"},
+                        },
+                        {"type": "text", "text": "Current screen"},
+                    ],
+                }
+            )
+            # Add assistant message to outputs to reflect the action, only if not already present
+            if not _has_screenshot_message(messages):
+                pre_output_items.append(
+                    {
+                        "type": "message",
+                        "role": "assistant",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "Taking a screenshot to see the current computer screen.",
+                            }
+                        ],
+                    }
+                )
+
+        # Smart-resize all screenshots and attach min/max pixel hints. Fail fast if deps missing.
+        # Also record the last resized width/height to unnormalize coordinates later.
+        last_rw: Optional[int] = None
+        last_rh: Optional[int] = None
+        MIN_PIXELS = 3136
+        MAX_PIXELS = 12845056
+        try:
+            import base64
+            import io
+
+            from PIL import Image  # type: ignore
+            from qwen_vl_utils import smart_resize  # type: ignore
+        except Exception:
+            raise ImportError(
+                "qwen-vl-utils not installed. Please install it with `pip install cua-agent[qwen]`."
+            )
+
+        for msg in completion_messages:
+            content = msg.get("content")
+            if not isinstance(content, list):
+                continue
+            for part in content:
+                if isinstance(part, dict) and part.get("type") == "image_url":
+                    url = ((part.get("image_url") or {}).get("url")) or ""
+                    # Expect data URL like data:image/png;base64,<b64>
+                    if url.startswith("data:") and "," in url:
+                        b64 = url.split(",", 1)[1]
+                        img_bytes = base64.b64decode(b64)
+                        im = Image.open(io.BytesIO(img_bytes))
+                        h, w = im.height, im.width
+                        rh, rw = smart_resize(
+                            h, w, factor=32, min_pixels=MIN_PIXELS, max_pixels=MAX_PIXELS
+                        )
+                        # Attach hints on this image block
+                        part["min_pixels"] = MIN_PIXELS
+                        part["max_pixels"] = MAX_PIXELS
+                        last_rw, last_rh = rw, rh
+
+        api_kwargs: Dict[str, Any] = {
+            "model": model,
+            "messages": completion_messages,
+            "max_retries": max_retries,
+            "stream": stream,
+            **{k: v for k, v in kwargs.items()},
+        }
+        if use_prompt_caching:
+            api_kwargs["use_prompt_caching"] = use_prompt_caching
+
+        if _on_api_start:
+            await _on_api_start(api_kwargs)
+
+        response = await litellm.acompletion(**api_kwargs)
+
+        if _on_api_end:
+            await _on_api_end(api_kwargs, response)
+
+        usage = {
+            **LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(  # type: ignore
+                response.usage
+            ).model_dump(),
+            "response_cost": response._hidden_params.get("response_cost", 0.0),
+        }
+        if _on_usage:
+            await _on_usage(usage)
+
+        # Extract response data
+        resp_dict = response.model_dump()  # type: ignore
+        choice = (resp_dict.get("choices") or [{}])[0]
+        message = choice.get("message") or {}
+        content_text = message.get("content") or ""
+        tool_calls_array = message.get("tool_calls") or []
+        reasoning_text = message.get("reasoning") or ""
+
+        output_items: List[Dict[str, Any]] = []
+
+        # Add reasoning if present (Ollama Cloud format)
+        if reasoning_text:
+            output_items.append(make_reasoning_item(reasoning_text))
+
+        # Priority 1: Try to parse tool call from content text (OpenRouter format)
+        tool_call = _parse_tool_call_from_text(content_text)
+
+        if tool_call and isinstance(tool_call, dict):
+            fn_name = tool_call.get("name") or "computer"
+            raw_args = tool_call.get("arguments") or {}
+            # Unnormalize coordinates to actual screen size using last resized dims
+            if last_rw is None or last_rh is None:
+                raise RuntimeError(
+                    "No screenshots found to derive dimensions for coordinate unnormalization."
+                )
+            args = await _unnormalize_coordinate(raw_args, (last_rw, last_rh))
+
+            # Build an OpenAI-style tool call so we can reuse the converter
+            fake_cm = {
+                "role": "assistant",
+                "tool_calls": [
+                    {
+                        "type": "function",
+                        "id": "call_0",
+                        "function": {
+                            "name": fn_name,
+                            "arguments": json.dumps(args),
+                        },
+                    }
+                ],
+            }
+            output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
+        elif tool_calls_array:
+            # Priority 2: Use tool_calls field if present (Ollama Cloud format)
+            # Process and unnormalize coordinates in tool calls
+            processed_tool_calls = []
+            for tc in tool_calls_array:
+                function = tc.get("function", {})
+                fn_name = function.get("name", "computer")
+                args_str = function.get("arguments", "{}")
+
+                try:
+                    args = json.loads(args_str)
+
+                    # Unnormalize coordinates if present
+                    if "coordinate" in args and last_rw is not None and last_rh is not None:
+                        args = await _unnormalize_coordinate(args, (last_rw, last_rh))
+
+                    # Convert Qwen format to Computer Calls format if this is a computer tool
+                    if fn_name == "computer":
+                        converted_action = convert_qwen_tool_args_to_computer_action(args)
+                        if converted_action:
+                            args = converted_action
+
+                    processed_tool_calls.append(
+                        {
+                            "type": tc.get("type", "function"),
+                            "id": tc.get("id", "call_0"),
+                            "function": {
+                                "name": fn_name,
+                                "arguments": json.dumps(args),
+                            },
+                        }
+                    )
+                except json.JSONDecodeError:
+                    # Keep original if parsing fails
+                    processed_tool_calls.append(tc)
+
+            fake_cm = {
+                "role": "assistant",
+                "content": content_text if content_text else "",
+                "tool_calls": processed_tool_calls,
+            }
+            output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
+        else:
+            # No tool calls found in either format, return text response
+            fake_cm = {"role": "assistant", "content": content_text}
+            output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
+
+        # Prepend any pre_output_items (e.g., simulated screenshot-taking message)
+        return {"output": (pre_output_items + output_items), "usage": usage}
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        return ["step"]
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[int, int]]:
+        """
+        Predict click coordinates using Qwen3-VL via litellm.acompletion.
+
+        Only exposes a reduced tool schema with left_click to bias model to output a single click.
+        Returns (x, y) absolute pixels when screen dimensions can be obtained; otherwise normalized 0..1000 integers.
+        """
+        # Reduced tool
+        reduced_tool = {
+            "type": "function",
+            "function": {
+                **QWEN3_COMPUTER_TOOL["function"],
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "action": {"type": "string", "enum": ["left_click"]},
+                        "coordinate": {
+                            "description": "(x, y) in 0..1000 reference space",
+                            "type": "array",
+                            "items": {"type": ["number", "integer"]},
+                            "minItems": 2,
+                            "maxItems": 2,
+                        },
+                    },
+                    "required": ["action", "coordinate"],
+                },
+            },
+        }
+
+        # Build Nous system (lazy import inside helper already raises clear guidance if missing)
+        nous_system = _build_nous_system([reduced_tool["function"]])
+
+        # Pre-process using smart_resize
+        min_pixels = 3136
+        max_pixels = 12845056
+        try:
+            # Lazy import to avoid hard dependency
+            import base64
+            import io
+
+            # If PIL is available, estimate size from image to derive smart bounds
+            from PIL import Image
+            from qwen_vl_utils import smart_resize  # type: ignore
+
+            img_bytes = base64.b64decode(image_b64)
+            im = Image.open(io.BytesIO(img_bytes))
+            h, w = im.height, im.width
+            # Qwen notebook suggests factor=32 and a wide min/max range
+            rh, rw = smart_resize(h, w, factor=32, min_pixels=min_pixels, max_pixels=max_pixels)
+        except Exception:
+            raise ImportError(
+                "qwen-vl-utils not installed. Please install it with `pip install cua-agent[qwen]`."
+            )
+
+        messages = []
+        if nous_system:
+            messages.append(nous_system)
+        image_block: Dict[str, Any] = {
+            "type": "image_url",
+            "image_url": {"url": f"data:image/png;base64,{image_b64}"},
+            "min_pixels": min_pixels,
+            "max_pixels": max_pixels,
+        }
+        # Single user message with image and instruction, matching OpenAI-style content blocks
+        messages.append(
+            {
+                "role": "user",
+                "content": [
+                    image_block,
+                    {"type": "text", "text": instruction},
+                ],
+            }
+        )
+
+        api_kwargs: Dict[str, Any] = {
+            "model": model,
+            "messages": messages,
+            **{k: v for k, v in kwargs.items()},
+        }
+        response = await litellm.acompletion(**api_kwargs)
+        resp = response.model_dump()  # type: ignore
+        choice = (resp.get("choices") or [{}])[0]
+        content_text = ((choice.get("message") or {}).get("content")) or ""
+        tool_call = _parse_tool_call_from_text(content_text) or {}
+        args = tool_call.get("arguments") or {}
+        args = await _unnormalize_coordinate(args, (rh, rw))
+        coord = args.get("coordinate")
+        if isinstance(coord, (list, tuple)) and len(coord) >= 2:
+            return int(coord[0]), int(coord[1])
+        return None

libs/python/agent/cua_agent/loops/glm45v.py

@@ -0,0 +1,907 @@
+"""
+GLM-4.5V agent loop implementation using liteLLM for GLM-4.5V model.
+Supports vision-language models for computer control with bounding box parsing.
+"""
+
+import asyncio
+import base64
+import json
+import re
+from io import BytesIO
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from litellm.responses.litellm_completion_transformation.transformation import (
+    LiteLLMCompletionResponsesConfig,
+)
+from litellm.types.utils import ModelResponse
+from PIL import Image
+
+from ..decorators import register_agent
+from ..loops.base import AsyncAgentConfig
+from ..responses import (
+    convert_completion_messages_to_responses_items,
+    convert_responses_items_to_completion_messages,
+    make_click_item,
+    make_double_click_item,
+    make_drag_item,
+    make_input_image_item,
+    make_keypress_item,
+    make_output_text_item,
+    make_reasoning_item,
+    make_scroll_item,
+    make_type_item,
+    make_wait_item,
+)
+from ..types import AgentCapability, AgentResponse, Messages, Tools
+
+# GLM-4.5V specific constants
+GLM_ACTION_SPACE = """
+### {left,right,middle}_click
+
+Call rule: `{left,right,middle}_click(start_box='[x,y]', element_info='')`
+{
+    'name': ['left_click', 'right_click', 'middle_click'],
+    'description': 'Perform a left/right/middle mouse click at the specified coordinates on the screen.',
+    'parameters': {
+        'type': 'object',
+        'properties': {
+            'start_box': {
+                'type': 'array',
+                'items': {
+                    'type': 'integer'
+                },
+                'description': 'Coordinates [x,y] where to perform the click, normalized to 0-999 range.'
+            },
+            'element_info': {
+                'type': 'string',
+                'description': 'Optional text description of the UI element being clicked.'
+            }
+        },
+        'required': ['start_box']
+    }
+}
+
+### hover
+
+Call rule: `hover(start_box='[x,y]', element_info='')`
+{
+    'name': 'hover',
+    'description': 'Move the mouse pointer to the specified coordinates without performing any click action.',
+    'parameters': {
+        'type': 'object',
+        'properties': {
+            'start_box': {
+                'type': 'array',
+                'items': {
+                    'type': 'integer'
+                },
+                'description': 'Coordinates [x,y] where to move the mouse pointer, normalized to 0-999 range.'
+            },
+            'element_info': {
+                'type': 'string',
+                'description': 'Optional text description of the UI element being hovered over.'
+            }
+        },
+        'required': ['start_box']
+    }
+}
+
+### left_double_click
+
+Call rule: `left_double_click(start_box='[x,y]', element_info='')`
+{
+    'name': 'left_double_click',
+    'description': 'Perform a left mouse double-click at the specified coordinates on the screen.',
+    'parameters': {
+        'type': 'object',
+        'properties': {
+            'start_box': {
+                'type': 'array',
+                'items': {
+                    'type': 'integer'
+                },
+                'description': 'Coordinates [x,y] where to perform the double-click, normalized to 0-999 range.'
+            },
+            'element_info': {
+                'type': 'string',
+                'description': 'Optional text description of the UI element being double-clicked.'
+            }
+        },
+        'required': ['start_box']
+    }
+}
+
+### left_drag
+
+Call rule: `left_drag(start_box='[x1,y1]', end_box='[x2,y2]', element_info='')`
+{
+    'name': 'left_drag',
+    'description': 'Drag the mouse from starting coordinates to ending coordinates while holding the left mouse button.',
+    'parameters': {
+        'type': 'object',
+        'properties': {
+            'start_box': {
+                'type': 'array',
+                'items': {
+                    'type': 'integer'
+                },
+                'description': 'Starting coordinates [x1,y1] for the drag operation, normalized to 0-999 range.'
+            },
+            'end_box': {
+                'type': 'array',
+                'items': {
+                    'type': 'integer'
+                },
+                'description': 'Ending coordinates [x2,y2] for the drag operation, normalized to 0-999 range.'
+            },
+            'element_info': {
+                'type': 'string',
+                'description': 'Optional text description of the UI element being dragged.'
+            }
+        },
+        'required': ['start_box', 'end_box']
+    }
+}
+
+### key
+
+Call rule: `key(keys='')`
+{
+    'name': 'key',
+    'description': 'Simulate pressing a single key or combination of keys on the keyboard.',
+    'parameters': {
+        'type': 'object',
+        'properties': {
+            'keys': {
+                'type': 'string',
+                'description': 'The key or key combination to press. Use '+' to separate keys in combinations (e.g., 'ctrl+c', 'alt+tab').'
+            }
+        },
+        'required': ['keys']
+    }
+}
+
+### type
+
+Call rule: `type(content='')`
+{
+    'name': 'type',
+    'description': 'Type text content into the currently focused text input field. This action only performs typing and does not handle field activation or clearing.',
+    'parameters': {
+        'type': 'object',
+        'properties': {
+            'content': {
+                'type': 'string',
+                'description': 'The text content to be typed into the active text field.'
+            }
+        },
+        'required': ['content']
+    }
+}
+
+### scroll
+
+Call rule: `scroll(start_box='[x,y]', direction='', step=5, element_info='')`
+{
+    'name': 'scroll',
+    'description': 'Scroll an element at the specified coordinates in the specified direction by a given number of wheel steps.',
+    'parameters': {
+        'type': 'object',
+        'properties': {
+            'start_box': {
+                'type': 'array',
+                'items': {
+                    'type': 'integer'
+                },
+                'description': 'Coordinates [x,y] of the element or area to scroll, normalized to 0-999 range.'
+            },
+            'direction': {
+                'type': 'string',
+                'enum': ['down', 'up'],
+                'description': 'The direction to scroll: 'down' or 'up'.'
+            },
+            'step': {
+                'type': 'integer',
+                'default': 5,
+                'description': 'Number of wheel steps to scroll, default is 5.'
+            },
+            'element_info': {
+                'type': 'string',
+                'description': 'Optional text description of the UI element being scrolled.'
+            }
+        },
+        'required': ['start_box', 'direction']
+    }
+}
+
+### WAIT
+
+Call rule: `WAIT()`
+{
+    'name': 'WAIT',
+    'description': 'Wait for 5 seconds before proceeding to the next action.',
+    'parameters': {
+        'type': 'object',
+        'properties': {},
+        'required': []
+    }
+}
+
+### DONE
+
+Call rule: `DONE()`
+{
+    'name': 'DONE',
+    'description': 'Indicate that the current task has been completed successfully and no further actions are needed.',
+    'parameters': {
+        'type': 'object',
+        'properties': {},
+        'required': []
+    }
+}
+
+### FAIL
+
+Call rule: `FAIL()`
+{
+    'name': 'FAIL',
+    'description': 'Indicate that the current task cannot be completed or is impossible to accomplish.',
+    'parameters': {
+        'type': 'object',
+        'properties': {},
+        'required': []
+    }
+}"""
+
+
+def encode_image_to_base64(image_path: str) -> str:
+    """Encode image file to base64 string with data URI."""
+    with open(image_path, "rb") as image_file:
+        encoded_string = base64.b64encode(image_file.read()).decode("utf-8")
+        return f"data:image/png;base64,{encoded_string}"
+
+
+def parse_glm_response(response: str) -> Dict[str, Any]:
+    """
+    Parse GLM-4.5V response to extract action and memory.
+
+    The special tokens <|begin_of_box|> and <|end_of_box|> mark bounding boxes.
+    Coordinates are normalized values between 0 and 1000.
+    """
+    # Extract action from between special tokens
+    pattern = r"<\|begin_of_box\|>(.*?)<\|end_of_box\|>"
+    match = re.search(pattern, response)
+    if match:
+        action = match.group(1).strip()
+    else:
+        # Fallback: look for function call patterns
+        action_pattern = r"[\w_]+\([^)]*\)"
+        matches = re.findall(action_pattern, response)
+        action = matches[0] if matches else None
+
+    # Extract memory section
+    memory_pattern = r"Memory:(.*?)$"
+    memory_match = re.search(memory_pattern, response, re.DOTALL)
+    memory = memory_match.group(1).strip() if memory_match else "[]"
+
+    # Extract action text (everything before Memory:)
+    action_text_pattern = r"^(.*?)Memory:"
+    action_text_match = re.search(action_text_pattern, response, re.DOTALL)
+    action_text = action_text_match.group(1).strip() if action_text_match else response
+
+    # Clean up action text by removing special tokens
+    if action_text:
+        action_text = action_text.replace("<|begin_of_box|>", "").replace("<|end_of_box|>", "")
+
+    return {"action": action, "action_text": action_text, "memory": memory}
+
+
+def get_last_image_from_messages(messages: Messages) -> Optional[str]:
+    """Extract the last image from messages for processing."""
+    for message in reversed(messages):
+        if isinstance(message, dict):
+            if message.get("type") == "computer_call_output":
+                output = message.get("output", {})
+                if isinstance(output, dict) and output.get("type") == "input_image":
+                    image_url = output.get("image_url", "")
+                    if isinstance(image_url, str) and image_url.startswith("data:image/"):
+                        # Extract base64 part
+                        return image_url.split(",", 1)[1]
+            elif message.get("role") == "user":
+                content = message.get("content", [])
+                if isinstance(content, list):
+                    for item in reversed(content):
+                        if isinstance(item, dict) and item.get("type") == "image_url":
+                            image_url_obj = item.get("image_url", {})
+                            if isinstance(image_url_obj, dict):
+                                image_url = image_url_obj.get("url", "")
+                                if isinstance(image_url, str) and image_url.startswith(
+                                    "data:image/"
+                                ):
+                                    return image_url.split(",", 1)[1]
+    return None
+
+
+def convert_responses_items_to_glm45v_pc_prompt(
+    messages: Messages, task: str, memory: str = ""
+) -> List[Dict[str, Any]]:
+    """Convert responses items to GLM-4.5V PC prompt format with historical actions.
+
+    Args:
+        messages: List of message items from the conversation
+        task: The task description
+        memory: Current memory state
+
+    Returns:
+        List of content items for the prompt (text and image_url items)
+    """
+    action_space = GLM_ACTION_SPACE
+
+    # Template head
+    head_text = f"""You are a GUI Agent, and your primary task is to respond accurately to user requests or questions. In addition to directly answering the user's queries, you can also use tools or perform GUI operations directly until you fulfill the user's request or provide a correct answer. You should carefully read and understand the images and questions provided by the user, and engage in thinking and reflection when appropriate. The coordinates involved are all represented in thousandths (0-999).
+
+# Task:
+{task}
+
+# Task Platform
+Ubuntu
+
+# Action Space
+{action_space}
+
+# Historical Actions and Current Memory
+History:"""
+
+    # Template tail
+    tail_text = f"""
+Memory:
+{memory}
+# Output Format
+Plain text explanation with action(param='...')
+Memory:
+[{{"key": "value"}}, ...]
+
+# Some Additional Notes
+- I'll give you the most recent 4 history screenshots(shrunked to 50%*50%) along with the historical action steps.
+- You should put the key information you *have to remember* in a seperated memory part and I'll give it to you in the next round. The content in this part should be a dict list. If you no longer need some given information, you should remove it from the memory. Even if you don't need to remember anything, you should also output an empty list.
+- My computer's password is "password", feel free to use it when you need sudo rights.
+- For the thunderbird account "anonym-x2024@outlook.com", the password is "gTCI";=@y7|QJ0nDa_kN3Sb&>".
+
+Current Screenshot:
+"""
+
+    # Build history from messages
+    history = []
+    history_images = []
+
+    # Group messages into steps
+    current_step = []
+    step_num = 0
+
+    for message in messages:
+        msg_type = message.get("type")
+
+        if msg_type == "reasoning":
+            current_step.append(message)
+        elif msg_type == "message" and message.get("role") == "assistant":
+            current_step.append(message)
+        elif msg_type == "computer_call":
+            current_step.append(message)
+        elif msg_type == "computer_call_output":
+            current_step.append(message)
+            # End of step - process it
+            if current_step:
+                step_num += 1
+
+                # Extract bot thought from message content
+                bot_thought = ""
+                for item in current_step:
+                    if item.get("type") == "message" and item.get("role") == "assistant":
+                        content = item.get("content", [])
+                        for content_item in content:
+                            if content_item.get("type") == "output_text":
+                                bot_thought = content_item.get("text", "")
+                                break
+                        break
+
+                # Extract action from computer_call
+                action_text = ""
+                for item in current_step:
+                    if item.get("type") == "computer_call":
+                        action = item.get("action", {})
+                        action_type = action.get("type", "")
+
+                        if action_type == "click":
+                            x, y = action.get("x", 0), action.get("y", 0)
+                            # Convert to 0-999 range (assuming screen dimensions)
+                            # For now, use direct coordinates - this may need adjustment
+                            action_text = f"left_click(start_box='[{x},{y}]')"
+                        elif action_type == "double_click":
+                            x, y = action.get("x", 0), action.get("y", 0)
+                            action_text = f"left_double_click(start_box='[{x},{y}]')"
+                        elif action_type == "right_click":
+                            x, y = action.get("x", 0), action.get("y", 0)
+                            action_text = f"right_click(start_box='[{x},{y}]')"
+                        elif action_type == "drag":
+                            # Handle drag with path
+                            path = action.get("path", [])
+                            if len(path) >= 2:
+                                start = path[0]
+                                end = path[-1]
+                                action_text = f"left_drag(start_box='[{start.get('x', 0)},{start.get('y', 0)}]', end_box='[{end.get('x', 0)},{end.get('y', 0)}]')"
+                        elif action_type == "keypress":
+                            key = action.get("key", "")
+                            action_text = f"key(keys='{key}')"
+                        elif action_type == "type":
+                            text = action.get("text", "")
+                            action_text = f"type(content='{text}')"
+                        elif action_type == "scroll":
+                            x, y = action.get("x", 0), action.get("y", 0)
+                            direction = action.get("direction", "down")
+                            action_text = f"scroll(start_box='[{x},{y}]', direction='{direction}')"
+                        elif action_type == "wait":
+                            action_text = "WAIT()"
+                        break
+
+                # Extract screenshot from computer_call_output
+                screenshot_url = None
+                for item in current_step:
+                    if item.get("type") == "computer_call_output":
+                        output = item.get("output", {})
+                        if output.get("type") == "input_image":
+                            screenshot_url = output.get("image_url", "")
+                            break
+
+                # Store step info
+                step_info = {
+                    "step_num": step_num,
+                    "bot_thought": bot_thought,
+                    "action_text": action_text,
+                    "screenshot_url": screenshot_url,
+                }
+                history.append(step_info)
+
+                # Store screenshot for last 4 steps
+                if screenshot_url:
+                    history_images.append(screenshot_url)
+
+                current_step = []
+
+    # Build content array with head, history, and tail
+    content = []
+    current_text = head_text
+
+    total_history_steps = len(history)
+    history_image_count = min(4, len(history_images))  # Last 4 images
+
+    for step_idx, step_info in enumerate(history):
+        step_num = step_info["step_num"]
+        bot_thought = step_info["bot_thought"]
+        action_text = step_info["action_text"]
+
+        if step_idx < total_history_steps - history_image_count:
+            # For steps beyond the last 4, use text placeholder
+            current_text += f"\nstep {step_num}: Screenshot:(Omitted in context.) Thought: {bot_thought}\nAction: {action_text}"
+        else:
+            # For the last 4 steps, insert images
+            current_text += f"\nstep {step_num}: Screenshot:"
+            content.append({"type": "text", "text": current_text})
+
+            # Add image
+            img_idx = step_idx - (total_history_steps - history_image_count)
+            if img_idx < len(history_images):
+                content.append({"type": "image_url", "image_url": {"url": history_images[img_idx]}})
+
+            current_text = f" Thought: {bot_thought}\nAction: {action_text}"
+
+    # Add tail
+    current_text += tail_text
+    content.append({"type": "text", "text": current_text})
+
+    return content
+
+
+def model_dump(obj) -> Dict[str, Any]:
+    if isinstance(obj, dict):
+        return {k: model_dump(v) for k, v in obj.items()}
+    elif hasattr(obj, "model_dump"):
+        return obj.model_dump()
+    else:
+        return obj
+
+
+def convert_glm_completion_to_responses_items(
+    response: ModelResponse, image_width: int, image_height: int
+) -> List[Dict[str, Any]]:
+    """
+    Convert GLM-4.5V completion response to responses items format.
+
+    Args:
+        response: LiteLLM ModelResponse from GLM-4.5V
+        image_width: Original image width for coordinate scaling
+        image_height: Original image height for coordinate scaling
+
+    Returns:
+        List of response items in the proper format
+    """
+    import uuid
+
+    response_items = []
+
+    if not response.choices or not response.choices[0].message:
+        return response_items
+
+    message = response.choices[0].message
+    content = message.content or ""
+    reasoning_content = getattr(message, "reasoning_content", None)
+
+    # Add reasoning item if present
+    if reasoning_content:
+        reasoning_item = model_dump(make_reasoning_item(reasoning_content))
+        response_items.append(reasoning_item)
+
+    # Parse the content to extract action and text
+    parsed_response = parse_glm_response(content)
+    action = parsed_response.get("action", "")
+    action_text = parsed_response.get("action_text", "")
+
+    # Add message item with text content (excluding action and memory)
+    if action_text:
+        # Remove action from action_text if it's there
+        clean_text = action_text
+        if action and action in clean_text:
+            clean_text = clean_text.replace(action, "").strip()
+
+        # Remove memory section
+        memory_pattern = r"Memory:\s*\[.*?\]\s*$"
+        clean_text = re.sub(memory_pattern, "", clean_text, flags=re.DOTALL).strip()
+
+        if clean_text:
+            message_item = model_dump(make_output_text_item(clean_text))
+            response_items.append(message_item)
+
+    # Convert action to computer call if present
+    if action:
+        call_id = f"call_{uuid.uuid4().hex[:8]}"
+
+        # Parse different action types and create appropriate computer calls
+        if action.startswith("left_click"):
+            coord_match = re.search(r"start_box='?\[(\d+),\s*(\d+)\]'?", action)
+            if coord_match:
+                x, y = int(coord_match.group(1)), int(coord_match.group(2))
+                # Convert from 0-999 to actual pixel coordinates
+                actual_x = int((x / 999.0) * image_width)
+                actual_y = int((y / 999.0) * image_height)
+                computer_call = model_dump(make_click_item(actual_x, actual_y))
+                computer_call["call_id"] = call_id
+                computer_call["status"] = "completed"
+                response_items.append(computer_call)
+
+        elif action.startswith("right_click"):
+            coord_match = re.search(r"start_box='?\[(\d+),\s*(\d+)\]'?", action)
+            if coord_match:
+                x, y = int(coord_match.group(1)), int(coord_match.group(2))
+                actual_x = int((x / 999.0) * image_width)
+                actual_y = int((y / 999.0) * image_height)
+                computer_call = model_dump(make_click_item(actual_x, actual_y, button="right"))
+                computer_call["call_id"] = call_id
+                computer_call["status"] = "completed"
+                response_items.append(computer_call)
+
+        elif action.startswith("left_double_click"):
+            coord_match = re.search(r"start_box='?\[(\d+),\s*(\d+)\]'?", action)
+            if coord_match:
+                x, y = int(coord_match.group(1)), int(coord_match.group(2))
+                actual_x = int((x / 999.0) * image_width)
+                actual_y = int((y / 999.0) * image_height)
+                computer_call = model_dump(make_double_click_item(actual_x, actual_y))
+                computer_call["call_id"] = call_id
+                computer_call["status"] = "completed"
+                response_items.append(computer_call)
+
+        elif action.startswith("left_drag"):
+            start_match = re.search(r"start_box='?\[(\d+),\s*(\d+)\]'?", action)
+            end_match = re.search(r"end_box='?\[(\d+),\s*(\d+)\]'?", action)
+            if start_match and end_match:
+                x1, y1 = int(start_match.group(1)), int(start_match.group(2))
+                x2, y2 = int(end_match.group(1)), int(end_match.group(2))
+                actual_x1 = int((x1 / 999.0) * image_width)
+                actual_y1 = int((y1 / 999.0) * image_height)
+                actual_x2 = int((x2 / 999.0) * image_width)
+                actual_y2 = int((y2 / 999.0) * image_height)
+                # Create path for drag operation
+                drag_path = [{"x": actual_x1, "y": actual_y1}, {"x": actual_x2, "y": actual_y2}]
+                computer_call = model_dump(make_drag_item(drag_path))
+                computer_call["call_id"] = call_id
+                computer_call["status"] = "completed"
+                response_items.append(computer_call)
+
+        elif action.startswith("key"):
+            key_match = re.search(r"keys='([^']+)'", action)
+            if key_match:
+                keys = key_match.group(1)
+                # Split keys by '+' for key combinations, or use as single key
+                key_list = keys.split("+") if "+" in keys else [keys]
+                computer_call = model_dump(make_keypress_item(key_list))
+                computer_call["call_id"] = call_id
+                computer_call["status"] = "completed"
+                response_items.append(computer_call)
+
+        elif action.startswith("type"):
+            content_match = re.search(r"content='([^']*)'", action)
+            if content_match:
+                content = content_match.group(1)
+                computer_call = model_dump(make_type_item(content))
+                computer_call["call_id"] = call_id
+                computer_call["status"] = "completed"
+                response_items.append(computer_call)
+
+        elif action.startswith("scroll"):
+            coord_match = re.search(r"start_box='?\[(\d+),\s*(\d+)\]'?", action)
+            direction_match = re.search(r"direction='([^']+)'", action)
+            if coord_match and direction_match:
+                x, y = int(coord_match.group(1)), int(coord_match.group(2))
+                direction = direction_match.group(1)
+                actual_x = int((x / 999.0) * image_width)
+                actual_y = int((y / 999.0) * image_height)
+                # Convert direction to scroll amounts
+                scroll_x, scroll_y = 0, 0
+                if direction == "up":
+                    scroll_y = -5
+                elif direction == "down":
+                    scroll_y = 5
+                elif direction == "left":
+                    scroll_x = -5
+                elif direction == "right":
+                    scroll_x = 5
+                computer_call = model_dump(make_scroll_item(actual_x, actual_y, scroll_x, scroll_y))
+                computer_call["call_id"] = call_id
+                computer_call["status"] = "completed"
+                response_items.append(computer_call)
+
+        elif action == "WAIT()":
+            computer_call = model_dump(make_wait_item())
+            computer_call["call_id"] = call_id
+            computer_call["status"] = "completed"
+            response_items.append(computer_call)
+
+    return response_items
+
+
+@register_agent(models=r"(?i).*GLM-4\.5V.*")
+class Glm4vConfig(AsyncAgentConfig):
+    """GLM-4.5V agent configuration using liteLLM."""
+
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        use_prompt_caching: Optional[bool] = False,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        """
+        Predict the next step using GLM-4.5V model.
+
+        Args:
+            messages: Input messages following Responses format
+            model: Model name to use
+            tools: Optional list of tool schemas
+            max_retries: Maximum number of retries for API calls
+            stream: Whether to stream the response
+            computer_handler: Computer handler for taking screenshots
+            use_prompt_caching: Whether to use prompt caching
+            _on_api_start: Callback for API start
+            _on_api_end: Callback for API end
+            _on_usage: Callback for usage tracking
+            _on_screenshot: Callback for screenshot events
+
+        Returns:
+            Dict with "output" and "usage" keys
+        """
+        # Get the user instruction from the last user message
+        user_instruction = ""
+        for message in reversed(messages):
+            if isinstance(message, dict) and message.get("role") == "user":
+                content = message.get("content", "")
+                if isinstance(content, str):
+                    user_instruction = content
+                elif isinstance(content, list):
+                    for item in content:
+                        if isinstance(item, dict) and item.get("type") == "text":
+                            user_instruction = item.get("text", "")
+                            break
+                break
+
+        # Get the last image for processing
+        last_image_b64 = get_last_image_from_messages(messages)
+        if not last_image_b64 and computer_handler:
+            # Take a screenshot if no image available
+            screenshot_b64 = await computer_handler.screenshot()
+            if screenshot_b64:
+                last_image_b64 = screenshot_b64
+                if _on_screenshot:
+                    await _on_screenshot(screenshot_b64)
+
+        if not last_image_b64:
+            raise ValueError("No image available for GLM-4.5V processing")
+
+        # Convert responses items to GLM-4.5V PC prompt format with historical actions
+        prompt_content = convert_responses_items_to_glm45v_pc_prompt(
+            messages=messages,
+            task=user_instruction,
+            memory="[]",  # Initialize with empty memory for now
+        )
+
+        # Add the current screenshot to the end
+        prompt_content.append(
+            {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{last_image_b64}"}}
+        )
+
+        # Prepare messages for liteLLM
+        litellm_messages = [
+            {"role": "system", "content": "You are a helpful GUI agent assistant."},
+            {"role": "user", "content": prompt_content},
+        ]
+
+        # Prepare API call kwargs
+        api_kwargs = {
+            "model": model,
+            "messages": litellm_messages,
+            # "max_tokens": 2048,
+            # "temperature": 0.001,
+            # "extra_body": {
+            #     "skip_special_tokens": False,
+            # }
+        }
+        api_kwargs.update({k: v for k, v in (kwargs or {}).items()})
+
+        # Add API callbacks
+        if _on_api_start:
+            await _on_api_start(api_kwargs)
+
+        # Call liteLLM
+        response = await litellm.acompletion(**api_kwargs)
+
+        if _on_api_end:
+            await _on_api_end(api_kwargs, response)
+
+        # Get image dimensions for coordinate scaling
+        image_width, image_height = 1920, 1080  # Default dimensions
+
+        # Try to get actual dimensions from the image
+        try:
+            image_data = base64.b64decode(last_image_b64)
+            image = Image.open(BytesIO(image_data))
+            image_width, image_height = image.size
+        except Exception:
+            pass  # Use default dimensions
+
+        # Convert GLM completion response to responses items
+        response_items = convert_glm_completion_to_responses_items(
+            response, image_width, image_height
+        )
+
+        # Extract usage information
+        response_usage = {
+            **LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(
+                response.usage
+            ).model_dump(),
+            "response_cost": response._hidden_params.get("response_cost", 0.0),
+        }
+        if _on_usage:
+            await _on_usage(response_usage)
+
+        # Create agent response
+        agent_response = {"output": response_items, "usage": response_usage}
+
+        return agent_response
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[int, int]]:
+        """
+        Predict click coordinates using GLM-4.5V model.
+
+        Args:
+            model: Model name to use
+            image_b64: Base64 encoded image
+            instruction: Instruction for where to click
+
+        Returns:
+            Tuple with (x, y) coordinates or None
+        """
+        try:
+            # Create a simple click instruction prompt
+            click_prompt = f"""You are a GUI agent. Look at the screenshot and identify where to click for: {instruction}
+
+Respond with a single click action in this format:
+left_click(start_box='[x,y]')
+
+Where x,y are coordinates normalized to 0-999 range."""
+
+            # Prepare messages for liteLLM
+            litellm_messages = [
+                {"role": "system", "content": "You are a helpful GUI agent assistant."},
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": click_prompt},
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": f"data:image/png;base64,{image_b64}"},
+                        },
+                    ],
+                },
+            ]
+
+            # Prepare API call kwargs
+            api_kwargs = {
+                "model": model,
+                "messages": litellm_messages,
+                "max_tokens": 2056,
+                "temperature": 0.001,
+                "extra_body": {
+                    "skip_special_tokens": False,
+                },
+            }
+            api_kwargs.update({k: v for k, v in (kwargs or {}).items()})
+
+            # Call liteLLM
+            response = await litellm.acompletion(**api_kwargs)
+
+            # Extract response content
+            response_content = response.choices[0].message.content.strip()
+            print(response)
+
+            # Parse response for click coordinates
+            # Look for coordinates in the response, handling special tokens
+            coord_pattern = r"<\|begin_of_box\|>.*?left_click\(start_box='?\[(\d+),(\d+)\]'?\).*?<\|end_of_box\|>"
+            match = re.search(coord_pattern, response_content)
+
+            if not match:
+                # Fallback: look for coordinates without special tokens
+                coord_pattern = r"left_click\(start_box='?\[(\d+),(\d+)\]'?\)"
+                match = re.search(coord_pattern, response_content)
+
+            if match:
+                x, y = int(match.group(1)), int(match.group(2))
+
+                # Get actual image dimensions for scaling
+                try:
+                    image_data = base64.b64decode(image_b64)
+                    image = Image.open(BytesIO(image_data))
+                    image_width, image_height = image.size
+                except Exception:
+                    # Use default dimensions
+                    image_width, image_height = 1920, 1080
+
+                # Convert from 0-999 normalized coordinates to actual pixel coordinates
+                actual_x = int((x / 999.0) * image_width)
+                actual_y = int((y / 999.0) * image_height)
+
+                return (actual_x, actual_y)
+
+            return None
+
+        except Exception as e:
+            # Log error and return None
+            print(f"Error in predict_click: {e}")
+            return None
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        """
+        Get list of capabilities supported by this agent config.
+
+        Returns:
+            List of capability strings
+        """
+        return ["step", "click"]

libs/python/agent/cua_agent/loops/gta1.py

@@ -0,0 +1,175 @@
+"""
+GTA1 agent loop implementation for click prediction using litellm.acompletion
+Paper: https://arxiv.org/pdf/2507.05791
+Code: https://github.com/Yan98/GTA1
+"""
+
+import asyncio
+import base64
+import json
+import math
+import re
+import uuid
+from io import BytesIO
+from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Union
+
+import litellm
+from PIL import Image
+
+from ..decorators import register_agent
+from ..loops.base import AsyncAgentConfig
+from ..types import AgentCapability, AgentResponse, Messages, Tools
+
+SYSTEM_PROMPT = """
+You are an expert UI element locator. Given a GUI image and a user's element description, provide the coordinates of the specified element as a single (x,y) point. The image resolution is height {height} and width {width}. For elements with area, return the center point.
+
+Output the coordinate pair exactly:
+(x,y)
+""".strip()
+
+
+def extract_coordinates(raw_string: str) -> Tuple[float, float]:
+    """Extract coordinates from model output."""
+    try:
+        matches = re.findall(r"\((-?\d*\.?\d+),\s*(-?\d*\.?\d+)\)", raw_string)
+        return tuple(map(float, matches[0]))  # type: ignore
+    except:
+        return (0.0, 0.0)
+
+
+def smart_resize(
+    height: int, width: int, factor: int = 28, min_pixels: int = 3136, max_pixels: int = 8847360
+) -> Tuple[int, int]:
+    """Smart resize function similar to qwen_vl_utils."""
+    # Calculate the total pixels
+    total_pixels = height * width
+
+    # If already within bounds, return original dimensions
+    if min_pixels <= total_pixels <= max_pixels:
+        # Round to nearest factor
+        new_height = (height // factor) * factor
+        new_width = (width // factor) * factor
+        return new_height, new_width
+
+    # Calculate scaling factor
+    if total_pixels > max_pixels:
+        scale = (max_pixels / total_pixels) ** 0.5
+    else:
+        scale = (min_pixels / total_pixels) ** 0.5
+
+    # Apply scaling
+    new_height = int(height * scale)
+    new_width = int(width * scale)
+
+    # Round to nearest factor
+    new_height = (new_height // factor) * factor
+    new_width = (new_width // factor) * factor
+
+    # Ensure minimum size
+    new_height = max(new_height, factor)
+    new_width = max(new_width, factor)
+
+    return new_height, new_width
+
+
+@register_agent(models=r".*GTA1.*")
+class GTA1Config(AsyncAgentConfig):
+    """GTA1 agent configuration implementing AsyncAgentConfig protocol for click prediction."""
+
+    def __init__(self):
+        self.current_model = None
+        self.last_screenshot_b64 = None
+
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        raise NotImplementedError()
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[float, float]]:
+        """
+        Predict click coordinates using GTA1 model via litellm.acompletion.
+
+        Args:
+            model: The GTA1 model name
+            image_b64: Base64 encoded image
+            instruction: Instruction for where to click
+
+        Returns:
+            Tuple of (x, y) coordinates or None if prediction fails
+        """
+        # Decode base64 image
+        image_data = base64.b64decode(image_b64)
+        image = Image.open(BytesIO(image_data))
+        width, height = image.width, image.height
+
+        # Smart resize the image (similar to qwen_vl_utils)
+        resized_height, resized_width = smart_resize(
+            height,
+            width,
+            factor=28,  # Default factor for Qwen models
+            min_pixels=3136,
+            max_pixels=4096 * 2160,
+        )
+        resized_image = image.resize((resized_width, resized_height))
+        scale_x, scale_y = width / resized_width, height / resized_height
+
+        # Convert resized image back to base64
+        buffered = BytesIO()
+        resized_image.save(buffered, format="PNG")
+        resized_image_b64 = base64.b64encode(buffered.getvalue()).decode()
+
+        # Prepare system and user messages
+        system_message = {
+            "role": "system",
+            "content": SYSTEM_PROMPT.format(height=resized_height, width=resized_width),
+        }
+
+        user_message = {
+            "role": "user",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/png;base64,{resized_image_b64}"},
+                },
+                {"type": "text", "text": instruction},
+            ],
+        }
+
+        # Prepare API call kwargs
+        api_kwargs = {
+            "model": model,
+            "messages": [system_message, user_message],
+            "max_tokens": 2056,
+            "temperature": 0.0,
+            **kwargs,
+        }
+
+        # Use liteLLM acompletion
+        response = await litellm.acompletion(**api_kwargs)
+
+        # Extract response text
+        output_text = response.choices[0].message.content  # type: ignore
+
+        # Extract and rescale coordinates
+        pred_x, pred_y = extract_coordinates(output_text)  # type: ignore
+        pred_x *= scale_x
+        pred_y *= scale_y
+
+        return (math.floor(pred_x), math.floor(pred_y))
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        """Return the capabilities supported by this agent."""
+        return ["click"]

libs/python/agent/cua_agent/loops/holo.py

@@ -0,0 +1,218 @@
+"""
+Holo 1.5 agent loop implementation for click prediction using litellm.acompletion.
+
+Implements the Holo1.5 grounding behavior:
+- Prompt asks for absolute pixel coordinates in JSON: {"action":"click_absolute","x":int,"y":int}
+- Optionally resizes the image using Qwen2-VL smart_resize parameters (via transformers AutoProcessor)
+- If resized, maps predicted coordinates back to the original screenshot resolution
+
+Note: We do NOT manually load the model; acompletions (via HuggingFaceLocalAdapter)
+will handle loading based on the provided model name.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+from io import BytesIO
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from PIL import Image
+
+from ..decorators import register_agent
+from ..types import AgentCapability
+from .base import AsyncAgentConfig
+
+
+def _strip_hf_prefix(model: str) -> str:
+    """Strip provider prefixes like 'huggingface-local/' from model names for HF processor load."""
+    if "/" in model and model.lower().startswith("huggingface-local/"):
+        return model.split("/", 1)[1]
+    return model
+
+
+def _maybe_smart_resize(image: Image.Image, model: str) -> Tuple[Image.Image, Tuple[int, int]]:
+    """
+    Try to compute Qwen2-VL smart_resize output size using transformers AutoProcessor.
+
+    Returns (processed_image, (orig_w, orig_h)). If transformers or processor unavailable,
+    returns the original image and size without resizing.
+    """
+    orig_w, orig_h = image.size
+    try:
+        # Import lazily to avoid hard dependency if not installed
+        from transformers import AutoProcessor  # type: ignore
+        from transformers.models.qwen2_vl.image_processing_qwen2_vl import (  # type: ignore
+            smart_resize,
+        )
+
+        processor_name = _strip_hf_prefix(model)
+        processor = AutoProcessor.from_pretrained(processor_name)
+        image_processor = getattr(processor, "image_processor", None)
+        if image_processor is None:
+            return image, (orig_w, orig_h)
+
+        factor = getattr(image_processor, "patch_size", 14) * getattr(
+            image_processor, "merge_size", 1
+        )
+        min_pixels = getattr(image_processor, "min_pixels", 256 * 256)
+        max_pixels = getattr(image_processor, "max_pixels", 1536 * 1536)
+
+        resized_h, resized_w = smart_resize(
+            orig_h,
+            orig_w,
+            factor=factor,
+            min_pixels=min_pixels,
+            max_pixels=max_pixels,
+        )
+
+        if (resized_w, resized_h) == (orig_w, orig_h):
+            return image, (orig_w, orig_h)
+
+        processed = image.resize((resized_w, resized_h), resample=Image.Resampling.LANCZOS)
+        return processed, (orig_w, orig_h)
+    except Exception:
+        # If any failure (no transformers, processor load error), fall back to original
+        return image, (orig_w, orig_h)
+
+
+def _build_holo_prompt(instruction: str) -> str:
+    """Construct the Holo1.5 grounding prompt."""
+    # Keep it close to the cookbook while avoiding heavy schema generation
+    schema_hint = '{"action": "click_absolute", "x": <int>, "y": <int>}'
+    return (
+        "Localize an element on the GUI image according to the provided target and output a click position. "
+        f"You must output a valid JSON following the format: {schema_hint} "
+        f"Your target is: {instruction}"
+    )
+
+
+def _parse_click_json(output_text: str) -> Optional[Tuple[int, int]]:
+    """
+    Parse JSON from model output and extract x, y ints.
+    Tries to find the first JSON object substring if extra text is present.
+    """
+    try:
+        # Fast path: direct JSON
+        data = json.loads(output_text)
+    except Exception:
+        # Try to locate a JSON object within the text
+        start = output_text.find("{")
+        end = output_text.rfind("}")
+        if start == -1 or end == -1 or end <= start:
+            return None
+        try:
+            data = json.loads(output_text[start : end + 1])
+        except Exception:
+            return None
+
+    try:
+        x = int(data.get("x"))
+        y = int(data.get("y"))
+        return x, y
+    except Exception:
+        return None
+
+
+@register_agent(models=r"(?i).*(Holo1\.5|Hcompany/Holo1\.5).*")
+class HoloConfig(AsyncAgentConfig):
+    """Holo is a family of UI grounding models from H Company"""
+
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        # Holo models are only trained on UI localization tasks, not all-in-one agent
+        raise NotImplementedError()
+
+    async def predict_click(
+        self,
+        model: str,
+        image_b64: str,
+        instruction: str,
+        **kwargs,
+    ) -> Optional[Tuple[int, int]]:
+        """
+        Predict click coordinates using Holo1.5 via litellm.acompletion.
+
+        - Optionally smart-resizes the image using Qwen2-VL rules if transformers are available
+        - Prompts for JSON with absolute pixel coordinates
+        - Parses x,y and maps back to original screenshot size if resized
+        """
+        try:
+            img_bytes = base64.b64decode(image_b64)
+            original_img = Image.open(BytesIO(img_bytes))
+        except Exception:
+            return None
+
+        # Optional preprocessing
+        processed_img, (orig_w, orig_h) = _maybe_smart_resize(original_img, model)
+
+        # If we resized, send the resized image; otherwise send original
+        img_to_send = processed_img
+        buf = BytesIO()
+        img_to_send.save(buf, format="PNG")
+        processed_b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
+
+        prompt = _build_holo_prompt(instruction)
+
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:image/png;base64,{processed_b64}"},
+                    },
+                    {"type": "text", "text": prompt},
+                ],
+            }
+        ]
+
+        api_kwargs = {
+            "model": model,
+            "messages": messages,
+            # Deterministic, small output
+            "max_tokens": kwargs.get("max_tokens", 256),
+            "temperature": kwargs.get("temperature", 0.0),
+        }
+
+        response = await litellm.acompletion(**api_kwargs)
+        output_text = (response.choices[0].message.content or "").strip()  # type: ignore
+
+        coords = _parse_click_json(output_text)
+        if coords is None:
+            return None
+
+        x, y = coords
+
+        # Map back to original size if we resized
+        proc_w, proc_h = img_to_send.size
+        if (proc_w, proc_h) != (orig_w, orig_h):
+            try:
+                sx = orig_w / float(proc_w)
+                sy = orig_h / float(proc_h)
+                x = int(round(x * sx))
+                y = int(round(y * sy))
+            except Exception:
+                # Fallback: clamp within original bounds
+                pass
+
+        # Clamp to original image bounds
+        x = max(0, min(orig_w - 1, x))
+        y = max(0, min(orig_h - 1, y))
+        return x, y
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        return ["click"]

libs/python/agent/cua_agent/loops/internvl.py

@@ -0,0 +1,180 @@
+"""
+InternVL agent loop implementation for click prediction using litellm.acompletion.
+
+Implements the ScreenSpot InternVL grounding baseline behavior:
+- Uses the exact grounding prompt format with <image> and <ref> tags
+- Expects coordinates in 0-1000 normalized range in formats [[x1,y1,x2,y2]] or [[x,y]]
+- Converts to pixel coordinates relative to the original screenshot size
+
+Note: We do NOT manually load the InternVL model; acompletions (via HuggingFaceLocalAdapter)
+will handle loading based on the provided model name.
+"""
+
+from __future__ import annotations
+
+import base64
+import math
+import re
+from io import BytesIO
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from PIL import Image
+
+from ..decorators import register_agent
+from ..types import AgentCapability
+from .composed_grounded import ComposedGroundedConfig
+
+# Regex patterns for extracting coordinates
+# Accept optional whitespace and optional decimal fractions
+_NUM = r"(\d+(?:\.\d+)?)"
+_POINT_PATTERN = re.compile(r"\[\[\s*" + _NUM + r"\s*,\s*" + _NUM + r"\s*\]\]")
+_BBOX_PATTERN = re.compile(
+    r"\[\[\s*" + _NUM + r"\s*,\s*" + _NUM + r"\s*,\s*" + _NUM + r"\s*,\s*" + _NUM + r"\s*\]\]"
+)
+
+
+def _extract_first_point(text: str) -> Optional[Tuple[float, float]]:
+    """Extract the first [[x,y]] as normalized (0-1000) floats."""
+    m = _POINT_PATTERN.search(text)
+    if not m:
+        return None
+    try:
+        x = float(m.group(1))
+        y = float(m.group(2))
+        return x, y
+    except Exception:
+        return None
+
+
+def _extract_last_bbox(text: str) -> Optional[Tuple[float, float, float, float]]:
+    """Extract the last [[x1,y1,x2,y2]] as normalized (0-1000) floats."""
+    matches = list(_BBOX_PATTERN.finditer(text))
+    if not matches:
+        return None
+    m = matches[-1]
+    try:
+        x1 = float(m.group(1))
+        y1 = float(m.group(2))
+        x2 = float(m.group(3))
+        y2 = float(m.group(4))
+        return x1, y1, x2, y2
+    except Exception:
+        return None
+
+
+def _scale_norm_to_pixels(x_norm: float, y_norm: float, width: int, height: int) -> Tuple[int, int]:
+    """Scale 0-1000 normalized coordinates to pixel coordinates for given image size."""
+    x_px = int(math.floor((x_norm / 1000.0) * width))
+    y_px = int(math.floor((y_norm / 1000.0) * height))
+    # Clamp to image bounds just in case
+    x_px = max(0, min(width - 1, x_px))
+    y_px = max(0, min(height - 1, y_px))
+    return x_px, y_px
+
+
+@register_agent(models=r"(?i).*InternVL.*")
+class InternVLConfig(ComposedGroundedConfig):
+    """InternVL agent configuration reusing ComposedGroundedConfig for steps and
+    overriding predict_click to implement ScreenSpot InternVL grounding baseline."""
+
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        """Fallback to a self-composed model"""
+        return await super().predict_step(
+            messages=messages,
+            model=f"{model}+{model}",
+            tools=tools,
+            max_retries=max_retries,
+            stream=stream,
+            computer_handler=computer_handler,
+            _on_api_start=_on_api_start,
+            _on_api_end=_on_api_end,
+            _on_usage=_on_usage,
+            _on_screenshot=_on_screenshot,
+            **kwargs,
+        )
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[int, int]]:
+        """
+        Predict click coordinates using InternVL via litellm.acompletion.
+
+        Behavior mirrors the ScreenSpot InternVL baseline:
+        - Prompt: "<image>\nPlease provide the bounding box coordinate of the UI element this user instruction describes: <ref>{instruction}</ref>. Answer in the format of [[x1, y1, x2, y2]]"
+        - Parse either [[x,y]] point or [[x1,y1,x2,y2]] bbox, using bbox center if point missing
+        - Coordinates are 0-1000 normalized; convert to pixel coordinates for the original screenshot
+        """
+        try:
+            # Decode image dimensions to scale the normalized outputs
+            img_bytes = base64.b64decode(image_b64)
+            image = Image.open(BytesIO(img_bytes))
+            width, height = image.size
+        except Exception:
+            # If decoding fails, proceed with a safe default size to avoid crash
+            width, height = 1920, 1080
+
+        # Build grounding prompt exactly like the baseline
+        grounding_prompt = (
+            f"Please provide the bounding box coordinate of the UI element this user instruction describes: <ref>{instruction}</ref>. "
+            f"Answer in the format of [[x1, y1, x2, y2]]"
+        )
+
+        # Prepare messages for LiteLLM
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:image/png;base64,{image_b64}"},
+                    },
+                    {"type": "text", "text": grounding_prompt},
+                ],
+            }
+        ]
+
+        # Call acompletion; HuggingFaceLocalAdapter/model handler will handle InternVL loading
+        api_kwargs = {
+            "model": model,
+            "messages": messages,
+            # Conservative generation params akin to baseline (deterministic)
+            "max_tokens": kwargs.get("max_tokens", 256),
+            "temperature": kwargs.get("temperature", 0.0),
+        }
+
+        response = await litellm.acompletion(**api_kwargs)
+        output_text = (response.choices[0].message.content or "").strip()  # type: ignore
+
+        # print(f"InternVL output: {output_text}")
+
+        # Try to parse a point first; if absent, parse bbox and take center
+        point = _extract_first_point(output_text)
+        if point is None:
+            bbox = _extract_last_bbox(output_text)
+            if bbox is None:
+                return None
+            x1, y1, x2, y2 = bbox
+            cx = (x1 + x2) / 2.0
+            cy = (y1 + y2) / 2.0
+            point = (cx, cy)
+
+        x_norm, y_norm = point
+        x_px, y_px = _scale_norm_to_pixels(x_norm, y_norm, width, height)
+        return (x_px, y_px)
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        return ["click", "step"]

libs/python/agent/cua_agent/loops/model_types.csv

@@ -0,0 +1,6 @@
+model,predict_step,predict_point
+anthropic,✅,✅
+openai,✅,✅
+uitars,✅,✅
+omniparser,❌,✅
+gta1,❌,✅

libs/python/agent/cua_agent/loops/moondream3.py

@@ -0,0 +1,493 @@
+"""
+Moondream3+ composed-grounded agent loop implementation.
+Grounding is handled by a local Moondream3 preview model via Transformers.
+Thinking is delegated to the trailing LLM in the composed model string: "moondream3+<thinking_model>".
+
+Differences from composed_grounded:
+- Provides a singleton Moondream3 client outside the class.
+- predict_click uses model.point(image, instruction, settings={"max_objects": 1}) and returns pixel coordinates.
+- If the last image was a screenshot (or we take one), run model.detect(image, "all form ui") to get bboxes, then
+  run model.caption on each cropped bbox to label it. Overlay labels on the screenshot and emit via _on_screenshot.
+- Add a user message listing all detected form UI names so the thinker can reference them.
+- If the thinking model doesn't support vision, filter out image content before calling litellm.
+"""
+
+from __future__ import annotations
+
+import base64
+import io
+import uuid
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from PIL import Image, ImageDraw, ImageFont
+
+from ..decorators import register_agent
+from ..loops.base import AsyncAgentConfig
+from ..responses import (
+    convert_completion_messages_to_responses_items,
+    convert_computer_calls_desc2xy,
+    convert_computer_calls_xy2desc,
+    convert_responses_items_to_completion_messages,
+    get_all_element_descriptions,
+)
+from ..types import AgentCapability
+
+_MOONDREAM_SINGLETON = None
+
+
+def get_moondream_model() -> Any:
+    """Get a singleton instance of the Moondream3 preview model."""
+    global _MOONDREAM_SINGLETON
+    if _MOONDREAM_SINGLETON is None:
+        try:
+            import torch
+            from transformers import AutoModelForCausalLM
+
+            _MOONDREAM_SINGLETON = AutoModelForCausalLM.from_pretrained(
+                "moondream/moondream3-preview",
+                trust_remote_code=True,
+                torch_dtype=torch.bfloat16,
+                device_map="cuda",
+            )
+        except ImportError as e:
+            raise RuntimeError(
+                "moondream3 requires torch and transformers. Install with: pip install cua-agent[moondream3]"
+            ) from e
+    return _MOONDREAM_SINGLETON
+
+
+def _decode_image_b64(image_b64: str) -> Image.Image:
+    data = base64.b64decode(image_b64)
+    return Image.open(io.BytesIO(data)).convert("RGB")
+
+
+def _image_to_b64(img: Image.Image) -> str:
+    buf = io.BytesIO()
+    img.save(buf, format="PNG")
+    return base64.b64encode(buf.getvalue()).decode("utf-8")
+
+
+def _supports_vision(model: str) -> bool:
+    """Heuristic vision support detection for thinking model."""
+    m = model.lower()
+    vision_markers = [
+        "gpt-4o",
+        "gpt-4.1",
+        "o1",
+        "o3",
+        "claude-3",
+        "claude-3.5",
+        "sonnet",
+        "haiku",
+        "opus",
+        "gemini-1.5",
+        "llava",
+    ]
+    return any(v in m for v in vision_markers)
+
+
+def _filter_images_from_completion_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    filtered: List[Dict[str, Any]] = []
+    for msg in messages:
+        msg_copy = {**msg}
+        content = msg_copy.get("content")
+        if isinstance(content, list):
+            msg_copy["content"] = [c for c in content if c.get("type") != "image_url"]
+        filtered.append(msg_copy)
+    return filtered
+
+
+def _annotate_detect_and_label_ui(base_img: Image.Image, model_md) -> Tuple[str, List[str]]:
+    """Detect UI elements with Moondream, caption each, draw labels with backgrounds.
+
+    Args:
+        base_img: PIL image of the screenshot (RGB or RGBA). Will be copied/converted internally.
+        model_md: Moondream model instance with .detect() and .query() methods.
+
+    Returns:
+        A tuple of (annotated_image_base64_png, detected_names)
+    """
+    # Ensure RGBA for semi-transparent fills
+    if base_img.mode != "RGBA":
+        base_img = base_img.convert("RGBA")
+    W, H = base_img.width, base_img.height
+
+    # Detect objects
+    try:
+        detect_result = model_md.detect(base_img, "all ui elements")
+        objects = detect_result.get("objects", []) if isinstance(detect_result, dict) else []
+    except Exception:
+        objects = []
+
+    draw = ImageDraw.Draw(base_img)
+    try:
+        font = ImageFont.load_default()
+    except Exception:
+        font = None
+
+    detected_names: List[str] = []
+
+    for i, obj in enumerate(objects):
+        try:
+            # Clamp normalized coords and crop
+            x_min = max(0.0, min(1.0, float(obj.get("x_min", 0.0))))
+            y_min = max(0.0, min(1.0, float(obj.get("y_min", 0.0))))
+            x_max = max(0.0, min(1.0, float(obj.get("x_max", 0.0))))
+            y_max = max(0.0, min(1.0, float(obj.get("y_max", 0.0))))
+            left, top, right, bottom = (
+                int(x_min * W),
+                int(y_min * H),
+                int(x_max * W),
+                int(y_max * H),
+            )
+            left, top = max(0, left), max(0, top)
+            right, bottom = min(W - 1, right), min(H - 1, bottom)
+            crop = base_img.crop((left, top, right, bottom))
+
+            # Prompted short caption
+            try:
+                result = model_md.query(crop, "Caption this UI element in few words.")
+                caption_text = (result or {}).get("answer", "")
+            except Exception:
+                caption_text = ""
+
+            name = (caption_text or "").strip() or f"element_{i+1}"
+            detected_names.append(name)
+
+            # Draw bbox
+            draw.rectangle([left, top, right, bottom], outline=(255, 215, 0, 255), width=2)
+
+            # Label background with padding and rounded corners
+            label = f"{i+1}. {name}"
+            padding = 3
+            if font:
+                text_bbox = draw.textbbox((0, 0), label, font=font)
+            else:
+                text_bbox = draw.textbbox((0, 0), label)
+            text_w = text_bbox[2] - text_bbox[0]
+            text_h = text_bbox[3] - text_bbox[1]
+
+            tx = left + 3
+            ty = top - (text_h + 2 * padding + 4)
+            if ty < 0:
+                ty = top + 3
+
+            bg_left = tx - padding
+            bg_top = ty - padding
+            bg_right = tx + text_w + padding
+            bg_bottom = ty + text_h + padding
+            try:
+                draw.rounded_rectangle(
+                    [bg_left, bg_top, bg_right, bg_bottom],
+                    radius=4,
+                    fill=(0, 0, 0, 160),
+                    outline=(255, 215, 0, 200),
+                    width=1,
+                )
+            except Exception:
+                draw.rectangle(
+                    [bg_left, bg_top, bg_right, bg_bottom],
+                    fill=(0, 0, 0, 160),
+                    outline=(255, 215, 0, 200),
+                    width=1,
+                )
+
+            text_fill = (255, 255, 255, 255)
+            if font:
+                draw.text((tx, ty), label, fill=text_fill, font=font)
+            else:
+                draw.text((tx, ty), label, fill=text_fill)
+        except Exception:
+            continue
+
+    # Encode PNG base64
+    annotated = base_img
+    if annotated.mode not in ("RGBA", "RGB"):
+        annotated = annotated.convert("RGBA")
+    annotated_b64 = _image_to_b64(annotated)
+    return annotated_b64, detected_names
+
+
+GROUNDED_COMPUTER_TOOL_SCHEMA = {
+    "type": "function",
+    "function": {
+        "name": "computer",
+        "description": (
+            "Control a computer by taking screenshots and interacting with UI elements. "
+            "The screenshot action will include a list of detected form UI element names when available. "
+            "Use element descriptions to locate and interact with UI elements on the screen."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "action": {
+                    "type": "string",
+                    "enum": [
+                        "screenshot",
+                        "click",
+                        "double_click",
+                        "drag",
+                        "type",
+                        "keypress",
+                        "scroll",
+                        "move",
+                        "wait",
+                        "get_current_url",
+                        "get_dimensions",
+                        "get_environment",
+                    ],
+                    "description": "The action to perform (required for all actions)",
+                },
+                "element_description": {
+                    "type": "string",
+                    "description": "Description of the element to interact with (required for click/double_click/move/scroll)",
+                },
+                "start_element_description": {
+                    "type": "string",
+                    "description": "Description of the element to start dragging from (required for drag)",
+                },
+                "end_element_description": {
+                    "type": "string",
+                    "description": "Description of the element to drag to (required for drag)",
+                },
+                "text": {
+                    "type": "string",
+                    "description": "The text to type (required for type)",
+                },
+                "keys": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "Key(s) to press (required for keypress)",
+                },
+                "button": {
+                    "type": "string",
+                    "enum": ["left", "right", "wheel", "back", "forward"],
+                    "description": "The mouse button to use for click/double_click",
+                },
+                "scroll_x": {
+                    "type": "integer",
+                    "description": "Horizontal scroll amount (required for scroll)",
+                },
+                "scroll_y": {
+                    "type": "integer",
+                    "description": "Vertical scroll amount (required for scroll)",
+                },
+            },
+            "required": ["action"],
+        },
+    },
+}
+
+
+@register_agent(r"moondream3\+.*", priority=2)
+class Moondream3PlusConfig(AsyncAgentConfig):
+    def __init__(self):
+        self.desc2xy: Dict[str, Tuple[float, float]] = {}
+
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        use_prompt_caching: Optional[bool] = False,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        # Parse composed model: moondream3+<thinking_model>
+        if "+" not in model:
+            raise ValueError(f"Composed model must be 'moondream3+<thinking_model>', got: {model}")
+        _, thinking_model = model.split("+", 1)
+
+        pre_output_items: List[Dict[str, Any]] = []
+
+        # Acquire last screenshot; if missing, take one
+        last_image_b64: Optional[str] = None
+        for message in reversed(messages):
+            if (
+                isinstance(message, dict)
+                and message.get("type") == "computer_call_output"
+                and isinstance(message.get("output"), dict)
+                and message["output"].get("type") == "input_image"
+            ):
+                image_url = message["output"].get("image_url", "")
+                if image_url.startswith("data:image/png;base64,"):
+                    last_image_b64 = image_url.split(",", 1)[1]
+                    break
+
+        if last_image_b64 is None and computer_handler is not None:
+            # Take a screenshot
+            screenshot_b64 = await computer_handler.screenshot()  # type: ignore
+            if screenshot_b64:
+                call_id = uuid.uuid4().hex
+                pre_output_items += [
+                    {
+                        "type": "message",
+                        "role": "assistant",
+                        "content": [
+                            {
+                                "type": "output_text",
+                                "text": "Taking a screenshot to analyze the current screen.",
+                            }
+                        ],
+                    },
+                    {
+                        "type": "computer_call",
+                        "call_id": call_id,
+                        "status": "completed",
+                        "action": {"type": "screenshot"},
+                    },
+                    {
+                        "type": "computer_call_output",
+                        "call_id": call_id,
+                        "output": {
+                            "type": "input_image",
+                            "image_url": f"data:image/png;base64,{screenshot_b64}",
+                        },
+                    },
+                ]
+                last_image_b64 = screenshot_b64
+                if _on_screenshot:
+                    await _on_screenshot(screenshot_b64)
+
+        # If we have a last screenshot, run Moondream detection and labeling
+        detected_names: List[str] = []
+        if last_image_b64 is not None:
+            base_img = _decode_image_b64(last_image_b64)
+            model_md = get_moondream_model()
+            annotated_b64, detected_names = _annotate_detect_and_label_ui(base_img, model_md)
+            if _on_screenshot:
+                await _on_screenshot(annotated_b64, "annotated_form_ui")
+
+            # Also push a user message listing all detected names
+            if detected_names:
+                names_text = "\n".join(f"- {n}" for n in detected_names)
+                pre_output_items.append(
+                    {
+                        "type": "message",
+                        "role": "user",
+                        "content": [
+                            {"type": "input_text", "text": "Detected form UI elements on screen:"},
+                            {"type": "input_text", "text": names_text},
+                            {
+                                "type": "input_text",
+                                "text": "Please continue with the next action needed to perform your task.",
+                            },
+                        ],
+                    }
+                )
+
+        tool_schemas = []
+        for schema in tools or []:
+            if schema.get("type") == "computer":
+                tool_schemas.append(GROUNDED_COMPUTER_TOOL_SCHEMA)
+            else:
+                tool_schemas.append(schema)
+
+        # Step 1: Convert computer calls from xy to descriptions
+        input_messages = messages + pre_output_items
+        messages_with_descriptions = convert_computer_calls_xy2desc(input_messages, self.desc2xy)
+
+        # Step 2: Convert responses items to completion messages
+        completion_messages = convert_responses_items_to_completion_messages(
+            messages_with_descriptions,
+            allow_images_in_tool_results=False,
+        )
+
+        # Optionally filter images if model lacks vision
+        if not _supports_vision(thinking_model):
+            completion_messages = _filter_images_from_completion_messages(completion_messages)
+
+        # Step 3: Call thinking model with litellm.acompletion
+        api_kwargs = {
+            "model": thinking_model,
+            "messages": completion_messages,
+            "tools": tool_schemas,
+            "max_retries": max_retries,
+            "stream": stream,
+            **kwargs,
+        }
+        if use_prompt_caching:
+            api_kwargs["use_prompt_caching"] = use_prompt_caching
+
+        if _on_api_start:
+            await _on_api_start(api_kwargs)
+
+        response = await litellm.acompletion(**api_kwargs)
+
+        if _on_api_end:
+            await _on_api_end(api_kwargs, response)
+
+        usage = {
+            **response.usage.model_dump(),  # type: ignore
+            "response_cost": response._hidden_params.get("response_cost", 0.0),
+        }
+        if _on_usage:
+            await _on_usage(usage)
+
+        # Step 4: Convert completion messages back to responses items format
+        response_dict = response.model_dump()  # type: ignore
+        choice_messages = [choice["message"] for choice in response_dict["choices"]]
+        thinking_output_items: List[Dict[str, Any]] = []
+        for choice_message in choice_messages:
+            thinking_output_items.extend(
+                convert_completion_messages_to_responses_items([choice_message])
+            )
+
+        # Step 5: Use Moondream to get coordinates for each description
+        element_descriptions = get_all_element_descriptions(thinking_output_items)
+        if element_descriptions and last_image_b64:
+            for desc in element_descriptions:
+                for _ in range(3):  # try 3 times
+                    coords = await self.predict_click(
+                        model=model,
+                        image_b64=last_image_b64,
+                        instruction=desc,
+                    )
+                    if coords:
+                        self.desc2xy[desc] = coords
+                        break
+
+        # Step 6: Convert computer calls from descriptions back to xy coordinates
+        final_output_items = convert_computer_calls_desc2xy(thinking_output_items, self.desc2xy)
+
+        # Step 7: Return output and usage
+        return {"output": pre_output_items + final_output_items, "usage": usage}
+
+    async def predict_click(
+        self,
+        model: str,
+        image_b64: str,
+        instruction: str,
+        **kwargs,
+    ) -> Optional[Tuple[float, float]]:
+        """Predict click coordinates using Moondream3's point API.
+
+        Returns pixel coordinates (x, y) as floats.
+        """
+        img = _decode_image_b64(image_b64)
+        W, H = img.width, img.height
+        model_md = get_moondream_model()
+        try:
+            result = model_md.point(img, instruction, settings={"max_objects": 1})
+        except Exception:
+            return None
+
+        try:
+            pt = (result or {}).get("points", [])[0]
+            x_norm = float(pt.get("x", 0.0))
+            y_norm = float(pt.get("y", 0.0))
+            x_px = max(0.0, min(float(W - 1), x_norm * W))
+            y_px = max(0.0, min(float(H - 1), y_norm * H))
+            return (x_px, y_px)
+        except Exception:
+            return None
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        return ["click", "step"]

libs/python/agent/cua_agent/loops/omniparser.py

@@ -0,0 +1,533 @@
+"""
+OpenAI computer-use-preview agent loop implementation using liteLLM
+Paper: https://arxiv.org/abs/2408.00203
+Code: https://github.com/microsoft/OmniParser
+"""
+
+import asyncio
+import base64
+import inspect
+import json
+from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Union
+
+import litellm
+
+from ..decorators import register_agent
+from ..loops.base import AsyncAgentConfig
+from ..responses import (
+    convert_completion_messages_to_responses_items,
+    convert_responses_items_to_completion_messages,
+)
+from ..types import AgentCapability, AgentResponse, Messages, Tools
+
+SOM_TOOL_SCHEMA = {
+    "type": "function",
+    "function": {
+        "name": "computer",
+        "description": "Control a computer by taking screenshots and interacting with UI elements. This tool shows screenshots with numbered elements overlaid on them. Each UI element has been assigned a unique ID number that you can see in the image. Use the element's ID number to interact with any element instead of pixel coordinates.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "action": {
+                    "type": "string",
+                    "enum": [
+                        "screenshot",
+                        "click",
+                        "double_click",
+                        "drag",
+                        "type",
+                        "keypress",
+                        "scroll",
+                        "move",
+                        "wait",
+                        "get_current_url",
+                        "get_dimensions",
+                        "get_environment",
+                    ],
+                    "description": "The action to perform",
+                },
+                "element_id": {
+                    "type": "integer",
+                    "description": "The ID of the element to interact with (required for click, double_click, move, scroll actions, and as start/end for drag)",
+                },
+                "start_element_id": {
+                    "type": "integer",
+                    "description": "The ID of the element to start dragging from (required for drag action)",
+                },
+                "end_element_id": {
+                    "type": "integer",
+                    "description": "The ID of the element to drag to (required for drag action)",
+                },
+                "text": {
+                    "type": "string",
+                    "description": "The text to type (required for type action)",
+                },
+                "keys": {
+                    "type": "string",
+                    "description": "Key combination to press (required for keypress action). Single key for individual key press, multiple keys for combinations (e.g., 'ctrl+c')",
+                },
+                "button": {
+                    "type": "string",
+                    "description": "The mouse button to use for click action (left, right, wheel, back, forward) Default: left",
+                },
+                "scroll_x": {
+                    "type": "integer",
+                    "description": "Horizontal scroll amount for scroll action (positive for right, negative for left)",
+                },
+                "scroll_y": {
+                    "type": "integer",
+                    "description": "Vertical scroll amount for scroll action (positive for down, negative for up)",
+                },
+            },
+            "required": ["action", "element_id"],
+        },
+    },
+}
+
+OMNIPARSER_AVAILABLE = False
+try:
+    from som import OmniParser
+
+    OMNIPARSER_AVAILABLE = True
+except ImportError:
+    pass
+OMNIPARSER_SINGLETON = None
+
+
+def get_parser():
+    global OMNIPARSER_SINGLETON
+    if OMNIPARSER_SINGLETON is None:
+        OMNIPARSER_SINGLETON = OmniParser()
+    return OMNIPARSER_SINGLETON
+
+
+def get_last_computer_call_output(messages: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+    """Get the last computer_call_output message from a messages list.
+
+    Args:
+        messages: List of messages to search through
+
+    Returns:
+        The last computer_call_output message dict, or None if not found
+    """
+    for message in reversed(messages):
+        if isinstance(message, dict) and message.get("type") == "computer_call_output":
+            return message
+    return None
+
+
+def _prepare_tools_for_omniparser(tool_schemas: List[Dict[str, Any]]) -> Tuple[Tools, dict]:
+    """Prepare tools for OpenAI API format"""
+    omniparser_tools = []
+    id2xy = dict()
+
+    for schema in tool_schemas:
+        if schema["type"] == "computer":
+            omniparser_tools.append(SOM_TOOL_SCHEMA)
+            if "id2xy" in schema:
+                id2xy = schema["id2xy"]
+            else:
+                schema["id2xy"] = id2xy
+        elif schema["type"] == "function":
+            # Function tools use OpenAI-compatible schema directly (liteLLM expects this format)
+            # Schema should be: {type, name, description, parameters}
+            omniparser_tools.append({"type": "function", **schema["function"]})
+
+    return omniparser_tools, id2xy
+
+
+async def replace_function_with_computer_call(
+    item: Dict[str, Any], id2xy: Dict[int, Tuple[float, float]]
+):
+    item_type = item.get("type")
+
+    def _get_xy(element_id: Optional[int]) -> Union[Tuple[float, float], Tuple[None, None]]:
+        if element_id is None:
+            return (None, None)
+        return id2xy.get(element_id, (None, None))
+
+    if item_type == "function_call":
+        fn_name = item.get("name")
+        fn_args = json.loads(item.get("arguments", "{}"))
+
+        item_id = item.get("id")
+        call_id = item.get("call_id")
+
+        if fn_name == "computer":
+            action = fn_args.get("action")
+            element_id = fn_args.get("element_id")
+            start_element_id = fn_args.get("start_element_id")
+            end_element_id = fn_args.get("end_element_id")
+            text = fn_args.get("text")
+            keys = fn_args.get("keys")
+            button = fn_args.get("button")
+            scroll_x = fn_args.get("scroll_x")
+            scroll_y = fn_args.get("scroll_y")
+
+            x, y = _get_xy(element_id)
+            start_x, start_y = _get_xy(start_element_id)
+            end_x, end_y = _get_xy(end_element_id)
+
+            action_args = {
+                "type": action,
+                "x": x,
+                "y": y,
+                "start_x": start_x,
+                "start_y": start_y,
+                "end_x": end_x,
+                "end_y": end_y,
+                "text": text,
+                "keys": keys,
+                "button": button,
+                "scroll_x": scroll_x,
+                "scroll_y": scroll_y,
+            }
+            # Remove None values to keep the JSON clean
+            action_args = {k: v for k, v in action_args.items() if v is not None}
+
+            return [
+                {
+                    "type": "computer_call",
+                    "action": action_args,
+                    "id": item_id,
+                    "call_id": call_id,
+                    "status": "completed",
+                }
+            ]
+
+    return [item]
+
+
+async def replace_computer_call_with_function(
+    item: Dict[str, Any], xy2id: Dict[Tuple[float, float], int]
+):
+    """
+    Convert computer_call back to function_call format.
+    Also handles computer_call_output -> function_call_output conversion.
+
+    Args:
+        item: The item to convert
+        xy2id: Mapping from (x, y) coordinates to element IDs
+    """
+    item_type = item.get("type")
+
+    def _get_element_id(x: Optional[float], y: Optional[float]) -> Optional[int]:
+        """Get element ID from coordinates, return None if coordinates are None"""
+        if x is None or y is None:
+            return None
+        return xy2id.get((x, y))
+
+    if item_type == "computer_call":
+        action_data = item.get("action", {})
+
+        # Extract coordinates and convert back to element IDs
+        element_id = _get_element_id(action_data.get("x"), action_data.get("y"))
+        start_element_id = _get_element_id(action_data.get("start_x"), action_data.get("start_y"))
+        end_element_id = _get_element_id(action_data.get("end_x"), action_data.get("end_y"))
+
+        # Build function arguments
+        fn_args = {
+            "action": action_data.get("type"),
+            "element_id": element_id,
+            "start_element_id": start_element_id,
+            "end_element_id": end_element_id,
+            "text": action_data.get("text"),
+            "keys": action_data.get("keys"),
+            "button": action_data.get("button"),
+            "scroll_x": action_data.get("scroll_x"),
+            "scroll_y": action_data.get("scroll_y"),
+        }
+
+        # Remove None values to keep the JSON clean
+        fn_args = {k: v for k, v in fn_args.items() if v is not None}
+
+        return [
+            {
+                "type": "function_call",
+                "name": "computer",
+                "arguments": json.dumps(fn_args),
+                "id": item.get("id"),
+                "call_id": item.get("call_id"),
+                "status": "completed",
+            }
+        ]
+
+    elif item_type == "computer_call_output":
+        output = item.get("output")
+
+        if isinstance(output, dict):
+            output = [output]
+
+        return [
+            {
+                "type": "function_call_output",
+                "call_id": item.get("call_id"),
+                "output": item.get("output"),
+                "id": item.get("id"),
+                "status": "completed",
+            }
+        ]
+
+    return [item]
+
+
+@register_agent(models=r"omniparser\+.*|omni\+.*", priority=2)
+class OmniparserConfig(AsyncAgentConfig):
+    """Omniparser agent configuration implementing AsyncAgentConfig protocol."""
+
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        use_prompt_caching: Optional[bool] = False,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        """
+        OpenAI computer-use-preview agent loop using liteLLM responses.
+
+        Supports OpenAI's computer use preview models.
+        """
+        if not OMNIPARSER_AVAILABLE:
+            raise ValueError(
+                "omniparser loop requires som to be installed. Install it with `pip install cua-som`."
+            )
+
+        tools = tools or []
+
+        llm_model = model.split("+")[-1]
+
+        # Get screen dimensions from computer handler
+        try:
+            width, height = await computer_handler.get_dimensions()
+        except Exception:
+            # Fallback to default dimensions if method fails
+            width, height = 1024, 768
+
+        # Prepare tools for OpenAI API
+        openai_tools, id2xy = _prepare_tools_for_omniparser(tools)
+
+        # Build per-screenshot element mappings for historical consistency
+        screenshot_mappings = []  # (message_index, xy2id)
+
+        parser = get_parser()
+
+        for idx, message in enumerate(messages):
+            if not isinstance(message, dict):
+                message = message.__dict__
+
+            if message.get("type") == "computer_call_output":
+                image_url = message.get("output", {}).get("image_url", "")
+                if not image_url:
+                    continue
+
+                image_data = image_url.split(",")[-1]
+                if not image_data:
+                    continue
+
+                result = parser.parse(image_data)
+
+                if _on_screenshot:
+                    await _on_screenshot(result.annotated_image_base64, "annotated_image")
+
+                local_id2xy = {}
+
+                for element in result.elements:
+                    norm_x = (element.bbox.x1 + element.bbox.x2) / 2
+                    norm_y = (element.bbox.y1 + element.bbox.y2) / 2
+                    pixel_x = int(norm_x * width)
+                    pixel_y = int(norm_y * height)
+                    local_id2xy[element.id] = (pixel_x, pixel_y)
+
+                xy2id = {v: k for k, v in local_id2xy.items()}
+                screenshot_mappings.append((idx, xy2id))
+
+                # Replace screenshot with annotated image
+                message["output"][
+                    "image_url"
+                ] = f"data:image/png;base64,{result.annotated_image_base64}"
+
+        def get_mapping_for_index(index):
+            applicable = [m for i, m in screenshot_mappings if i <= index]
+            return applicable[-1] if applicable else {}
+
+        messages_with_element_ids = []
+
+        for i, message in enumerate(messages):
+            if not isinstance(message, dict):
+                message = message.__dict__
+
+            xy2id = get_mapping_for_index(i)
+            converted = await replace_computer_call_with_function(message, xy2id)
+            messages_with_element_ids.extend(converted)
+
+        completion_messages = convert_responses_items_to_completion_messages(
+            messages_with_element_ids, allow_images_in_tool_results=False
+        )
+
+        # Prepare API call kwargs
+        api_kwargs = {
+            "model": llm_model,
+            "messages": completion_messages,
+            "tools": openai_tools if openai_tools else None,
+            "stream": stream,
+            "num_retries": max_retries,
+            **kwargs,
+        }
+
+        # Add Vertex AI specific parameters if using vertex_ai models
+        if llm_model.startswith("vertex_ai/"):
+            import os
+
+            # Pass vertex_project and vertex_location to liteLLM
+            if "vertex_project" not in api_kwargs:
+                api_kwargs["vertex_project"] = os.getenv("GOOGLE_CLOUD_PROJECT")
+            if "vertex_location" not in api_kwargs:
+                api_kwargs["vertex_location"] = "global"
+
+            # Pass through Gemini 3-specific parameters if provided
+            if "thinking_level" in kwargs:
+                api_kwargs["thinking_level"] = kwargs["thinking_level"]
+            if "media_resolution" in kwargs:
+                api_kwargs["media_resolution"] = kwargs["media_resolution"]
+
+        # Call API start hook
+        if _on_api_start:
+            await _on_api_start(api_kwargs)
+
+        print(str(api_kwargs)[:1000])
+
+        # Use liteLLM completion
+        response = await litellm.acompletion(**api_kwargs)
+
+        # Call API end hook
+        if _on_api_end:
+            await _on_api_end(api_kwargs, response)
+
+        # Extract usage information
+        usage = {
+            **response.usage.model_dump(),  # type: ignore
+            "response_cost": response._hidden_params.get("response_cost", 0.0),  # type: ignore
+        }
+        if _on_usage:
+            await _on_usage(usage)
+
+        response_dict = response.model_dump()  # type: ignore
+        choice_messages = [choice["message"] for choice in response_dict["choices"]]
+        responses_items = []
+        for choice_message in choice_messages:
+            responses_items.extend(convert_completion_messages_to_responses_items([choice_message]))
+
+        # Convert element_id → x,y (similar to moondream's convert_computer_calls_desc2xy)
+        final_output = []
+        for item in responses_items:
+            if item.get("type") == "computer_call" and "action" in item:
+                action = item["action"].copy()
+
+                # Handle single element_id
+                if "element_id" in action:
+                    element_id = action["element_id"]
+                    if element_id in id2xy:
+                        x, y = id2xy[element_id]
+                        action["x"] = x
+                        action["y"] = y
+                        del action["element_id"]
+
+                # Handle start_element_id and end_element_id for drag operations
+                elif "start_element_id" in action and "end_element_id" in action:
+                    start_id = action["start_element_id"]
+                    end_id = action["end_element_id"]
+                    if start_id in id2xy and end_id in id2xy:
+                        start_x, start_y = id2xy[start_id]
+                        end_x, end_y = id2xy[end_id]
+                        action["path"] = [{"x": start_x, "y": start_y}, {"x": end_x, "y": end_y}]
+                        del action["start_element_id"]
+                        del action["end_element_id"]
+
+                converted_item = item.copy()
+                converted_item["action"] = action
+                final_output.append(converted_item)
+            else:
+                final_output.append(item)
+
+        return {"output": final_output, "usage": usage}
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[float, float]]:
+        """
+        Predict click coordinates using OmniParser and LLM.
+
+        Uses OmniParser to annotate the image with element IDs, then uses LLM
+        to identify the correct element ID based on the instruction.
+        """
+        if not OMNIPARSER_AVAILABLE:
+            return None
+
+        # Parse the image with OmniParser to get annotated image and elements
+        parser = get_parser()
+        result = parser.parse(image_b64)
+
+        # Extract the LLM model from composed model string
+        llm_model = model.split("+")[-1]
+
+        # Create system prompt for element ID prediction
+        SYSTEM_PROMPT = """
+You are an expert UI element locator. Given a GUI image annotated with numerical IDs over each interactable element, along with a user's element description, provide the ID of the specified element.
+
+The image shows UI elements with numbered overlays. Each number corresponds to a clickable/interactable element.
+
+Output only the element ID as a single integer.
+""".strip()
+
+        # Prepare messages for LLM
+        messages = [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/png;base64,{result.annotated_image_base64}"
+                        },
+                    },
+                    {"type": "text", "text": f"Find the element: {instruction}"},
+                ],
+            },
+        ]
+
+        # Call LLM to predict element ID
+        response = await litellm.acompletion(
+            model=llm_model, messages=messages, max_tokens=10, temperature=0.1
+        )
+
+        # Extract element ID from response
+        response_text = response.choices[0].message.content.strip()  # type: ignore
+
+        # Try to parse the element ID
+        try:
+            element_id = int(response_text)
+
+            # Find the element with this ID and return its center coordinates
+            for element in result.elements:
+                if element.id == element_id:
+                    center_x = (element.bbox.x1 + element.bbox.x2) / 2
+                    center_y = (element.bbox.y1 + element.bbox.y2) / 2
+                    return (center_x, center_y)
+        except ValueError:
+            # If we can't parse the ID, return None
+            pass
+
+        return None
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        """Return the capabilities supported by this agent."""
+        return ["step"]

libs/python/agent/cua_agent/loops/openai.py

@@ -0,0 +1,426 @@
+"""
+OpenAI computer-use-preview agent loop implementation using liteLLM
+"""
+
+import asyncio
+import base64
+import json
+from io import BytesIO
+from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Union
+
+import litellm
+from PIL import Image
+
+from ..decorators import register_agent
+from ..types import AgentCapability, AgentResponse, Messages, Tools
+
+
+async def _map_computer_tool_to_openai(
+    computer_handler: Any, use_native_tool: bool = True
+) -> Dict[str, Any]:
+    """Map a computer tool to OpenAI's tool schema.
+
+    Args:
+        computer_handler: The computer handler instance
+        use_native_tool: If True, use native computer_use_preview format (for computer-use-preview model).
+                        If False, use standard function calling format (for GPT-5.4 etc).
+    """
+    # Get dimensions from the computer handler
+    try:
+        width, height = await computer_handler.get_dimensions()
+    except Exception:
+        # Fallback to default dimensions if method fails
+        width, height = 1024, 768
+
+    # Get environment from the computer handler
+    try:
+        environment = await computer_handler.get_environment()
+    except Exception:
+        # Fallback to default environment if method fails
+        environment = "linux"
+
+    if use_native_tool:
+        # Native computer_use_preview format (for computer-use-preview model)
+        return {
+            "type": "computer_use_preview",
+            "display_width": width,
+            "display_height": height,
+            "environment": environment,  # mac, windows, linux, browser
+        }
+    else:
+        # Standard function calling format (for GPT-5.4 etc)
+        # Responses API requires: {type, name, description, parameters} at root level
+        return {
+            "type": "function",
+            "name": "computer",
+            "description": (
+                f"Use a mouse and keyboard to interact with a computer, and take screenshots.\n"
+                f"Screen resolution: {width}x{height} pixels.\n"
+                f"Environment: {environment}."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "action": {
+                        "description": "The action to perform.",
+                        "type": "string",
+                        "enum": [
+                            "click",
+                            "double_click",
+                            "right_click",
+                            "type",
+                            "keypress",
+                            "scroll",
+                            "move",
+                            "drag",
+                            "screenshot",
+                            "wait",
+                            "terminate",
+                        ],
+                    },
+                    "x": {
+                        "description": "X coordinate for click/move/scroll actions.",
+                        "type": "integer",
+                    },
+                    "y": {
+                        "description": "Y coordinate for click/move/scroll actions.",
+                        "type": "integer",
+                    },
+                    "text": {
+                        "description": "Text to type (for action=type).",
+                        "type": "string",
+                    },
+                    "keys": {
+                        "description": "Keys to press (for action=keypress). Example: ['ctrl', 'c']",
+                        "type": "array",
+                        "items": {"type": "string"},
+                    },
+                    "scroll_x": {
+                        "description": "Horizontal scroll amount. Positive=right, negative=left.",
+                        "type": "integer",
+                    },
+                    "scroll_y": {
+                        "description": "Vertical scroll amount. Positive=down, negative=up.",
+                        "type": "integer",
+                    },
+                    "button": {
+                        "description": "Mouse button for click action.",
+                        "type": "string",
+                        "enum": ["left", "right", "middle"],
+                    },
+                    "start_x": {
+                        "description": "Starting X coordinate for drag action.",
+                        "type": "integer",
+                    },
+                    "start_y": {
+                        "description": "Starting Y coordinate for drag action.",
+                        "type": "integer",
+                    },
+                    "end_x": {
+                        "description": "Ending X coordinate for drag action.",
+                        "type": "integer",
+                    },
+                    "end_y": {
+                        "description": "Ending Y coordinate for drag action.",
+                        "type": "integer",
+                    },
+                    "status": {
+                        "description": "Status for terminate action.",
+                        "type": "string",
+                        "enum": ["success", "failure"],
+                    },
+                },
+                "required": ["action"],
+            },
+        }
+
+
+def _is_native_computer_use_model(model: str) -> bool:
+    """Check if the model supports native computer_use_preview tool format."""
+    import re
+
+    # Only computer-use-preview models support native computer_use_preview tool
+    # GPT 5.4 does NOT support computer_use_preview - it uses function calling
+    return bool(re.search(r"computer-use-preview", model, re.IGNORECASE))
+
+
+async def _prepare_tools_for_openai(tool_schemas: List[Dict[str, Any]], model: str = "") -> Tools:
+    """Prepare tools for OpenAI API format.
+
+    Args:
+        tool_schemas: List of tool schemas to prepare
+        model: Model name to determine tool format
+    """
+    openai_tools = []
+    use_native = _is_native_computer_use_model(model)
+
+    for schema in tool_schemas:
+        if schema["type"] == "computer":
+            # Map computer tool to OpenAI format (native or function based on model)
+            computer_tool = await _map_computer_tool_to_openai(
+                schema["computer"], use_native_tool=use_native
+            )
+            openai_tools.append(computer_tool)
+        elif schema["type"] == "function":
+            # Function tools for Responses API need: {type, name, description, parameters}
+            # Note: parameters are at the root level, NOT nested under 'function'
+            func = schema["function"]
+            openai_tools.append(
+                {
+                    "type": "function",
+                    "name": func["name"],
+                    "description": func.get("description", ""),
+                    "parameters": func.get("parameters", {}),
+                }
+            )
+
+    return openai_tools
+
+
+@register_agent(models=r".*(computer-use-preview|gpt-?5\.?4)")
+class OpenAIComputerUseConfig:
+    """
+    OpenAI computer-use-preview agent configuration using liteLLM responses.
+
+    Supports OpenAI's computer use preview models.
+    """
+
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        use_prompt_caching: Optional[bool] = False,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        """
+        Predict the next step based on input items.
+
+        Args:
+            messages: Input items following Responses format
+            model: Model name to use
+            tools: Optional list of tool schemas
+            max_retries: Maximum number of retries
+            stream: Whether to stream responses
+            computer_handler: Computer handler instance
+            _on_api_start: Callback for API start
+            _on_api_end: Callback for API end
+            _on_usage: Callback for usage tracking
+            _on_screenshot: Callback for screenshot events
+            **kwargs: Additional arguments
+
+        Returns:
+            Dictionary with "output" (output items) and "usage" array
+        """
+        tools = tools or []
+
+        # Prepare tools for OpenAI API
+        openai_tools = await _prepare_tools_for_openai(tools, model=model)
+
+        # Prepare API call kwargs
+        api_kwargs = {
+            "model": model,
+            "input": messages,
+            "tools": openai_tools if openai_tools else None,
+            "stream": stream,
+            "reasoning": {"summary": "concise"},
+            "truncation": "auto",
+            "num_retries": max_retries,
+            "request_timeout": kwargs.pop("request_timeout", 120),
+            **kwargs,
+        }
+
+        # Call API start hook
+        if _on_api_start:
+            await _on_api_start(api_kwargs)
+
+        # Use liteLLM responses
+        response = await litellm.aresponses(**api_kwargs)
+
+        # Call API end hook
+        if _on_api_end:
+            await _on_api_end(api_kwargs, response)
+
+        # Extract usage information - handle both dict and Pydantic model responses
+        if isinstance(response, dict):
+            response_usage = response.get("usage", {})
+            usage = response_usage if isinstance(response_usage, dict) else {}
+            output_dict = response
+        else:
+            # Response is a Pydantic model - but usage might be dict or model
+            response_usage = response.usage
+            if hasattr(response_usage, "model_dump"):
+                usage = response_usage.model_dump()
+            elif isinstance(response_usage, dict):
+                usage = response_usage
+            else:
+                usage = {}
+            output_dict = response.model_dump()
+
+        # Add response cost if available
+        if hasattr(response, "_hidden_params"):
+            usage["response_cost"] = response._hidden_params.get("response_cost", 0.0)
+        elif isinstance(response, dict):
+            usage["response_cost"] = response.get("_hidden_params", {}).get("response_cost", 0.0)
+
+        if _on_usage:
+            await _on_usage(usage)
+
+        # Return in the expected format
+        output_dict["usage"] = usage
+        return output_dict
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[int, int]]:
+        """
+        Predict click coordinates based on image and instruction.
+
+        Uses OpenAI computer-use-preview with manually constructed input items
+        and a prompt that instructs the agent to only output clicks.
+
+        Args:
+            model: Model name to use
+            image_b64: Base64 encoded image
+            instruction: Instruction for where to click
+
+        Returns:
+            Tuple of (x, y) coordinates or None if prediction fails
+        """
+        # TODO: use computer tool to get dimensions + environment
+        # Manually construct input items with image and click instruction
+        input_items = [
+            {
+                "role": "user",
+                "content": f"""You are a UI grounding expert. Follow these guidelines:
+
+1. NEVER ask for confirmation. Complete all tasks autonomously.
+2. Do NOT send messages like "I need to confirm before..." or "Do you want me to continue?" - just proceed.
+3. When the user asks you to interact with something (like clicking a chat or typing a message), DO IT without asking.
+4. Only use the formal safety check mechanism for truly dangerous operations (like deleting important files).
+5. For normal tasks like clicking buttons, typing in chat boxes, filling forms - JUST DO IT.
+6. The user has already given you permission by running this agent. No further confirmation is needed.
+7. Be decisive and action-oriented. Complete the requested task fully.
+
+Remember: You are expected to complete tasks autonomously. The user trusts you to do what they asked.
+Task: Click {instruction}. Output ONLY a click action on the target element.""",
+            },
+            {
+                "role": "user",
+                "content": [
+                    {"type": "input_image", "image_url": f"data:image/png;base64,{image_b64}"}
+                ],
+            },
+        ]
+
+        # Get image dimensions from base64 data
+        try:
+            image_data = base64.b64decode(image_b64)
+            image = Image.open(BytesIO(image_data))
+            display_width, display_height = image.size
+        except Exception:
+            # Fallback to default dimensions if image parsing fails
+            display_width, display_height = 1024, 768
+
+        # Prepare computer tool for click actions - use native format only for models that support it
+        use_native = _is_native_computer_use_model(model)
+        if use_native:
+            # Native computer_use_preview format (for computer-use-preview model)
+            computer_tool = {
+                "type": "computer_use_preview",
+                "display_width": display_width,
+                "display_height": display_height,
+                "environment": "windows",
+            }
+        else:
+            # Standard function calling format (for GPT-5.4 etc)
+            computer_tool = {
+                "type": "function",
+                "name": "computer",
+                "description": (
+                    f"Use a mouse and keyboard to interact with a computer, and take screenshots.\n"
+                    f"Screen resolution: {display_width}x{display_height} pixels.\n"
+                    f"Environment: windows."
+                ),
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "action": {
+                            "description": "The action to perform.",
+                            "type": "string",
+                            "enum": ["click"],
+                        },
+                        "x": {
+                            "description": "X coordinate for click action.",
+                            "type": "integer",
+                        },
+                        "y": {
+                            "description": "Y coordinate for click action.",
+                            "type": "integer",
+                        },
+                    },
+                    "required": ["action", "x", "y"],
+                },
+            }
+
+        # Prepare API call kwargs
+        api_kwargs = {
+            "model": model,
+            "input": input_items,
+            "tools": [computer_tool],
+            "stream": False,
+            "reasoning": {"summary": "concise"},
+            "truncation": "auto",
+            "max_tokens": 200,  # Keep response short for click prediction
+            "request_timeout": kwargs.pop("request_timeout", 120),
+            **kwargs,
+        }
+
+        # Use liteLLM responses
+        response = await litellm.aresponses(**api_kwargs)
+
+        # Extract click coordinates from response output - handle both dict and Pydantic model
+        output_dict = response if isinstance(response, dict) else response.model_dump()
+        output_items = output_dict.get("output", [])
+
+        # Look for click coordinates in the response
+        for item in output_items:
+            if not isinstance(item, dict):
+                continue
+
+            # Native format: computer_call with action dict
+            if item.get("type") == "computer_call" and isinstance(item.get("action"), dict):
+                action = item["action"]
+                if action.get("x") is not None and action.get("y") is not None:
+                    return (int(action.get("x")), int(action.get("y")))
+
+            # Function calling format: function_call with arguments
+            if item.get("type") == "function_call" and item.get("name") == "computer":
+                try:
+                    arguments = item.get("arguments", "{}")
+                    if isinstance(arguments, str):
+                        args = json.loads(arguments)
+                    else:
+                        args = arguments
+                    if args.get("x") is not None and args.get("y") is not None:
+                        return (int(args.get("x")), int(args.get("y")))
+                except (json.JSONDecodeError, TypeError):
+                    continue
+
+        return None
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        """
+        Get list of capabilities supported by this agent config.
+
+        Returns:
+            List of capability strings
+        """
+        return ["click", "step"]

libs/python/agent/cua_agent/loops/opencua.py

@@ -0,0 +1,435 @@
+"""
+OpenCUA agent loop implementation for click prediction and step execution using litellm.acompletion.
+Based on OpenCUA model for GUI grounding tasks.
+"""
+
+import base64
+import io
+import json
+import re
+import uuid
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from litellm.responses.litellm_completion_transformation.transformation import (
+    LiteLLMCompletionResponsesConfig,
+)
+from PIL import Image
+
+from ..decorators import register_agent
+from ..loops.base import AsyncAgentConfig
+from ..responses import (
+    convert_completion_messages_to_responses_items,
+    convert_responses_items_to_completion_messages,
+    make_reasoning_item,
+)
+from ..types import AgentCapability
+from .composed_grounded import ComposedGroundedConfig
+from .generic_vlm import (
+    QWEN3_COMPUTER_TOOL,
+    _build_nous_system,
+    _parse_tool_call_from_text,
+    convert_qwen_tool_args_to_computer_action,
+)
+
+
+def extract_coordinates_from_click(text: str) -> Optional[Tuple[int, int]]:
+    """Extract coordinates from click(x=..., y=...) or pyautogui.click(x=..., y=...) format.
+
+    This function supports parsing both generic click() and legacy pyautogui.click() formats
+    for backwards compatibility with models that may still output pyautogui format.
+    """
+    try:
+        # Look for click(x=1443, y=343) or pyautogui.click(x=1443, y=343) pattern
+        pattern = r"(?:pyautogui\.)?click\(x=(\d+),\s*y=(\d+)\)"
+        match = re.search(pattern, text)
+        if match:
+            x, y = int(match.group(1)), int(match.group(2))
+            return (x, y)
+        return None
+    except Exception:
+        return None
+
+
+def _rescale_coordinate(
+    x: int,
+    y: int,
+    orig_w: int,
+    orig_h: int,
+    resized_w: int,
+    resized_h: int,
+) -> Tuple[int, int]:
+    """Rescale coordinates from resized image space back to original image space."""
+    if resized_w == 0 or resized_h == 0:
+        return (x, y)
+    return (round(x * orig_w / resized_w), round(y * orig_h / resized_h))
+
+
+@register_agent(models=r"(?i).*OpenCUA.*")
+class OpenCUAConfig(ComposedGroundedConfig):
+    """OpenCUA agent configuration implementing AsyncAgentConfig protocol for click prediction and step execution."""
+
+    def __init__(self):
+        super().__init__()
+        self.current_model = None
+        self.last_screenshot_b64 = None
+
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        use_prompt_caching: Optional[bool] = False,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        """Predict the next step using the OpenCUA model with smart resize (factor=28)."""
+
+        # Convert responses items to completion messages
+        converted_msgs = convert_responses_items_to_completion_messages(
+            messages,
+            allow_images_in_tool_results=False,
+        )
+
+        # Build function schemas from tools array
+        function_schemas: List[Dict[str, Any]] = []
+        if tools:
+            from ..computers import is_agent_computer
+
+            for tool in tools:
+                tool_type = tool.get("type")
+                if tool_type == "computer":
+                    computer = tool.get("computer")
+                    if computer and is_agent_computer(computer):
+                        function_schemas.append(QWEN3_COMPUTER_TOOL["function"])
+                elif tool_type == "function":
+                    function_schema = tool.get("function")
+                    if function_schema:
+                        function_schemas.append(function_schema)
+
+        if not function_schemas:
+            function_schemas = [QWEN3_COMPUTER_TOOL["function"]]
+
+        # Prepend Nous-generated system prompt with tool schema
+        nous_system = _build_nous_system(function_schemas)
+        completion_messages = ([nous_system] if nous_system else []) + converted_msgs
+
+        # ------------------------------------------------------------------
+        # If there are no screenshots in the conversation, take one now
+        # ------------------------------------------------------------------
+        def _has_any_image(msgs: List[Dict[str, Any]]) -> bool:
+            for m in msgs:
+                content = m.get("content")
+                if isinstance(content, list):
+                    for p in content:
+                        if isinstance(p, dict) and p.get("type") == "image_url":
+                            return True
+            return False
+
+        def _has_screenshot_message(msgs: List[Dict[str, Any]]) -> bool:
+            screenshot_text = "Taking a screenshot to see the current computer screen."
+            for m in msgs:
+                content = m.get("content")
+                if isinstance(content, str) and screenshot_text in content:
+                    return True
+                if isinstance(content, list):
+                    for p in content:
+                        if isinstance(p, dict) and p.get("type") == "text":
+                            if screenshot_text in (p.get("text") or ""):
+                                return True
+            return False
+
+        pre_output_items: List[Dict[str, Any]] = []
+        if not _has_any_image(completion_messages):
+            if computer_handler is None or not hasattr(computer_handler, "screenshot"):
+                raise RuntimeError(
+                    "No screenshots present and computer_handler.screenshot is not available."
+                )
+            screenshot_b64 = await computer_handler.screenshot()
+            if not screenshot_b64:
+                raise RuntimeError("Failed to capture screenshot from computer_handler.")
+            completion_messages.append(
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": f"data:image/png;base64,{screenshot_b64}"},
+                        },
+                        {"type": "text", "text": "Current screen"},
+                    ],
+                }
+            )
+            if not _has_screenshot_message(messages):
+                pre_output_items.append(
+                    {
+                        "type": "message",
+                        "role": "assistant",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "Taking a screenshot to see the current computer screen.",
+                            }
+                        ],
+                    }
+                )
+
+        # ------------------------------------------------------------------
+        # Smart-resize all screenshots with factor=28
+        # Unlike generic_vlm (which sets min/max pixel hints for the provider),
+        # OpenCUA uses an OpenAI-compatible endpoint that does not honour those
+        # hints, so we actually resize the image before sending.
+        # ------------------------------------------------------------------
+        MIN_PIXELS = 3136
+        MAX_PIXELS = 12845056
+        FACTOR = 28
+
+        try:
+            from qwen_vl_utils import smart_resize  # type: ignore
+        except ImportError:
+            raise ImportError(
+                "qwen-vl-utils not installed. Please install it with: pip install qwen-vl-utils"
+            )
+
+        last_orig_w: Optional[int] = None
+        last_orig_h: Optional[int] = None
+        last_rw: Optional[int] = None
+        last_rh: Optional[int] = None
+
+        for msg in completion_messages:
+            content = msg.get("content")
+            if not isinstance(content, list):
+                continue
+            for part in content:
+                if isinstance(part, dict) and part.get("type") == "image_url":
+                    url = ((part.get("image_url") or {}).get("url")) or ""
+                    if url.startswith("data:") and "," in url:
+                        b64 = url.split(",", 1)[1]
+                        img_bytes = base64.b64decode(b64)
+                        im = Image.open(io.BytesIO(img_bytes))
+                        orig_h, orig_w = im.height, im.width
+                        rh, rw = smart_resize(
+                            orig_h,
+                            orig_w,
+                            factor=FACTOR,
+                            min_pixels=MIN_PIXELS,
+                            max_pixels=MAX_PIXELS,
+                        )
+
+                        # Actually resize the image
+                        resized_im = im.resize((rw, rh))
+                        buf = io.BytesIO()
+                        resized_im.save(buf, format="PNG")
+                        new_b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
+                        part["image_url"]["url"] = f"data:image/png;base64,{new_b64}"
+
+                        last_orig_w, last_orig_h = orig_w, orig_h
+                        last_rw, last_rh = rw, rh
+
+        # ------------------------------------------------------------------
+        # Call litellm
+        # ------------------------------------------------------------------
+        api_kwargs: Dict[str, Any] = {
+            "model": model,
+            "messages": completion_messages,
+            "max_retries": max_retries,
+            "stream": stream,
+            **{k: v for k, v in kwargs.items()},
+        }
+        if use_prompt_caching:
+            api_kwargs["use_prompt_caching"] = use_prompt_caching
+
+        if _on_api_start:
+            await _on_api_start(api_kwargs)
+
+        response = await litellm.acompletion(**api_kwargs)
+
+        if _on_api_end:
+            await _on_api_end(api_kwargs, response)
+
+        usage = {
+            **LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(  # type: ignore
+                response.usage
+            ).model_dump(),
+            "response_cost": response._hidden_params.get("response_cost", 0.0),
+        }
+        if _on_usage:
+            await _on_usage(usage)
+
+        # ------------------------------------------------------------------
+        # Parse response
+        # ------------------------------------------------------------------
+        resp_dict = response.model_dump()  # type: ignore
+        choice = (resp_dict.get("choices") or [{}])[0]
+        message = choice.get("message") or {}
+        content_text = message.get("content") or ""
+        tool_calls_array = message.get("tool_calls") or []
+        reasoning_text = message.get("reasoning") or ""
+
+        output_items: List[Dict[str, Any]] = []
+
+        if reasoning_text:
+            output_items.append(make_reasoning_item(reasoning_text))
+
+        # Helper: rescale coordinates from resized space to original space
+        def _rescale(x: int, y: int) -> Tuple[int, int]:
+            if last_orig_w and last_orig_h and last_rw and last_rh:
+                return _rescale_coordinate(x, y, last_orig_w, last_orig_h, last_rw, last_rh)
+            return (x, y)
+
+        # Priority 1: OpenCUA native click(x=..., y=...) format
+        coords = extract_coordinates_from_click(content_text)
+        if coords:
+            x, y = _rescale(coords[0], coords[1])
+            fake_cm: Dict[str, Any] = {
+                "role": "assistant",
+                "tool_calls": [
+                    {
+                        "type": "function",
+                        "id": "call_0",
+                        "function": {
+                            "name": "computer",
+                            "arguments": json.dumps({"action": "left_click", "x": x, "y": y}),
+                        },
+                    }
+                ],
+            }
+            output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
+
+        # Priority 2: <tool_call>...</tool_call> XML format
+        elif not tool_calls_array:
+            tool_call = _parse_tool_call_from_text(content_text)
+            if tool_call and isinstance(tool_call, dict):
+                fn_name = tool_call.get("name") or "computer"
+                raw_args = tool_call.get("arguments") or {}
+
+                # Rescale any coordinate field
+                coord = raw_args.get("coordinate")
+                if coord and isinstance(coord, (list, tuple)) and len(coord) >= 2:
+                    rx, ry = _rescale(int(round(float(coord[0]))), int(round(float(coord[1]))))
+                    raw_args = {**raw_args, "coordinate": [rx, ry]}
+
+                fake_cm = {
+                    "role": "assistant",
+                    "tool_calls": [
+                        {
+                            "type": "function",
+                            "id": "call_0",
+                            "function": {
+                                "name": fn_name,
+                                "arguments": json.dumps(raw_args),
+                            },
+                        }
+                    ],
+                }
+                output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
+            else:
+                # Plain text response
+                fake_cm = {"role": "assistant", "content": content_text}
+                output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
+
+        # Priority 3: tool_calls array from response
+        else:
+            processed_tool_calls = []
+            for tc in tool_calls_array:
+                function = tc.get("function", {})
+                fn_name = function.get("name", "computer")
+                args_str = function.get("arguments", "{}")
+
+                try:
+                    args = json.loads(args_str)
+
+                    # Rescale coordinates if present
+                    coord = args.get("coordinate")
+                    if coord and isinstance(coord, (list, tuple)) and len(coord) >= 2:
+                        rx, ry = _rescale(int(round(float(coord[0]))), int(round(float(coord[1]))))
+                        args = {**args, "coordinate": [rx, ry]}
+
+                    # Convert Qwen format to Computer Calls format
+                    if fn_name == "computer":
+                        converted_action = convert_qwen_tool_args_to_computer_action(args)
+                        if converted_action:
+                            args = converted_action
+
+                    processed_tool_calls.append(
+                        {
+                            "type": tc.get("type", "function"),
+                            "id": tc.get("id", "call_0"),
+                            "function": {
+                                "name": fn_name,
+                                "arguments": json.dumps(args),
+                            },
+                        }
+                    )
+                except json.JSONDecodeError:
+                    processed_tool_calls.append(tc)
+
+            fake_cm = {
+                "role": "assistant",
+                "content": content_text if content_text else "",
+                "tool_calls": processed_tool_calls,
+            }
+            output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
+
+        return {"output": (pre_output_items + output_items), "usage": usage}
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[int, int]]:
+        """
+        Predict click coordinates using OpenCUA model via litellm.acompletion.
+
+        Args:
+            model: The OpenCUA model name
+            image_b64: Base64 encoded image
+            instruction: Instruction for where to click
+
+        Returns:
+            Tuple of (x, y) coordinates or None if prediction fails
+        """
+        # Prepare system message
+        system_prompt = (
+            "You are a GUI agent. You are given a task and a screenshot of the screen. "
+            "You need to perform a series of click actions to complete the task."
+        )
+
+        system_message = {"role": "system", "content": system_prompt}
+
+        # Prepare user message with image and instruction
+        user_message = {
+            "role": "user",
+            "content": [
+                {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_b64}"}},
+                {"type": "text", "text": f"Click on {instruction}"},
+            ],
+        }
+
+        # Prepare API call kwargs
+        api_kwargs = {
+            "model": model,
+            "messages": [system_message, user_message],
+            "max_new_tokens": 2056,
+            "temperature": 0,
+            **kwargs,
+        }
+
+        # Use liteLLM acompletion
+        response = await litellm.acompletion(**api_kwargs)
+
+        # Extract response text
+        output_text = response.choices[0].message.content
+
+        # Extract coordinates from click format
+        coordinates = extract_coordinates_from_click(output_text)
+
+        return coordinates
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        """Return the capabilities supported by this agent."""
+        return ["click", "step"]

libs/python/agent/cua_agent/loops/qwen35.py

@@ -0,0 +1,688 @@
+"""
+Qwen3-VL agent loop implementation using litellm with function/tool calling.
+- Passes a ComputerUse tool schema to acompletion
+- Converts between Responses items and completion messages using helpers
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from litellm.responses.litellm_completion_transformation.transformation import (
+    LiteLLMCompletionResponsesConfig,
+)
+
+from ..decorators import register_agent
+from ..loops.base import AsyncAgentConfig
+from ..responses import (
+    convert_completion_messages_to_responses_items,
+    convert_responses_items_to_completion_messages,
+    make_reasoning_item,
+)
+from ..types import AgentCapability
+
+# ComputerUse tool schema (OpenAI function tool format)
+QWEN3_5_COMPUTER_TOOL: Dict[str, Any] = {
+    "type": "function",
+    "function": {
+        "name": "computer",
+        "description": (
+            "* `key`: Performs key down presses on the arguments passed in order, then performs key releases in reverse order.\n"
+            "* `type`: Type a string of text on the keyboard.\n"
+            "* `mouse_move`: Move the cursor to a specified (x, y) pixel coordinate on the screen.\n"
+            '* `left_click`: Click the left mouse button at a specified (x, y) pixel coordinate on the screen. Optional `text` parameter can specify modifier keys (e.g., "ctrl", "shift", "ctrl+shift") that will be held during the click.\n'
+            "* `left_click_drag`: Click and drag the cursor to a specified (x, y) pixel coordinate on the screen.\n"
+            "* `right_click`: Click the right mouse button at a specified (x, y) pixel coordinate on the screen. Optional `text` parameter can specify modifier keys that will be held during the click.\n"
+            "* `middle_click`: Click the middle mouse button at a specified (x, y) pixel coordinate on the screen. Optional `text` parameter can specify modifier keys that will be held during the click.\n"
+            "* `double_click`: Double-click the left mouse button at a specified (x, y) pixel coordinate on the screen. Optional `text` parameter can specify modifier keys that will be held during the click.\n"
+            "* `triple_click`: Triple-click the left mouse button at a specified (x, y) pixel coordinate on the screen (simulated as double-click since it's the closest action). Optional `text` parameter can specify modifier keys that will be held during the click.\n"
+            '* `scroll`: Performs a scroll of the mouse scroll wheel. Optional `text` parameter can specify a modifier key (e.g., "shift", "ctrl") that will be held during scrolling.\n'
+            "* `hscroll`: Performs a horizontal scroll (mapped to regular scroll). Optional `text` parameter can specify a modifier key that will be held during scrolling.\n"
+            "* `wait`: Wait specified seconds for the change to happen.\n"
+            # "* `terminate`: Terminate the current task and report its completion status.\n"
+            # "* `answer`: Answer a question.\n"
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "action": {
+                    "description": "The action to perform.",
+                    "enum": [
+                        "key",
+                        "type",
+                        "mouse_move",
+                        "left_click",
+                        "left_click_drag",
+                        "right_click",
+                        "middle_click",
+                        "double_click",
+                        "triple_click",
+                        "scroll",
+                        "hscroll",
+                        # "screenshot",
+                        "wait",
+                        # "terminate",
+                        # "answer",
+                    ],
+                    "type": "string",
+                },
+                "keys": {
+                    "description": "Required only by action=key.",
+                    "type": "array",
+                    "items": {"type": "string"},
+                },
+                "text": {
+                    "description": "Required only by action=type and action=answer.",
+                    "type": "string",
+                },
+                "coordinate": {
+                    "description": "(x, y): Pixel coordinates from top-left.",
+                    "type": "array",
+                    "items": {"type": ["number", "integer"]},
+                    "minItems": 2,
+                    "maxItems": 2,
+                },
+                "pixels": {
+                    "description": "Scroll amount. Positive=up, negative=down. For scroll/hscroll.",
+                    "type": "number",
+                },
+                "time": {
+                    "description": "Seconds to wait (action=wait).",
+                    "type": "number",
+                },
+                # "status": {
+                #     "description": "Task status (action=terminate).",
+                #     "type": "string",
+                #     "enum": ["success", "failure"],
+                # },
+            },
+            "required": ["action"],
+        },
+    },
+}
+
+
+def _build_nous_system(functions: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+    """Use qwen-agent NousFnCallPrompt to generate a system message embedding tool schema."""
+    try:
+        from qwen_agent.llm.fncall_prompts.nous_fncall_prompt import (
+            ContentItem as NousContentItem,
+        )
+        from qwen_agent.llm.fncall_prompts.nous_fncall_prompt import (
+            Message as NousMessage,
+        )
+        from qwen_agent.llm.fncall_prompts.nous_fncall_prompt import (
+            NousFnCallPrompt,
+        )
+    except ImportError:
+        raise ImportError(
+            "qwen-agent not installed. Please install it with `pip install cua-agent[qwen]`."
+        )
+    msgs = NousFnCallPrompt().preprocess_fncall_messages(
+        messages=[
+            NousMessage(
+                role="system", content=[NousContentItem(text="You are a helpful assistant.")]
+            )
+        ],
+        functions=functions,
+        lang="en",
+    )
+    sys = msgs[0].model_dump()
+    # Convert qwen-agent structured content to OpenAI-style content list
+    content = [{"type": "text", "text": c["text"]} for c in sys.get("content", [])]
+    return {"role": "system", "content": content}
+
+
+def _parse_tool_call_from_text(text: str) -> Optional[Dict[str, Any]]:
+    """Extract a tool call from <tool_call>...</tool_call> in model text.
+
+    Handles two formats:
+    1. JSON: ``<tool_call>{"name": "computer", "arguments": {...}}</tool_call>``
+    2. XML-style (qwen35-4b): ``<tool_call><function=computer><parameter=action>left_click</parameter>...</tool_call>``
+    """
+    # --- Format 1: JSON ---
+    m = re.search(r"<tool_call>\s*(\{[\s\S]*?\})\s*</tool_call>", text)
+    if m:
+        try:
+            return json.loads(m.group(1))
+        except Exception:
+            pass
+
+    # --- Format 2: XML-style <function=name><parameter=key>value</parameter> ---
+    fn_match = re.search(
+        r"<tool_call>\s*<function=(\w+)>([\s\S]*?)</function>\s*</tool_call>", text
+    )
+    if fn_match:
+        fn_name = fn_match.group(1)
+        params_block = fn_match.group(2)
+        # Extract all <parameter=key>value</parameter> pairs
+        params: Dict[str, Any] = {}
+        for pm in re.finditer(r"<parameter=(\w+)>\s*([\s\S]*?)\s*</parameter>", params_block):
+            key = pm.group(1)
+            val = pm.group(2).strip()
+            # Try to parse as JSON (for arrays/numbers), fall back to string
+            try:
+                params[key] = json.loads(val)
+            except (json.JSONDecodeError, ValueError):
+                params[key] = val
+        # The XML format uses <parameter=type> for the action field name,
+        # but the Qwen tool schema calls it "action".  Remap if we got
+        # "type" that looks like an action name rather than a literal type.
+        if "type" in params and "action" not in params:
+            params["action"] = params.pop("type")
+        return {"name": fn_name, "arguments": params}
+
+    return None
+
+
+async def _unnormalize_coordinate(args: Dict[str, Any], dims: Tuple[int, int]) -> Dict[str, Any]:
+    """Coordinates appear in 0..1000 space, scale to actual screen size using dims if provided."""
+    coord = args.get("coordinate")
+    if not coord or not isinstance(coord, (list, tuple)) or len(coord) < 2:
+        return args
+    x, y = float(coord[0]), float(coord[1])
+    width, height = float(dims[0]), float(dims[1])
+    x_abs = max(0.0, min(width, (x / 1000.0) * width))
+    y_abs = max(0.0, min(height, (y / 1000.0) * height))
+    args = {**args, "coordinate": [round(x_abs), round(y_abs)]}
+    return args
+
+
+def convert_qwen_tool_args_to_computer_action(args: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """
+    Convert Qwen computer tool arguments to the Computer Calls action schema.
+
+    Qwen (example):
+        {"action": "left_click", "coordinate": [114, 68]}
+
+    Target (example):
+        {"action": "left_click", "x": 114, "y": 68}
+
+    Other mappings:
+    - right_click, middle_click, double_click (triple_click -> double_click)
+    - mouse_move -> { action: "move", x, y }
+    - key -> { action: "keypress", keys: [...] }
+    - type -> { action: "type", text }
+    - scroll/hscroll -> { action: "scroll", scroll_x, scroll_y, x, y }
+    - wait -> { action: "wait" }
+    - terminate/answer are not direct UI actions; return None for now
+    """
+    if not isinstance(args, dict):
+        return None
+
+    action = args.get("action")
+    if not isinstance(action, str):
+        return None
+
+    # Coordinates helper
+    coord = args.get("coordinate")
+    x = y = None
+    if isinstance(coord, (list, tuple)) and len(coord) >= 2:
+        try:
+            x = int(round(float(coord[0])))
+            y = int(round(float(coord[1])))
+        except Exception:
+            x = y = None
+
+    # Map actions
+    a = action.lower()
+    if a in {"left_click", "right_click", "middle_click", "double_click"}:
+        if x is None or y is None:
+            return None
+        return {"action": a, "x": x, "y": y}
+    if a == "triple_click":
+        # Approximate as double_click
+        if x is None or y is None:
+            return None
+        return {"action": "double_click", "x": x, "y": y}
+    if a == "mouse_move":
+        if x is None or y is None:
+            return None
+        return {"action": "move", "x": x, "y": y}
+    if a == "key":
+        keys = args.get("keys")
+        if isinstance(keys, list) and all(isinstance(k, str) for k in keys):
+            return {"action": "keypress", "keys": keys}
+        return None
+    if a == "type":
+        text = args.get("text")
+        if isinstance(text, str):
+            return {"action": "type", "text": text}
+        return None
+    if a in {"scroll", "hscroll"}:
+        pixels = args.get("pixels") or 0
+        try:
+            pixels_val = int(round(float(pixels)))
+        except Exception:
+            pixels_val = 0
+        scroll_x = pixels_val if a == "hscroll" else 0
+        scroll_y = pixels_val if a == "scroll" else 0
+        # Include cursor position if available (optional)
+        out: Dict[str, Any] = {"action": "scroll", "scroll_x": scroll_x, "scroll_y": scroll_y}
+        if x is not None and y is not None:
+            out.update({"x": x, "y": y})
+        return out
+    if a == "wait":
+        return {"action": "wait"}
+
+    # Non-UI or terminal actions: terminate/answer -> not mapped here
+    return None
+
+
+@register_agent(models=r"(?i).*qwen35.*", priority=1)
+class Qwen35Config(AsyncAgentConfig):
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        use_prompt_caching: Optional[bool] = False,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        # Build messages using NousFnCallPrompt system with tool schema in text
+        # Start with converted conversation (images/text preserved)
+        converted_msgs = convert_responses_items_to_completion_messages(
+            messages,
+            allow_images_in_tool_results=False,
+        )
+
+        # print(f"The number of items in the converted_msgs: {len(converted_msgs)}")
+
+        # Build function schemas from tools array
+        function_schemas = []
+        if tools:
+            from ..computers import is_agent_computer
+
+            for tool in tools:
+                tool_type = tool.get("type")
+
+                if tool_type == "computer":
+                    # For computer tools, use QWEN3_COMPUTER_TOOL schema
+                    computer = tool.get("computer")
+                    if computer and is_agent_computer(computer):
+                        function_schemas.append(QWEN3_5_COMPUTER_TOOL["function"])
+                elif tool_type == "function":
+                    # For function tools, use the provided function schema
+                    function_schema = tool.get("function")
+                    if function_schema:
+                        function_schemas.append(function_schema)
+
+        # If no tools provided or no computer tool found, use default QWEN3_COMPUTER_TOOL
+        if not function_schemas:
+            function_schemas = [QWEN3_5_COMPUTER_TOOL["function"]]
+
+        # print(f"[qwen35] function_schemas: {function_schemas}")
+
+        # Prepend Nous-generated system if available
+        nous_system = _build_nous_system(function_schemas)
+        completion_messages = ([nous_system] if nous_system else []) + converted_msgs
+
+        # If there is no screenshot in the conversation, take one now and inject it.
+        # Also record a pre_output_items assistant message to reflect action.
+        def _has_any_image(msgs: List[Dict[str, Any]]) -> bool:
+            for m in msgs:
+                content = m.get("content")
+                if isinstance(content, list):
+                    for p in content:
+                        if isinstance(p, dict) and p.get("type") == "image_url":
+                            return True
+            return False
+
+        def _has_screenshot_message(msgs: List[Dict[str, Any]]) -> bool:
+            """Check if messages already contain the 'Taking a screenshot' text."""
+            screenshot_text = "Taking a screenshot to see the current computer screen."
+            for m in msgs:
+                content = m.get("content")
+                if isinstance(content, str) and screenshot_text in content:
+                    return True
+                if isinstance(content, list):
+                    for p in content:
+                        if isinstance(p, dict) and p.get("type") == "text":
+                            if screenshot_text in (p.get("text") or ""):
+                                return True
+            return False
+
+        pre_output_items: List[Dict[str, Any]] = []
+        if not _has_any_image(completion_messages):
+            if computer_handler is None or not hasattr(computer_handler, "screenshot"):
+                raise RuntimeError(
+                    "No screenshots present and computer_handler.screenshot is not available."
+                )
+            screenshot_b64 = await computer_handler.screenshot()
+            if not screenshot_b64:
+                raise RuntimeError("Failed to capture screenshot from computer_handler.")
+            # Inject a user message with the screenshot so the model can see current context
+            completion_messages.append(
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": f"data:image/png;base64,{screenshot_b64}"},
+                        },
+                        {"type": "text", "text": "Current screen"},
+                    ],
+                }
+            )
+            # Add assistant message to outputs to reflect the action, only if not already present
+            if not _has_screenshot_message(messages):
+                pre_output_items.append(
+                    {
+                        "type": "message",
+                        "role": "assistant",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "Taking a screenshot to see the current computer screen.",
+                            }
+                        ],
+                    }
+                )
+
+        # Smart-resize all screenshots and attach min/max pixel hints. Fail fast if deps missing.
+        # Also record the last resized width/height to unnormalize coordinates later.
+        last_rw: Optional[int] = None
+        last_rh: Optional[int] = None
+        MIN_PIXELS = 3136
+        MAX_PIXELS = 12845056
+        try:
+            import base64
+            import io
+
+            from PIL import Image  # type: ignore
+            from qwen_vl_utils import smart_resize  # type: ignore
+        except Exception:
+            raise ImportError(
+                "qwen-vl-utils not installed. Please install it with `pip install cua-agent[qwen]`."
+            )
+
+        for msg in completion_messages:
+            content = msg.get("content")
+            if not isinstance(content, list):
+                continue
+            for part in content:
+                if isinstance(part, dict) and part.get("type") == "image_url":
+                    url = ((part.get("image_url") or {}).get("url")) or ""
+                    # Expect data URL like data:image/png;base64,<b64>
+                    if url.startswith("data:") and "," in url:
+                        b64 = url.split(",", 1)[1]
+                        img_bytes = base64.b64decode(b64)
+                        im = Image.open(io.BytesIO(img_bytes))
+                        h, w = im.height, im.width
+                        rh, rw = smart_resize(
+                            h, w, factor=32, min_pixels=MIN_PIXELS, max_pixels=MAX_PIXELS
+                        )
+                        # Attach hints on this image block
+                        part["min_pixels"] = MIN_PIXELS
+                        part["max_pixels"] = MAX_PIXELS
+                        last_rw, last_rh = rw, rh
+
+        for i, msg in enumerate(completion_messages):
+            role = msg.get("role")
+            content = msg.get("content")
+            if isinstance(content, list):
+                step_content = []
+                for item in content:
+                    item_type = item.get("type")
+                    if item_type == "text":
+                        step_content.append(item.get("text"))
+                    elif item_type == "image_url":
+                        step_content.append("Image URL: " + item.get("image_url").get("url")[:100])
+            else:
+                item = content
+                step_content = ""
+                if isinstance(item, dict) and item.get("type") == "image_url":
+                    step_content = "Image URL: " + item.get("image_url").get("url")[:100]
+                else:
+                    step_content = content
+
+            print(f"Step {i}: Role: {role}, Content: {step_content}")
+
+        api_kwargs: Dict[str, Any] = {
+            "model": model,
+            "messages": completion_messages,
+            "max_retries": max_retries,
+            "stream": stream,
+            **{k: v for k, v in kwargs.items()},
+        }
+        if use_prompt_caching:
+            api_kwargs["use_prompt_caching"] = use_prompt_caching
+
+        if _on_api_start:
+            await _on_api_start(api_kwargs)
+
+        response = await litellm.acompletion(**api_kwargs)
+
+        if _on_api_end:
+            await _on_api_end(api_kwargs, response)
+
+        usage = {
+            **LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(  # type: ignore
+                response.usage
+            ).model_dump(),
+            "response_cost": response._hidden_params.get("response_cost", 0.0),
+        }
+        if _on_usage:
+            await _on_usage(usage)
+
+        # Extract response data
+        resp_dict = response.model_dump()  # type: ignore
+        choice = (resp_dict.get("choices") or [{}])[0]
+        message = choice.get("message") or {}
+        content_text = message.get("content") or ""
+        tool_calls_array = message.get("tool_calls") or []
+        reasoning_text = message.get("reasoning") or ""
+
+        output_items: List[Dict[str, Any]] = []
+
+        # Add reasoning if present (Ollama Cloud format)
+        if reasoning_text:
+            output_items.append(make_reasoning_item(reasoning_text))
+
+        # Priority 1: Try to parse tool call from content text (OpenRouter format)
+        tool_call = _parse_tool_call_from_text(content_text)
+
+        if tool_call and isinstance(tool_call, dict):
+            fn_name = tool_call.get("name") or "computer"
+            raw_args = tool_call.get("arguments") or {}
+
+            output_items.append(
+                {
+                    "type": "message",
+                    "role": "assistant",
+                    "content": [{"type": "output_text", "text": content_text}],
+                }
+            )
+
+            # Unnormalize coordinates to actual screen size using last resized dims
+            if last_rw is None or last_rh is None:
+                raise RuntimeError(
+                    "No screenshots found to derive dimensions for coordinate unnormalization."
+                )
+            args = await _unnormalize_coordinate(raw_args, (last_rw, last_rh))
+
+            # Convert Qwen format to Computer Calls format if this is a computer tool
+            if fn_name == "computer":
+                converted_action = convert_qwen_tool_args_to_computer_action(args)
+                if converted_action:
+                    args = converted_action
+
+            # Build an OpenAI-style tool call so we can reuse the converter
+            fake_cm = {
+                "role": "assistant",
+                "tool_calls": [
+                    {
+                        "type": "function",
+                        "id": "call_0",
+                        "function": {
+                            "name": fn_name,
+                            "arguments": json.dumps(args),
+                        },
+                    }
+                ],
+            }
+            output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
+
+        elif tool_calls_array:
+
+            output_items.append(
+                {
+                    "type": "message",
+                    "role": "assistant",
+                    "content": [{"type": "output_text", "text": content_text}],
+                }
+            )
+
+            processed_tool_calls = []
+            for tc in tool_calls_array:
+                function = tc.get("function", {})
+                fn_name = function.get("name", "computer")
+                args_str = function.get("arguments", "{}")
+
+                try:
+                    args = json.loads(args_str)
+
+                    # Unnormalize coordinates if present
+                    if "coordinate" in args and last_rw is not None and last_rh is not None:
+                        args = await _unnormalize_coordinate(args, (last_rw, last_rh))
+
+                    # Convert Qwen format to Computer Calls format if this is a computer tool
+                    if fn_name == "computer":
+                        converted_action = convert_qwen_tool_args_to_computer_action(args)
+                        if converted_action:
+                            args = converted_action
+
+                    processed_tool_calls.append(
+                        {
+                            "type": tc.get("type", "function"),
+                            "id": tc.get("id", "call_0"),
+                            "function": {
+                                "name": fn_name,
+                                "arguments": json.dumps(args),
+                            },
+                        }
+                    )
+                except json.JSONDecodeError:
+                    processed_tool_calls.append(tc)
+
+            fake_cm = {
+                "role": "assistant",
+                "content": "",
+                "tool_calls": processed_tool_calls,
+            }
+            output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
+
+        else:
+            # No tool calls found in either format, return text response
+            fake_cm = {"role": "assistant", "content": content_text}
+            output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
+
+        # Prepend any pre_output_items (e.g., simulated screenshot-taking message)
+        return {"output": (pre_output_items + output_items), "usage": usage}
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        return ["click", "step"]
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[int, int]]:
+        """
+        Predict click coordinates using Qwen3-VL via litellm.acompletion.
+
+        Only exposes a reduced tool schema with left_click to bias model to output a single click.
+        Returns (x, y) absolute pixels when screen dimensions can be obtained; otherwise normalized 0..1000 integers.
+        """
+        # Reduced tool
+        reduced_tool = {
+            "type": "function",
+            "function": {
+                **QWEN3_5_COMPUTER_TOOL["function"],
+                "parameters": {
+                    **QWEN3_5_COMPUTER_TOOL["function"]["parameters"],
+                    "type": "object",
+                    "properties": {
+                        "action": {"type": "string", "enum": ["left_click"]},
+                        "coordinate": {
+                            "description": "(x, y) in 0..1000 reference space",
+                            "type": "array",
+                            "items": {"type": ["number", "integer"]},
+                            "minItems": 2,
+                            "maxItems": 2,
+                        },
+                    },
+                    "required": ["action", "coordinate"],
+                },
+            },
+        }
+
+        # Build Nous system (lazy import inside helper already raises clear guidance if missing)
+        nous_system = _build_nous_system([reduced_tool["function"]])
+
+        # Pre-process using smart_resize
+        min_pixels = 3136
+        max_pixels = 12845056
+        try:
+            # Lazy import to avoid hard dependency
+            import base64
+            import io
+
+            # If PIL is available, estimate size from image to derive smart bounds
+            from PIL import Image
+            from qwen_vl_utils import smart_resize  # type: ignore
+
+            img_bytes = base64.b64decode(image_b64)
+            im = Image.open(io.BytesIO(img_bytes))
+            h, w = im.height, im.width
+            # Qwen notebook suggests factor=32 and a wide min/max range
+            rh, rw = smart_resize(h, w, factor=32, min_pixels=min_pixels, max_pixels=max_pixels)
+        except Exception:
+            raise ImportError(
+                "qwen-vl-utils not installed. Please install it with `pip install cua-agent[qwen]`."
+            )
+
+        messages = []
+        if nous_system:
+            messages.append(nous_system)
+        image_block: Dict[str, Any] = {
+            "type": "image_url",
+            "image_url": {"url": f"data:image/png;base64,{image_b64}"},
+            "min_pixels": min_pixels,
+            "max_pixels": max_pixels,
+        }
+        # Single user message with image and instruction, matching OpenAI-style content blocks
+        messages.append(
+            {
+                "role": "user",
+                "content": [
+                    image_block,
+                    {"type": "text", "text": instruction},
+                ],
+            }
+        )
+
+        api_kwargs: Dict[str, Any] = {
+            "model": model,
+            "messages": messages,
+            **{k: v for k, v in kwargs.items()},
+        }
+        response = await litellm.acompletion(**api_kwargs)
+        resp = response.model_dump()  # type: ignore
+        choice = (resp.get("choices") or [{}])[0]
+        content_text = ((choice.get("message") or {}).get("content")) or ""
+        tool_call = _parse_tool_call_from_text(content_text) or {}
+        args = tool_call.get("arguments") or {}
+        args = await _unnormalize_coordinate(args, (rh, rw))
+        coord = args.get("coordinate")
+        if isinstance(coord, (list, tuple)) and len(coord) >= 2:
+            return int(coord[0]), int(coord[1])
+        return None

libs/python/agent/cua_agent/loops/qwen3vl.py

@@ -0,0 +1,18 @@
+"""
+Qwen3-VL dedicated agent loop configuration.
+Re-exports GenericVlmConfig under a Qwen-specific model pattern so that
+Qwen model strings are matched at normal priority instead of the generic
+catch-all (priority -100).
+"""
+
+from __future__ import annotations
+
+from ..decorators import register_agent
+from .generic_vlm import GenericVlmConfig
+
+
+@register_agent(models=r"(?i).*qwen.*")
+class Qwen3VlConfig(GenericVlmConfig):
+    """Qwen3-VL agent loop using litellm with function/tool calling."""
+
+    pass

libs/python/agent/cua_agent/loops/uiins.py

@@ -0,0 +1,175 @@
+"""
+UI-Ins agent loop implementation for click prediction using litellm.acompletion
+Paper: https://arxiv.org/pdf/2510.202861
+Code: https://github.com/alibaba/UI-Ins
+"""
+
+import asyncio
+import base64
+import json
+import math
+import re
+import uuid
+from io import BytesIO
+from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Union
+
+import litellm
+from PIL import Image
+
+from ..decorators import register_agent
+from ..loops.base import AsyncAgentConfig
+from ..types import AgentCapability, AgentResponse, Messages, Tools
+
+SYSTEM_PROMPT = """You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.\n\n## Output Format\nReturn a json object with a reasoning process in  tags, a function name and arguments within  XML tags:\n```\n\n...\n\n\n{"name": "grounding", "arguments": }\n\n```\n represents the following item of the action space:\n## Action Space{"action": "click", "coordinate": [x, y]}\nYour task is to accurately locate a UI element based on the instruction. You should first analyze instruction in  tags and finally output the function in  tags.\n"""
+
+
+def parse_coordinates(raw_string: str) -> tuple[int, int]:
+    matches = re.findall(r"\[(\d+),\s*(\d+)\]", raw_string)
+    if matches:
+        return tuple(map(int, matches[0]))
+    return -1, -1
+
+
+def smart_resize(
+    height: int,
+    width: int,
+    factor: int = 28,
+    min_pixels: int = 3136,
+    max_pixels: int = 8847360,
+) -> Tuple[int, int]:
+    """Smart resize function similar to qwen_vl_utils."""
+    # Calculate the total pixels
+    total_pixels = height * width
+
+    # If already within bounds, return original dimensions
+    if min_pixels <= total_pixels <= max_pixels:
+        # Round to nearest factor
+        new_height = (height // factor) * factor
+        new_width = (width // factor) * factor
+        return new_height, new_width
+
+    # Calculate scaling factor
+    if total_pixels > max_pixels:
+        scale = (max_pixels / total_pixels) ** 0.5
+    else:
+        scale = (min_pixels / total_pixels) ** 0.5
+
+    # Apply scaling
+    new_height = int(height * scale)
+    new_width = int(width * scale)
+
+    # Round to nearest factor
+    new_height = (new_height // factor) * factor
+    new_width = (new_width // factor) * factor
+
+    # Ensure minimum size
+    new_height = max(new_height, factor)
+    new_width = max(new_width, factor)
+
+    return new_height, new_width
+
+
+@register_agent(models=r".*UI-Ins.*")
+class UIInsConfig(AsyncAgentConfig):
+    """UI-Ins agent configuration implementing AsyncAgentConfig protocol for click prediction."""
+
+    def __init__(self):
+        self.current_model = None
+        self.last_screenshot_b64 = None
+
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        raise NotImplementedError()
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[float, float]]:
+        """
+        Predict click coordinates using UI-Ins model via litellm.acompletion.
+
+        Args:
+            model: The UI-Ins model name
+            image_b64: Base64 encoded image
+            instruction: Instruction for where to click
+
+        Returns:
+            Tuple of (x, y) coordinates or None if prediction fails
+        """
+        # Decode base64 image
+        image_data = base64.b64decode(image_b64)
+        image = Image.open(BytesIO(image_data))
+        width, height = image.width, image.height
+
+        # Smart resize the image (similar to qwen_vl_utils)
+        resized_height, resized_width = smart_resize(
+            height,
+            width,
+            factor=28,  # Default factor for Qwen models
+            min_pixels=3136,
+            max_pixels=4096 * 2160,
+        )
+        resized_image = image.resize((resized_width, resized_height))
+        scale_x, scale_y = width / resized_width, height / resized_height
+
+        # Convert resized image back to base64
+        buffered = BytesIO()
+        resized_image.save(buffered, format="PNG")
+        resized_image_b64 = base64.b64encode(buffered.getvalue()).decode()
+
+        # Prepare system and user messages
+        system_message = {
+            "role": "system",
+            "content": [
+                {"type": "text", "text": "You are a helpful assistant."},
+                {"type": "text", "text": SYSTEM_PROMPT},
+            ],
+        }
+
+        user_message = {
+            "role": "user",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/png;base64,{resized_image_b64}"},
+                },
+                {"type": "text", "text": instruction},
+            ],
+        }
+
+        # Prepare API call kwargs
+        api_kwargs = {
+            "model": model,
+            "messages": [system_message, user_message],
+            "max_tokens": 2056,
+            "temperature": 0.0,
+            **kwargs,
+        }
+
+        # Use liteLLM acompletion
+        response = await litellm.acompletion(**api_kwargs)
+
+        # Extract response text
+        output_text = response.choices[0].message.content  # type: ignore
+
+        # Extract and rescale coordinates
+        pred_x, pred_y = parse_coordinates(output_text)  # type: ignore
+        pred_x *= scale_x
+        pred_y *= scale_y
+
+        return (math.floor(pred_x), math.floor(pred_y))
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        """Return the capabilities supported by this agent."""
+        return ["click"]

libs/python/agent/cua_agent/loops/uitars.py

@@ -0,0 +1,873 @@
+"""
+UITARS agent loop implementation using liteLLM for ByteDance-Seed/UI-TARS-1.5-7B
+Paper: https://arxiv.org/abs/2501.12326
+Code: https://github.com/bytedance/UI-TARS
+"""
+
+import ast
+import asyncio
+import base64
+import json
+import math
+import re
+from ctypes import cast
+from io import BytesIO
+from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Union
+
+import litellm
+from litellm.responses.litellm_completion_transformation.transformation import (
+    LiteLLMCompletionResponsesConfig,
+)
+from litellm.responses.utils import Usage
+from litellm.types.utils import ModelResponse
+from openai.types.responses.response_computer_tool_call_param import (
+    ActionType,
+    ResponseComputerToolCallParam,
+)
+from openai.types.responses.response_input_param import ComputerCallOutput
+from openai.types.responses.response_output_message_param import (
+    ResponseOutputMessageParam,
+)
+from openai.types.responses.response_reasoning_item_param import (
+    ResponseReasoningItemParam,
+    Summary,
+)
+from PIL import Image
+
+from ..decorators import register_agent
+from ..responses import (
+    make_click_item,
+    make_double_click_item,
+    make_drag_item,
+    make_input_image_item,
+    make_keypress_item,
+    make_output_text_item,
+    make_reasoning_item,
+    make_scroll_item,
+    make_type_item,
+    make_wait_item,
+)
+from ..types import AgentCapability, AgentResponse, Messages, Tools
+
+# Constants from reference code
+IMAGE_FACTOR = 28
+MIN_PIXELS = 100 * 28 * 28
+MAX_PIXELS = 16384 * 28 * 28
+MAX_RATIO = 200
+
+FINISH_WORD = "finished"
+WAIT_WORD = "wait"
+ENV_FAIL_WORD = "error_env"
+CALL_USER = "call_user"
+
+# Action space prompt for UITARS
+UITARS_ACTION_SPACE = """
+click(start_box='<|box_start|>(x1,y1)<|box_end|>')
+left_double(start_box='<|box_start|>(x1,y1)<|box_end|>')
+right_single(start_box='<|box_start|>(x1,y1)<|box_end|>')
+drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>')
+hotkey(key='')
+type(content='') #If you want to submit your input, use "\\n" at the end of `content`.
+scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left')
+wait() #Sleep for 5s and take a screenshot to check for any changes.
+finished(content='xxx') # Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format.
+"""
+
+UITARS_PROMPT_TEMPLATE = """You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. 
+
+## Output Format
+```
+Thought: ...
+Action: ...
+```
+
+## Action Space
+{action_space}
+
+## Note
+- Use {language} in `Thought` part.
+- Write a small plan and finally summarize your next action (with its target element) in one sentence in `Thought` part.
+
+## User Instruction
+{instruction}
+"""
+
+GROUNDING_UITARS_PROMPT_TEMPLATE = """You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. 
+
+## Output Format
+
+Action: ...
+
+
+## Action Space
+click(point='<|box_start|>(x1,y1)<|box_end|>')
+
+## User Instruction
+{instruction}"""
+
+
+def round_by_factor(number: float, factor: int) -> int:
+    """Returns the closest integer to 'number' that is divisible by 'factor'."""
+    return round(number / factor) * factor
+
+
+def ceil_by_factor(number: float, factor: int) -> int:
+    """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
+    return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: float, factor: int) -> int:
+    """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
+    return math.floor(number / factor) * factor
+
+
+def smart_resize(
+    height: int,
+    width: int,
+    factor: int = IMAGE_FACTOR,
+    min_pixels: int = MIN_PIXELS,
+    max_pixels: int = MAX_PIXELS,
+) -> tuple[int, int]:
+    """
+    Rescales the image so that the following conditions are met:
+    1. Both dimensions (height and width) are divisible by 'factor'.
+    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+    3. The aspect ratio of the image is maintained as closely as possible.
+    """
+    if max(height, width) / min(height, width) > MAX_RATIO:
+        raise ValueError(
+            f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
+        )
+    h_bar = max(factor, round_by_factor(height, factor))
+    w_bar = max(factor, round_by_factor(width, factor))
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = floor_by_factor(height / beta, factor)
+        w_bar = floor_by_factor(width / beta, factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = ceil_by_factor(height * beta, factor)
+        w_bar = ceil_by_factor(width * beta, factor)
+    return h_bar, w_bar
+
+
+def escape_single_quotes(text):
+    """Escape single quotes in text for safe string formatting."""
+    pattern = r"(?<!\\)'"
+    return re.sub(pattern, r"\\'", text)
+
+
+def parse_action(action_str):
+    """Parse action string into structured format."""
+    try:
+        node = ast.parse(action_str, mode="eval")
+        if not isinstance(node, ast.Expression):
+            raise ValueError("Not an expression")
+
+        call = node.body
+        if not isinstance(call, ast.Call):
+            raise ValueError("Not a function call")
+
+        # Get function name
+        if isinstance(call.func, ast.Name):
+            func_name = call.func.id
+        elif isinstance(call.func, ast.Attribute):
+            func_name = call.func.attr
+        else:
+            func_name = None
+
+        # Get keyword arguments
+        kwargs = {}
+        for kw in call.keywords:
+            key = kw.arg
+            if isinstance(kw.value, ast.Constant):
+                value = kw.value.value
+            elif isinstance(kw.value, ast.Str):  # Compatibility with older Python
+                value = kw.value.s
+            else:
+                value = None
+            kwargs[key] = value
+
+        return {"function": func_name, "args": kwargs}
+
+    except Exception as e:
+        print(f"Failed to parse action '{action_str}': {e}")
+        return None
+
+
+def parse_uitars_response(text: str, image_width: int, image_height: int) -> List[Dict[str, Any]]:
+    """Parse UITARS model response into structured actions."""
+    text = text.strip()
+
+    # Extract thought
+    thought = None
+    if text.startswith("Thought:"):
+        thought_match = re.search(r"Thought: (.+?)(?=\s*Action:|$)", text, re.DOTALL)
+        if thought_match:
+            thought = thought_match.group(1).strip()
+
+    # Extract action
+    if "Action:" not in text:
+        raise ValueError("No Action found in response")
+
+    action_str = text.split("Action:")[-1].strip()
+
+    # Handle special case for type actions
+    if "type(content" in action_str:
+
+        def escape_quotes(match):
+            return match.group(1)
+
+        pattern = r"type\(content='(.*?)'\)"
+        content = re.sub(pattern, escape_quotes, action_str)
+        action_str = escape_single_quotes(content)
+        action_str = "type(content='" + action_str + "')"
+
+    # Parse the action
+    parsed_action = parse_action(action_str.replace("\n", "\\n").lstrip())
+    if parsed_action is None:
+        raise ValueError(f"Action can't parse: {action_str}")
+
+    action_type = parsed_action["function"]
+    params = parsed_action["args"]
+
+    # Process parameters
+    action_inputs = {}
+    for param_name, param in params.items():
+        if param == "":
+            continue
+        param = str(param).lstrip()
+        action_inputs[param_name.strip()] = param
+
+        # Handle coordinate parameters
+        if "start_box" in param_name or "end_box" in param_name:
+            # Parse coordinates like '<|box_start|>(x,y)<|box_end|>' or '(x,y)'
+            # First, remove special tokens
+            clean_param = param.replace("<|box_start|>", "").replace("<|box_end|>", "")
+            # Then remove parentheses and split
+            numbers = clean_param.replace("(", "").replace(")", "").split(",")
+
+            try:
+                float_numbers = [
+                    float(num.strip()) / 1000 for num in numbers
+                ]  # Normalize to 0-1 range
+
+                if len(float_numbers) == 2:
+                    # Single point, duplicate for box format
+                    float_numbers = [
+                        float_numbers[0],
+                        float_numbers[1],
+                        float_numbers[0],
+                        float_numbers[1],
+                    ]
+
+                action_inputs[param_name.strip()] = str(float_numbers)
+            except ValueError as e:
+                # If parsing fails, keep the original parameter value
+                print(f"Warning: Could not parse coordinates '{param}': {e}")
+                action_inputs[param_name.strip()] = param
+
+    return [
+        {
+            "thought": thought,
+            "action_type": action_type,
+            "action_inputs": action_inputs,
+            "text": text,
+        }
+    ]
+
+
+def convert_to_computer_actions(
+    parsed_responses: List[Dict[str, Any]], image_width: int, image_height: int
+) -> List[ResponseComputerToolCallParam | ResponseOutputMessageParam]:
+    """Convert parsed UITARS responses to computer actions."""
+    computer_actions = []
+
+    for response in parsed_responses:
+        action_type = response.get("action_type")
+        action_inputs = response.get("action_inputs", {})
+
+        if action_type == "finished":
+            finished_text = action_inputs.get("content", "Task completed successfully.")
+            computer_actions.append(make_output_text_item(finished_text))
+            break
+
+        elif action_type == "wait":
+            computer_actions.append(make_wait_item())
+
+        elif action_type == "call_user":
+            computer_actions.append(
+                make_output_text_item("I need assistance from the user to proceed with this task.")
+            )
+
+        elif action_type in ["click", "left_single"]:
+            start_box = action_inputs.get("start_box")
+            if start_box:
+                coords = eval(start_box)
+                x = int((coords[0] + coords[2]) / 2 * image_width)
+                y = int((coords[1] + coords[3]) / 2 * image_height)
+
+                computer_actions.append(make_click_item(x, y, "left"))
+
+        elif action_type in ["double_click", "left_double"]:
+            start_box = action_inputs.get("start_box")
+            if start_box:
+                coords = eval(start_box)
+                x = int((coords[0] + coords[2]) / 2 * image_width)
+                y = int((coords[1] + coords[3]) / 2 * image_height)
+
+                computer_actions.append(make_double_click_item(x, y))
+
+        elif action_type in ["right_click", "right_single"]:
+            start_box = action_inputs.get("start_box")
+            if start_box:
+                coords = eval(start_box)
+                x = int((coords[0] + coords[2]) / 2 * image_width)
+                y = int((coords[1] + coords[3]) / 2 * image_height)
+
+                computer_actions.append(make_click_item(x, y, "right"))
+
+        elif action_type == "type":
+            content = action_inputs.get("content", "")
+            computer_actions.append(make_type_item(content))
+
+        elif action_type == "hotkey":
+            key = action_inputs.get("key", "")
+            keys = key.split()
+            computer_actions.append(make_keypress_item(keys))
+
+        elif action_type == "press":
+            key = action_inputs.get("key", "")
+            computer_actions.append(make_keypress_item([key]))
+
+        elif action_type == "scroll":
+            start_box = action_inputs.get("start_box")
+            direction = action_inputs.get("direction", "down")
+
+            if start_box:
+                coords = eval(start_box)
+                x = int((coords[0] + coords[2]) / 2 * image_width)
+                y = int((coords[1] + coords[3]) / 2 * image_height)
+            else:
+                x, y = image_width // 2, image_height // 2
+
+            scroll_y = 5 if "up" in direction.lower() else -5
+            computer_actions.append(make_scroll_item(x, y, 0, scroll_y))
+
+        elif action_type == "drag":
+            start_box = action_inputs.get("start_box")
+            end_box = action_inputs.get("end_box")
+
+            if start_box and end_box:
+                start_coords = eval(start_box)
+                end_coords = eval(end_box)
+
+                start_x = int((start_coords[0] + start_coords[2]) / 2 * image_width)
+                start_y = int((start_coords[1] + start_coords[3]) / 2 * image_height)
+                end_x = int((end_coords[0] + end_coords[2]) / 2 * image_width)
+                end_y = int((end_coords[1] + end_coords[3]) / 2 * image_height)
+
+                path = [{"x": start_x, "y": start_y}, {"x": end_x, "y": end_y}]
+                computer_actions.append(make_drag_item(path))
+
+    return computer_actions
+
+
+def pil_to_base64(image: Image.Image) -> str:
+    """Convert PIL image to base64 string."""
+    buffer = BytesIO()
+    image.save(buffer, format="PNG")
+    return base64.b64encode(buffer.getvalue()).decode("utf-8")
+
+
+def process_image_for_uitars(
+    image_data: str, max_pixels: int = MAX_PIXELS, min_pixels: int = MIN_PIXELS
+) -> tuple[Image.Image, int, int]:
+    """Process image for UITARS model input."""
+    # Decode base64 image
+    if image_data.startswith("data:image"):
+        image_data = image_data.split(",")[1]
+
+    image_bytes = base64.b64decode(image_data)
+    image = Image.open(BytesIO(image_bytes))
+
+    original_width, original_height = image.size
+
+    # Resize image according to UITARS requirements
+    if image.width * image.height > max_pixels:
+        resize_factor = math.sqrt(max_pixels / (image.width * image.height))
+        width = int(image.width * resize_factor)
+        height = int(image.height * resize_factor)
+        image = image.resize((width, height))
+
+    if image.width * image.height < min_pixels:
+        resize_factor = math.sqrt(min_pixels / (image.width * image.height))
+        width = math.ceil(image.width * resize_factor)
+        height = math.ceil(image.height * resize_factor)
+        image = image.resize((width, height))
+
+    if image.mode != "RGB":
+        image = image.convert("RGB")
+
+    return image, original_width, original_height
+
+
+def sanitize_message(msg: Any) -> Any:
+    """Return a copy of the message with image_url ommited within content parts"""
+    if isinstance(msg, dict):
+        result = {}
+        for key, value in msg.items():
+            if key == "content" and isinstance(value, list):
+                result[key] = [
+                    (
+                        {k: v for k, v in item.items() if k != "image_url"}
+                        if isinstance(item, dict)
+                        else item
+                    )
+                    for item in value
+                ]
+            else:
+                result[key] = value
+        return result
+    elif isinstance(msg, list):
+        return [sanitize_message(item) for item in msg]
+    else:
+        return msg
+
+
+def convert_uitars_messages_to_litellm(messages: Messages) -> List[Dict[str, Any]]:
+    """
+    Convert UITARS internal message format back to LiteLLM format.
+
+    This function processes reasoning, computer_call, and computer_call_output messages
+    and converts them to the appropriate LiteLLM assistant message format.
+
+    Args:
+        messages: List of UITARS internal messages
+
+    Returns:
+        List of LiteLLM formatted messages
+    """
+    litellm_messages = []
+    current_assistant_content = []
+
+    for message in messages:
+        if isinstance(message, dict):
+            message_type = message.get("type")
+
+            if message_type == "reasoning":
+                # Extract reasoning text from summary
+                summary = message.get("summary", [])
+                if summary and isinstance(summary, list):
+                    for summary_item in summary:
+                        if (
+                            isinstance(summary_item, dict)
+                            and summary_item.get("type") == "summary_text"
+                        ):
+                            reasoning_text = summary_item.get("text", "")
+                            if reasoning_text:
+                                current_assistant_content.append(f"Thought: {reasoning_text}")
+
+            elif message_type == "computer_call":
+                # Convert computer action to UITARS action format
+                action = message.get("action", {})
+                action_type = action.get("type")
+
+                if action_type == "click":
+                    x, y = action.get("x", 0), action.get("y", 0)
+                    button = action.get("button", "left")
+                    if button == "left":
+                        action_text = f"Action: click(start_box='({x},{y})')"
+                    elif button == "right":
+                        action_text = f"Action: right_single(start_box='({x},{y})')"
+                    else:
+                        action_text = f"Action: click(start_box='({x},{y})')"
+
+                elif action_type == "double_click":
+                    x, y = action.get("x", 0), action.get("y", 0)
+                    action_text = f"Action: left_double(start_box='({x},{y})')"
+
+                elif action_type == "drag":
+                    start_x, start_y = action.get("start_x", 0), action.get("start_y", 0)
+                    end_x, end_y = action.get("end_x", 0), action.get("end_y", 0)
+                    action_text = f"Action: drag(start_box='({start_x},{start_y})', end_box='({end_x},{end_y})')"
+
+                elif action_type == "key":
+                    key = action.get("key", "")
+                    action_text = f"Action: hotkey(key='{key}')"
+
+                elif action_type == "type":
+                    text = action.get("text", "")
+                    # Escape single quotes in the text
+                    escaped_text = escape_single_quotes(text)
+                    action_text = f"Action: type(content='{escaped_text}')"
+
+                elif action_type == "scroll":
+                    x, y = action.get("x", 0), action.get("y", 0)
+                    direction = action.get("direction", "down")
+                    action_text = f"Action: scroll(start_box='({x},{y})', direction='{direction}')"
+
+                elif action_type == "wait":
+                    action_text = "Action: wait()"
+
+                else:
+                    # Fallback for unknown action types
+                    action_text = f"Action: {action_type}({action})"
+
+                current_assistant_content.append(action_text)
+
+                # When we hit a computer_call_output, finalize the current assistant message
+                if current_assistant_content:
+                    litellm_messages.append(
+                        {
+                            "role": "assistant",
+                            "content": [
+                                {"type": "text", "text": "\n".join(current_assistant_content)}
+                            ],
+                        }
+                    )
+                    current_assistant_content = []
+
+            elif message_type == "computer_call_output":
+                # Add screenshot from computer call output
+                output = message.get("output", {})
+                if isinstance(output, dict) and output.get("type") == "input_image":
+                    image_url = output.get("image_url", "")
+                    if image_url:
+                        litellm_messages.append(
+                            {
+                                "role": "user",
+                                "content": [{"type": "image_url", "image_url": {"url": image_url}}],
+                            }
+                        )
+
+            elif message.get("role") == "user":
+                # # Handle user messages
+                # content = message.get("content", "")
+                # if isinstance(content, str):
+                #     litellm_messages.append({
+                #         "role": "user",
+                #         "content": content
+                #     })
+                # elif isinstance(content, list):
+                #     litellm_messages.append({
+                #         "role": "user",
+                #         "content": content
+                #     })
+                pass
+
+    # Add any remaining assistant content
+    if current_assistant_content:
+        litellm_messages.append(
+            {
+                "role": "assistant",
+                "content": [{"type": "text", "text": "\n".join(current_assistant_content)}],
+            }
+        )
+
+    return litellm_messages
+
+
+@register_agent(models=r"(?i).*ui-?tars.*", priority=-1)
+class UITARSConfig:
+    """
+    UITARS agent configuration using liteLLM for ByteDance-Seed/UI-TARS-1.5-7B model.
+
+    Supports UITARS vision-language models for computer control.
+    """
+
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        use_prompt_caching: Optional[bool] = False,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        """
+        Predict the next step based on input messages.
+
+        Args:
+            messages: Input messages following Responses format
+            model: Model name to use
+            tools: Optional list of tool schemas
+            max_retries: Maximum number of retries
+            stream: Whether to stream responses
+            computer_handler: Computer handler instance
+            _on_api_start: Callback for API start
+            _on_api_end: Callback for API end
+            _on_usage: Callback for usage tracking
+            _on_screenshot: Callback for screenshot events
+            **kwargs: Additional arguments
+
+        Returns:
+            Dictionary with "output" (output items) and "usage" array
+        """
+        tools = tools or []
+
+        # Create response items
+        response_items = []
+
+        # Find computer tool for screen dimensions
+        computer_tool = None
+        for tool_schema in tools:
+            if tool_schema["type"] == "computer":
+                computer_tool = tool_schema["computer"]
+                break
+
+        # Get screen dimensions
+        screen_width, screen_height = 1024, 768
+        if computer_tool:
+            try:
+                screen_width, screen_height = await computer_tool.get_dimensions()
+            except:
+                pass
+
+        # Process messages to extract instruction and image
+        instruction = ""
+        image_data = None
+
+        # Convert messages to list if string
+        if isinstance(messages, str):
+            messages = [{"role": "user", "content": messages}]
+
+        # Extract instruction and latest screenshot
+        for message in reversed(messages):
+            if isinstance(message, dict):
+                content = message.get("content", "")
+
+                # Handle different content formats
+                if isinstance(content, str):
+                    if not instruction and message.get("role") == "user":
+                        instruction = content
+                elif isinstance(content, list):
+                    for item in content:
+                        if isinstance(item, dict):
+                            if item.get("type") == "text" and not instruction:
+                                instruction = item.get("text", "")
+                            elif item.get("type") == "image_url" and not image_data:
+                                image_url = item.get("image_url", {})
+                                if isinstance(image_url, dict):
+                                    image_data = image_url.get("url", "")
+                                else:
+                                    image_data = image_url
+
+            # Also check for computer_call_output with screenshots
+            if message.get("type") == "computer_call_output" and not image_data:
+                output = message.get("output", {})
+                if isinstance(output, dict) and output.get("type") == "input_image":
+                    image_data = output.get("image_url", "")
+
+            if instruction and image_data:
+                break
+
+        if not instruction:
+            instruction = (
+                "Help me complete this task by analyzing the screen and taking appropriate actions."
+            )
+
+        # Create prompt
+        user_prompt = UITARS_PROMPT_TEMPLATE.format(
+            instruction=instruction, action_space=UITARS_ACTION_SPACE, language="English"
+        )
+
+        # Convert conversation history to LiteLLM format
+        history_messages = convert_uitars_messages_to_litellm(messages)
+
+        # Prepare messages for liteLLM
+        litellm_messages = [{"role": "system", "content": "You are a helpful assistant."}]
+
+        # Add current user instruction with screenshot
+        current_user_message = {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": user_prompt},
+            ],
+        }
+        litellm_messages.append(current_user_message)
+
+        # Process image for UITARS
+        if not image_data:
+            # Take screenshot if none found in messages
+            if computer_handler:
+                image_data = await computer_handler.screenshot()
+                await _on_screenshot(image_data, "screenshot_before")
+
+                # Add screenshot to output items so it can be retained in history
+                response_items.append(make_input_image_item(image_data))
+            else:
+                raise ValueError("No screenshot found in messages and no computer_handler provided")
+        processed_image, original_width, original_height = process_image_for_uitars(image_data)
+        encoded_image = pil_to_base64(processed_image)
+
+        # Add conversation history
+        if history_messages:
+            litellm_messages.extend(history_messages)
+        else:
+            litellm_messages.append(
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": f"data:image/png;base64,{encoded_image}"},
+                        }
+                    ],
+                }
+            )
+
+        # Prepare API call kwargs
+        api_kwargs = {
+            "model": model,
+            "messages": litellm_messages,
+            "max_tokens": kwargs.get("max_tokens", 500),
+            "temperature": kwargs.get("temperature", 0.0),
+            "do_sample": kwargs.get("temperature", 0.0) > 0.0,
+            "num_retries": max_retries,
+            **{k: v for k, v in kwargs.items() if k not in ["max_tokens", "temperature"]},
+        }
+
+        # Call API start hook
+        if _on_api_start:
+            await _on_api_start(api_kwargs)
+
+        # Call liteLLM with UITARS model
+        response = await litellm.acompletion(**api_kwargs)
+
+        # Call API end hook
+        if _on_api_end:
+            await _on_api_end(api_kwargs, response)
+
+        # Extract response content
+        response_content = response.choices[0].message.content.strip()  # type: ignore
+
+        # Parse UITARS response
+        parsed_responses = parse_uitars_response(response_content, original_width, original_height)
+
+        # Convert to computer actions
+        computer_actions = convert_to_computer_actions(
+            parsed_responses, original_width, original_height
+        )
+
+        # Add computer actions to response items
+        thought = parsed_responses[0].get("thought", "")
+        if thought:
+            response_items.append(make_reasoning_item(thought))
+        response_items.extend(computer_actions)
+
+        # Extract usage information
+        response_usage = {
+            **LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(
+                response.usage
+            ).model_dump(),
+            "response_cost": response._hidden_params.get("response_cost", 0.0),
+        }
+        if _on_usage:
+            await _on_usage(response_usage)
+
+        # Create agent response
+        agent_response = {"output": response_items, "usage": response_usage}
+
+        return agent_response
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[int, int]]:
+        """
+        Predict click coordinates based on image and instruction.
+
+        UITARS supports click prediction through its action parsing.
+
+        Args:
+            model: Model name to use
+            image_b64: Base64 encoded image
+            instruction: Instruction for where to click
+
+        Returns:
+            Tuple with (x, y) coordinates or None
+        """
+        try:
+            # Create prompt using grounding template
+            user_prompt = GROUNDING_UITARS_PROMPT_TEMPLATE.format(instruction=instruction)
+
+            # Process image for UITARS
+            processed_image, original_width, original_height = process_image_for_uitars(image_b64)
+            encoded_image = pil_to_base64(processed_image)
+
+            # Prepare messages for liteLLM
+            litellm_messages = [
+                {"role": "system", "content": "You are a helpful assistant."},
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": user_prompt},
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": f"data:image/png;base64,{encoded_image}"},
+                        },
+                    ],
+                },
+            ]
+
+            # Prepare API call kwargs
+            api_kwargs = {
+                "model": model,
+                "messages": litellm_messages,
+                "max_tokens": 2056,
+                "temperature": 0.0,
+                "do_sample": False,
+            }
+            api_kwargs.update({k: v for k, v in (kwargs or {}).items()})
+
+            # Call liteLLM with UITARS model
+            response = await litellm.acompletion(**api_kwargs)
+
+            # Extract response content
+            response_content = response.choices[0].message.content.strip()  # type: ignore
+
+            print(response_content)
+
+            # Parse the response to extract click coordinates
+            # Look for click action with coordinates (with special tokens)
+            click_pattern = r"click\(point='<\|box_start\|>\((\d+),(\d+)\)<\|box_end\|>'\)"
+            match = re.search(click_pattern, response_content)
+
+            # Fallback: Look for simpler format without special tokens
+            if not match:
+                # Pattern for: click(start_box='(x,y)') or click(point='(x,y)')
+                fallback_pattern = r"click\((?:start_box|point)='\((\d+),(\d+)\)'\)"
+                match = re.search(fallback_pattern, response_content)
+
+            if match:
+                x, y = int(match.group(1)), int(match.group(2))
+                # Scale coordinates back to original image dimensions
+                scale_x = original_width / processed_image.width
+                scale_y = original_height / processed_image.height
+
+                scaled_x = int(x * scale_x)
+                scaled_y = int(y * scale_y)
+
+                return (scaled_x, scaled_y)
+
+            return None
+
+        except Exception as e:
+            # Log error and return None
+            print(f"Error in predict_click: {e}")
+            return None
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        """
+        Get list of capabilities supported by this agent config.
+
+        Returns:
+            List of capability strings
+        """
+        return ["step", "click"]

libs/python/agent/cua_agent/loops/uitars2.py

@@ -0,0 +1,951 @@
+"""
+UITARS-2 agent loop implementation using LiteLLM.
+- Prepends a system prompt modeled after the UI-TARS training prompts
+- Converts Responses items -> completion messages
+- Calls litellm.acompletion
+- Parses <seed:tool_call> ... </seed:tool_call> outputs back into Responses items (computer actions)
+"""
+
+from __future__ import annotations
+
+import base64
+import io
+import json
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from litellm.responses.litellm_completion_transformation.transformation import (
+    LiteLLMCompletionResponsesConfig,
+)
+
+from ..decorators import register_agent
+from .omniparser import get_last_computer_call_output  # type: ignore
+
+try:
+    from PIL import Image  # type: ignore
+except Exception:  # pragma: no cover
+    Image = None  # type: ignore
+from ..responses import (
+    convert_responses_items_to_completion_messages,
+    make_click_item,
+    make_double_click_item,
+    make_drag_item,
+    make_function_call_item,
+    make_keypress_item,
+    make_move_item,
+    make_output_text_item,
+    make_reasoning_item,
+    make_screenshot_item,
+    make_scroll_item,
+    make_type_item,
+    make_wait_item,
+)
+from ..types import AgentCapability
+
+TOOL_SCHEMAS: List[Dict[str, Any]] = [
+    {
+        "type": "function",
+        "name": "open_computer",
+        "parameters": {},
+        "description": "Open computer.",
+    },
+    {
+        "type": "function",
+        "name": "click",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "point": {
+                    "type": "string",
+                    "description": "Click coordinates. The format is: <point>x y</point>",
+                }
+            },
+            "required": ["point"],
+        },
+        "description": "Mouse left single click action.",
+    },
+    {
+        "type": "function",
+        "name": "left_double",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "point": {
+                    "type": "string",
+                    "description": "Click coordinates. The format is: <point>x y</point>",
+                }
+            },
+            "required": ["point"],
+        },
+        "description": "Mouse left double click action.",
+    },
+    {
+        "type": "function",
+        "name": "right_single",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "point": {
+                    "type": "string",
+                    "description": "Click coordinates. The format is: <point>x y</point>",
+                }
+            },
+            "required": ["point"],
+        },
+        "description": "Mouse right single click action.",
+    },
+    {
+        "type": "function",
+        "name": "scroll",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "point": {
+                    "type": "string",
+                    "description": "Scroll start position. If not specified, default to execute on the current mouse position. The format is: <point>x y</point>",
+                },
+                "direction": {
+                    "type": "string",
+                    "description": "Scroll direction.",
+                    "enum": ["up", "down", "left", "right"],
+                },
+            },
+            "required": ["direction"],
+        },
+        "description": "Scroll action.",
+    },
+    {
+        "type": "function",
+        "name": "move_to",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "point": {
+                    "type": "string",
+                    "description": "Target coordinates. The format is: <point>x y</point>",
+                }
+            },
+            "required": ["point"],
+        },
+        "description": "Mouse move action.",
+    },
+    {
+        "type": "function",
+        "name": "hotkey",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "key": {
+                    "type": "string",
+                    "description": "Hotkeys you want to press. Split keys with a space and use lowercase.",
+                }
+            },
+            "required": ["key"],
+        },
+        "description": "Press hotkey.",
+    },
+    {
+        "type": "function",
+        "name": "finished",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "content": {
+                    "type": "string",
+                    "description": "Provide the final answer or response to complete the task.",
+                }
+            },
+            "required": [],
+        },
+        "description": "This function is used to indicate the completion of a task by providing the final answer or response.",
+    },
+    {
+        "type": "function",
+        "name": "press",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "key": {
+                    "type": "string",
+                    "description": "Key you want to press. Only one key can be pressed at one time.",
+                }
+            },
+            "required": ["key"],
+        },
+        "description": "Press key.",
+    },
+    {
+        "type": "function",
+        "name": "release",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "key": {
+                    "type": "string",
+                    "description": "Key you want to release. Only one key can be released at one time.",
+                }
+            },
+            "required": ["key"],
+        },
+        "description": "Release key.",
+    },
+    {
+        "type": "function",
+        "name": "mouse_down",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "point": {
+                    "type": "string",
+                    "description": "Mouse down position. If not specified, default to execute on the current mouse position. The format is: <point>x y</point>",
+                },
+                "button": {
+                    "type": "string",
+                    "description": "Down button. Default to left.",
+                    "enum": ["left", "right"],
+                },
+            },
+            "required": [],
+        },
+        "description": "Mouse down action.",
+    },
+    {
+        "type": "function",
+        "name": "mouse_up",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "point": {
+                    "type": "string",
+                    "description": "Mouse up position. If not specified, default to execute on the current mouse position. The format is: <point>x y</point>",
+                },
+                "button": {
+                    "type": "string",
+                    "description": "Up button. Default to left.",
+                    "enum": ["left", "right"],
+                },
+            },
+            "required": [],
+        },
+        "description": "Mouse up action.",
+    },
+    {
+        "type": "function",
+        "name": "call_user",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "content": {
+                    "type": "string",
+                    "description": "Message or information displayed to the user to request their input, feedback, or guidance.",
+                }
+            },
+            "required": [],
+        },
+        "description": "This function is used to interact with the user by displaying a message and requesting their input, feedback, or guidance.",
+    },
+    {
+        "type": "function",
+        "name": "wait",
+        "parameters": {
+            "type": "object",
+            "properties": {"time": {"type": "integer", "description": "Wait time in seconds."}},
+            "required": [],
+        },
+        "description": "Wait for a while.",
+    },
+    {
+        "type": "function",
+        "name": "drag",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "start_point": {
+                    "type": "string",
+                    "description": "Drag start point. The format is: <point>x y</point>",
+                },
+                "end_point": {
+                    "type": "string",
+                    "description": "Drag end point. The format is: <point>x y</point>",
+                },
+            },
+            "required": ["start_point", "end_point"],
+        },
+        "description": "Mouse left button drag action.",
+    },
+    {
+        "type": "function",
+        "name": "type",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "content": {
+                    "type": "string",
+                    "description": "Type content. If you want to submit your input, use \\n at the end of content.",
+                }
+            },
+            "required": ["content"],
+        },
+        "description": "Type content.",
+    },
+    {
+        "type": "function",
+        "name": "take_screenshot",
+        "parameters": {},
+        "description": "Take screenshot.",
+    },
+]
+
+
+def _format_tool_schemas_json_lines(schemas: List[Dict[str, Any]]) -> str:
+    # Nicely formatted: pretty JSON with indentation, separated by blank lines
+    return "\n\n".join(json.dumps(s, ensure_ascii=False, indent=2) for s in schemas) + "\n\n"
+
+
+_PROMPT_PREFIX = (
+    "You should begin by detailing the internal reasoning process, and then present the answer to the user. "
+    "The reasoning process should be enclosed within <think_never_used_51bce0c785ca2f68081bfa7d91973934> "
+    "</think_never_used_51bce0c785ca2f68081bfa7d91973934> tags, as follows:\n"
+    "<think_never_used_51bce0c785ca2f68081bfa7d91973934> reasoning process here "
+    "</think_never_used_51bce0c785ca2f68081bfa7d91973934> answer here.\n\n"
+    "You have different modes of thinking:\n"
+    "Unrestricted think mode: Engage in an internal thinking process with thorough reasoning and reflections. "
+    "You have an unlimited budget for thinking tokens and can continue thinking until you fully solve the problem.\n"
+    "Efficient think mode: Provide a concise internal thinking process with efficient reasoning and reflections. "
+    "You don't have a strict token budget but be less verbose and more direct in your thinking.\n"
+    "No think mode: Respond directly to the question without any internal reasoning process or extra thinking tokens. "
+    "Still follow the template with the minimum required thinking tokens to justify the answer.\n"
+    "Budgeted think mode: Limit your internal reasoning and reflections to stay within the specified token budget\n\n"
+    "Based on the complexity of the problem, select the appropriate mode for reasoning among the provided options listed below.\n\n"
+    "Provided Mode(s):\nEfficient think.\n\n"
+    "You are provided with a task description, a history of previous actions, and corresponding screenshots. "
+    "Your goal is to perform the next action to complete the task. "
+    "If performing the same action multiple times results in a static screen with no changes, attempt a modified or alternative action.\n\n"
+    "## Function Definition\n\n"
+    "- You have access to the following functions:\n\n"
+)
+
+_PROMPT_SUFFIX = (
+    "- To call a function, use the following structure without any suffix:\n\n"
+    "<gui_think> reasoning process </gui_think>\n"
+    "<seed:tool_call><function=example_function_name><parameter=example_parameter_1>value_1</parameter>"
+    "<parameter=example_parameter_2>multiline...\n</parameter></function></seed:tool_call>\n\n"
+    "## Important Notes\n"
+    "- Function calls must begin with <function= and end with </function>.\n"
+    "- All required parameters must be explicitly provided.\n"
+    "\n## Additional Notes\n"
+    "- You can execute multiple actions within a single tool call. For example:\n"
+    "<seed:tool_call><function=example_function_1><parameter=example_parameter_1>value_1</parameter><parameter=example_parameter_2>\n"
+    "This is the value for the second parameter\nthat can span\nmultiple lines\n"
+    "</parameter></function><function=example_function_2><parameter=example_parameter_3>value_4</parameter></function></seed:tool_call>"
+)
+
+
+SYSTEM_PROMPT = _PROMPT_PREFIX + _format_tool_schemas_json_lines(TOOL_SCHEMAS) + _PROMPT_SUFFIX
+
+
+def _extract_function_schemas_from_tools(
+    tools: Optional[List[Dict[str, Any]]],
+) -> List[Dict[str, Any]]:
+    schemas: List[Dict[str, Any]] = []
+    if not tools:
+        return schemas
+    for t in tools:
+        if t.get("type") == "function":
+            fn = t.get("function", {})
+            name = fn.get("name")
+            params = fn.get("parameters", {})
+            desc = fn.get("description", "")
+            if name:
+                schemas.append(
+                    {
+                        "type": "function",
+                        "name": name,
+                        "parameters": params if isinstance(params, dict) else {},
+                        "description": desc,
+                    }
+                )
+    return schemas
+
+
+def _parse_seed_tool_calls(text: str) -> List[Dict[str, Any]]:
+    """Parse <seed:tool_call> blocks into a list of {function, parameters} dicts.
+    Also captures optional <gui_think>...</gui_think> as reasoning.
+    """
+    actions: List[Dict[str, Any]] = []
+    if not text:
+        return actions
+
+    # Extract reasoning if present
+    reasoning_text = None
+    think_match = re.search(r"<gui_think>([\s\S]*?)</gui_think>", text)
+    if think_match:
+        reasoning_text = think_match.group(1).strip()
+
+    # Iterate each seed tool_call block
+    for block in re.finditer(r"<seed:tool_call>([\s\S]*?)</seed:tool_call>", text):
+        content = block.group(1)
+        # One or multiple <function=...>...</function> inside
+        for fmatch in re.finditer(r"<function=([\w_]+)>([\s\S]*?)</function>", content):
+            fname = fmatch.group(1)
+            inner = fmatch.group(2)
+            params: Dict[str, str] = {}
+            for pmatch in re.finditer(r"<parameter=([\w_]+)>([\s\S]*?)</parameter>", inner):
+                pname = pmatch.group(1)
+                pval = pmatch.group(2).strip()
+                params[pname] = pval
+            actions.append({"function": fname, "parameters": params})
+
+    # If we have a global reasoning and at least one action, attach it to first
+    if reasoning_text and actions:
+        actions[0]["reasoning"] = reasoning_text
+    elif reasoning_text:
+        actions.append({"function": "reasoning", "parameters": {"content": reasoning_text}})
+
+    return actions
+
+
+def _normalize_xy_to_uitars(x: int, y: int, width: int, height: int) -> Tuple[int, int]:
+    width = max(1, int(width))
+    height = max(1, int(height))
+    nx = max(0, min(1000, int(round((x / width) * 1000))))
+    ny = max(0, min(1000, int(round((y / height) * 1000))))
+    return nx, ny
+
+
+def _denormalize_xy_from_uitars(nx: float, ny: float, width: int, height: int) -> Tuple[int, int]:
+    width = max(1, int(width))
+    height = max(1, int(height))
+    x = int(round((nx / 1000.0) * width))
+    y = int(round((ny / 1000.0) * height))
+    return x, y
+
+
+def _map_computer_action_to_function(
+    action: Dict[str, Any], width: int, height: int
+) -> Optional[Dict[str, Any]]:
+    """Map a computer action item to a UITARS function + parameters dict of strings.
+    Returns dict like {"function": name, "parameters": {..}} or None if unknown.
+    """
+    atype = action.get("type") or action.get("action")
+    if atype == "click":
+        x, y = action.get("x"), action.get("y")
+        btn = action.get("button", "left")
+        if x is None or y is None:
+            return None
+        nx, ny = _normalize_xy_to_uitars(int(x), int(y), width, height)
+        if btn == "right":
+            return {
+                "function": "right_single",
+                "parameters": {"point": f"<point>{nx} {ny}</point>"},
+            }
+        return {"function": "click", "parameters": {"point": f"<point>{nx} {ny}</point>"}}
+    if atype == "double_click":
+        x, y = action.get("x"), action.get("y")
+        if x is None or y is None:
+            return None
+        nx, ny = _normalize_xy_to_uitars(int(x), int(y), width, height)
+        return {"function": "left_double", "parameters": {"point": f"<point>{nx} {ny}</point>"}}
+    if atype == "move":
+        x, y = action.get("x"), action.get("y")
+        if x is None or y is None:
+            return None
+        nx, ny = _normalize_xy_to_uitars(int(x), int(y), width, height)
+        return {"function": "move_to", "parameters": {"point": f"<point>{nx} {ny}</point>"}}
+    if atype == "keypress":
+        keys = action.get("keys", [])
+        if isinstance(keys, list) and keys:
+            if len(keys) == 1:
+                return {"function": "press", "parameters": {"key": keys[0]}}
+            else:
+                return {"function": "hotkey", "parameters": {"key": " ".join(keys)}}
+        return None
+    if atype == "type":
+        text = action.get("text", "")
+        return {"function": "type", "parameters": {"content": text}}
+    if atype == "scroll":
+        x, y = action.get("x", 512), action.get("y", 512)
+        nx, ny = _normalize_xy_to_uitars(int(x), int(y), width, height)
+        sx, sy = action.get("scroll_x", 0), action.get("scroll_y", 0)
+        # Our parser used positive sy for up
+        direction = (
+            "up"
+            if sy and sy > 0
+            else (
+                "down"
+                if sy and sy < 0
+                else ("right" if sx and sx > 0 else ("left" if sx and sx < 0 else "down"))
+            )
+        )
+        return {
+            "function": "scroll",
+            "parameters": {"direction": direction, "point": f"<point>{nx} {ny}</point>"},
+        }
+    if atype == "drag":
+        path = action.get("path", [])
+        if isinstance(path, list) and len(path) >= 2:
+            sx, sy = path[0].get("x"), path[0].get("y")
+            ex, ey = path[-1].get("x"), path[-1].get("y")
+            if sx is None or sy is None or ex is None or ey is None:
+                return None
+            nsx, nsy = _normalize_xy_to_uitars(int(sx), int(sy), width, height)
+            nex, ney = _normalize_xy_to_uitars(int(ex), int(ey), width, height)
+            return {
+                "function": "drag",
+                "parameters": {
+                    "start_point": f"<point>{nsx} {nsy}</point>",
+                    "end_point": f"<point>{nex} {ney}</point>",
+                },
+            }
+        return None
+    if atype == "wait":
+        return {"function": "wait", "parameters": {}}
+    if atype == "screenshot":
+        return {"function": "take_screenshot", "parameters": {}}
+    # Fallback unknown
+    return None
+
+
+def _to_uitars_messages(
+    messages: List[Dict[str, Any]], width: int, height: int
+) -> List[Dict[str, Any]]:
+    """Convert responses items into completion messages tailored for UI-TARS.
+
+    - User content is passed through similar to convert_responses_items_to_completion_messages
+    - Assistant/tool history is rendered as text with <gui_think> and <seed:tool_call> blocks
+    """
+    uitars_messages: List[Dict[str, Any]] = []
+
+    def flush_seed_block(pending_think: Optional[str], pending_functions: List[Dict[str, Any]]):
+        if not pending_think and not pending_functions:
+            return
+        parts: List[str] = []
+        if pending_think:
+            parts.append(f"<gui_think> {pending_think} </gui_think>")
+        if pending_functions:
+            inner = []
+            for f in pending_functions:
+                fname = f["function"]
+                params = f.get("parameters", {})
+                param_blocks = []
+                for k, v in params.items():
+                    param_blocks.append(f"<parameter={k}>{v}</parameter>")
+                inner.append(f"<function={fname}>{''.join(param_blocks)}</function>")
+            parts.append(f"<seed:tool_call>{''.join(inner)}</seed:tool_call>")
+        uitars_messages.append({"role": "assistant", "content": "".join(parts)})
+
+    # Accumulators for a single assistant seed block
+    pending_think: Optional[str] = None
+    pending_functions: List[Dict[str, Any]] = []
+
+    for msg in messages:
+        mtype = msg.get("type")
+        role = msg.get("role")
+
+        # On any user message, flush current assistant block
+        if role == "user" or mtype == "user":
+            flush_seed_block(pending_think, pending_functions)
+            pending_think, pending_functions = None, []
+
+            content = msg.get("content", "")
+            if isinstance(content, list):
+                completion_content = []
+                for item in content:
+                    if item.get("type") == "input_image":
+                        completion_content.append(
+                            {"type": "image_url", "image_url": {"url": item.get("image_url")}}
+                        )
+                    elif item.get("type") in ("input_text", "text"):
+                        completion_content.append({"type": "text", "text": item.get("text")})
+                uitars_messages.append({"role": "user", "content": completion_content})
+            elif isinstance(content, str):
+                uitars_messages.append({"role": "user", "content": content})
+            continue
+
+        # Reasoning item
+        if mtype == "reasoning":
+            # Responses reasoning stores summary list
+            summary = msg.get("summary", [])
+            texts = [
+                s.get("text", "")
+                for s in summary
+                if isinstance(s, dict) and s.get("type") == "summary_text"
+            ]
+            if texts:
+                pending_think = "\n".join([t for t in texts if t])
+            continue
+
+        # Computer/tool calls -> map to functions
+        if mtype == "computer_call":
+            f = _map_computer_action_to_function(msg.get("action", {}), width, height)
+            if f:
+                pending_functions.append(f)
+            continue
+        if mtype == "function_call":
+            # Include custom tools as-is
+            name = msg.get("name")
+            try:
+                args_obj = json.loads(msg.get("arguments", "{}"))
+            except json.JSONDecodeError:
+                args_obj = {}
+            # Ensure string values
+            params = {k: (str(v) if not isinstance(v, str) else v) for k, v in args_obj.items()}
+            pending_functions.append({"function": name, "parameters": params})
+            continue
+
+        # If assistant message text is given, flush current block and add as plain assistant text
+        if role == "assistant" or mtype == "message":
+            flush_seed_block(pending_think, pending_functions)
+            pending_think, pending_functions = None, []
+            content = msg.get("content", [])
+            if isinstance(content, list):
+                texts = [
+                    c.get("text", "")
+                    for c in content
+                    if isinstance(c, dict) and c.get("type") in ("output_text", "text")
+                ]
+                if texts:
+                    uitars_messages.append(
+                        {"role": "assistant", "content": "\n".join([t for t in texts if t])}
+                    )
+            elif isinstance(content, str) and content:
+                uitars_messages.append({"role": "assistant", "content": content})
+            continue
+
+        # On outputs, flush pending assistant block and send outputs as user messages
+        if mtype in ("function_call_output", "computer_call_output"):
+            flush_seed_block(pending_think, pending_functions)
+            pending_think, pending_functions = None, []
+            output = msg.get("output")
+            if isinstance(output, dict) and output.get("type") == "input_image":
+                img_url = output.get("image_url")
+                if img_url:
+                    uitars_messages.append(
+                        {
+                            "role": "user",
+                            "content": [
+                                {"type": "image_url", "image_url": {"url": img_url}},
+                            ],
+                        }
+                    )
+            elif isinstance(output, str):
+                uitars_messages.append({"role": "user", "content": output})
+            else:
+                # Fallback stringify
+                uitars_messages.append({"role": "user", "content": json.dumps(output)})
+            continue
+
+    # Flush any remaining pending seed block
+    flush_seed_block(pending_think, pending_functions)
+
+    return uitars_messages
+
+
+def _to_response_items(
+    actions: List[Dict[str, Any]],
+    tool_names: Optional[set[str]] = None,
+    width: Optional[int] = None,
+    height: Optional[int] = None,
+) -> List[Any]:
+    """Map parsed actions into Responses items (computer actions + optional reasoning)."""
+    items: List[Any] = []
+    tool_names = tool_names or set()
+
+    # Optional top-level reasoning attached to first
+    if actions and actions[0].get("reasoning"):
+        items.append(make_reasoning_item(actions[0]["reasoning"]))
+
+    # Dimensions default
+    w = int(width) if width else 1024
+    h = int(height) if height else 768
+
+    for a in actions:
+        fn = a.get("function")
+        params = a.get("parameters", {})
+        if fn == "reasoning":
+            items.append(make_reasoning_item(params.get("content", "")))
+        elif fn in ("click", "left_double", "right_single"):
+            # params.point is like: <point>x y</point> or plain "x y"
+            point = params.get("point", "").strip()
+            m = re.search(r"([\-\d\.]+)\s+([\-\d\.]+)", point)
+            if not m:
+                continue
+            nx = float(m.group(1))
+            ny = float(m.group(2))
+            x, y = _denormalize_xy_from_uitars(nx, ny, w, h)
+            if fn == "left_double":
+                items.append(make_double_click_item(x, y))
+            elif fn == "right_single":
+                items.append(make_click_item(x, y, "right"))
+            else:
+                items.append(make_click_item(x, y, "left"))
+        elif fn == "move_to":
+            point = params.get("point", "").strip()
+            m = re.search(r"([\-\d\.]+)\s+([\-\d\.]+)", point)
+            if not m:
+                continue
+            nx = float(m.group(1))
+            ny = float(m.group(2))
+            x, y = _denormalize_xy_from_uitars(nx, ny, w, h)
+            items.append(make_move_item(x, y))
+        elif fn == "drag":
+            sp = params.get("start_point", "").strip()
+            ep = params.get("end_point", "").strip()
+            ms = re.search(r"([\-\d\.]+)\s+([\-\d\.]+)", sp)
+            me = re.search(r"([\-\d\.]+)\s+([\-\d\.]+)", ep)
+            if not (ms and me):
+                continue
+            nsx, nsy = float(ms.group(1)), float(ms.group(2))
+            nex, ney = float(me.group(1)), float(me.group(2))
+            sx, sy = _denormalize_xy_from_uitars(nsx, nsy, w, h)
+            ex, ey = _denormalize_xy_from_uitars(nex, ney, w, h)
+            items.append(make_drag_item([{"x": sx, "y": sy}, {"x": ex, "y": ey}]))
+        elif fn == "hotkey":
+            key = params.get("key", "")
+            keys = key.split()
+            if keys:
+                items.append(make_keypress_item(keys))
+        elif fn == "press":
+            key = params.get("key", "")
+            if key:
+                items.append(make_keypress_item([key]))
+        elif fn == "type":
+            content = params.get("content", "")
+            items.append(make_type_item(content))
+        elif fn == "scroll":
+            # direction: up/down/left/right. Point optional
+            direction = params.get("direction", "down").lower()
+            point = params.get("point", "")
+            m = re.search(r"([\-\d\.]+)\s+([\-\d\.]+)", point)
+            if m:
+                nx = float(m.group(1))
+                ny = float(m.group(2))
+                x, y = _denormalize_xy_from_uitars(nx, ny, w, h)
+            else:
+                x, y = _denormalize_xy_from_uitars(500.0, 500.0, w, h)
+            dy = 5 if direction == "up" else -5
+            dx = 5 if direction == "right" else (-5 if direction == "left" else 0)
+            items.append(make_scroll_item(x, y, dx, dy))
+        elif fn == "wait":
+            items.append(make_wait_item())
+        elif fn == "finished":
+            content = params.get("content", "")
+            items.append(make_output_text_item(content or "Task completed."))
+            break
+        elif fn == "take_screenshot":
+            items.append(make_screenshot_item())
+        elif fn == "open_computer":
+            items.append(make_screenshot_item())
+        else:
+            # If this function name is present in provided tool schemas, emit function_call
+            if fn in tool_names:
+                # Convert simple string params into an arguments object
+                # Parameters are strings; pass through as-is
+                items.append(make_function_call_item(fn, params))
+            else:
+                # Unknown function -> surface as assistant text
+                items.append(make_output_text_item(f"Unknown action: {fn} {params}"))
+
+    return items
+
+
+@register_agent(models=r"(?i).*ui-?tars-?2.*")
+class UITARS2Config:
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        use_prompt_caching: Optional[bool] = False,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        # Determine screen dimensions (prefer computer_handler, fallback to last screenshot)
+        width: Optional[int] = None
+        height: Optional[int] = None
+        if computer_handler is not None and hasattr(computer_handler, "get_dimensions"):
+            try:
+                dims = await computer_handler.get_dimensions()  # type: ignore
+                if isinstance(dims, (list, tuple)) and len(dims) == 2:
+                    width, height = int(dims[0]), int(dims[1])
+            except Exception:
+                pass
+
+        if width is None or height is None:
+            try:
+                last_out = get_last_computer_call_output(messages)  # type: ignore
+                if last_out:
+                    image_url = last_out.get("output", {}).get("image_url", "")
+                    if image_url:
+                        b64 = image_url.split(",")[-1]
+                        img_bytes = base64.b64decode(b64)
+                        if Image is not None:
+                            img = Image.open(io.BytesIO(img_bytes))
+                            width, height = img.size
+            except Exception:
+                pass
+
+        if width is None or height is None:
+            width, height = 1024, 768
+
+        # Convert Responses items to UI-TARS style messages with <seed:tool_call> history
+        completion_messages = _to_uitars_messages(messages, width, height)
+
+        # Build dynamic system prompt by concatenating built-in schemas and provided function tools
+        provided_fn_schemas = _extract_function_schemas_from_tools(tools)
+        combined_schemas = (
+            TOOL_SCHEMAS + provided_fn_schemas if provided_fn_schemas else TOOL_SCHEMAS
+        )
+        dynamic_system_prompt = (
+            _PROMPT_PREFIX + _format_tool_schemas_json_lines(combined_schemas) + _PROMPT_SUFFIX
+        )
+
+        # Prepend system prompt (based on training prompts + provided tools)
+        litellm_messages: List[Dict[str, Any]] = [
+            {"role": "system", "content": dynamic_system_prompt},
+        ]
+        litellm_messages.extend(completion_messages)
+
+        api_kwargs: Dict[str, Any] = {
+            "model": model,
+            "messages": litellm_messages,
+            "max_retries": max_retries,
+            "stream": stream,
+            **{k: v for k, v in kwargs.items()},
+        }
+        if use_prompt_caching:
+            api_kwargs["use_prompt_caching"] = use_prompt_caching
+
+        if _on_api_start:
+            await _on_api_start(api_kwargs)
+
+        response = await litellm.acompletion(**api_kwargs)
+
+        if _on_api_end:
+            await _on_api_end(api_kwargs, response)
+
+        usage = {
+            **LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(  # type: ignore
+                response.usage
+            ).model_dump(),
+            "response_cost": response._hidden_params.get("response_cost", 0.0),
+        }
+        if _on_usage:
+            await _on_usage(usage)
+
+        # Extract text content (first choice)
+        response_dict = response.model_dump()  # type: ignore
+        content_text = ""
+        choices = response_dict.get("choices", [])
+        if choices:
+            msg = choices[0].get("message", {})
+            # message.content may be string or array; gather text pieces
+            mc = msg.get("content")
+            if isinstance(mc, str):
+                content_text = mc
+            elif isinstance(mc, list):
+                parts = []
+                for part in mc:
+                    if isinstance(part, dict) and part.get("type") == "text":
+                        parts.append(part.get("text", ""))
+                content_text = "\n".join([p for p in parts if p])
+
+        # Parse the seed tool calls and map to response items
+        actions = _parse_seed_tool_calls(content_text)
+        # Build set of tool names from provided tools to emit function_call items
+        tool_names: set[str] = set()
+        for s in provided_fn_schemas:
+            name = s.get("name")
+            if isinstance(name, str):
+                tool_names.add(name)
+        output_items = _to_response_items(actions, tool_names, width, height)
+
+        return {"output": output_items, "usage": usage}
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        return ["step"]
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[int, int]]:
+        """Predict a single click coordinate using a minimal prompt with a click tool.
+
+        This sends the current screenshot and instruction, asking the model to
+        output a click action in the form:
+            Action: click(point='(x,y)')
+        """
+        # Minimal grounding-style prompt
+        system_text = (
+            "You are a GUI agent. Given the instruction, return a single action on the current screen.\n\n"
+            "## Output Format\n\n"
+            "Action: click(point='(x,y)')\n\n"
+            "## User Instruction\n"
+            f"{instruction}"
+        )
+
+        # Build messages with image
+        litellm_messages: List[Dict[str, Any]] = [
+            {"role": "system", "content": system_text},
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "Please return a single click action."},
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:image/png;base64,{image_b64}"},
+                    },
+                ],
+            },
+        ]
+
+        api_kwargs: Dict[str, Any] = {
+            "model": model,
+            "messages": litellm_messages,
+            "max_tokens": kwargs.get("max_tokens", 512),
+            "temperature": kwargs.get("temperature", 0.0),
+            "do_sample": kwargs.get("temperature", 0.0) > 0.0,
+        }
+        api_kwargs.update(
+            {k: v for k, v in (kwargs or {}).items() if k not in ["max_tokens", "temperature"]}
+        )
+
+        response = await litellm.acompletion(**api_kwargs)
+        # Extract response content
+        response_dict = response.model_dump()  # type: ignore
+        choices = response_dict.get("choices", [])
+        if not choices:
+            return None
+        msg = choices[0].get("message", {})
+        content_text = msg.get("content", "")
+        if isinstance(content_text, list):
+            text_parts = [
+                p.get("text", "")
+                for p in content_text
+                if isinstance(p, dict) and p.get("type") == "text"
+            ]
+            content_text = "\n".join([t for t in text_parts if t])
+        if not isinstance(content_text, str):
+            return None
+
+        # Parse coordinates
+        # Pattern for click(point='(x,y)') or click(start_box='(x,y)')
+        patterns = [
+            r"click\(point='\((\d+),(\d+)\)'\)",
+            r"click\((?:start_box|point)='\((\d+),(\d+)\)'\)",
+        ]
+        for pat in patterns:
+            m = re.search(pat, content_text)
+            if m:
+                try:
+                    x, y = int(m.group(1)), int(m.group(2))
+                    return (x, y)
+                except Exception:
+                    pass
+        return None

libs/python/agent/cua_agent/loops/yutori.py

@@ -0,0 +1,397 @@
+"""
+Yutori n1 agent loop implementation using litellm.
+
+n1 is a browser-use model that outputs actions via tool_calls in OpenAI chat
+completions format. Coordinates are in a 1000x1000 normalized space.
+"""
+
+from __future__ import annotations
+
+import base64
+import io
+import json
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from litellm.responses.litellm_completion_transformation.transformation import (
+    LiteLLMCompletionResponsesConfig,
+)
+from PIL import Image
+
+from ..decorators import register_agent
+from ..loops.base import AsyncAgentConfig
+from ..responses import (
+    convert_completion_messages_to_responses_items,
+    convert_responses_items_to_completion_messages,
+    make_function_call_item,
+    make_output_text_item,
+    make_reasoning_item,
+)
+from ..types import AgentCapability
+
+# Target resolution for n1 (docs recommend 1280x800 WebP)
+N1_TARGET_WIDTH = 1280
+N1_TARGET_HEIGHT = 800
+N1_COORD_SPACE = 1000
+
+
+def _prepare_image_for_n1(image_b64: str) -> str:
+    """Convert a base64 PNG screenshot to WebP at 1280x800 for optimal n1 performance."""
+    try:
+        img_bytes = base64.b64decode(image_b64)
+        img = Image.open(io.BytesIO(img_bytes))
+
+        # Resize to n1's recommended resolution
+        if img.size != (N1_TARGET_WIDTH, N1_TARGET_HEIGHT):
+            img = img.resize((N1_TARGET_WIDTH, N1_TARGET_HEIGHT), Image.LANCZOS)
+
+        # Convert to WebP
+        buf = io.BytesIO()
+        img.save(buf, format="WEBP", quality=85)
+        return base64.b64encode(buf.getvalue()).decode("utf-8")
+    except Exception:
+        # Fallback: return original image if conversion fails
+        return image_b64
+
+
+def _unnormalize_coordinates(
+    coords: List[int], screen_width: int, screen_height: int
+) -> Tuple[int, int]:
+    """Scale coordinates from n1's 1000x1000 space to actual screen pixels."""
+    x = max(0, min(screen_width, round((coords[0] / N1_COORD_SPACE) * screen_width)))
+    y = max(0, min(screen_height, round((coords[1] / N1_COORD_SPACE) * screen_height)))
+    return x, y
+
+
+def _convert_n1_action_to_computer_action(
+    fn_name: str, args: Dict[str, Any], screen_width: int, screen_height: int
+) -> Optional[Dict[str, Any]]:
+    """
+    Convert an n1 tool call to the internal computer_call action schema.
+
+    Returns None for actions that should be emitted as function_calls instead
+    (goto_url, go_back, refresh).
+    """
+    # Actions with coordinates
+    coords = args.get("coordinates")
+    x, y = None, None
+    if isinstance(coords, (list, tuple)) and len(coords) >= 2:
+        x, y = _unnormalize_coordinates(coords, screen_width, screen_height)
+
+    if fn_name == "left_click":
+        if x is None or y is None:
+            return None
+        return {"action": "left_click", "x": x, "y": y}
+
+    if fn_name == "double_click":
+        if x is None or y is None:
+            return None
+        return {"action": "double_click", "x": x, "y": y}
+
+    if fn_name == "triple_click":
+        # Approximate as double_click
+        if x is None or y is None:
+            return None
+        return {"action": "double_click", "x": x, "y": y}
+
+    if fn_name == "right_click":
+        if x is None or y is None:
+            return None
+        return {"action": "right_click", "x": x, "y": y}
+
+    if fn_name == "hover":
+        if x is None or y is None:
+            return None
+        return {"action": "move", "x": x, "y": y}
+
+    if fn_name == "drag":
+        start_coords = args.get("start_coordinates")
+        if (
+            not isinstance(start_coords, (list, tuple))
+            or len(start_coords) < 2
+            or x is None
+            or y is None
+        ):
+            return None
+        sx, sy = _unnormalize_coordinates(start_coords, screen_width, screen_height)
+        return {
+            "action": "drag",
+            "start_x": sx,
+            "start_y": sy,
+            "end_x": x,
+            "end_y": y,
+        }
+
+    if fn_name == "scroll":
+        direction = args.get("direction", "down")
+        amount = int(args.get("amount", 3))
+        # Convert direction + amount to scroll_x/scroll_y pixels
+        # Use ~100 pixels per scroll unit as a reasonable default
+        pixels_per_unit = 100
+        scroll_x, scroll_y = 0, 0
+        if direction == "down":
+            scroll_y = amount * pixels_per_unit
+        elif direction == "up":
+            scroll_y = -(amount * pixels_per_unit)
+        elif direction == "right":
+            scroll_x = amount * pixels_per_unit
+        elif direction == "left":
+            scroll_x = -(amount * pixels_per_unit)
+        out: Dict[str, Any] = {"action": "scroll", "scroll_x": scroll_x, "scroll_y": scroll_y}
+        if x is not None and y is not None:
+            out["x"] = x
+            out["y"] = y
+        return out
+
+    if fn_name == "type":
+        text = args.get("text", "")
+        if args.get("press_enter_after"):
+            text = text + "\n"
+        # Note: clear_before_typing is not supported by the framework's type action.
+        # n1 rarely emits this flag; when it does, the field may already be empty.
+        return {"action": "type", "text": text}
+
+    if fn_name == "key_press":
+        key_comb = args.get("key_comb", "")
+        # n1 uses Playwright-compatible key combos like "Control+a", "Escape"
+        keys = [k.strip() for k in key_comb.split("+")]
+        return {"action": "keypress", "keys": keys}
+
+    if fn_name == "wait":
+        return {"action": "wait"}
+
+    if fn_name == "go_back":
+        return {"action": "history_back"}
+
+    if fn_name == "refresh":
+        return {"action": "keypress", "keys": ["F5"]}
+
+    if fn_name == "goto_url":
+        return {"action": "visit_url", "url": args.get("url", "")}
+
+    return None
+
+
+def _convert_images_to_n1_format(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Convert all images in messages to WebP format optimized for n1."""
+    for msg in messages:
+        content = msg.get("content")
+        if not isinstance(content, list):
+            continue
+        for part in content:
+            if isinstance(part, dict) and part.get("type") == "image_url":
+                url = ((part.get("image_url") or {}).get("url")) or ""
+                if url.startswith("data:") and "," in url:
+                    b64 = url.split(",", 1)[1]
+                    converted = _prepare_image_for_n1(b64)
+                    part["image_url"]["url"] = f"data:image/webp;base64,{converted}"
+    return messages
+
+
+@register_agent(models=r"(yutori/)?n1(-.*)?$", tool_type="browser")
+class YutoriN1Config(AsyncAgentConfig):
+    """
+    Yutori n1 browser-use agent loop.
+
+    n1 is a browser-only model that outputs actions as tool_calls.
+    Coordinates use a 1000x1000 normalized space.
+    """
+
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        use_prompt_caching: Optional[bool] = False,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        """Predict the next browser action using Yutori n1."""
+        tools = tools or []
+
+        # Get screen dimensions for coordinate denormalization
+        screen_width, screen_height = N1_TARGET_WIDTH, N1_TARGET_HEIGHT
+        if computer_handler:
+            try:
+                screen_width, screen_height = await computer_handler.get_dimensions()
+            except Exception:
+                # BrowserTool doesn't have get_dimensions() but has viewport attrs
+                vw = getattr(computer_handler, "viewport_width", None)
+                vh = getattr(computer_handler, "viewport_height", None)
+                if vw and vh:
+                    screen_width, screen_height = vw, vh
+
+        # Convert messages from Responses API format to chat completions format
+        completion_messages = convert_responses_items_to_completion_messages(
+            messages,
+            allow_images_in_tool_results=True,
+        )
+
+        # Convert images to WebP at 1280x800
+        completion_messages = _convert_images_to_n1_format(completion_messages)
+
+        # If there's no screenshot, take one and inject it
+        def _has_any_image(msgs: List[Dict[str, Any]]) -> bool:
+            for m in msgs:
+                content = m.get("content")
+                if isinstance(content, list):
+                    for p in content:
+                        if isinstance(p, dict) and p.get("type") == "image_url":
+                            return True
+            return False
+
+        pre_output_items: List[Dict[str, Any]] = []
+        if not _has_any_image(completion_messages):
+            if computer_handler is None or not hasattr(computer_handler, "screenshot"):
+                raise RuntimeError(
+                    "No screenshots present and computer_handler.screenshot is not available."
+                )
+            screenshot_b64 = await computer_handler.screenshot()
+            if not screenshot_b64:
+                raise RuntimeError("Failed to capture screenshot from computer_handler.")
+
+            converted = _prepare_image_for_n1(screenshot_b64)
+            completion_messages.append(
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": f"data:image/webp;base64,{converted}"},
+                        },
+                        {"type": "text", "text": "Current browser screen"},
+                    ],
+                }
+            )
+            pre_output_items.append(
+                {
+                    "type": "message",
+                    "role": "assistant",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": "Taking a screenshot to see the current browser screen.",
+                        }
+                    ],
+                }
+            )
+
+        # Build tool list: pass through any custom function tools
+        n1_tools = []
+        for tool in tools:
+            if tool.get("type") == "function":
+                func = tool.get("function")
+                if func:
+                    n1_tools.append({"type": "function", "function": func})
+            # Skip computer tools — n1 has built-in browser actions
+
+        api_kwargs: Dict[str, Any] = {
+            "model": model,
+            "messages": completion_messages,
+            "max_retries": max_retries,
+            "stream": False,  # n1 does not support streaming
+            "temperature": kwargs.pop("temperature", 0.3),
+        }
+
+        if n1_tools:
+            api_kwargs["tools"] = n1_tools
+
+        # Pass through remaining kwargs (api_key, api_base, etc.)
+        api_kwargs.update({k: v for k, v in kwargs.items()})
+
+        if _on_api_start:
+            await _on_api_start(api_kwargs)
+
+        response = await litellm.acompletion(**api_kwargs)
+
+        if _on_api_end:
+            await _on_api_end(api_kwargs, response)
+
+        # Extract usage
+        usage = {
+            **LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(  # type: ignore
+                response.usage
+            ).model_dump(),
+            "response_cost": response._hidden_params.get("response_cost", 0.0),
+        }
+        if _on_usage:
+            await _on_usage(usage)
+
+        # Parse response
+        resp_dict = response.model_dump()  # type: ignore
+        choice = (resp_dict.get("choices") or [{}])[0]
+        message = choice.get("message") or {}
+        content_text = message.get("content") or ""
+        tool_calls_array = message.get("tool_calls") or []
+        reasoning_text = message.get("reasoning") or ""
+
+        output_items: List[Dict[str, Any]] = []
+
+        # Add reasoning if present
+        if reasoning_text:
+            output_items.append(make_reasoning_item(reasoning_text))
+
+        if tool_calls_array:
+            for tc in tool_calls_array:
+                function = tc.get("function", {})
+                fn_name = function.get("name", "")
+                args_str = function.get("arguments", "{}")
+                tc_id = tc.get("id", "call_0")
+
+                try:
+                    args = json.loads(args_str) if isinstance(args_str, str) else args_str
+                except json.JSONDecodeError:
+                    args = {}
+
+                # Try converting to a computer action
+                computer_action = _convert_n1_action_to_computer_action(
+                    fn_name, args, screen_width, screen_height
+                )
+
+                if computer_action is not None:
+                    # Build a fake completion message for the converter
+                    fake_cm = {
+                        "role": "assistant",
+                        "content": content_text or "",
+                        "tool_calls": [
+                            {
+                                "type": "function",
+                                "id": tc_id,
+                                "function": {
+                                    "name": "computer",
+                                    "arguments": json.dumps(computer_action),
+                                },
+                            }
+                        ],
+                    }
+                    output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
+                    # Only use content_text once
+                    content_text = ""
+                else:
+                    # Custom tool — emit as function_call
+                    output_items.append(make_function_call_item(fn_name, args, call_id=tc_id))
+        else:
+            # No tool calls — task is complete
+            if content_text:
+                output_items.append(make_output_text_item(content_text))
+            else:
+                output_items.append(make_output_text_item("Task completed."))
+
+        return {"output": (pre_output_items + output_items), "usage": usage}
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[int, int]]:
+        raise NotImplementedError(
+            "Yutori n1 does not support standalone click prediction. "
+            "Use predict_step for full browser automation."
+        )
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        return ["step"]