chore: import upstream snapshot with attribution

libs/python/agent/cua_agent/loops/fara/__init__.py

@@ -0,0 +1,8 @@
+"""
+FARA-7B agent loop implementation.
+Original implementation from Microsoft: https://github.com/microsoft/Fara
+"""
+
+from .config import FaraVlmConfig
+
+__all__ = ("FaraVlmConfig",)

libs/python/agent/cua_agent/loops/fara/config.py

@@ -0,0 +1,661 @@
+"""FARA VLM agent configuration."""
+
+from __future__ import annotations
+
+import ast
+import json
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from litellm.responses.litellm_completion_transformation.transformation import (
+    LiteLLMCompletionResponsesConfig,
+)
+
+from ...decorators import register_agent
+from ...loops.base import AsyncAgentConfig
+from ...responses import (
+    make_click_item,
+    make_double_click_item,
+    make_drag_item,
+    make_keypress_item,
+    make_move_item,
+    make_output_text_item,
+    make_reasoning_item,
+    make_screenshot_item,
+    make_scroll_item,
+    make_type_item,
+    make_wait_item,
+)
+from ...types import AgentCapability
+from .helpers import (
+    _convert_responses_items_to_fara_messages,
+    build_nous_system,
+    parse_tool_call_from_text,
+)
+
+
+def _scale_fara_coordinates(
+    args: Dict[str, Any],
+    original_dims: Tuple[int, int],
+    resized_dims: Tuple[int, int],
+) -> Dict[str, Any]:
+    """
+    Scale FARA coordinates from resized image space to original viewport space.
+
+    FARA outputs pixel coordinates on the resized image (after smart_resize).
+    This scales them back to the original browser viewport, matching FARA's
+    convert_resized_coords_to_original() in fara_agent.py:
+        scale_x = og_w / rsz_w
+        return [coords[0] * scale_x, coords[1] * scale_y]
+
+    Args:
+        args: Action arguments containing "coordinate" key
+        original_dims: (width, height) of original browser viewport
+        resized_dims: (width, height) after smart_resize
+    """
+    coord = args.get("coordinate")
+    if not coord or not isinstance(coord, (list, tuple)) or len(coord) < 2:
+        return args
+
+    x, y = float(coord[0]), float(coord[1])
+    original_w, original_h = float(original_dims[0]), float(original_dims[1])
+    resized_w, resized_h = float(resized_dims[0]), float(resized_dims[1])
+
+    # Scale from resized to original: x_final = x * (original / resized)
+    scale_x = original_w / resized_w
+    scale_y = original_h / resized_h
+
+    x_scaled = max(0.0, min(original_w, x * scale_x))
+    y_scaled = max(0.0, min(original_h, y * scale_y))
+
+    return {**args, "coordinate": [round(x_scaled), round(y_scaled)]}
+
+
+def _fara_args_to_sdk_item(args: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """
+    Convert FARA model output args to SDK item using make_*_item helpers.
+
+    FARA format: {"action": "left_click", "coordinate": [100, 200]}
+    SDK format:  ResponseComputerToolCallParam with action={"type": "click", "x": 100, "y": 200}
+    """
+    action = args.get("action", "")
+    coordinate = args.get("coordinate", [0, 0])
+    x = coordinate[0] if len(coordinate) > 0 else 0
+    y = coordinate[1] if len(coordinate) > 1 else 0
+
+    # Click actions
+    if action in ("left_click", "click"):
+        return make_click_item(x=x, y=y, button="left")
+    if action == "right_click":
+        return make_click_item(x=x, y=y, button="right")
+    if action == "middle_click":
+        return make_click_item(x=x, y=y, button="wheel")
+    if action == "double_click":
+        return make_double_click_item(x=x, y=y)
+
+    # Type action
+    if action == "type":
+        return make_type_item(text=args.get("text", ""))
+
+    # Key action
+    if action in ("key", "keypress"):
+        keys = args.get("keys", [])
+        if isinstance(keys, str):
+            keys = keys.split("+")
+        return make_keypress_item(keys=keys)
+
+    # Move action
+    if action in ("mouse_move", "move"):
+        return make_move_item(x=x, y=y)
+
+    # Scroll action
+    if action == "scroll":
+        pixels = args.get("pixels") or 0  # Handle None explicitly
+        # FARA: positive = up, negative = down
+        scroll_y = -pixels  # SDK: positive = down
+        return make_scroll_item(x=x, y=y, scroll_x=0, scroll_y=scroll_y)
+
+    if action == "hscroll":
+        pixels = args.get("pixels") or 0  # Handle None explicitly
+        return make_scroll_item(x=x, y=y, scroll_x=pixels, scroll_y=0)
+
+    # Drag action
+    if action == "left_click_drag":
+        start_coord = args.get("start_coordinate", [0, 0])
+        end_coord = args.get("end_coordinate", [0, 0])
+        return make_drag_item(
+            path=[
+                {"x": start_coord[0], "y": start_coord[1]},
+                {"x": end_coord[0], "y": end_coord[1]},
+            ]
+        )
+
+    # Screenshot
+    if action == "screenshot":
+        return make_screenshot_item()
+
+    # Wait
+    if action == "wait":
+        return make_wait_item()
+
+    # Terminate - return None so no action is executed
+    # The caller checks for terminate action and adds an assistant message to stop the loop
+    if action == "terminate":
+        return None
+
+    # FARA browser-specific actions - create computer_call items directly
+    # agent.py uses getattr(computer, action_type) to call these methods
+    if action == "visit_url":
+        return {
+            "type": "computer_call",
+            "call_id": f"call_{id(args)}",
+            "action": {"type": "visit_url", "url": args.get("url", "")},
+            "pending_safety_checks": [],
+            "status": "completed",
+        }
+
+    if action == "web_search":
+        return {
+            "type": "computer_call",
+            "call_id": f"call_{id(args)}",
+            "action": {"type": "web_search", "query": args.get("query", "")},
+            "pending_safety_checks": [],
+            "status": "completed",
+        }
+
+    if action == "history_back":
+        return {
+            "type": "computer_call",
+            "call_id": f"call_{id(args)}",
+            "action": {"type": "history_back"},
+            "pending_safety_checks": [],
+            "status": "completed",
+        }
+
+    return None
+
+
+@register_agent(models=r"(?i).*fara-7b.*", tool_type="browser")
+class FaraVlmConfig(AsyncAgentConfig):
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        use_prompt_caching: Optional[bool] = False,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        # Check if the last message is a terminate function_call_output
+        # If so, return a final assistant message to stop the loop
+        if messages:
+            last_msg = messages[-1]
+            if last_msg.get("type") in ("function_call_output", "computer_call_output"):
+                output_data = last_msg.get("output")
+
+                # Parse string if needed (could be JSON or Python dict literal)
+                if isinstance(output_data, str):
+                    try:
+                        output_data = json.loads(output_data)
+                    except:
+                        try:
+                            output_data = ast.literal_eval(output_data)
+                        except:
+                            pass
+
+                # Check if it's a terminate action output (contains "terminated": True)
+                if isinstance(output_data, dict) and output_data.get("terminated") is True:
+                    return {
+                        "output": [
+                            {
+                                "type": "message",
+                                "role": "assistant",
+                                "content": [{"type": "output_text", "text": "Task completed."}],
+                            }
+                        ],
+                        "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0},
+                    }
+
+        # Build messages using FARA's dedicated conversion layer
+        # This converts SDK format to FARA's native format (action + coordinate)
+        converted_msgs = _convert_responses_items_to_fara_messages(
+            messages, allow_images_in_tool_results=False
+        )
+
+        # Build function schemas from tools array
+        function_schemas = []
+        if tools:
+            from ...computers import is_agent_computer
+
+            for tool in tools:
+                tool_type = tool.get("type")
+
+                if tool_type == "computer":
+                    # For computer tools, use FARA_COMPUTER_TOOL schema
+                    computer = tool.get("computer")
+                    if computer and is_agent_computer(computer):
+                        function_schemas.append(FARA_COMPUTER_TOOL["function"])
+                elif tool_type == "function":
+                    # For function tools, use the provided function schema
+                    function_schema = tool.get("function")
+                    if function_schema:
+                        function_schemas.append(function_schema)
+
+        # If no tools provided or no computer tool found, use default FARA_COMPUTER_TOOL
+        if not function_schemas:
+            function_schemas = [FARA_COMPUTER_TOOL["function"]]
+
+        # Prepend Nous-generated system if available
+        nous_system = build_nous_system(function_schemas)
+        completion_messages = ([nous_system] if nous_system else []) + converted_msgs
+
+        # If there is no screenshot in the conversation, take one now and inject it.
+        def _has_any_image(msgs: List[Dict[str, Any]]) -> bool:
+            for m in msgs:
+                content = m.get("content")
+                if isinstance(content, list):
+                    for p in content:
+                        if isinstance(p, dict) and p.get("type") == "image_url":
+                            return True
+            return False
+
+        pre_output_items: List[Dict[str, Any]] = []
+        if not _has_any_image(completion_messages):
+            if computer_handler is None or not hasattr(computer_handler, "screenshot"):
+                raise RuntimeError(
+                    "No screenshots present and computer_handler.screenshot is not available."
+                )
+            screenshot_b64 = await computer_handler.screenshot()
+            if not screenshot_b64:
+                raise RuntimeError("Failed to capture screenshot from computer_handler.")
+
+            await _on_screenshot(screenshot_b64, "screenshot_before")
+
+            # Check if computer_handler has get_current_url method
+            screenshot_text = "Here is the next screenshot. Think about what to do next."
+            if hasattr(computer_handler, "get_current_url"):
+                try:
+                    current_url = await computer_handler.get_current_url()
+                    screenshot_text = f"Current URL: {current_url[:100]}\nHere is the next screenshot. Think about what to do next."
+                except Exception:
+                    # If get_current_url fails, fall back to default text
+                    pass
+
+            # Inject a user message with the screenshot so the model can see current context
+            screenshot_msg = {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:image/png;base64,{screenshot_b64}"},
+                    },
+                    {"type": "text", "text": screenshot_text},
+                ],
+            }
+            completion_messages.append(screenshot_msg)
+
+        # Smart-resize all screenshots and attach min/max pixel hints. Fail fast if deps missing.
+        # Track both original and resized dimensions for coordinate scaling.
+        last_original_w: Optional[int] = None
+        last_original_h: Optional[int] = None
+        last_rw: Optional[int] = None
+        last_rh: Optional[int] = None
+        MIN_PIXELS = 3136
+        MAX_PIXELS = 12845056
+        try:
+            import base64
+            import io
+
+            from PIL import Image  # type: ignore
+            from qwen_vl_utils import smart_resize  # type: ignore
+        except Exception:
+            raise ImportError(
+                "qwen-vl-utils not installed. Please install it with `pip install cua-agent[qwen]`."
+            )
+
+        for msg in completion_messages:
+            content = msg.get("content")
+            if not isinstance(content, list):
+                continue
+            for part in content:
+                if isinstance(part, dict) and part.get("type") == "image_url":
+                    url = ((part.get("image_url") or {}).get("url")) or ""
+                    # Expect data URL like data:image/png;base64,<b64>
+                    if url.startswith("data:") and "," in url:
+                        b64 = url.split(",", 1)[1]
+                        img_bytes = base64.b64decode(b64)
+                        im = Image.open(io.BytesIO(img_bytes))
+                        h, w = im.height, im.width
+                        rh, rw = smart_resize(
+                            h, w, factor=28, min_pixels=MIN_PIXELS, max_pixels=MAX_PIXELS
+                        )
+                        # Attach hints on this image block
+                        part["min_pixels"] = MIN_PIXELS
+                        part["max_pixels"] = MAX_PIXELS
+                        # Track both original and resized dimensions
+                        last_original_w, last_original_h = w, h
+                        last_rw, last_rh = rw, rh
+
+        api_kwargs: Dict[str, Any] = {
+            "model": model,
+            "messages": completion_messages,
+            "max_retries": max_retries,
+            "stream": stream,
+            **{k: v for k, v in kwargs.items()},
+        }
+        if use_prompt_caching:
+            api_kwargs["use_prompt_caching"] = use_prompt_caching
+
+        if _on_api_start:
+            await _on_api_start(api_kwargs)
+
+        response = await litellm.acompletion(**api_kwargs)
+
+        if _on_api_end:
+            await _on_api_end(api_kwargs, response)
+
+        usage = {
+            **LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(  # type: ignore
+                response.usage
+            ).model_dump(),
+            "response_cost": response._hidden_params.get("response_cost", 0.0),
+        }
+        if _on_usage:
+            await _on_usage(usage)
+
+        # Extract response data
+        resp_dict = response.model_dump()  # type: ignore
+        choice = (resp_dict.get("choices") or [{}])[0]
+        message = choice.get("message") or {}
+        content_text = message.get("content") or ""
+        tool_calls_array = message.get("tool_calls") or []
+        reasoning_text = message.get("reasoning") or ""
+
+        output_items: List[Dict[str, Any]] = []
+        has_terminate = False
+
+        # Add reasoning if present (Ollama Cloud format)
+        if reasoning_text:
+            output_items.append(make_reasoning_item(reasoning_text))
+
+        # Extract thoughts (text before <tool_call> tag)
+        thoughts = ""
+        if "<tool_call>" in content_text:
+            thoughts = content_text.split("<tool_call>")[0].strip()
+
+        # Add thoughts as assistant message if present
+        if thoughts:
+            output_items.append(make_output_text_item(thoughts))
+
+        # Priority 1: Try to parse tool call from content text (OpenRouter format)
+        tool_call = parse_tool_call_from_text(content_text)
+
+        if tool_call and isinstance(tool_call, dict):
+            fn_name = tool_call.get("name") or "computer"
+            raw_args = tool_call.get("arguments") or {}
+
+            # Scale coordinates from resized image space to original viewport
+            if (
+                last_rw is None
+                or last_rh is None
+                or last_original_w is None
+                or last_original_h is None
+            ):
+                raise RuntimeError(
+                    "No screenshots found to derive dimensions for coordinate scaling."
+                )
+            args = _scale_fara_coordinates(
+                raw_args,
+                original_dims=(last_original_w, last_original_h),
+                resized_dims=(last_rw, last_rh),
+            )
+
+            # Convert FARA output to SDK format using make_*_item helpers
+            if fn_name in ("computer", "computer_use"):
+                item = _fara_args_to_sdk_item(args)
+                if item:
+                    output_items.append(item)
+                # Check for terminate (even if item is None)
+                if args.get("action") == "terminate":
+                    has_terminate = True
+
+        elif tool_calls_array:
+            # Priority 2: Use tool_calls field if present (Ollama Cloud format)
+            for tc in tool_calls_array:
+                function = tc.get("function", {})
+                fn_name = function.get("name", "computer")
+                args_str = function.get("arguments", "{}")
+
+                try:
+                    args = json.loads(args_str)
+
+                    # Scale coordinates from resized image space to original viewport
+                    if "coordinate" in args and last_rw is not None and last_rh is not None:
+                        if last_original_w is not None and last_original_h is not None:
+                            args = _scale_fara_coordinates(
+                                args,
+                                original_dims=(last_original_w, last_original_h),
+                                resized_dims=(last_rw, last_rh),
+                            )
+
+                    # Convert FARA output to SDK format
+                    if fn_name in ("computer", "computer_use"):
+                        item = _fara_args_to_sdk_item(args)
+                        if item:
+                            output_items.append(item)
+                        # Check for terminate (even if item is None)
+                        if args.get("action") == "terminate":
+                            has_terminate = True
+                except json.JSONDecodeError:
+                    pass
+
+        elif content_text:
+            # No tool calls found, return text response
+            output_items.append(make_output_text_item(content_text))
+
+        # If terminate detected, ensure LAST item is an assistant message to exit the loop
+        # The generic agent loop checks: while new_items[-1].get("role") != "assistant"
+        if has_terminate:
+            output_items.append(
+                {
+                    "type": "message",
+                    "role": "assistant",
+                    "content": [{"type": "output_text", "text": ""}],
+                }
+            )
+
+        # Prepend any pre_output_items (e.g., simulated screenshot-taking message)
+        return {"output": (pre_output_items + output_items), "usage": usage}
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        return ["step"]
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[int, int]]:
+        """
+        Predict click coordinates using Qwen3-VL via litellm.acompletion.
+
+        Only exposes a reduced tool schema with left_click to bias model to output a single click.
+        Returns (x, y) absolute pixels when screen dimensions can be obtained; otherwise normalized 0..1000 integers.
+        """
+        # Reduced tool
+        reduced_tool = {
+            "type": "function",
+            "function": {
+                **FARA_COMPUTER_TOOL["function"],
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "action": {"type": "string", "enum": ["left_click"]},
+                        "coordinate": {
+                            "description": "(x, y) in 0..1000 reference space",
+                            "type": "array",
+                            "items": {"type": ["number", "integer"]},
+                            "minItems": 2,
+                            "maxItems": 2,
+                        },
+                    },
+                    "required": ["action", "coordinate"],
+                },
+            },
+        }
+
+        # Build Nous system (lazy import inside helper already raises clear guidance if missing)
+        nous_system = build_nous_system([reduced_tool["function"]])
+
+        # Pre-process using smart_resize
+        min_pixels = 3136
+        max_pixels = 12845056
+        try:
+            # Lazy import to avoid hard dependency
+            import base64
+            import io
+
+            # If PIL is available, estimate size from image to derive smart bounds
+            from PIL import Image
+            from qwen_vl_utils import smart_resize  # type: ignore
+
+            img_bytes = base64.b64decode(image_b64)
+            im = Image.open(io.BytesIO(img_bytes))
+            h, w = im.height, im.width
+            rh, rw = smart_resize(h, w, factor=28, min_pixels=min_pixels, max_pixels=max_pixels)
+        except Exception:
+            raise ImportError(
+                "qwen-vl-utils not installed. Please install it with `pip install cua-agent[qwen]`."
+            )
+
+        messages = []
+        if nous_system:
+            messages.append(nous_system)
+        image_block: Dict[str, Any] = {
+            "type": "image_url",
+            "image_url": {"url": f"data:image/png;base64,{image_b64}"},
+            "min_pixels": min_pixels,
+            "max_pixels": max_pixels,
+        }
+        # Single user message with image and instruction, matching OpenAI-style content blocks
+        messages.append(
+            {
+                "role": "user",
+                "content": [
+                    image_block,
+                    {"type": "text", "text": instruction},
+                ],
+            }
+        )
+
+        api_kwargs: Dict[str, Any] = {
+            "model": model,
+            "messages": messages,
+            **{k: v for k, v in kwargs.items()},
+        }
+        response = await litellm.acompletion(**api_kwargs)
+        resp = response.model_dump()  # type: ignore
+        choice = (resp.get("choices") or [{}])[0]
+        content_text = ((choice.get("message") or {}).get("content")) or ""
+        tool_call = parse_tool_call_from_text(content_text) or {}
+        args = tool_call.get("arguments") or {}
+        # Scale from resized image space to original viewport
+        args = _scale_fara_coordinates(
+            args,
+            original_dims=(w, h),
+            resized_dims=(rw, rh),
+        )
+        coord = args.get("coordinate")
+        if isinstance(coord, (list, tuple)) and len(coord) >= 2:
+            return int(coord[0]), int(coord[1])
+        return None
+
+
+# FARA-specific ComputerUse tool schema (OpenAI function tool format)
+# This schema is tailored for FARA-7B model and includes browser-specific actions
+# NOTE: Tool name MUST be "computer_use" to match what FARA-7B was trained on
+FARA_COMPUTER_TOOL: dict[str, Any] = {
+    "type": "function",
+    "function": {
+        "name": "computer_use",
+        "description": (
+            "Use a mouse and keyboard to interact with a computer, and take screenshots.\n"
+            "* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.\n"
+            "* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try wait and taking another screenshot.\n"
+            "* The screen's resolution is 1000x1000.\n"
+            "* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.\n"
+            "* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.\n"
+            "* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges.\n"
+            "* Use terminate action when you have completed the task or cannot proceed further."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "action": {
+                    "description": "The action to perform.",
+                    "enum": [
+                        "key",
+                        "type",
+                        "mouse_move",
+                        "left_click",
+                        "left_click_drag",
+                        "right_click",
+                        "middle_click",
+                        "double_click",
+                        "triple_click",
+                        "scroll",
+                        "hscroll",
+                        "screenshot",
+                        "wait",
+                        "visit_url",
+                        "web_search",
+                        "history_back",
+                        "terminate",
+                    ],
+                    "type": "string",
+                },
+                "keys": {
+                    "description": "Required only by action=key.",
+                    "type": "array",
+                    "items": {"type": "string"},
+                },
+                "text": {
+                    "description": "Required only by action=type.",
+                    "type": "string",
+                },
+                "coordinate": {
+                    "description": "(x, y): Pixel coordinates from top-left.",
+                    "type": "array",
+                    "items": {"type": ["number", "integer"]},
+                    "minItems": 2,
+                    "maxItems": 2,
+                },
+                "pixels": {
+                    "description": "Scroll amount. Positive=up, negative=down. For scroll/hscroll.",
+                    "type": "number",
+                },
+                "time": {
+                    "description": "Seconds to wait (action=wait).",
+                    "type": "number",
+                },
+                "url": {
+                    "description": "The URL to visit. Required only by action=visit_url.",
+                    "type": "string",
+                },
+                "query": {
+                    "description": "The search query. Required only by action=web_search.",
+                    "type": "string",
+                },
+                "status": {
+                    "description": "Task completion status. Required only by action=terminate.",
+                    "type": "string",
+                    "enum": ["success", "failure"],
+                },
+            },
+            "required": ["action"],
+        },
+    },
+}

libs/python/agent/cua_agent/loops/fara/helpers.py

@@ -0,0 +1,817 @@
+# Source: https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/llm/fncall_prompts/nous_fncall_prompt.py
+
+import copy
+import json
+import os
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union
+
+from .schema import ContentItem, Message
+
+FN_CALL_TEMPLATE_QWEN = """# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{tool_descs}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{{"name": <function-name>, "arguments": <args-json-object>}}
+</tool_call>"""
+
+FN_CALL_TEMPLATE = """You are a web automation agent that performs actions on websites to fulfill user requests by calling various tools.
+* You should stop execution at Critical Points. A Critical Point would be encountered in tasks like 'Checkout', 'Book', 'Purchase', 'Call', 'Email', 'Order', etc where a binding transaction/agreement would require the user's permission/personal or sensitive information (name, email, credit card, address, payment information, resume, etc) in order to complete a transaction (purchase, reservation, sign-up etc), or to communicate in a way that a human would be expected to do (call, email, apply to a job, etc).
+* Solve the task as far as you can up until a Critical Point:
+    - For example, if the task is to "call a restaurant to make a reservation", you should not actually make the call but should navigate to the restaurant's page and find the phone number.
+    - Similarly, if the task is to "order new size 12 running shoes" you should not actually place the order but should instead search for the right shoes that meet the criteria and add them to the cart.
+    - Some tasks, like answering questions, may not encounter a Critical Point at all.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{tool_descs}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{{"name": <function-name>, "arguments": <args-json-object>}}
+</tool_call>"""
+
+
+SPECIAL_CODE_MODE = os.getenv("SPECIAL_CODE_MODE", "false").lower() == "true"
+CODE_TOOL_PATTERN = "code_interpreter"
+FN_CALL_TEMPLATE_WITH_CI = """# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{tool_descs}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{{"name": <function-name>, "arguments": <args-json-object>}}
+</tool_call>
+For code parameters, use placeholders first, and then put the code within <code></code> XML tags, such as:
+<tool_call>
+{{"name": <function-name>, "arguments": {{"code": ""}}}}
+<code>
+Here is the code.
+</code>
+</tool_call>"""
+
+
+class NousFnCallPrompt:
+    def __init__(self, template_name: str = "default"):
+        """Initialize NousFnCallPrompt with a specific template.
+
+        Args:
+            template_name: Name of the template to use. Options:
+                          "default", "qwen", "with_ci"
+        """
+        self.template_name = template_name
+        self.template_map = {
+            "default": FN_CALL_TEMPLATE,
+            "qwen": FN_CALL_TEMPLATE_QWEN,
+            "with_ci": FN_CALL_TEMPLATE_WITH_CI,
+        }
+
+        if template_name not in self.template_map:
+            raise ValueError(
+                f"Unknown template_name: {template_name}. "
+                f"Available options: {list(self.template_map.keys())}"
+            )
+
+    def preprocess_fncall_messages(
+        self,
+        messages: List[Message],
+        functions: List[dict],
+        lang: Literal["en", "zh"],
+        parallel_function_calls: bool = True,
+        function_choice: Union[Literal["auto"], str] = "auto",
+    ) -> List[Message]:
+        del lang  # ignored
+        del parallel_function_calls  # ignored
+        if function_choice != "auto":
+            raise NotImplementedError
+
+        ori_messages = messages
+
+        # Change function_call responses to plaintext responses:
+        messages = []
+        for msg in copy.deepcopy(ori_messages):
+            role, content, reasoning_content = (
+                msg.role,
+                msg.content,
+                msg.reasoning_content,
+            )
+            if role in ("system", "user"):
+                messages.append(msg)
+            elif role == "assistant":
+                content = content or []
+                fn_call = msg.function_call
+                if fn_call:
+                    if (not SPECIAL_CODE_MODE) or (CODE_TOOL_PATTERN not in fn_call.name):
+                        fc = {
+                            "name": fn_call.name,
+                            "arguments": json.loads(fn_call.arguments),
+                        }
+                        fc = json.dumps(fc, ensure_ascii=False)
+                        fc = f"<tool_call>\n{fc}\n</tool_call>"
+                    else:
+                        para = json.loads(fn_call.arguments)
+                        code = para["code"]
+                        para["code"] = ""
+                        fc = {"name": fn_call.name, "arguments": para}
+                        fc = json.dumps(fc, ensure_ascii=False)
+                        fc = f"<tool_call>\n{fc}\n<code>\n{code}\n</code>\n</tool_call>"
+
+                    content.append(ContentItem(text=fc))
+                if messages[-1].role == "assistant":
+                    messages[-1].content.append(ContentItem(text="\n"))
+                    messages[-1].content.extend(content)
+                else:
+                    # TODO: Assuming there will only be one continuous reasoning_content here
+                    messages.append(
+                        Message(
+                            role=role,
+                            content=content,
+                            reasoning_content=reasoning_content,
+                        )
+                    )
+            elif role == "function":
+                assert isinstance(content, list)
+                assert len(content) == 1
+                assert content[0].text
+                fc = f"<tool_response>\n{content[0].text}\n</tool_response>"
+                content = [ContentItem(text=fc)]
+                if messages[-1].role == "user":
+                    messages[-1].content.append(ContentItem(text="\n"))
+                    messages[-1].content.extend(content)
+                else:
+                    messages.append(Message(role="user", content=content))
+            else:
+                raise TypeError
+
+        tool_descs = [{"type": "function", "function": f} for f in functions]
+        tool_names = [
+            function.get("name_for_model", function.get("name", "")) for function in functions
+        ]
+        tool_descs = "\n".join([json.dumps(f, ensure_ascii=False) for f in tool_descs])
+
+        # Select template based on configuration
+        if SPECIAL_CODE_MODE and any([CODE_TOOL_PATTERN in x for x in tool_names]):
+            selected_template = FN_CALL_TEMPLATE_WITH_CI
+        else:
+            selected_template = self.template_map[self.template_name]
+
+        tool_system = selected_template.format(tool_descs=tool_descs)
+        if messages[0].role == "system":
+            messages[0].content.append(ContentItem(text="\n\n" + tool_system))
+        else:
+            messages = [Message(role="system", content=[ContentItem(text=tool_system)])] + messages
+        return messages
+
+
+# Mainly for removing incomplete special tokens when streaming the output
+# This assumes that '<tool_call>\n{"name": "' is the special token for the NousFnCallPrompt
+def remove_incomplete_special_tokens(text: str) -> str:
+    if text in '<tool_call>\n{"name": "':
+        text = ""
+    return text
+
+
+def extract_fn(text: str):
+    fn_name, fn_args = "", ""
+    fn_name_s = '"name": "'
+    fn_name_e = '", "'
+    fn_args_s = '"arguments": '
+    i = text.find(fn_name_s)
+    k = text.find(fn_args_s)
+    if i > 0:
+        _text = text[i + len(fn_name_s) :]
+        j = _text.find(fn_name_e)
+        if j > -1:
+            fn_name = _text[:j]
+    if k > 0:
+        fn_args = text[k + len(fn_args_s) :]
+
+    if len(fn_args) > 5:
+        fn_args = fn_args[:-5]
+    else:
+        fn_args = ""
+    return fn_name, fn_args
+
+
+def build_nous_system(functions: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+    """Use original FARA NousFnCallPrompt to generate a system message embedding tool schema."""
+    from .schema import ContentItem as NousContentItem
+    from .schema import Message as NousMessage
+
+    msgs = NousFnCallPrompt().preprocess_fncall_messages(
+        messages=[
+            NousMessage(
+                role="system", content=[NousContentItem(text="You are a helpful assistant.")]
+            )
+        ],
+        functions=functions,
+        lang="en",
+    )
+    sys = msgs[0].model_dump()
+    # Convert structured content to OpenAI-style content list
+    content = [{"type": "text", "text": c["text"]} for c in sys.get("content", [])]
+    return {"role": "system", "content": content}
+
+
+def fix_fara_tool_call_format(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """
+    Fix tool call format in conversation history for FARA compatibility.
+
+    The shared `convert_responses_items_to_completion_messages` function outputs:
+    - Tool name as "computer" (should be "computer_use")
+    - Action key as "type" (should be "action")
+
+    This function post-processes assistant messages to fix these issues.
+    """
+    import re
+
+    # Valid FARA action types
+    valid_actions = {
+        "left_click",
+        "right_click",
+        "middle_click",
+        "double_click",
+        "triple_click",
+        "click",
+        "type",
+        "key",
+        "scroll",
+        "hscroll",
+        "mouse_move",
+        "wait",
+        "visit_url",
+        "web_search",
+        "history_back",
+        "screenshot",
+        "terminate",
+    }
+
+    fixed_messages = []
+    for msg in messages:
+        if msg.get("role") != "assistant":
+            fixed_messages.append(msg)
+            continue
+
+        content = msg.get("content", "")
+        if not isinstance(content, str) or "<tool_call>" not in content:
+            fixed_messages.append(msg)
+            continue
+
+        # Find and fix all tool calls in the content
+        def fix_tool_call(match):
+            tool_call_content = match.group(1)
+            try:
+                tool_call = json.loads(tool_call_content)
+
+                # Fix tool name: "computer" -> "computer_use"
+                if tool_call.get("name") == "computer":
+                    tool_call["name"] = "computer_use"
+
+                # Fix arguments: "type" -> "action" and x/y -> coordinate
+                args = tool_call.get("arguments", {})
+                if isinstance(args, dict):
+                    # If "type" contains a valid action, rename to "action"
+                    if "type" in args and args["type"] in valid_actions:
+                        args["action"] = args.pop("type")
+
+                    # Convert internal x/y format back to FARA coordinate format
+                    if "x" in args and "y" in args and "coordinate" not in args:
+                        args["coordinate"] = [args.pop("x"), args.pop("y")]
+
+                    # Normalize action names: "click" -> "left_click"
+                    if args.get("action") == "click":
+                        args["action"] = "left_click"
+
+                    # Remove "button" field - FARA doesn't use it (action name implies button)
+                    args.pop("button", None)
+
+                    # If "action" is empty but we can infer from other keys
+                    if args.get("action") == "" and "coordinate" in args:
+                        args["action"] = "left_click"
+
+                    tool_call["arguments"] = args
+
+                return f"<tool_call>\n{json.dumps(tool_call)}\n</tool_call>"
+            except (json.JSONDecodeError, TypeError):
+                return match.group(0)  # Return original if parsing fails
+
+        # Match <tool_call>...</tool_call> or <tool_call>...</tool_call>
+        fixed_content = re.sub(
+            r"<tool_call>\s*(\{.*?\})\s*</tool_call>", fix_tool_call, content, flags=re.DOTALL
+        )
+
+        # Also handle malformed closing tags like <tool_call> used as closing
+        fixed_content = re.sub(
+            r"<tool_call>(\{.*?\})<tool_call>", fix_tool_call, fixed_content, flags=re.DOTALL
+        )
+
+        fixed_messages.append({**msg, "content": fixed_content})
+
+    return fixed_messages
+
+
+def parse_tool_call_from_text(text: str) -> Optional[Dict[str, Any]]:
+    """Extract JSON object within <tool_call>...</tool_call> from model text.
+
+    Accepts both </tool_call> and <tool_call> as closing tags for robustness.
+    Handles nested braces in JSON objects.
+    """
+    # Find the opening tag
+    start_idx = text.find("<tool_call>")
+    if start_idx == -1:
+        return None
+
+    # Find the start of JSON (first '{' after opening tag)
+    json_start = text.find("{", start_idx)
+    if json_start == -1:
+        return None
+
+    # Extract JSON by counting braces
+    brace_count = 0
+    json_end = json_start
+    for i in range(json_start, len(text)):
+        if text[i] == "{":
+            brace_count += 1
+        elif text[i] == "}":
+            brace_count -= 1
+            if brace_count == 0:
+                json_end = i + 1
+                break
+
+    if brace_count != 0:
+        return None
+
+    json_str = text[json_start:json_end]
+    try:
+        return json.loads(json_str)
+    except Exception:
+        return None
+
+
+async def unnormalize_coordinate(args: Dict[str, Any], dims: Tuple[int, int]) -> Dict[str, Any]:
+    """Coordinates appear in 0..1000 space, scale to actual screen size using dims if provided."""
+    coord = args.get("coordinate")
+    if not coord or not isinstance(coord, (list, tuple)) or len(coord) < 2:
+        return args
+    x, y = float(coord[0]), float(coord[1])
+    width, height = float(dims[0]), float(dims[1])
+    x_abs = max(0.0, min(width, (x / 1000.0) * width))
+    y_abs = max(0.0, min(height, (y / 1000.0) * height))
+    args = {**args, "coordinate": [round(x_abs), round(y_abs)]}
+    return args
+
+
+def convert_qwen_tool_args_to_computer_action(args: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """
+    Convert Qwen computer tool arguments to the Computer Calls action schema.
+
+    Qwen (example):
+        {"action": "left_click", "coordinate": [114, 68]}
+
+    Target (example):
+        {"action": "left_click", "x": 114, "y": 68}
+
+    Other mappings:
+    - right_click, middle_click, double_click (triple_click -> double_click)
+    - mouse_move -> { action: "move", x, y }
+    - key -> { action: "keypress", keys: [...] }
+    - type -> { action: "type", text }
+    - scroll/hscroll -> { action: "scroll", scroll_x, scroll_y, x, y }
+    - wait -> { action: "wait" }
+    - terminate/answer are not direct UI actions; return None for now
+    """
+    if not isinstance(args, dict):
+        return None
+
+    action = args.get("action")
+    if not isinstance(action, str):
+        return None
+
+    # Coordinates helper
+    coord = args.get("coordinate")
+    x = y = None
+    if isinstance(coord, (list, tuple)) and len(coord) >= 2:
+        try:
+            x = int(round(float(coord[0])))
+            y = int(round(float(coord[1])))
+        except Exception:
+            x = y = None
+
+    # Map actions
+    a = action.lower()
+    if a in {"left_click", "right_click", "middle_click", "double_click"}:
+        if x is None or y is None:
+            return None
+        return {"action": a, "x": x, "y": y}
+    if a == "triple_click":
+        # Approximate as double_click
+        if x is None or y is None:
+            return None
+        return {"action": "double_click", "x": x, "y": y}
+    if a == "mouse_move":
+        if x is None or y is None:
+            return None
+        return {"action": "move", "x": x, "y": y}
+    if a == "key":
+        keys = args.get("keys")
+        if isinstance(keys, list) and all(isinstance(k, str) for k in keys):
+            return {"action": "keypress", "keys": keys}
+        return None
+    if a == "type":
+        text = args.get("text")
+        if isinstance(text, str):
+            return {"action": "type", "text": text}
+        return None
+    if a in {"scroll", "hscroll"}:
+        pixels = args.get("pixels") or 0
+        try:
+            pixels_val = int(round(float(pixels)))
+        except Exception:
+            pixels_val = 0
+        scroll_x = pixels_val if a == "hscroll" else 0
+        scroll_y = pixels_val if a == "scroll" else 0
+        # Include cursor position if available (optional)
+        out: Dict[str, Any] = {"action": "scroll", "scroll_x": scroll_x, "scroll_y": scroll_y}
+        if x is not None and y is not None:
+            out.update({"x": x, "y": y})
+        return out
+    if a == "wait":
+        return {"action": "wait"}
+
+    # Non-UI or terminal actions: terminate/answer -> not mapped here
+    return None
+
+
+def convert_fara_args_to_browser_tool_format(args: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Convert FARA model output format to BrowserTool compatible format.
+
+    FARA model may output extra parameters that BrowserTool methods don't accept.
+    This function cleans up the arguments and maps them to the correct format.
+
+    Examples:
+        Input:  {"action": "click", "button": "left", "x": 378, "y": 144}
+        Output: {"action": "left_click", "coordinate": [378, 144]}
+
+        Input:  {"action": "visit_url", "url": "https://...", "text": "..."}
+        Output: {"action": "visit_url", "url": "https://..."}
+
+        Input:  {"action": "terminate", "url": "...", "text": "...", "status": "success"}
+        Output: {"action": "terminate", "status": "success"}
+    """
+    if not isinstance(args, dict):
+        return args
+
+    action = args.get("action", "")
+    if not isinstance(action, str):
+        return args
+
+    a = action.lower()
+    result: Dict[str, Any] = {"action": a}
+
+    # Handle coordinate-based actions
+    # Check for both coordinate array and separate x/y fields
+    coord = args.get("coordinate")
+    x = args.get("x")
+    y = args.get("y")
+
+    if coord and isinstance(coord, (list, tuple)) and len(coord) >= 2:
+        x, y = coord[0], coord[1]
+
+    # Click actions - normalize to left_click with coordinate
+    if a in {"click", "left_click"}:
+        if x is not None and y is not None:
+            result["action"] = "left_click"
+            result["coordinate"] = [x, y]
+        return result
+
+    if a in {"right_click", "middle_click", "double_click", "triple_click"}:
+        if x is not None and y is not None:
+            result["coordinate"] = [x, y]
+        return result
+
+    if a == "mouse_move":
+        if x is not None and y is not None:
+            result["coordinate"] = [x, y]
+        return result
+
+    if a == "left_click_drag":
+        if x is not None and y is not None:
+            result["coordinate"] = [x, y]
+        # Also handle start/end coordinates if present
+        start_coord = args.get("start_coordinate")
+        end_coord = args.get("end_coordinate")
+        if start_coord:
+            result["start_coordinate"] = start_coord
+        if end_coord:
+            result["end_coordinate"] = end_coord
+        return result
+
+    # Keyboard actions
+    if a == "key":
+        keys = args.get("keys")
+        if keys:
+            result["keys"] = keys
+        return result
+
+    if a == "type":
+        text = args.get("text")
+        if text:
+            result["text"] = text
+        # Include coordinate if typing at a specific location
+        if x is not None and y is not None:
+            result["coordinate"] = [x, y]
+        return result
+
+    # Scroll actions
+    if a in {"scroll", "hscroll"}:
+        pixels = args.get("pixels")
+        if pixels is not None:
+            result["pixels"] = pixels
+        if x is not None and y is not None:
+            result["coordinate"] = [x, y]
+        return result
+
+    # Browser-specific actions
+    if a == "visit_url":
+        url = args.get("url")
+        if url:
+            result["url"] = url
+        return result
+
+    if a == "web_search":
+        query = args.get("query")
+        if query:
+            result["query"] = query
+        return result
+
+    if a == "history_back":
+        return result
+
+    # Wait action
+    if a == "wait":
+        time_val = args.get("time")
+        if time_val is not None:
+            result["time"] = time_val
+        return result
+
+    # Screenshot action
+    if a == "screenshot":
+        return result
+
+    # Terminate action
+    if a == "terminate":
+        status = args.get("status", "success")
+        result["status"] = status
+        return result
+
+    # For any other action, return cleaned args (just action + known fields)
+    return result
+
+
+def _convert_responses_items_to_fara_messages(
+    messages: List[Dict[str, Any]],
+    allow_images_in_tool_results: bool = False,
+) -> List[Dict[str, Any]]:
+    """
+    Convert SDK responses_items format to FARA-compatible completion messages.
+
+    This is FARA's dedicated conversion layer (similar to Anthropic's pattern).
+    It handles the conversion from SDK's OpenAI-style format to FARA's native format:
+
+    SDK format:
+        {"type": "click", "x": 100, "y": 200, "button": "left"}
+
+    FARA format (in XML tool_call):
+        {"name": "computer_use", "arguments": {"action": "left_click", "coordinate": [100, 200]}}
+    """
+    completion_messages: List[Dict[str, Any]] = []
+
+    for message in messages:
+        msg_type = message.get("type")
+        role = message.get("role")
+
+        # Handle user messages
+        if role == "user" or msg_type == "user":
+            content = message.get("content", "")
+            if isinstance(content, list):
+                converted_content = []
+                for item in content:
+                    if isinstance(item, dict):
+                        item_type = item.get("type")
+                        if item_type == "input_image":
+                            image_url = item.get("image_url", "")
+                            if image_url and image_url != "[omitted]":
+                                converted_content.append(
+                                    {"type": "image_url", "image_url": {"url": image_url}}
+                                )
+                        elif item_type == "input_text":
+                            converted_content.append({"type": "text", "text": item.get("text", "")})
+                        elif item_type == "image_url":
+                            # Already in correct format
+                            converted_content.append(item)
+                        elif item_type == "text":
+                            converted_content.append(item)
+                        else:
+                            converted_content.append(item)
+                    else:
+                        converted_content.append({"type": "text", "text": str(item)})
+                completion_messages.append({"role": "user", "content": converted_content})
+            else:
+                completion_messages.append({"role": "user", "content": content})
+
+        # Handle assistant messages
+        elif role == "assistant" and msg_type == "message":
+            content = message.get("content", [])
+            if isinstance(content, str):
+                completion_messages.append({"role": "assistant", "content": content})
+            elif isinstance(content, list):
+                text_parts = []
+                for item in content:
+                    if isinstance(item, dict) and item.get("type") == "output_text":
+                        text_parts.append(item.get("text", ""))
+                completion_messages.append({"role": "assistant", "content": "\n".join(text_parts)})
+
+        # Handle reasoning
+        elif msg_type == "reasoning":
+            summary = message.get("summary", [])
+            reasoning_text = ""
+            if isinstance(summary, list) and summary:
+                for item in summary:
+                    if isinstance(item, dict) and item.get("type") == "summary_text":
+                        reasoning_text = item.get("text", "")
+                        break
+            if reasoning_text:
+                completion_messages.append({"role": "assistant", "content": reasoning_text})
+
+        # Handle computer_call - convert SDK format to FARA's XML tool_call format
+        elif msg_type == "computer_call":
+            action = message.get("action", {})
+            action_type = action.get("type")
+
+            # Convert SDK action to FARA format
+            fara_args = _sdk_action_to_fara_args(action)
+
+            # Build FARA's XML tool_call format
+            tool_call_json = json.dumps({"name": "computer_use", "arguments": fara_args})
+            tool_call_text = f"<tool_call>\n{tool_call_json}\n</tool_call>"
+
+            # Append to last assistant message or create new one
+            if completion_messages and completion_messages[-1].get("role") == "assistant":
+                prev_content = completion_messages[-1].get("content", "")
+                completion_messages[-1]["content"] = f"{prev_content}\n{tool_call_text}".strip()
+            else:
+                completion_messages.append({"role": "assistant", "content": tool_call_text})
+
+        # Handle computer_call_output - convert to FARA's tool_response format
+        elif msg_type == "computer_call_output":
+            output = message.get("output", {})
+
+            # Build response content
+            if isinstance(output, dict) and output.get("type") == "input_image":
+                image_url = output.get("image_url", "")
+                response_text = "<tool_response>\nAction executed successfully. Here is the next screenshot.\n</tool_response>"
+
+                # Add as user message with image
+                if allow_images_in_tool_results and image_url and image_url != "[omitted]":
+                    completion_messages.append(
+                        {
+                            "role": "user",
+                            "content": [
+                                {"type": "text", "text": response_text},
+                                {"type": "image_url", "image_url": {"url": image_url}},
+                            ],
+                        }
+                    )
+                else:
+                    completion_messages.append(
+                        {
+                            "role": "user",
+                            "content": [
+                                {"type": "text", "text": response_text},
+                            ],
+                        }
+                    )
+            elif isinstance(output, dict) and output.get("terminated"):
+                response_text = "<tool_response>\nTask terminated.\n</tool_response>"
+                completion_messages.append({"role": "user", "content": response_text})
+            else:
+                response_text = f"<tool_response>\n{json.dumps(output) if isinstance(output, dict) else str(output)}\n</tool_response>"
+                completion_messages.append({"role": "user", "content": response_text})
+
+        # Handle function_call (non-computer tools)
+        elif msg_type == "function_call":
+            fn_name = message.get("name", "")
+            fn_args = message.get("arguments", "{}")
+
+            tool_call_json = json.dumps(
+                {
+                    "name": fn_name,
+                    "arguments": json.loads(fn_args) if isinstance(fn_args, str) else fn_args,
+                }
+            )
+            tool_call_text = f"<tool_call>\n{tool_call_json}\n</tool_call>"
+
+            if completion_messages and completion_messages[-1].get("role") == "assistant":
+                prev_content = completion_messages[-1].get("content", "")
+                completion_messages[-1]["content"] = f"{prev_content}\n{tool_call_text}".strip()
+            else:
+                completion_messages.append({"role": "assistant", "content": tool_call_text})
+
+        # Handle function_call_output
+        elif msg_type == "function_call_output":
+            output = message.get("output", "")
+            response_text = f"<tool_response>\n{output}\n</tool_response>"
+            completion_messages.append({"role": "user", "content": response_text})
+
+    return completion_messages
+
+
+def _sdk_action_to_fara_args(action: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Convert SDK action format to FARA arguments format.
+
+    SDK format:  {"type": "click", "x": 100, "y": 200, "button": "left"}
+    FARA format: {"action": "left_click", "coordinate": [100, 200]}
+    """
+    action_type = action.get("type", "")
+
+    # Click actions
+    if action_type == "click":
+        button = action.get("button", "left")
+        action_name = {
+            "left": "left_click",
+            "right": "right_click",
+            "wheel": "middle_click",
+            "middle": "middle_click",
+        }.get(button, "left_click")
+        return {"action": action_name, "coordinate": [action.get("x", 0), action.get("y", 0)]}
+
+    if action_type == "double_click":
+        return {"action": "double_click", "coordinate": [action.get("x", 0), action.get("y", 0)]}
+
+    # Type action
+    if action_type == "type":
+        result = {"action": "type", "text": action.get("text", "")}
+        # Include coordinate if present (for click-then-type)
+        if "x" in action and "y" in action:
+            result["coordinate"] = [action.get("x", 0), action.get("y", 0)]
+        return result
+
+    # Keypress action
+    if action_type == "keypress":
+        keys = action.get("keys", [])
+        return {"action": "key", "keys": keys}
+
+    # Move action
+    if action_type in ("move", "mouse_move"):
+        return {"action": "mouse_move", "coordinate": [action.get("x", 0), action.get("y", 0)]}
+
+    # Scroll action
+    if action_type == "scroll":
+        scroll_x = action.get("scroll_x", 0)
+        scroll_y = action.get("scroll_y", 0)
+        # FARA uses pixels (positive = up/left, negative = down/right)
+        pixels = scroll_y if scroll_y != 0 else scroll_x
+        result = {"action": "scroll", "pixels": pixels}
+        if "x" in action and "y" in action:
+            result["coordinate"] = [action.get("x", 0), action.get("y", 0)]
+        return result
+
+    # Drag action
+    if action_type == "drag":
+        path = action.get("path", [])
+        if len(path) >= 2:
+            return {
+                "action": "left_click_drag",
+                "start_coordinate": [path[0].get("x", 0), path[0].get("y", 0)],
+                "end_coordinate": [path[-1].get("x", 0), path[-1].get("y", 0)],
+            }
+        return {"action": "left_click_drag"}
+
+    # Screenshot
+    if action_type == "screenshot":
+        return {"action": "screenshot"}
+
+    # Wait
+    if action_type == "wait":
+        return {"action": "wait"}
+
+    # Terminate
+    if action_type == "terminate":
+        return {"action": "terminate", "status": action.get("status", "success")}
+
+    # Fallback - return as-is with type renamed to action
+    return {"action": action_type, **{k: v for k, v in action.items() if k != "type"}}

libs/python/agent/cua_agent/loops/fara/schema.py

@@ -0,0 +1,143 @@
+# Source: https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/llm/schema.py
+
+from typing import List, Literal, Optional, Tuple, Union
+
+from pydantic import BaseModel, field_validator, model_validator
+
+
+class BaseModelCompatibleDict(BaseModel):
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def model_dump(self, **kwargs):
+        if "exclude_none" not in kwargs:
+            kwargs["exclude_none"] = True
+        return super().model_dump(**kwargs)
+
+    def model_dump_json(self, **kwargs):
+        if "exclude_none" not in kwargs:
+            kwargs["exclude_none"] = True
+        return super().model_dump_json(**kwargs)
+
+    def get(self, key, default=None):
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            return default
+
+    def __str__(self):
+        return f"{self.model_dump()}"
+
+
+class FunctionCall(BaseModelCompatibleDict):
+    name: str
+    arguments: str
+
+    def __init__(self, name: str, arguments: str):
+        super().__init__(name=name, arguments=arguments)
+
+    def __repr__(self):
+        return f"FunctionCall({self.model_dump()})"
+
+
+class ContentItem(BaseModelCompatibleDict):
+    text: Optional[str] = None
+    image: Optional[str] = None
+    file: Optional[str] = None
+    audio: Optional[Union[str, dict]] = None
+    video: Optional[Union[str, list]] = None
+
+    def __init__(
+        self,
+        text: Optional[str] = None,
+        image: Optional[str] = None,
+        file: Optional[str] = None,
+        audio: Optional[Union[str, dict]] = None,
+        video: Optional[Union[str, list]] = None,
+    ):
+        super().__init__(text=text, image=image, file=file, audio=audio, video=video)
+
+    @model_validator(mode="after")
+    def check_exclusivity(self):
+        provided_fields = 0
+        if self.text is not None:
+            provided_fields += 1
+        if self.image:
+            provided_fields += 1
+        if self.file:
+            provided_fields += 1
+        if self.audio:
+            provided_fields += 1
+        if self.video:
+            provided_fields += 1
+
+        if provided_fields != 1:
+            raise ValueError(
+                "Exactly one of 'text', 'image', 'file', 'audio', or 'video' must be provided."
+            )
+        return self
+
+    def __repr__(self):
+        return f"ContentItem({self.model_dump()})"
+
+    def get_type_and_value(
+        self,
+    ) -> Tuple[Literal["text", "image", "file", "audio", "video"], str]:
+        ((t, v),) = self.model_dump().items()
+        assert t in ("text", "image", "file", "audio", "video")
+        return t, v
+
+    @property
+    def type(self) -> Literal["text", "image", "file", "audio", "video"]:
+        t, _ = self.get_type_and_value()
+        return t
+
+    @property
+    def value(self) -> str:
+        _, v = self.get_type_and_value()
+        return v
+
+
+class Message(BaseModelCompatibleDict):
+    role: str
+    content: Union[str, List[ContentItem]]
+    reasoning_content: Optional[Union[str, List[ContentItem]]] = None
+    name: Optional[str] = None
+    function_call: Optional[FunctionCall] = None
+    extra: Optional[dict] = None
+
+    def __init__(
+        self,
+        role: str,
+        content: Union[str, List[ContentItem]],
+        reasoning_content: Optional[Union[str, List[ContentItem]]] = None,
+        name: Optional[str] = None,
+        function_call: Optional[FunctionCall] = None,
+        extra: Optional[dict] = None,
+        **kwargs,
+    ):
+        if content is None:
+            content = ""
+        if reasoning_content is None:
+            reasoning_content = ""
+        super().__init__(
+            role=role,
+            content=content,
+            reasoning_content=reasoning_content,
+            name=name,
+            function_call=function_call,
+            extra=extra,
+        )
+
+    def __repr__(self):
+        return f"Message({self.model_dump()})"
+
+    @field_validator("role")
+    def role_checker(cls, value: str) -> str:
+        values = ["system", "user", "assistant", "function"]
+        if value not in values:
+            raise ValueError(f'{value} must be one of {",".join(values)}')
+        return value