trycua--cua/libs/python/agent/cua_agent/adapters/mlxvlm_adapter.py

import asyncio
import base64
import functools
import io
import math
import re
import warnings
from concurrent.futures import ThreadPoolExecutor
from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Tuple, cast

from litellm import acompletion, completion
from litellm.llms.custom_llm import CustomLLM
from litellm.types.utils import GenericStreamingChunk, ModelResponse
from PIL import Image

# Try to import MLX dependencies
try:
    import mlx.core as mx
    from mlx_vlm import generate, load
    from mlx_vlm.prompt_utils import apply_chat_template
    from mlx_vlm.utils import load_config
    from transformers.tokenization_utils import PreTrainedTokenizer

    MLX_AVAILABLE = True
except ImportError:
    MLX_AVAILABLE = False

# Constants for smart_resize
IMAGE_FACTOR = 28
MIN_PIXELS = 100 * 28 * 28
MAX_PIXELS = 16384 * 28 * 28
MAX_RATIO = 200


def round_by_factor(number: float, factor: int) -> int:
    """Returns the closest integer to 'number' that is divisible by 'factor'."""
    return round(number / factor) * factor


def ceil_by_factor(number: float, factor: int) -> int:
    """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
    return math.ceil(number / factor) * factor


def floor_by_factor(number: float, factor: int) -> int:
    """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
    return math.floor(number / factor) * factor


def smart_resize(
    height: int,
    width: int,
    factor: int = IMAGE_FACTOR,
    min_pixels: int = MIN_PIXELS,
    max_pixels: int = MAX_PIXELS,
) -> tuple[int, int]:
    """
    Rescales the image so that the following conditions are met:

    1. Both dimensions (height and width) are divisible by 'factor'.
    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
    3. The aspect ratio of the image is maintained as closely as possible.
    """
    if max(height, width) / min(height, width) > MAX_RATIO:
        raise ValueError(
            f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
        )
    h_bar = max(factor, round_by_factor(height, factor))
    w_bar = max(factor, round_by_factor(width, factor))
    if h_bar * w_bar > max_pixels:
        beta = math.sqrt((height * width) / max_pixels)
        h_bar = floor_by_factor(height / beta, factor)
        w_bar = floor_by_factor(width / beta, factor)
    elif h_bar * w_bar < min_pixels:
        beta = math.sqrt(min_pixels / (height * width))
        h_bar = ceil_by_factor(height * beta, factor)
        w_bar = ceil_by_factor(width * beta, factor)
    return h_bar, w_bar


class MLXVLMAdapter(CustomLLM):
    """MLX VLM Adapter for running vision-language models locally using MLX."""

    def __init__(self, **kwargs):
        """Initialize the adapter.

        Args:
            **kwargs: Additional arguments
        """
        super().__init__()

        self.models = {}  # Cache for loaded models
        self.processors = {}  # Cache for loaded processors
        self.configs = {}  # Cache for loaded configs
        self._executor = ThreadPoolExecutor(max_workers=1)  # Single thread pool

    def _load_model_and_processor(self, model_name: str):
        """Load model and processor if not already cached.

        Args:
            model_name: Name of the model to load

        Returns:
            Tuple of (model, processor, config)
        """
        if not MLX_AVAILABLE:
            raise ImportError("MLX VLM dependencies not available. Please install mlx-vlm.")

        if model_name not in self.models:
            # Load model and processor
            model_obj, processor = load(
                model_name, processor_kwargs={"min_pixels": MIN_PIXELS, "max_pixels": MAX_PIXELS}
            )
            config = load_config(model_name)

            # Cache them
            self.models[model_name] = model_obj
            self.processors[model_name] = processor
            self.configs[model_name] = config

        return self.models[model_name], self.processors[model_name], self.configs[model_name]

    def _process_coordinates(
        self, text: str, original_size: Tuple[int, int], model_size: Tuple[int, int]
    ) -> str:
        """Process coordinates in box tokens based on image resizing using smart_resize approach.

        Args:
            text: Text containing box tokens
            original_size: Original image size (width, height)
            model_size: Model processed image size (width, height)

        Returns:
            Text with processed coordinates
        """
        # Find all box tokens
        box_pattern = r"<\|box_start\|>\((\d+),\s*(\d+)\)<\|box_end\|>"

        def process_coords(match):
            model_x, model_y = int(match.group(1)), int(match.group(2))
            # Scale coordinates from model space to original image space
            # Both original_size and model_size are in (width, height) format
            new_x = int(model_x * original_size[0] / model_size[0])  # Width
            new_y = int(model_y * original_size[1] / model_size[1])  # Height
            return f"<|box_start|>({new_x},{new_y})<|box_end|>"

        return re.sub(box_pattern, process_coords, text)

    def _convert_messages(self, messages: List[Dict[str, Any]]) -> Tuple[
        List[Dict[str, Any]],
        List[Image.Image],
        Dict[int, Tuple[int, int]],
        Dict[int, Tuple[int, int]],
    ]:
        """Convert OpenAI format messages to MLX VLM format and extract images.

        Args:
            messages: Messages in OpenAI format

        Returns:
            Tuple of (processed_messages, images, original_sizes, model_sizes)
        """
        processed_messages = []
        images = []
        original_sizes = {}  # Track original sizes of images for coordinate mapping
        model_sizes = {}  # Track model processed sizes
        image_index = 0

        for message in messages:
            processed_message = {"role": message["role"], "content": []}

            content = message.get("content", [])
            if isinstance(content, str):
                # Simple text content
                processed_message["content"] = content
            elif isinstance(content, list):
                # Multi-modal content
                processed_content = []
                for item in content:
                    if item.get("type") == "text":
                        processed_content.append({"type": "text", "text": item.get("text", "")})
                    elif item.get("type") == "image_url":
                        image_url = item.get("image_url", {}).get("url", "")
                        pil_image = None

                        if image_url.startswith("data:image/"):
                            # Extract base64 data
                            base64_data = image_url.split(",")[1]
                            # Convert base64 to PIL Image
                            image_data = base64.b64decode(base64_data)
                            pil_image = Image.open(io.BytesIO(image_data))
                        else:
                            # Handle file path or URL
                            pil_image = Image.open(image_url)

                        # Store original image size for coordinate mapping
                        original_size = pil_image.size
                        original_sizes[image_index] = original_size

                        # Use smart_resize to determine model size
                        # Note: smart_resize expects (height, width) but PIL gives (width, height)
                        height, width = original_size[1], original_size[0]
                        new_height, new_width = smart_resize(height, width)
                        # Store model size in (width, height) format for consistent coordinate processing
                        model_sizes[image_index] = (new_width, new_height)

                        # Resize the image using the calculated dimensions from smart_resize
                        resized_image = pil_image.resize((new_width, new_height))
                        images.append(resized_image)

                        # Add image placeholder to content
                        processed_content.append({"type": "image"})

                        image_index += 1

                processed_message["content"] = processed_content

            processed_messages.append(processed_message)

        return processed_messages, images, original_sizes, model_sizes

    def _generate(self, **kwargs) -> str:
        """Generate response using the local MLX VLM model.

        Args:
            **kwargs: Keyword arguments containing messages and model info

        Returns:
            Generated text response
        """
        messages = kwargs.get("messages", [])
        model_name = kwargs.get("model", "mlx-community/UI-TARS-1.5-7B-4bit")
        max_tokens = kwargs.get("max_tokens", 128)

        # Warn about ignored kwargs
        ignored_kwargs = set(kwargs.keys()) - {"messages", "model", "max_tokens"}
        if ignored_kwargs:
            warnings.warn(f"Ignoring unsupported kwargs: {ignored_kwargs}")

        # Load model and processor
        model, processor, config = self._load_model_and_processor(model_name)

        # Convert messages and extract images
        processed_messages, images, original_sizes, model_sizes = self._convert_messages(messages)

        # Process user text input with box coordinates after image processing
        # Swap original_size and model_size arguments for inverse transformation
        for msg_idx, msg in enumerate(processed_messages):
            if msg.get("role") == "user" and isinstance(msg.get("content"), str):
                content = msg.get("content", "")
                if (
                    "<|box_start|>" in content
                    and original_sizes
                    and model_sizes
                    and 0 in original_sizes
                    and 0 in model_sizes
                ):
                    orig_size = original_sizes[0]
                    model_size = model_sizes[0]
                    # Swap arguments to perform inverse transformation for user input
                    processed_messages[msg_idx]["content"] = self._process_coordinates(
                        content, model_size, orig_size
                    )

        try:
            # Format prompt according to model requirements using the processor directly
            prompt = processor.apply_chat_template(
                processed_messages, tokenize=False, add_generation_prompt=True, return_tensors="pt"
            )
            tokenizer = cast(PreTrainedTokenizer, processor)

            # Generate response
            text_content, usage = generate(
                model,
                tokenizer,
                str(prompt),
                images,  # type: ignore
                verbose=False,
                max_tokens=max_tokens,
            )

        except Exception as e:
            raise RuntimeError(f"Error generating response: {str(e)}") from e

        # Process coordinates in the response back to original image space
        if original_sizes and model_sizes and 0 in original_sizes and 0 in model_sizes:
            # Get original image size and model size (using the first image)
            orig_size = original_sizes[0]
            model_size = model_sizes[0]

            # Check if output contains box tokens that need processing
            if "<|box_start|>" in text_content:
                # Process coordinates from model space back to original image space
                text_content = self._process_coordinates(text_content, orig_size, model_size)

        return text_content

    def completion(self, *args, **kwargs) -> ModelResponse:
        """Synchronous completion method.

        Returns:
            ModelResponse with generated text
        """
        generated_text = self._generate(**kwargs)

        result = completion(
            model=f"mlx/{kwargs.get('model', 'mlx-community/UI-TARS-1.5-7B-4bit')}",
            mock_response=generated_text,
        )
        return cast(ModelResponse, result)

    async def acompletion(self, *args, **kwargs) -> ModelResponse:
        """Asynchronous completion method.

        Returns:
            ModelResponse with generated text
        """
        # Run _generate in thread pool to avoid blocking
        loop = asyncio.get_event_loop()
        generated_text = await loop.run_in_executor(
            self._executor, functools.partial(self._generate, **kwargs)
        )

        result = await acompletion(
            model=f"mlx/{kwargs.get('model', 'mlx-community/UI-TARS-1.5-7B-4bit')}",
            mock_response=generated_text,
        )
        return cast(ModelResponse, result)

    def streaming(self, *args, **kwargs) -> Iterator[GenericStreamingChunk]:
        """Synchronous streaming method.

        Returns:
            Iterator of GenericStreamingChunk
        """
        generated_text = self._generate(**kwargs)

        generic_streaming_chunk: GenericStreamingChunk = {
            "finish_reason": "stop",
            "index": 0,
            "is_finished": True,
            "text": generated_text,
            "tool_use": None,
            "usage": {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0},
        }

        yield generic_streaming_chunk

    async def astreaming(self, *args, **kwargs) -> AsyncIterator[GenericStreamingChunk]:
        """Asynchronous streaming method.

        Returns:
            AsyncIterator of GenericStreamingChunk
        """
        # Run _generate in thread pool to avoid blocking
        loop = asyncio.get_event_loop()
        generated_text = await loop.run_in_executor(
            self._executor, functools.partial(self._generate, **kwargs)
        )

        generic_streaming_chunk: GenericStreamingChunk = {
            "finish_reason": "stop",
            "index": 0,
            "is_finished": True,
            "text": generated_text,
            "tool_use": None,
            "usage": {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0},
        }

        yield generic_streaming_chunk