sandbox init

app/daytona/README.md

@@ -0,0 +1,45 @@
+# Agent Sandbox
+
+This directory contains the agent sandbox implementation - a Docker-based virtual environment that agents use as their own computer to execute tasks, access the web, and manipulate files.
+
+## Overview
+
+The sandbox provides a complete containerized Linux environment with:
+- Chrome browser for web interactions
+- VNC server for accessing the Web User
+- Web server for serving content (port 8080) -> loading html files from the /workspace directory
+- Full file system access
+- Full sudo access
+
+## Customizing the Sandbox
+
+You can modify the sandbox environment for development or to add new capabilities:
+
+1. Edit files in the `docker/` directory
+2. Build a custom image:
+   ```
+   cd backend/sandbox/docker
+   docker compose build
+   docker push kortix/suna:0.1.3
+   ```
+3. Test your changes locally using docker-compose
+
+## Using a Custom Image
+
+To use your custom sandbox image:
+
+1. Change the `image` parameter in `docker-compose.yml` (that defines the image name `kortix/suna:___`)
+2. Update the same image name in `backend/sandbox/sandbox.py` in the `create_sandbox` function
+3. If using Daytona for deployment, update the image reference there as well
+
+## Publishing New Versions
+
+When publishing a new version of the sandbox:
+
+1. Update the version number in `docker-compose.yml` (e.g., from `0.1.2` to `0.1.3`)
+2. Build the new image: `docker compose build`
+3. Push the new version: `docker push kortix/suna:0.1.3`
+4. Update all references to the image version in:
+   - `backend/utils/config.py`
+   - Daytona images
+   - Any other services using this image

app/daytona/api.py

@@ -0,0 +1,390 @@
+import os
+import urllib.parse
+from typing import Optional
+
+from fastapi import FastAPI, UploadFile, File, HTTPException, APIRouter, Form, Depends, Request
+from fastapi.responses import Response
+from pydantic import BaseModel
+
+from sandbox.sandbox import get_or_start_sandbox, delete_sandbox
+from utils.logger import logger
+from utils.auth_utils import get_optional_user_id
+from services.supabase import DBConnection
+
+# Initialize shared resources
+router = APIRouter(tags=["sandbox"])
+db = None
+
+def initialize(_db: DBConnection):
+    """Initialize the sandbox API with resources from the main API."""
+    global db
+    db = _db
+    logger.info("Initialized sandbox API with database connection")
+
+class FileInfo(BaseModel):
+    """Model for file information"""
+    name: str
+    path: str
+    is_dir: bool
+    size: int
+    mod_time: str
+    permissions: Optional[str] = None
+
+def normalize_path(path: str) -> str:
+    """
+    Normalize a path to ensure proper UTF-8 encoding and handling.
+    
+    Args:
+        path: The file path, potentially containing URL-encoded characters
+        
+    Returns:
+        Normalized path with proper UTF-8 encoding
+    """
+    try:
+        # First, ensure the path is properly URL-decoded
+        decoded_path = urllib.parse.unquote(path)
+        
+        # Handle Unicode escape sequences like \u0308
+        try:
+            # Replace Python-style Unicode escapes (\u0308) with actual characters
+            # This handles cases where the Unicode escape sequence is part of the URL
+            import re
+            unicode_pattern = re.compile(r'\\u([0-9a-fA-F]{4})')
+            
+            def replace_unicode(match):
+                hex_val = match.group(1)
+                return chr(int(hex_val, 16))
+            
+            decoded_path = unicode_pattern.sub(replace_unicode, decoded_path)
+        except Exception as unicode_err:
+            logger.warning(f"Error processing Unicode escapes in path '{path}': {str(unicode_err)}")
+        
+        logger.debug(f"Normalized path from '{path}' to '{decoded_path}'")
+        return decoded_path
+    except Exception as e:
+        logger.error(f"Error normalizing path '{path}': {str(e)}")
+        return path  # Return original path if decoding fails
+
+async def verify_sandbox_access(client, sandbox_id: str, user_id: Optional[str] = None):
+    """
+    Verify that a user has access to a specific sandbox based on account membership.
+    
+    Args:
+        client: The Supabase client
+        sandbox_id: The sandbox ID to check access for
+        user_id: The user ID to check permissions for. Can be None for public resource access.
+        
+    Returns:
+        dict: Project data containing sandbox information
+        
+    Raises:
+        HTTPException: If the user doesn't have access to the sandbox or sandbox doesn't exist
+    """
+    # Find the project that owns this sandbox
+    project_result = await client.table('projects').select('*').filter('sandbox->>id', 'eq', sandbox_id).execute()
+    
+    if not project_result.data or len(project_result.data) == 0:
+        raise HTTPException(status_code=404, detail="Sandbox not found")
+    
+    project_data = project_result.data[0]
+
+    if project_data.get('is_public'):
+        return project_data
+    
+    # For private projects, we must have a user_id
+    if not user_id:
+        raise HTTPException(status_code=401, detail="Authentication required for this resource")
+    
+    account_id = project_data.get('account_id')
+    
+    # Verify account membership
+    if account_id:
+        account_user_result = await client.schema('basejump').from_('account_user').select('account_role').eq('user_id', user_id).eq('account_id', account_id).execute()
+        if account_user_result.data and len(account_user_result.data) > 0:
+            return project_data
+    
+    raise HTTPException(status_code=403, detail="Not authorized to access this sandbox")
+
+async def get_sandbox_by_id_safely(client, sandbox_id: str):
+    """
+    Safely retrieve a sandbox object by its ID, using the project that owns it.
+    
+    Args:
+        client: The Supabase client
+        sandbox_id: The sandbox ID to retrieve
+    
+    Returns:
+        Sandbox: The sandbox object
+        
+    Raises:
+        HTTPException: If the sandbox doesn't exist or can't be retrieved
+    """
+    # Find the project that owns this sandbox
+    project_result = await client.table('projects').select('project_id').filter('sandbox->>id', 'eq', sandbox_id).execute()
+    
+    if not project_result.data or len(project_result.data) == 0:
+        logger.error(f"No project found for sandbox ID: {sandbox_id}")
+        raise HTTPException(status_code=404, detail="Sandbox not found - no project owns this sandbox ID")
+    
+    # project_id = project_result.data[0]['project_id']
+    # logger.debug(f"Found project {project_id} for sandbox {sandbox_id}")
+    
+    try:
+        # Get the sandbox
+        sandbox = await get_or_start_sandbox(sandbox_id)
+        # Extract just the sandbox object from the tuple (sandbox, sandbox_id, sandbox_pass)
+        # sandbox = sandbox_tuple[0]
+            
+        return sandbox
+    except Exception as e:
+        logger.error(f"Error retrieving sandbox {sandbox_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Failed to retrieve sandbox: {str(e)}")
+
+@router.post("/sandboxes/{sandbox_id}/files")
+async def create_file(
+    sandbox_id: str, 
+    path: str = Form(...),
+    file: UploadFile = File(...),
+    request: Request = None,
+    user_id: Optional[str] = Depends(get_optional_user_id)
+):
+    """Create a file in the sandbox using direct file upload"""
+    # Normalize the path to handle UTF-8 encoding correctly
+    path = normalize_path(path)
+    
+    logger.info(f"Received file upload request for sandbox {sandbox_id}, path: {path}, user_id: {user_id}")
+    client = await db.client
+    
+    # Verify the user has access to this sandbox
+    await verify_sandbox_access(client, sandbox_id, user_id)
+    
+    try:
+        # Get sandbox using the safer method
+        sandbox = await get_sandbox_by_id_safely(client, sandbox_id)
+        
+        # Read file content directly from the uploaded file
+        content = await file.read()
+        
+        # Create file using raw binary content
+        sandbox.fs.upload_file(content, path)
+        logger.info(f"File created at {path} in sandbox {sandbox_id}")
+        
+        return {"status": "success", "created": True, "path": path}
+    except Exception as e:
+        logger.error(f"Error creating file in sandbox {sandbox_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/sandboxes/{sandbox_id}/files")
+async def list_files(
+    sandbox_id: str, 
+    path: str,
+    request: Request = None,
+    user_id: Optional[str] = Depends(get_optional_user_id)
+):
+    """List files and directories at the specified path"""
+    # Normalize the path to handle UTF-8 encoding correctly
+    path = normalize_path(path)
+    
+    logger.info(f"Received list files request for sandbox {sandbox_id}, path: {path}, user_id: {user_id}")
+    client = await db.client
+    
+    # Verify the user has access to this sandbox
+    await verify_sandbox_access(client, sandbox_id, user_id)
+    
+    try:
+        # Get sandbox using the safer method
+        sandbox = await get_sandbox_by_id_safely(client, sandbox_id)
+        
+        # List files
+        files = sandbox.fs.list_files(path)
+        result = []
+        
+        for file in files:
+            # Convert file information to our model
+            # Ensure forward slashes are used for paths, regardless of OS
+            full_path = f"{path.rstrip('/')}/{file.name}" if path != '/' else f"/{file.name}"
+            file_info = FileInfo(
+                name=file.name,
+                path=full_path, # Use the constructed path
+                is_dir=file.is_dir,
+                size=file.size,
+                mod_time=str(file.mod_time),
+                permissions=getattr(file, 'permissions', None)
+            )
+            result.append(file_info)
+        
+        logger.info(f"Successfully listed {len(result)} files in sandbox {sandbox_id}")
+        return {"files": [file.dict() for file in result]}
+    except Exception as e:
+        logger.error(f"Error listing files in sandbox {sandbox_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/sandboxes/{sandbox_id}/files/content")
+async def read_file(
+    sandbox_id: str, 
+    path: str,
+    request: Request = None,
+    user_id: Optional[str] = Depends(get_optional_user_id)
+):
+    """Read a file from the sandbox"""
+    # Normalize the path to handle UTF-8 encoding correctly
+    original_path = path
+    path = normalize_path(path)
+    
+    logger.info(f"Received file read request for sandbox {sandbox_id}, path: {path}, user_id: {user_id}")
+    if original_path != path:
+        logger.info(f"Normalized path from '{original_path}' to '{path}'")
+    
+    client = await db.client
+    
+    # Verify the user has access to this sandbox
+    await verify_sandbox_access(client, sandbox_id, user_id)
+    
+    try:
+        # Get sandbox using the safer method
+        sandbox = await get_sandbox_by_id_safely(client, sandbox_id)
+        
+        # Read file directly - don't check existence first with a separate call
+        try:
+            content = sandbox.fs.download_file(path)
+        except Exception as download_err:
+            logger.error(f"Error downloading file {path} from sandbox {sandbox_id}: {str(download_err)}")
+            raise HTTPException(
+                status_code=404, 
+                detail=f"Failed to download file: {str(download_err)}"
+            )
+        
+        # Return a Response object with the content directly
+        filename = os.path.basename(path)
+        logger.info(f"Successfully read file {filename} from sandbox {sandbox_id}")
+        
+        # Ensure proper encoding by explicitly using UTF-8 for the filename in Content-Disposition header
+        # This applies RFC 5987 encoding for the filename to support non-ASCII characters
+        encoded_filename = filename.encode('utf-8').decode('latin-1')
+        content_disposition = f"attachment; filename*=UTF-8''{encoded_filename}"
+        
+        return Response(
+            content=content,
+            media_type="application/octet-stream",
+            headers={"Content-Disposition": content_disposition}
+        )
+    except HTTPException:
+        # Re-raise HTTP exceptions without wrapping
+        raise
+    except Exception as e:
+        logger.error(f"Error reading file in sandbox {sandbox_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.delete("/sandboxes/{sandbox_id}/files")
+async def delete_file(
+    sandbox_id: str, 
+    path: str,
+    request: Request = None,
+    user_id: Optional[str] = Depends(get_optional_user_id)
+):
+    """Delete a file from the sandbox"""
+    # Normalize the path to handle UTF-8 encoding correctly
+    path = normalize_path(path)
+    
+    logger.info(f"Received file delete request for sandbox {sandbox_id}, path: {path}, user_id: {user_id}")
+    client = await db.client
+    
+    # Verify the user has access to this sandbox
+    await verify_sandbox_access(client, sandbox_id, user_id)
+    
+    try:
+        # Get sandbox using the safer method
+        sandbox = await get_sandbox_by_id_safely(client, sandbox_id)
+        
+        # Delete file
+        sandbox.fs.delete_file(path)
+        logger.info(f"File deleted at {path} in sandbox {sandbox_id}")
+        
+        return {"status": "success", "deleted": True, "path": path}
+    except Exception as e:
+        logger.error(f"Error deleting file in sandbox {sandbox_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.delete("/sandboxes/{sandbox_id}")
+async def delete_sandbox_route(
+    sandbox_id: str,
+    request: Request = None,
+    user_id: Optional[str] = Depends(get_optional_user_id)
+):
+    """Delete an entire sandbox"""
+    logger.info(f"Received sandbox delete request for sandbox {sandbox_id}, user_id: {user_id}")
+    client = await db.client
+    
+    # Verify the user has access to this sandbox
+    await verify_sandbox_access(client, sandbox_id, user_id)
+    
+    try:
+        # Delete the sandbox using the sandbox module function
+        await delete_sandbox(sandbox_id)
+        
+        return {"status": "success", "deleted": True, "sandbox_id": sandbox_id}
+    except Exception as e:
+        logger.error(f"Error deleting sandbox {sandbox_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Should happen on server-side fully
+@router.post("/project/{project_id}/sandbox/ensure-active")
+async def ensure_project_sandbox_active(
+    project_id: str,
+    request: Request = None,
+    user_id: Optional[str] = Depends(get_optional_user_id)
+):
+    """
+    Ensure that a project's sandbox is active and running.
+    Checks the sandbox status and starts it if it's not running.
+    """
+    logger.info(f"Received ensure sandbox active request for project {project_id}, user_id: {user_id}")
+    client = await db.client
+    
+    # Find the project and sandbox information
+    project_result = await client.table('projects').select('*').eq('project_id', project_id).execute()
+    
+    if not project_result.data or len(project_result.data) == 0:
+        logger.error(f"Project not found: {project_id}")
+        raise HTTPException(status_code=404, detail="Project not found")
+    
+    project_data = project_result.data[0]
+    
+    # For public projects, no authentication is needed
+    if not project_data.get('is_public'):
+        # For private projects, we must have a user_id
+        if not user_id:
+            logger.error(f"Authentication required for private project {project_id}")
+            raise HTTPException(status_code=401, detail="Authentication required for this resource")
+            
+        account_id = project_data.get('account_id')
+        
+        # Verify account membership
+        if account_id:
+            account_user_result = await client.schema('basejump').from_('account_user').select('account_role').eq('user_id', user_id).eq('account_id', account_id).execute()
+            if not (account_user_result.data and len(account_user_result.data) > 0):
+                logger.error(f"User {user_id} not authorized to access project {project_id}")
+                raise HTTPException(status_code=403, detail="Not authorized to access this project")
+    
+    try:
+        # Get sandbox ID from project data
+        sandbox_info = project_data.get('sandbox', {})
+        if not sandbox_info.get('id'):
+            raise HTTPException(status_code=404, detail="No sandbox found for this project")
+            
+        sandbox_id = sandbox_info['id']
+        
+        # Get or start the sandbox
+        logger.info(f"Ensuring sandbox is active for project {project_id}")
+        sandbox = await get_or_start_sandbox(sandbox_id)
+        
+        logger.info(f"Successfully ensured sandbox {sandbox_id} is active for project {project_id}")
+        
+        return {
+            "status": "success", 
+            "sandbox_id": sandbox_id,
+            "message": "Sandbox is active"
+        }
+    except Exception as e:
+        logger.error(f"Error ensuring sandbox is active for project {project_id}: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))

app/daytona/docker/Dockerfile

@@ -0,0 +1,133 @@
+FROM python:3.11-slim
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    wget \
+    netcat-traditional \
+    gnupg \
+    curl \
+    unzip \
+    zip \
+    xvfb \
+    libgconf-2-4 \
+    libxss1 \
+    libnss3 \
+    libnspr4 \
+    libasound2 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    libcups2 \
+    libdbus-1-3 \
+    libdrm2 \
+    libgbm1 \
+    libgtk-3-0 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxfixes3 \
+    libxrandr2 \
+    xdg-utils \
+    fonts-liberation \
+    dbus \
+    xauth \
+    xvfb \
+    x11vnc \
+    tigervnc-tools \
+    supervisor \
+    net-tools \
+    procps \
+    git \
+    python3-numpy \
+    fontconfig \
+    fonts-dejavu \
+    fonts-dejavu-core \
+    fonts-dejavu-extra \
+    tmux \
+    # PDF Processing Tools
+    poppler-utils \
+    wkhtmltopdf \
+    # Document Processing Tools
+    antiword \
+    unrtf \
+    catdoc \
+    # Text Processing Tools
+    grep \
+    gawk \
+    sed \
+    # File Analysis Tools
+    file \
+    # Data Processing Tools
+    jq \
+    csvkit \
+    xmlstarlet \
+    # Additional Utilities
+    less \
+    vim \
+    tree \
+    rsync \
+    lsof \
+    iputils-ping \
+    dnsutils \
+    sudo \
+    # OCR Tools
+    tesseract-ocr \
+    tesseract-ocr-eng \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Node.js and npm
+RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+    && apt-get install -y nodejs \
+    && npm install -g npm@latest
+
+# Install Cloudflare Wrangler CLI globally
+RUN npm install -g wrangler
+
+# Install noVNC
+RUN git clone https://github.com/novnc/noVNC.git /opt/novnc \
+    && git clone https://github.com/novnc/websockify /opt/novnc/utils/websockify \
+    && ln -s /opt/novnc/vnc.html /opt/novnc/index.html
+
+# Set platform for ARM64 compatibility
+ARG TARGETPLATFORM=linux/amd64
+
+# Set up working directory
+WORKDIR /app
+
+# Copy requirements and install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Install Playwright and browsers with system dependencies
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
+# Install Playwright package first
+RUN pip install playwright
+# Then install dependencies and browsers
+RUN playwright install-deps
+RUN playwright install chromium
+# Verify installation
+RUN python -c "from playwright.sync_api import sync_playwright; print('Playwright installation verified')"
+
+# Copy server script
+COPY . /app
+COPY server.py /app/server.py
+COPY browser_api.py /app/browser_api.py
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1
+ENV CHROME_PATH=/ms-playwright/chromium-*/chrome-linux/chrome
+ENV ANONYMIZED_TELEMETRY=false
+ENV DISPLAY=:99
+ENV RESOLUTION=1024x768x24
+ENV VNC_PASSWORD=vncpassword
+ENV CHROME_PERSISTENT_SESSION=true
+ENV RESOLUTION_WIDTH=1024
+ENV RESOLUTION_HEIGHT=768
+# Add Chrome flags to prevent multiple tabs/windows
+ENV CHROME_FLAGS="--single-process --no-first-run --no-default-browser-check --disable-background-networking --disable-background-timer-throttling --disable-backgrounding-occluded-windows --disable-breakpad --disable-component-extensions-with-background-pages --disable-dev-shm-usage --disable-extensions --disable-features=TranslateUI --disable-ipc-flooding-protection --disable-renderer-backgrounding --enable-features=NetworkServiceInProcess2 --force-color-profile=srgb --metrics-recording-only --mute-audio --no-sandbox --disable-gpu"
+
+# Set up supervisor configuration
+RUN mkdir -p /var/log/supervisor
+COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+EXPOSE 7788 6080 5901 8000 8080
+
+CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]

app/daytona/docker/README.md

@@ -0,0 +1 @@
+# Sandbox

app/daytona/docker/docker-compose.yml

@@ -0,0 +1,45 @@
+services:
+  kortix-suna:
+    platform: linux/amd64
+    build:
+      context: .
+      dockerfile: ${DOCKERFILE:-Dockerfile}
+      args:
+        TARGETPLATFORM: ${TARGETPLATFORM:-linux/amd64}
+    image: kortix/suna:0.1.3
+    ports:
+      - "6080:6080"  # noVNC web interface
+      - "5901:5901"  # VNC port
+      - "9222:9222"  # Chrome remote debugging port
+      - "8003:8003"  # API server port
+      - "8080:8080"  # HTTP server port
+    environment:
+      - ANONYMIZED_TELEMETRY=${ANONYMIZED_TELEMETRY:-false}
+      - CHROME_PATH=/usr/bin/google-chrome
+      - CHROME_USER_DATA=/app/data/chrome_data
+      - CHROME_PERSISTENT_SESSION=${CHROME_PERSISTENT_SESSION:-false}
+      - CHROME_CDP=${CHROME_CDP:-http://localhost:9222}
+      - DISPLAY=:99
+      - PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
+      - RESOLUTION=${RESOLUTION:-1024x768x24}
+      - RESOLUTION_WIDTH=${RESOLUTION_WIDTH:-1024}
+      - RESOLUTION_HEIGHT=${RESOLUTION_HEIGHT:-768}
+      - VNC_PASSWORD=${VNC_PASSWORD:-vncpassword}
+      - CHROME_DEBUGGING_PORT=9222
+      - CHROME_DEBUGGING_HOST=localhost
+      - CHROME_FLAGS=${CHROME_FLAGS:-"--single-process --no-first-run --no-default-browser-check --disable-background-networking --disable-background-timer-throttling --disable-backgrounding-occluded-windows --disable-breakpad --disable-component-extensions-with-background-pages --disable-dev-shm-usage --disable-extensions --disable-features=TranslateUI --disable-ipc-flooding-protection --disable-renderer-backgrounding --enable-features=NetworkServiceInProcess2 --force-color-profile=srgb --metrics-recording-only --mute-audio --no-sandbox --disable-gpu"}
+    volumes:
+      - /tmp/.X11-unix:/tmp/.X11-unix
+    restart: unless-stopped
+    shm_size: '2gb'
+    cap_add:
+      - SYS_ADMIN
+    security_opt:
+      - seccomp=unconfined
+    tmpfs:
+      - /tmp
+    healthcheck:
+      test: ["CMD", "nc", "-z", "localhost", "5901"]
+      interval: 10s
+      timeout: 5s
+      retries: 3

app/daytona/docker/entrypoint.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# Start supervisord in the foreground to properly manage child processes
+exec /usr/bin/supervisord -n -c /etc/supervisor/conf.d/supervisord.conf

app/daytona/docker/requirements.txt

@@ -0,0 +1,6 @@
+fastapi==0.115.12
+uvicorn==0.34.0
+pyautogui==0.9.54
+pillow==10.2.0
+pydantic==2.6.1
+pytesseract==0.3.13

app/daytona/docker/server.py

@@ -0,0 +1,29 @@
+from fastapi import FastAPI, Request
+from fastapi.staticfiles import StaticFiles
+from starlette.middleware.base import BaseHTTPMiddleware
+import uvicorn
+import os
+
+# Ensure we're serving from the /workspace directory
+workspace_dir = "/workspace"
+
+class WorkspaceDirMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):
+        # Check if workspace directory exists and recreate if deleted
+        if not os.path.exists(workspace_dir):
+            print(f"Workspace directory {workspace_dir} not found, recreating...")
+            os.makedirs(workspace_dir, exist_ok=True)
+        return await call_next(request)
+
+app = FastAPI()
+app.add_middleware(WorkspaceDirMiddleware)
+
+# Initial directory creation
+os.makedirs(workspace_dir, exist_ok=True)
+app.mount('/', StaticFiles(directory=workspace_dir, html=True), name='site')
+
+# This is needed for the import string approach with uvicorn
+if __name__ == '__main__':
+    print(f"Starting server with auto-reload, serving files from: {workspace_dir}")
+    # Don't use reload directly in the run call
+    uvicorn.run("server:app", host="0.0.0.0", port=8080, reload=True) 

app/daytona/docker/supervisord.conf

@@ -0,0 +1,94 @@
+[supervisord]
+user=root
+nodaemon=true
+logfile=/dev/stdout
+logfile_maxbytes=0
+loglevel=debug
+
+[program:xvfb]
+command=Xvfb :99 -screen 0 %(ENV_RESOLUTION)s -ac +extension GLX +render -noreset
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+priority=100
+startsecs=3
+stopsignal=TERM
+stopwaitsecs=10
+
+[program:vnc_setup]
+command=bash -c "mkdir -p ~/.vnc && echo '%(ENV_VNC_PASSWORD)s' | vncpasswd -f > ~/.vnc/passwd && chmod 600 ~/.vnc/passwd && ls -la ~/.vnc/passwd"
+autorestart=false
+startsecs=0
+priority=150
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:x11vnc]
+command=bash -c "mkdir -p /var/log && touch /var/log/x11vnc.log && chmod 666 /var/log/x11vnc.log && sleep 5 && DISPLAY=:99 x11vnc -display :99 -forever -shared -rfbauth /root/.vnc/passwd -rfbport 5901 -o /var/log/x11vnc.log"
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+priority=200
+startretries=10
+startsecs=10
+stopsignal=TERM
+stopwaitsecs=10
+depends_on=vnc_setup,xvfb
+
+[program:x11vnc_log]
+command=bash -c "mkdir -p /var/log && touch /var/log/x11vnc.log && tail -f /var/log/x11vnc.log"
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+priority=250
+stopsignal=TERM
+stopwaitsecs=5
+depends_on=x11vnc
+
+[program:novnc]
+command=bash -c "sleep 5 && cd /opt/novnc && ./utils/novnc_proxy --vnc localhost:5901 --listen 0.0.0.0:6080 --web /opt/novnc"
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+priority=300
+startretries=5
+startsecs=3
+depends_on=x11vnc
+
+[program:http_server]
+command=python /app/server.py
+directory=/app
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+priority=400
+startretries=5
+startsecs=5
+stopsignal=TERM
+stopwaitsecs=10
+
+[program:browser_api]
+command=python /app/browser_api.py
+directory=/app
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+priority=400
+startretries=5
+startsecs=5
+stopsignal=TERM
+stopwaitsecs=10

app/daytona/sandbox.py

@@ -0,0 +1,144 @@
+from daytona_sdk import Daytona, DaytonaConfig, CreateSandboxFromImageParams, Sandbox, SessionExecuteRequest, Resources, SandboxState
+from dotenv import load_dotenv
+from utils.logger import logger
+from utils.config import config
+from utils.config import Configuration
+
+load_dotenv()
+
+logger.debug("Initializing Daytona sandbox configuration")
+daytona_config = DaytonaConfig(
+    api_key=config.DAYTONA_API_KEY,
+    server_url=config.DAYTONA_SERVER_URL,
+    target=config.DAYTONA_TARGET
+)
+
+if daytona_config.api_key:
+    logger.debug("Daytona API key configured successfully")
+else:
+    logger.warning("No Daytona API key found in environment variables")
+
+if daytona_config.server_url:
+    logger.debug(f"Daytona server URL set to: {daytona_config.server_url}")
+else:
+    logger.warning("No Daytona server URL found in environment variables")
+
+if daytona_config.target:
+    logger.debug(f"Daytona target set to: {daytona_config.target}")
+else:
+    logger.warning("No Daytona target found in environment variables")
+
+daytona = Daytona(daytona_config)
+logger.debug("Daytona client initialized")
+
+async def get_or_start_sandbox(sandbox_id: str):
+    """Retrieve a sandbox by ID, check its state, and start it if needed."""
+    
+    logger.info(f"Getting or starting sandbox with ID: {sandbox_id}")
+    
+    try:
+        sandbox = daytona.get(sandbox_id)
+        
+        # Check if sandbox needs to be started
+        if sandbox.state == SandboxState.ARCHIVED or sandbox.state == SandboxState.STOPPED:
+            logger.info(f"Sandbox is in {sandbox.state} state. Starting...")
+            try:
+                daytona.start(sandbox)
+                # Wait a moment for the sandbox to initialize
+                # sleep(5)
+                # Refresh sandbox state after starting
+                sandbox = daytona.get(sandbox_id)
+                
+                # Start supervisord in a session when restarting
+                start_supervisord_session(sandbox)
+            except Exception as e:
+                logger.error(f"Error starting sandbox: {e}")
+                raise e
+        
+        logger.info(f"Sandbox {sandbox_id} is ready")
+        return sandbox
+        
+    except Exception as e:
+        logger.error(f"Error retrieving or starting sandbox: {str(e)}")
+        raise e
+
+def start_supervisord_session(sandbox: Sandbox):
+    """Start supervisord in a session."""
+    session_id = "supervisord-session"
+    try:
+        logger.info(f"Creating session {session_id} for supervisord")
+        sandbox.process.create_session(session_id)
+        
+        # Execute supervisord command
+        sandbox.process.execute_session_command(session_id, SessionExecuteRequest(
+            command="exec /usr/bin/supervisord -n -c /etc/supervisor/conf.d/supervisord.conf",
+            var_async=True
+        ))
+        logger.info(f"Supervisord started in session {session_id}")
+    except Exception as e:
+        logger.error(f"Error starting supervisord session: {str(e)}")
+        raise e
+
+def create_sandbox(password: str, project_id: str = None):
+    """Create a new sandbox with all required services configured and running."""
+    
+    logger.debug("Creating new Daytona sandbox environment")
+    logger.debug("Configuring sandbox with browser-use image and environment variables")
+    
+    labels = None
+    if project_id:
+        logger.debug(f"Using sandbox_id as label: {project_id}")
+        labels = {'id': project_id}
+        
+    params = CreateSandboxFromImageParams(
+        image=Configuration.SANDBOX_IMAGE_NAME,
+        public=True,
+        labels=labels,
+        env_vars={
+            "CHROME_PERSISTENT_SESSION": "true",
+            "RESOLUTION": "1024x768x24",
+            "RESOLUTION_WIDTH": "1024",
+            "RESOLUTION_HEIGHT": "768",
+            "VNC_PASSWORD": password,
+            "ANONYMIZED_TELEMETRY": "false",
+            "CHROME_PATH": "",
+            "CHROME_USER_DATA": "",
+            "CHROME_DEBUGGING_PORT": "9222",
+            "CHROME_DEBUGGING_HOST": "localhost",
+            "CHROME_CDP": ""
+        },
+        resources=Resources(
+            cpu=2,
+            memory=4,
+            disk=5,
+        ),
+        auto_stop_interval=15,
+        auto_archive_interval=24 * 60,
+    )
+    
+    # Create the sandbox
+    sandbox = daytona.create(params)
+    logger.debug(f"Sandbox created with ID: {sandbox.id}")
+    
+    # Start supervisord in a session for new sandbox
+    start_supervisord_session(sandbox)
+    
+    logger.debug(f"Sandbox environment successfully initialized")
+    return sandbox
+
+async def delete_sandbox(sandbox_id: str):
+    """Delete a sandbox by its ID."""
+    logger.info(f"Deleting sandbox with ID: {sandbox_id}")
+    
+    try:
+        # Get the sandbox
+        sandbox = daytona.get(sandbox_id)
+        
+        # Delete the sandbox
+        daytona.delete(sandbox)
+        
+        logger.info(f"Successfully deleted sandbox {sandbox_id}")
+        return True
+    except Exception as e:
+        logger.error(f"Error deleting sandbox {sandbox_id}: {str(e)}")
+        raise e

app/daytona/tool_base.py

@@ -0,0 +1,89 @@
+
+from typing import Optional
+
+from agentpress.thread_manager import ThreadManager
+from app.tool.base import BaseTool, ToolResult
+from daytona_sdk import Sandbox
+from sandbox.sandbox import get_or_start_sandbox
+from utils.logger import logger
+from utils.files_utils import clean_path
+
+class SandboxToolsBase(BaseTool):
+    """Base class for all sandbox tools that provides project-based sandbox access."""
+
+    # Class variable to track if sandbox URLs have been printed
+    _urls_printed = False
+
+    def __init__(self, project_id: str, thread_manager: Optional[ThreadManager] = None):
+        super().__init__()
+        self.project_id = project_id
+        self.thread_manager = thread_manager
+        self.workspace_path = "/workspace"
+        self._sandbox = None
+        self._sandbox_id = None
+        self._sandbox_pass = None
+
+    async def _ensure_sandbox(self) -> Sandbox:
+        """Ensure we have a valid sandbox instance, retrieving it from the project if needed."""
+        if self._sandbox is None:
+            try:
+                # Get database client
+                client = await self.thread_manager.db.client
+
+                # Get project data
+                project = await client.table('projects').select('*').eq('project_id', self.project_id).execute()
+                if not project.data or len(project.data) == 0:
+                    raise ValueError(f"Project {self.project_id} not found")
+
+                project_data = project.data[0]
+                sandbox_info = project_data.get('sandbox', {})
+
+                if not sandbox_info.get('id'):
+                    raise ValueError(f"No sandbox found for project {self.project_id}")
+
+                # Store sandbox info
+                self._sandbox_id = sandbox_info['id']
+                self._sandbox_pass = sandbox_info.get('pass')
+
+                # Get or start the sandbox
+                self._sandbox = await get_or_start_sandbox(self._sandbox_id)
+
+                # # Log URLs if not already printed
+                # if not SandboxToolsBase._urls_printed:
+                #     vnc_link = self._sandbox.get_preview_link(6080)
+                #     website_link = self._sandbox.get_preview_link(8080)
+
+                #     vnc_url = vnc_link.url if hasattr(vnc_link, 'url') else str(vnc_link)
+                #     website_url = website_link.url if hasattr(website_link, 'url') else str(website_link)
+
+                #     print("\033[95m***")
+                #     print(f"VNC URL: {vnc_url}")
+                #     print(f"Website URL: {website_url}")
+                #     print("***\033[0m")
+                #     SandboxToolsBase._urls_printed = True
+
+            except Exception as e:
+                logger.error(f"Error retrieving sandbox for project {self.project_id}: {str(e)}", exc_info=True)
+                raise e
+
+        return self._sandbox
+
+    @property
+    def sandbox(self) -> Sandbox:
+        """Get the sandbox instance, ensuring it exists."""
+        if self._sandbox is None:
+            raise RuntimeError("Sandbox not initialized. Call _ensure_sandbox() first.")
+        return self._sandbox
+
+    @property
+    def sandbox_id(self) -> str:
+        """Get the sandbox ID, ensuring it exists."""
+        if self._sandbox_id is None:
+            raise RuntimeError("Sandbox ID not initialized. Call _ensure_sandbox() first.")
+        return self._sandbox_id
+
+    def clean_path(self, path: str) -> str:
+        """Clean and normalize a path to be relative to /workspace."""
+        cleaned_path = clean_path(path, self.workspace_path)
+        logger.debug(f"Cleaned path: {path} -> {cleaned_path}")
+        return cleaned_path

app/tool/computer_use_tool.py

@@ -0,0 +1,660 @@
+import os
+import time
+import base64
+import aiohttp
+import asyncio
+import logging
+from typing import Optional, Dict
+import os
+
+# from agentpress.tool import Tool, ToolResult, openapi_schema, xml_schema
+from daytona.tool_base import SandboxToolsBase, Sandbox
+
+KEYBOARD_KEYS = [
+    'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
+    'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
+    'enter', 'esc', 'backspace', 'tab', 'space', 'delete',
+    'ctrl', 'alt', 'shift', 'win',
+    'up', 'down', 'left', 'right',
+    'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12',
+    'ctrl+c', 'ctrl+v', 'ctrl+x', 'ctrl+z', 'ctrl+a', 'ctrl+s',
+    'alt+tab', 'alt+f4', 'ctrl+alt+delete'
+]
+
+class ComputerUseTool(SandboxToolsBase):
+    """Computer automation tool for controlling the sandbox browser and GUI."""
+
+    def __init__(self, sandbox: Sandbox):
+        """Initialize automation tool with sandbox connection."""
+        super().__init__(sandbox)
+        self.session = None
+        self.mouse_x = 0  # Track current mouse position
+        self.mouse_y = 0
+        # Get automation service URL using port 8000
+        self.api_base_url = self.sandbox.get_preview_link(8000)
+        logging.info(f"Initialized Computer Use Tool with API URL: {self.api_base_url}")
+
+    async def _get_session(self) -> aiohttp.ClientSession:
+        """Get or create aiohttp session for API requests."""
+        if self.session is None or self.session.closed:
+            self.session = aiohttp.ClientSession()
+        return self.session
+
+    async def _api_request(self, method: str, endpoint: str, data: Optional[Dict] = None) -> Dict:
+        """Send request to automation service API."""
+        try:
+            session = await self._get_session()
+            url = f"{self.api_base_url}/api{endpoint}"
+
+            logging.debug(f"API request: {method} {url} {data}")
+
+            if method.upper() == "GET":
+                async with session.get(url) as response:
+                    result = await response.json()
+            else:  # POST
+                async with session.post(url, json=data) as response:
+                    result = await response.json()
+
+            logging.debug(f"API response: {result}")
+            return result
+
+        except Exception as e:
+            logging.error(f"API request failed: {str(e)}")
+            return {"success": False, "error": str(e)}
+
+    async def cleanup(self):
+        """Clean up resources."""
+        if self.session and not self.session.closed:
+            await self.session.close()
+            self.session = None
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "move_to",
+            "description": "Move cursor to specified position",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "x": {
+                        "type": "number",
+                        "description": "X coordinate"
+                    },
+                    "y": {
+                        "type": "number",
+                        "description": "Y coordinate"
+                    }
+                },
+                "required": ["x", "y"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="move-to",
+        mappings=[
+            {"param_name": "x", "node_type": "attribute", "path": "."},
+            {"param_name": "y", "node_type": "attribute", "path": "."}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="move_to">
+        <parameter name="x">100</parameter>
+        <parameter name="y">200</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def move_to(self, x: float, y: float) -> ToolResult:
+        """Move cursor to specified position."""
+        try:
+            x_int = int(round(float(x)))
+            y_int = int(round(float(y)))
+
+            result = await self._api_request("POST", "/automation/mouse/move", {
+                "x": x_int,
+                "y": y_int
+            })
+
+            if result.get("success", False):
+                self.mouse_x = x_int
+                self.mouse_y = y_int
+                return ToolResult(success=True, output=f"Moved to ({x_int}, {y_int})")
+            else:
+                return ToolResult(success=False, output=f"Failed to move: {result.get('error', 'Unknown error')}")
+
+        except Exception as e:
+            return ToolResult(success=False, output=f"Failed to move: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "click",
+            "description": "Click at current or specified position",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "button": {
+                        "type": "string",
+                        "description": "Mouse button to click",
+                        "enum": ["left", "right", "middle"],
+                        "default": "left"
+                    },
+                    "x": {
+                        "type": "number",
+                        "description": "Optional X coordinate"
+                    },
+                    "y": {
+                        "type": "number",
+                        "description": "Optional Y coordinate"
+                    },
+                    "num_clicks": {
+                        "type": "integer",
+                        "description": "Number of clicks",
+                        "enum": [1, 2, 3],
+                        "default": 1
+                    }
+                }
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="click",
+        mappings=[
+            {"param_name": "x", "node_type": "attribute", "path": "x"},
+            {"param_name": "y", "node_type": "attribute", "path": "y"},
+            {"param_name": "button", "node_type": "attribute", "path": "button"},
+            {"param_name": "num_clicks", "node_type": "attribute", "path": "num_clicks"}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="click">
+        <parameter name="x">100</parameter>
+        <parameter name="y">200</parameter>
+        <parameter name="button">left</parameter>
+        <parameter name="num_clicks">1</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def click(self, x: Optional[float] = None, y: Optional[float] = None,
+                   button: str = "left", num_clicks: int = 1) -> ToolResult:
+        """Click at current or specified position."""
+        try:
+            x_val = x if x is not None else self.mouse_x
+            y_val = y if y is not None else self.mouse_y
+
+            x_int = int(round(float(x_val)))
+            y_int = int(round(float(y_val)))
+            num_clicks = int(num_clicks)
+
+            result = await self._api_request("POST", "/automation/mouse/click", {
+                "x": x_int,
+                "y": y_int,
+                "clicks": num_clicks,
+                "button": button.lower()
+            })
+
+            if result.get("success", False):
+                self.mouse_x = x_int
+                self.mouse_y = y_int
+                return ToolResult(success=True,
+                                output=f"{num_clicks} {button} click(s) performed at ({x_int}, {y_int})")
+            else:
+                return ToolResult(success=False, output=f"Failed to click: {result.get('error', 'Unknown error')}")
+        except Exception as e:
+            return ToolResult(success=False, output=f"Failed to click: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "scroll",
+            "description": "Scroll the mouse wheel at current position",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "amount": {
+                        "type": "integer",
+                        "description": "Scroll amount (positive for up, negative for down)",
+                        "minimum": -10,
+                        "maximum": 10
+                    }
+                },
+                "required": ["amount"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="scroll",
+        mappings=[
+            {"param_name": "amount", "node_type": "attribute", "path": "amount"}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="scroll">
+        <parameter name="amount">-3</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def scroll(self, amount: int) -> ToolResult:
+        """
+        Scroll the mouse wheel at current position.
+        Positive values scroll up, negative values scroll down.
+        """
+        try:
+            amount = int(float(amount))
+            amount = max(-10, min(10, amount))
+
+            result = await self._api_request("POST", "/automation/mouse/scroll", {
+                "clicks": amount,
+                "x": self.mouse_x,
+                "y": self.mouse_y
+            })
+
+            if result.get("success", False):
+                direction = "up" if amount > 0 else "down"
+                steps = abs(amount)
+                return ToolResult(success=True,
+                                output=f"Scrolled {direction} {steps} step(s) at position ({self.mouse_x}, {self.mouse_y})")
+            else:
+                return ToolResult(success=False, output=f"Failed to scroll: {result.get('error', 'Unknown error')}")
+        except Exception as e:
+            return ToolResult(success=False, output=f"Failed to scroll: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "typing",
+            "description": "Type specified text",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "text": {
+                        "type": "string",
+                        "description": "Text to type"
+                    }
+                },
+                "required": ["text"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="typing",
+        mappings=[
+            {"param_name": "text", "node_type": "content", "path": "text"}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="typing">
+        <parameter name="text">Hello World!</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def typing(self, text: str) -> ToolResult:
+        """Type specified text."""
+        try:
+            text = str(text)
+
+            result = await self._api_request("POST", "/automation/keyboard/write", {
+                "message": text,
+                "interval": 0.01
+            })
+
+            if result.get("success", False):
+                return ToolResult(success=True, output=f"Typed: {text}")
+            else:
+                return ToolResult(success=False, output=f"Failed to type: {result.get('error', 'Unknown error')}")
+        except Exception as e:
+            return ToolResult(success=False, output=f"Failed to type: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "press",
+            "description": "Press and release a key",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "key": {
+                        "type": "string",
+                        "description": "Key to press",
+                        "enum": KEYBOARD_KEYS
+                    }
+                },
+                "required": ["key"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="press",
+        mappings=[
+            {"param_name": "key", "node_type": "attribute", "path": "key"}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="press">
+        <parameter name="key">enter</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def press(self, key: str) -> ToolResult:
+        """Press and release a key."""
+        try:
+            key = str(key).lower()
+
+            result = await self._api_request("POST", "/automation/keyboard/press", {
+                "keys": key,
+                "presses": 1
+            })
+
+            if result.get("success", False):
+                return ToolResult(success=True, output=f"Pressed key: {key}")
+            else:
+                return ToolResult(success=False, output=f"Failed to press key: {result.get('error', 'Unknown error')}")
+        except Exception as e:
+            return ToolResult(success=False, output=f"Failed to press key: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "wait",
+            "description": "Wait for specified duration",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "duration": {
+                        "type": "number",
+                        "description": "Duration in seconds",
+                        "default": 0.5
+                    }
+                }
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="wait",
+        mappings=[
+            {"param_name": "duration", "node_type": "attribute", "path": "duration"}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="wait">
+        <parameter name="duration">1.5</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def wait(self, duration: float = 0.5) -> ToolResult:
+        """Wait for specified duration."""
+        try:
+            duration = float(duration)
+            duration = max(0, min(10, duration))
+            await asyncio.sleep(duration)
+            return ToolResult(success=True, output=f"Waited {duration} seconds")
+        except Exception as e:
+            return ToolResult(success=False, output=f"Failed to wait: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "mouse_down",
+            "description": "Press a mouse button",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "button": {
+                        "type": "string",
+                        "description": "Mouse button to press",
+                        "enum": ["left", "right", "middle"],
+                        "default": "left"
+                    }
+                }
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="mouse-down",
+        mappings=[
+            {"param_name": "button", "node_type": "attribute", "path": "button"}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="mouse_down">
+        <parameter name="button">left</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def mouse_down(self, button: str = "left", x: Optional[float] = None, y: Optional[float] = None) -> ToolResult:
+        """Press a mouse button at current or specified position."""
+        try:
+            x_val = x if x is not None else self.mouse_x
+            y_val = y if y is not None else self.mouse_y
+
+            x_int = int(round(float(x_val)))
+            y_int = int(round(float(y_val)))
+
+            result = await self._api_request("POST", "/automation/mouse/down", {
+                "x": x_int,
+                "y": y_int,
+                "button": button.lower()
+            })
+
+            if result.get("success", False):
+                self.mouse_x = x_int
+                self.mouse_y = y_int
+                return ToolResult(success=True, output=f"{button} button pressed at ({x_int}, {y_int})")
+            else:
+                return ToolResult(success=False, output=f"Failed to press button: {result.get('error', 'Unknown error')}")
+        except Exception as e:
+            return ToolResult(success=False, output=f"Failed to press button: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "mouse_up",
+            "description": "Release a mouse button",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "button": {
+                        "type": "string",
+                        "description": "Mouse button to release",
+                        "enum": ["left", "right", "middle"],
+                        "default": "left"
+                    }
+                }
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="mouse-up",
+        mappings=[
+            {"param_name": "button", "node_type": "attribute", "path": "button"}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="mouse_up">
+        <parameter name="button">left</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def mouse_up(self, button: str = "left", x: Optional[float] = None, y: Optional[float] = None) -> ToolResult:
+        """Release a mouse button at current or specified position."""
+        try:
+            x_val = x if x is not None else self.mouse_x
+            y_val = y if y is not None else self.mouse_y
+
+            x_int = int(round(float(x_val)))
+            y_int = int(round(float(y_val)))
+
+            result = await self._api_request("POST", "/automation/mouse/up", {
+                "x": x_int,
+                "y": y_int,
+                "button": button.lower()
+            })
+
+            if result.get("success", False):
+                self.mouse_x = x_int
+                self.mouse_y = y_int
+                return ToolResult(success=True, output=f"{button} button released at ({x_int}, {y_int})")
+            else:
+                return ToolResult(success=False, output=f"Failed to release button: {result.get('error', 'Unknown error')}")
+        except Exception as e:
+            return ToolResult(success=False, output=f"Failed to release button: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "drag_to",
+            "description": "Drag cursor to specified position",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "x": {
+                        "type": "number",
+                        "description": "Target X coordinate"
+                    },
+                    "y": {
+                        "type": "number",
+                        "description": "Target Y coordinate"
+                    }
+                },
+                "required": ["x", "y"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="drag-to",
+        mappings=[
+            {"param_name": "x", "node_type": "attribute", "path": "x"},
+            {"param_name": "y", "node_type": "attribute", "path": "y"}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="drag_to">
+        <parameter name="x">500</parameter>
+        <parameter name="y">50</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def drag_to(self, x: float, y: float) -> ToolResult:
+        """Click and drag from current position to target position."""
+        try:
+            target_x = int(round(float(x)))
+            target_y = int(round(float(y)))
+            start_x = self.mouse_x
+            start_y = self.mouse_y
+
+            result = await self._api_request("POST", "/automation/mouse/drag", {
+                "x": target_x,
+                "y": target_y,
+                "duration": 0.3,
+                "button": "left"
+            })
+
+            if result.get("success", False):
+                self.mouse_x = target_x
+                self.mouse_y = target_y
+                return ToolResult(success=True,
+                                output=f"Dragged from ({start_x}, {start_y}) to ({target_x}, {target_y})")
+            else:
+                return ToolResult(success=False, output=f"Failed to drag: {result.get('error', 'Unknown error')}")
+        except Exception as e:
+            return ToolResult(success=False, output=f"Failed to drag: {str(e)}")
+
+    async def get_screenshot_base64(self) -> Optional[dict]:
+        """Capture screen and return as base64 encoded image."""
+        try:
+            result = await self._api_request("POST", "/automation/screenshot")
+
+            if "image" in result:
+                base64_str = result["image"]
+                timestamp = time.strftime("%Y%m%d_%H%M%S")
+
+                # Save screenshot to file
+                screenshots_dir = "screenshots"
+                if not os.path.exists(screenshots_dir):
+                    os.makedirs(screenshots_dir)
+
+                timestamped_filename = os.path.join(screenshots_dir, f"screenshot_{timestamp}.png")
+                latest_filename = "latest_screenshot.png"
+
+                # Decode base64 string and save to file
+                img_data = base64.b64decode(base64_str)
+                with open(timestamped_filename, 'wb') as f:
+                    f.write(img_data)
+
+                # Save a copy as the latest screenshot
+                with open(latest_filename, 'wb') as f:
+                    f.write(img_data)
+
+                return {
+                    "content_type": "image/png",
+                    "base64": base64_str,
+                    "timestamp": timestamp,
+                    "filename": timestamped_filename
+                }
+            else:
+                return None
+
+        except Exception as e:
+            print(f"[Screenshot] Error during screenshot process: {str(e)}")
+            return None
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "hotkey",
+            "description": "Press a key combination",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "keys": {
+                        "type": "string",
+                        "description": "Key combination to press",
+                        "enum": KEYBOARD_KEYS
+                    }
+                },
+                "required": ["keys"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="hotkey",
+        mappings=[
+            {"param_name": "keys", "node_type": "attribute", "path": "keys"}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="hotkey">
+        <parameter name="keys">ctrl+a</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def hotkey(self, keys: str) -> ToolResult:
+        """Press a key combination."""
+        try:
+            keys = str(keys).lower().strip()
+            key_sequence = keys.split('+')
+
+            result = await self._api_request("POST", "/automation/keyboard/hotkey", {
+                "keys": key_sequence,
+                "interval": 0.01
+            })
+
+            if result.get("success", False):
+                return ToolResult(success=True, output=f"Pressed key combination: {keys}")
+            else:
+                return ToolResult(success=False, output=f"Failed to press keys: {result.get('error', 'Unknown error')}")
+        except Exception as e:
+            return ToolResult(success=False, output=f"Failed to press keys: {str(e)}")
+
+if __name__ == "__main__":
+    print("This module should be imported, not run directly.")

app/tool/data_providers/ActiveJobsProvider.py

@@ -0,0 +1,57 @@
+from typing import Dict
+
+from agent.tools.data_providers.RapidDataProviderBase import RapidDataProviderBase, EndpointSchema
+
+
+class ActiveJobsProvider(RapidDataProviderBase):
+    def __init__(self):
+        endpoints: Dict[str, EndpointSchema] = {
+            "active_jobs": {
+                "route": "/active-ats-7d",
+                "method": "GET",
+                "name": "Active Jobs Search",
+                "description": "Get active job listings with various filter options.",
+                "payload": {
+                    "limit": "Optional. Number of jobs per API call (10-100). Default is 100.",
+                    "offset": "Optional. Offset for pagination. Default is 0.",
+                    "title_filter": "Optional. Search terms for job title.",
+                    "advanced_title_filter": "Optional. Advanced title filter with operators (can't be used with title_filter).",
+                    "location_filter": "Optional. Filter by location(s). Use full names like 'United States' not 'US'.",
+                    "description_filter": "Optional. Filter on job description content.",
+                    "organization_filter": "Optional. Filter by company name(s).",
+                    "description_type": "Optional. Return format for description: 'text' or 'html'. Leave empty to exclude descriptions.",
+                    "source": "Optional. Filter by ATS source.",
+                    "date_filter": "Optional. Filter by posting date (greater than).",
+                    "ai_employment_type_filter": "Optional. Filter by employment type (FULL_TIME, PART_TIME, etc).",
+                    "ai_work_arrangement_filter": "Optional. Filter by work arrangement (On-site, Hybrid, Remote OK, Remote Solely).",
+                    "ai_experience_level_filter": "Optional. Filter by experience level (0-2, 2-5, 5-10, 10+).",
+                    "li_organization_slug_filter": "Optional. Filter by LinkedIn company slug.",
+                    "li_organization_slug_exclusion_filter": "Optional. Exclude LinkedIn company slugs.",
+                    "li_industry_filter": "Optional. Filter by LinkedIn industry.",
+                    "li_organization_specialties_filter": "Optional. Filter by LinkedIn company specialties.",
+                    "li_organization_description_filter": "Optional. Filter by LinkedIn company description."
+                }
+            }
+        }
+           
+        base_url = "https://active-jobs-db.p.rapidapi.com"
+        super().__init__(base_url, endpoints)
+
+
+if __name__ == "__main__":
+    from dotenv import load_dotenv
+    load_dotenv()
+    tool = ActiveJobsProvider()
+
+    # Example for searching active jobs
+    jobs = tool.call_endpoint(
+        route="active_jobs",
+        payload={
+            "limit": "10",
+            "offset": "0",
+            "title_filter": "\"Data Engineer\"",
+            "location_filter": "\"United States\" OR \"United Kingdom\"",
+            "description_type": "text"
+        }
+    )
+    print("Active Jobs:", jobs)

app/tool/data_providers/AmazonProvider.py

@@ -0,0 +1,191 @@
+from typing import Dict
+
+from agent.tools.data_providers.RapidDataProviderBase import RapidDataProviderBase, EndpointSchema
+
+
+class AmazonProvider(RapidDataProviderBase):
+    def __init__(self):
+        endpoints: Dict[str, EndpointSchema] = {
+            "search": {
+                "route": "/search",
+                "method": "GET",
+                "name": "Amazon Product Search",
+                "description": "Search for products on Amazon with various filters and parameters.",
+                "payload": {
+                    "query": "Search query (supports both free-form text queries or a product asin)",
+                    "page": "Results page to return (default: 1)",
+                    "country": "Sets the Amazon domain, marketplace country, language and currency (default: US)",
+                    "sort_by": "Return the results in a specific sort order (RELEVANCE, LOWEST_PRICE, HIGHEST_PRICE, REVIEWS, NEWEST, BEST_SELLERS)",
+                    "product_condition": "Return products in a specific condition (ALL, NEW, USED, RENEWED, COLLECTIBLE)",
+                    "is_prime": "Only return prime products (boolean)",
+                    "deals_and_discounts": "Return deals and discounts in a specific condition (NONE, ALL_DISCOUNTS, TODAYS_DEALS)",
+                    "category_id": "Find products in a specific category / department (optional)",
+                    "category": "Filter by specific numeric Amazon category (optional)",
+                    "min_price": "Only return product offers with price greater than a certain value (optional)",
+                    "max_price": "Only return product offers with price lower than a certain value (optional)",
+                    "brand": "Find products with a specific brand (optional)",
+                    "seller_id": "Find products sold by specific seller (optional)",
+                    "four_stars_and_up": "Return product listings with ratings of 4 stars & up (optional)",
+                    "additional_filters": "Any filters available on the Amazon page but not part of this endpoint's parameters (optional)"
+                }
+            },
+            "product-details": {
+                "route": "/product-details",
+                "method": "GET",
+                "name": "Amazon Product Details",
+                "description": "Get detailed information about specific Amazon products by ASIN.",
+                "payload": {
+                    "asin": "Product ASIN for which to get details. Supports batching of up to 10 ASINs in a single request, separated by comma.",
+                    "country": "Sets the Amazon domain, marketplace country, language and currency (default: US)",
+                    "more_info_query": "A query to search and get more info about the product as part of Product Information, Customer Q&As, and Customer Reviews (optional)",
+                    "fields": "A comma separated list of product fields to include in the response (field projection). By default all fields are returned. (optional)"
+                }
+            },
+            "products-by-category": {
+                "route": "/products-by-category",
+                "method": "GET",
+                "name": "Amazon Products by Category",
+                "description": "Get products from a specific Amazon category.",
+                "payload": {
+                    "category_id": "The Amazon category for which to return results. Multiple category values can be separated by comma.",
+                    "page": "Page to return (default: 1)",
+                    "country": "Sets the Amazon domain, marketplace country, language and currency (default: US)",
+                    "sort_by": "Return the results in a specific sort order (RELEVANCE, LOWEST_PRICE, HIGHEST_PRICE, REVIEWS, NEWEST, BEST_SELLERS)",
+                    "min_price": "Only return product offers with price greater than a certain value (optional)",
+                    "max_price": "Only return product offers with price lower than a certain value (optional)",
+                    "product_condition": "Return products in a specific condition (ALL, NEW, USED, RENEWED, COLLECTIBLE)",
+                    "brand": "Only return products of a specific brand. Multiple brands can be specified as a comma separated list (optional)",
+                    "is_prime": "Only return prime products (boolean)",
+                    "deals_and_discounts": "Return deals and discounts in a specific condition (NONE, ALL_DISCOUNTS, TODAYS_DEALS)",
+                    "four_stars_and_up": "Return product listings with ratings of 4 stars & up (optional)",
+                    "additional_filters": "Any filters available on the Amazon page but not part of this endpoint's parameters (optional)"
+                }
+            },
+            "product-reviews": {
+                "route": "/product-reviews",
+                "method": "GET",
+                "name": "Amazon Product Reviews",
+                "description": "Get customer reviews for a specific Amazon product by ASIN.",
+                "payload": {
+                    "asin": "Product asin for which to get reviews.",
+                    "country": "Sets the Amazon domain, marketplace country, language and currency (default: US)",
+                    "page": "Results page to return (default: 1)",
+                    "sort_by": "Return reviews in a specific sort order (TOP_REVIEWS, MOST_RECENT)",
+                    "star_rating": "Only return reviews with a specific star rating (ALL, 5_STARS, 4_STARS, 3_STARS, 2_STARS, 1_STARS, POSITIVE, CRITICAL)",
+                    "verified_purchases_only": "Only return reviews by reviewers who made a verified purchase (boolean)",
+                    "images_or_videos_only": "Only return reviews containing images and / or videos (boolean)",
+                    "current_format_only": "Only return reviews of the current format (product variant - e.g. Color) (boolean)"
+                }
+            },
+            "seller-profile": {
+                "route": "/seller-profile",
+                "method": "GET",
+                "name": "Amazon Seller Profile",
+                "description": "Get detailed information about a specific Amazon seller by Seller ID.",
+                "payload": {
+                    "seller_id": "The Amazon Seller ID for which to get seller profile details",
+                    "country": "Sets the Amazon domain, marketplace country, language and currency (default: US)",
+                    "fields": "A comma separated list of seller profile fields to include in the response (field projection). By default all fields are returned. (optional)"
+                }
+            },
+            "seller-reviews": {
+                "route": "/seller-reviews",
+                "method": "GET",
+                "name": "Amazon Seller Reviews",
+                "description": "Get customer reviews for a specific Amazon seller by Seller ID.",
+                "payload": {
+                    "seller_id": "The Amazon Seller ID for which to get seller reviews",
+                    "country": "Sets the Amazon domain, marketplace country, language and currency (default: US)",
+                    "star_rating": "Only return reviews with a specific star rating or positive / negative sentiment (ALL, 5_STARS, 4_STARS, 3_STARS, 2_STARS, 1_STARS, POSITIVE, CRITICAL)",
+                    "page": "The page of seller feedback results to retrieve (default: 1)",
+                    "fields": "A comma separated list of seller review fields to include in the response (field projection). By default all fields are returned. (optional)"
+                }
+            }
+        }
+        base_url = "https://real-time-amazon-data.p.rapidapi.com"
+        super().__init__(base_url, endpoints)
+
+
+if __name__ == "__main__":
+    from dotenv import load_dotenv
+    load_dotenv()
+    tool = AmazonProvider()
+
+    # Example for product search
+    search_result = tool.call_endpoint(
+        route="search",
+        payload={
+            "query": "Phone",
+            "page": 1,
+            "country": "US",
+            "sort_by": "RELEVANCE",
+            "product_condition": "ALL",
+            "is_prime": False,
+            "deals_and_discounts": "NONE"
+        }
+    )
+    print("Search Result:", search_result)
+    
+    # Example for product details
+    details_result = tool.call_endpoint(
+        route="product-details",
+        payload={
+            "asin": "B07ZPKBL9V",
+            "country": "US"
+        }
+    )
+    print("Product Details:", details_result)
+    
+    # Example for products by category
+    category_result = tool.call_endpoint(
+        route="products-by-category",
+        payload={
+            "category_id": "2478868012",
+            "page": 1,
+            "country": "US",
+            "sort_by": "RELEVANCE",
+            "product_condition": "ALL",
+            "is_prime": False,
+            "deals_and_discounts": "NONE"
+        }
+    )
+    print("Category Products:", category_result)
+    
+    # Example for product reviews
+    reviews_result = tool.call_endpoint(
+        route="product-reviews",
+        payload={
+            "asin": "B07ZPKN6YR",
+            "country": "US",
+            "page": 1,
+            "sort_by": "TOP_REVIEWS",
+            "star_rating": "ALL",
+            "verified_purchases_only": False,
+            "images_or_videos_only": False,
+            "current_format_only": False
+        }
+    )
+    print("Product Reviews:", reviews_result)
+    
+    # Example for seller profile
+    seller_result = tool.call_endpoint(
+        route="seller-profile",
+        payload={
+            "seller_id": "A02211013Q5HP3OMSZC7W",
+            "country": "US"
+        }
+    )
+    print("Seller Profile:", seller_result)
+    
+    # Example for seller reviews
+    seller_reviews_result = tool.call_endpoint(
+        route="seller-reviews",
+        payload={
+            "seller_id": "A02211013Q5HP3OMSZC7W",
+            "country": "US",
+            "star_rating": "ALL",
+            "page": 1
+        }
+    )
+    print("Seller Reviews:", seller_reviews_result)
+

app/tool/data_providers/LinkedinProvider.py

@@ -0,0 +1,250 @@
+from typing import Dict
+
+from agent.tools.data_providers.RapidDataProviderBase import RapidDataProviderBase, EndpointSchema
+
+
+class LinkedinProvider(RapidDataProviderBase):
+    def __init__(self):
+        endpoints: Dict[str, EndpointSchema] = {
+            "person": {
+                "route": "/person",
+                "method": "POST",
+                "name": "Person Data",
+                "description": "Fetches any Linkedin profiles data including skills, certificates, experiences, qualifications and much more.",
+                "payload": {
+                    "link": "LinkedIn Profile URL"
+                }
+            },
+            "person_urn": {
+                "route": "/person_urn",
+                "method": "POST",
+                "name": "Person Data (Using Urn)",
+                "description": "It takes profile urn instead of profile public identifier in input",
+                "payload": {
+                    "link": "LinkedIn Profile URL or URN"
+                }
+            },
+            "person_deep": {
+                "route": "/person_deep",
+                "method": "POST",
+                "name": "Person Data (Deep)",
+                "description": "Fetches all experiences, educations, skills, languages, publications... related to a profile.",
+                "payload": {
+                    "link": "LinkedIn Profile URL"
+                }
+            },
+            "profile_updates": {
+                "route": "/profile_updates",
+                "method": "GET",
+                "name": "Person Posts (WITH PAGINATION)",
+                "description": "Fetches posts of a linkedin profile alongwith reactions, comments, postLink and reposts data.",
+                "payload": {
+                    "profile_url": "LinkedIn Profile URL",
+                    "page": "Page number",
+                    "reposts": "Include reposts (1 or 0)",
+                    "comments": "Include comments (1 or 0)"
+                }
+            },
+            "profile_recent_comments": {
+                "route": "/profile_recent_comments",
+                "method": "POST",
+                "name": "Person Recent Activity (Comments on Posts)",
+                "description": "Fetches 20 most recent comments posted by a linkedin user (per page).",
+                "payload": {
+                    "profile_url": "LinkedIn Profile URL",
+                    "page": "Page number",
+                    "paginationToken": "Token for pagination"
+                }
+            },
+            "comments_from_recent_activity": {
+                "route": "/comments_from_recent_activity",
+                "method": "GET",
+                "name": "Comments from recent activity",
+                "description": "Fetches recent comments posted by a person as per his recent activity tab.",
+                "payload": {
+                    "profile_url": "LinkedIn Profile URL",
+                    "page": "Page number"
+                }
+            },
+            "person_skills": {
+                "route": "/person_skills",
+                "method": "POST",
+                "name": "Person Skills",
+                "description": "Scraper all skills of a linkedin user",
+                "payload": {
+                    "link": "LinkedIn Profile URL"
+                }
+            },
+            "email_to_linkedin_profile": {
+                "route": "/email_to_linkedin_profile",
+                "method": "POST",
+                "name": "Email to LinkedIn Profile",
+                "description": "Finds LinkedIn profile associated with an email address",
+                "payload": {
+                    "email": "Email address to search"
+                }
+            },
+            "company": {
+                "route": "/company",
+                "method": "POST",
+                "name": "Company Data",
+                "description": "Fetches LinkedIn company profile data",
+                "payload": {
+                    "link": "LinkedIn Company URL"
+                }
+            },
+            "web_domain": {
+                "route": "/web-domain",
+                "method": "POST",
+                "name": "Web Domain to Company",
+                "description": "Fetches LinkedIn company profile data from a web domain",
+                "payload": {
+                    "link": "Website domain (e.g., huzzle.app)"
+                }
+            },
+            "similar_profiles": {
+                "route": "/similar_profiles",
+                "method": "GET",
+                "name": "Similar Profiles",
+                "description": "Fetches profiles similar to a given LinkedIn profile",
+                "payload": {
+                    "profileUrl": "LinkedIn Profile URL"
+                }
+            },
+            "company_jobs": {
+                "route": "/company_jobs",
+                "method": "POST",
+                "name": "Company Jobs",
+                "description": "Fetches job listings from a LinkedIn company page",
+                "payload": {
+                    "company_url": "LinkedIn Company URL",
+                    "count": "Number of job listings to fetch"
+                }
+            },
+            "company_updates": {
+                "route": "/company_updates",
+                "method": "GET",
+                "name": "Company Posts",
+                "description": "Fetches posts from a LinkedIn company page",
+                "payload": {
+                    "company_url": "LinkedIn Company URL",
+                    "page": "Page number",
+                    "reposts": "Include reposts (0, 1, or 2)",
+                    "comments": "Include comments (0, 1, or 2)"
+                }
+            },
+            "company_employee": {
+                "route": "/company_employee",
+                "method": "GET",
+                "name": "Company Employees",
+                "description": "Fetches employees of a LinkedIn company using company ID",
+                "payload": {
+                    "companyId": "LinkedIn Company ID",
+                    "page": "Page number"
+                }
+            },
+            "company_updates_post": {
+                "route": "/company_updates",
+                "method": "POST",
+                "name": "Company Posts (POST)",
+                "description": "Fetches posts from a LinkedIn company page with specific count parameters",
+                "payload": {
+                    "company_url": "LinkedIn Company URL",
+                    "posts": "Number of posts to fetch",
+                    "comments": "Number of comments to fetch per post",
+                    "reposts": "Number of reposts to fetch"
+                }
+            },
+            "search_posts_with_filters": {
+                "route": "/search_posts_with_filters",
+                "method": "GET",
+                "name": "Search Posts With Filters",
+                "description": "Searches LinkedIn posts with various filtering options",
+                "payload": {
+                    "query": "Keywords/Search terms (text you put in LinkedIn search bar)",
+                    "page": "Page number (1-100, each page contains 20 results)",
+                    "sort_by": "Sort method: 'relevance' (Top match) or 'date_posted' (Latest)",
+                    "author_job_title": "Filter by job title of author (e.g., CEO)",
+                    "content_type": "Type of content post contains (photos, videos, liveVideos, collaborativeArticles, documents)",
+                    "from_member": "URN of person who posted (comma-separated for multiple)",
+                    "from_organization": "ID of organization who posted (comma-separated for multiple)",
+                    "author_company": "ID of company author works for (comma-separated for multiple)",
+                    "author_industry": "URN of industry author is connected with (comma-separated for multiple)",
+                    "mentions_member": "URN of person mentioned in post (comma-separated for multiple)",
+                    "mentions_organization": "ID of organization mentioned in post (comma-separated for multiple)"
+                }
+            },
+            "search_jobs": {
+                "route": "/search_jobs",
+                "method": "GET",
+                "name": "Search Jobs",
+                "description": "Searches LinkedIn jobs with various filtering options",
+                "payload": {
+                    "query": "Job search keywords (e.g., Software developer)",
+                    "page": "Page number",
+                    "searchLocationId": "Location ID for job search (get from Suggestion location endpoint)",
+                    "easyApply": "Filter for easy apply jobs (true or false)",
+                    "experience": "Experience level required (1=Internship, 2=Entry level, 3=Associate, 4=Mid senior, 5=Director, 6=Executive, comma-separated)",
+                    "jobType": "Job type (F=Full time, P=Part time, C=Contract, T=Temporary, V=Volunteer, I=Internship, O=Other, comma-separated)",
+                    "postedAgo": "Time jobs were posted in seconds (e.g., 3600 for past hour)",
+                    "workplaceType": "Workplace type (1=On-Site, 2=Remote, 3=Hybrid, comma-separated)",
+                    "sortBy": "Sort method (DD=most recent, R=most relevant)",
+                    "companyIdsList": "List of company IDs, comma-separated",
+                    "industryIdsList": "List of industry IDs, comma-separated",
+                    "functionIdsList": "List of function IDs, comma-separated",
+                    "titleIdsList": "List of job title IDs, comma-separated",
+                    "locationIdsList": "List of location IDs within specified searchLocationId country, comma-separated"
+                }
+            },
+            "search_people_with_filters": {
+                "route": "/search_people_with_filters",
+                "method": "POST",
+                "name": "Search People With Filters",
+                "description": "Searches LinkedIn profiles with detailed filtering options",
+                "payload": {
+                    "keyword": "General search keyword",
+                    "page": "Page number",
+                    "title_free_text": "Job title to filter by (e.g., CEO)",
+                    "company_free_text": "Company name to filter by",
+                    "first_name": "First name of person",
+                    "last_name": "Last name of person",
+                    "current_company_list": "List of current companies (comma-separated IDs)",
+                    "past_company_list": "List of past companies (comma-separated IDs)",
+                    "location_list": "List of locations (comma-separated IDs)",
+                    "language_list": "List of languages (comma-separated)",
+                    "service_catagory_list": "List of service categories (comma-separated)",
+                    "school_free_text": "School name to filter by",
+                    "industry_list": "List of industries (comma-separated IDs)",
+                    "school_list": "List of schools (comma-separated IDs)"
+                }
+            },
+            "search_company_with_filters": {
+                "route": "/search_company_with_filters",
+                "method": "POST",
+                "name": "Search Company With Filters",
+                "description": "Searches LinkedIn companies with detailed filtering options",
+                "payload": {
+                    "keyword": "General search keyword",
+                    "page": "Page number",
+                    "company_size_list": "List of company sizes (comma-separated, e.g., A,D)",
+                    "hasJobs": "Filter companies with jobs (true or false)",
+                    "location_list": "List of location IDs (comma-separated)",
+                    "industry_list": "List of industry IDs (comma-separated)"
+                }
+            }
+        }
+        base_url = "https://linkedin-data-scraper.p.rapidapi.com"
+        super().__init__(base_url, endpoints)
+
+
+if __name__ == "__main__":
+    from dotenv import load_dotenv
+    load_dotenv()
+    tool = LinkedinProvider()
+
+    result = tool.call_endpoint(
+        route="comments_from_recent_activity",
+        payload={"profile_url": "https://www.linkedin.com/in/adamcohenhillel/", "page": 1}
+    )
+    print(result)
+

app/tool/data_providers/RapidDataProviderBase.py

@@ -0,0 +1,61 @@
+import os
+import requests
+from typing import Dict, Any, Optional, TypedDict, Literal
+
+
+class EndpointSchema(TypedDict):
+    route: str
+    method: Literal['GET', 'POST']
+    name: str
+    description: str
+    payload: Dict[str, Any]
+
+
+class RapidDataProviderBase:
+    def __init__(self, base_url: str, endpoints: Dict[str, EndpointSchema]):
+        self.base_url = base_url
+        self.endpoints = endpoints
+    
+    def get_endpoints(self):
+        return self.endpoints
+    
+    def call_endpoint(
+            self,
+            route: str,
+            payload: Optional[Dict[str, Any]] = None
+    ):
+        """
+        Call an API endpoint with the given parameters and data.
+        
+        Args:
+            endpoint (EndpointSchema): The endpoint configuration dictionary
+            params (dict, optional): Query parameters for GET requests
+            payload (dict, optional): JSON payload for POST requests
+            
+        Returns:
+            dict: The JSON response from the API
+        """
+        if route.startswith("/"):
+            route = route[1:]
+
+        endpoint = self.endpoints.get(route)
+        if not endpoint:
+            raise ValueError(f"Endpoint {route} not found")
+        
+        url = f"{self.base_url}{endpoint['route']}"
+        
+        headers = {
+            "x-rapidapi-key": os.getenv("RAPID_API_KEY"),
+            "x-rapidapi-host": url.split("//")[1].split("/")[0],
+            "Content-Type": "application/json"
+        }
+
+        method = endpoint.get('method', 'GET').upper()
+        
+        if method == 'GET':
+            response = requests.get(url, params=payload, headers=headers)
+        elif method == 'POST':
+            response = requests.post(url, json=payload, headers=headers)
+        else:
+            raise ValueError(f"Unsupported HTTP method: {method}")
+        return response.json()

app/tool/data_providers/TwitterProvider.py

@@ -0,0 +1,240 @@
+from typing import Dict
+
+from agent.tools.data_providers.RapidDataProviderBase import RapidDataProviderBase, EndpointSchema
+
+
+class TwitterProvider(RapidDataProviderBase):
+    def __init__(self):
+        endpoints: Dict[str, EndpointSchema] = {
+            "user_info": {
+                "route": "/screenname.php",
+                "method": "GET",
+                "name": "Twitter User Info",
+                "description": "Get information about a Twitter user by screenname or user ID.",
+                "payload": {
+                    "screenname": "Twitter username without the @ symbol",
+                    "rest_id": "Optional Twitter user's ID. If provided, overwrites screenname parameter."
+                }
+            },
+            "timeline": {
+                "route": "/timeline.php",
+                "method": "GET",
+                "name": "User Timeline",
+                "description": "Get tweets from a user's timeline.",
+                "payload": {
+                    "screenname": "Twitter username without the @ symbol",
+                    "rest_id": "Optional parameter that overwrites the screenname",
+                    "cursor": "Optional pagination cursor"
+                }
+            },
+            "following": {
+                "route": "/following.php",
+                "method": "GET",
+                "name": "User Following",
+                "description": "Get users that a specific user follows.",
+                "payload": {
+                    "screenname": "Twitter username without the @ symbol",
+                    "rest_id": "Optional parameter that overwrites the screenname",
+                    "cursor": "Optional pagination cursor"
+                }
+            },
+            "followers": {
+                "route": "/followers.php",
+                "method": "GET",
+                "name": "User Followers",
+                "description": "Get followers of a specific user.",
+                "payload": {
+                    "screenname": "Twitter username without the @ symbol",
+                    "cursor": "Optional pagination cursor"
+                }
+            },
+            "search": {
+                "route": "/search.php",
+                "method": "GET",
+                "name": "Twitter Search",
+                "description": "Search for tweets with a specific query.",
+                "payload": {
+                    "query": "Search query string",
+                    "cursor": "Optional pagination cursor",
+                    "search_type": "Optional search type (e.g. 'Top')"
+                }
+            },
+            "replies": {
+                "route": "/replies.php",
+                "method": "GET",
+                "name": "User Replies",
+                "description": "Get replies made by a user.",
+                "payload": {
+                    "screenname": "Twitter username without the @ symbol",
+                    "cursor": "Optional pagination cursor"
+                }
+            },
+            "check_retweet": {
+                "route": "/checkretweet.php",
+                "method": "GET",
+                "name": "Check Retweet",
+                "description": "Check if a user has retweeted a specific tweet.",
+                "payload": {
+                    "screenname": "Twitter username without the @ symbol",
+                    "tweet_id": "ID of the tweet to check"
+                }
+            },
+            "tweet": {
+                "route": "/tweet.php",
+                "method": "GET",
+                "name": "Get Tweet",
+                "description": "Get details of a specific tweet by ID.",
+                "payload": {
+                    "id": "ID of the tweet"
+                }
+            },
+            "tweet_thread": {
+                "route": "/tweet_thread.php",
+                "method": "GET",
+                "name": "Get Tweet Thread",
+                "description": "Get a thread of tweets starting from a specific tweet ID.",
+                "payload": {
+                    "id": "ID of the tweet",
+                    "cursor": "Optional pagination cursor"
+                }
+            },
+            "retweets": {
+                "route": "/retweets.php",
+                "method": "GET",
+                "name": "Get Retweets",
+                "description": "Get users who retweeted a specific tweet.",
+                "payload": {
+                    "id": "ID of the tweet",
+                    "cursor": "Optional pagination cursor"
+                }
+            },
+            "latest_replies": {
+                "route": "/latest_replies.php",
+                "method": "GET",
+                "name": "Get Latest Replies",
+                "description": "Get the latest replies to a specific tweet.",
+                "payload": {
+                    "id": "ID of the tweet",
+                    "cursor": "Optional pagination cursor"
+                }
+            }
+        }
+        base_url = "https://twitter-api45.p.rapidapi.com"
+        super().__init__(base_url, endpoints)
+
+
+if __name__ == "__main__":
+    from dotenv import load_dotenv
+    load_dotenv()
+    tool = TwitterProvider()
+
+    # Example for getting user info
+    user_info = tool.call_endpoint(
+        route="user_info",
+        payload={
+            "screenname": "elonmusk",
+            # "rest_id": "44196397"  # Optional, uncomment to use user ID instead of screenname
+        }
+    )
+    print("User Info:", user_info)
+    
+    # Example for getting user timeline
+    timeline = tool.call_endpoint(
+        route="timeline",
+        payload={
+            "screenname": "elonmusk",
+            # "cursor": "optional-cursor-value"  # Optional for pagination
+        }
+    )
+    print("Timeline:", timeline)
+    
+    # Example for getting user following
+    following = tool.call_endpoint(
+        route="following",
+        payload={
+            "screenname": "elonmusk",
+            # "cursor": "optional-cursor-value"  # Optional for pagination
+        }
+    )
+    print("Following:", following)
+    
+    # Example for getting user followers
+    followers = tool.call_endpoint(
+        route="followers",
+        payload={
+            "screenname": "elonmusk",
+            # "cursor": "optional-cursor-value"  # Optional for pagination
+        }
+    )
+    print("Followers:", followers)
+    
+    # Example for searching tweets
+    search_results = tool.call_endpoint(
+        route="search",
+        payload={
+            "query": "cybertruck",
+            "search_type": "Top"  # Optional, defaults to Top
+            # "cursor": "optional-cursor-value"  # Optional for pagination
+        }
+    )
+    print("Search Results:", search_results)
+    
+    # Example for getting user replies
+    replies = tool.call_endpoint(
+        route="replies",
+        payload={
+            "screenname": "elonmusk",
+            # "cursor": "optional-cursor-value"  # Optional for pagination
+        }
+    )
+    print("Replies:", replies)
+    
+    # Example for checking if user retweeted a tweet
+    check_retweet = tool.call_endpoint(
+        route="check_retweet",
+        payload={
+            "screenname": "elonmusk",
+            "tweet_id": "1671370010743263233"
+        }
+    )
+    print("Check Retweet:", check_retweet)
+    
+    # Example for getting tweet details
+    tweet = tool.call_endpoint(
+        route="tweet",
+        payload={
+            "id": "1671370010743263233"
+        }
+    )
+    print("Tweet:", tweet)
+    
+    # Example for getting a tweet thread
+    tweet_thread = tool.call_endpoint(
+        route="tweet_thread",
+        payload={
+            "id": "1738106896777699464",
+            # "cursor": "optional-cursor-value"  # Optional for pagination
+        }
+    )
+    print("Tweet Thread:", tweet_thread)
+    
+    # Example for getting retweets of a tweet
+    retweets = tool.call_endpoint(
+        route="retweets",
+        payload={
+            "id": "1700199139470942473",
+            # "cursor": "optional-cursor-value"  # Optional for pagination
+        }
+    )
+    print("Retweets:", retweets)
+    
+    # Example for getting latest replies to a tweet
+    latest_replies = tool.call_endpoint(
+        route="latest_replies",
+        payload={
+            "id": "1738106896777699464",
+            # "cursor": "optional-cursor-value"  # Optional for pagination
+        }
+    )
+    print("Latest Replies:", latest_replies)
+  

app/tool/data_providers/YahooFinanceProvider.py

@@ -0,0 +1,190 @@
+from typing import Dict
+
+from agent.tools.data_providers.RapidDataProviderBase import RapidDataProviderBase, EndpointSchema
+
+
+class YahooFinanceProvider(RapidDataProviderBase):
+    def __init__(self):
+        endpoints: Dict[str, EndpointSchema] = {
+            "get_tickers": {
+                "route": "/v2/markets/tickers",
+                "method": "GET",
+                "name": "Yahoo Finance Tickers",
+                "description": "Get financial tickers from Yahoo Finance with various filters and parameters.",
+                "payload": {
+                    "page": "Page number for pagination (optional, default: 1)",
+                    "type": "Asset class type (required): STOCKS, ETF, MUTUALFUNDS, or FUTURES",
+                }
+            },
+            "search": {
+                "route": "/v1/markets/search",
+                "method": "GET",
+                "name": "Yahoo Finance Search",
+                "description": "Search for financial instruments on Yahoo Finance",
+                "payload": {
+                    "search": "Search term (required)",
+                }
+            },
+            "get_news": {
+                "route": "/v2/markets/news",
+                "method": "GET",
+                "name": "Yahoo Finance News",
+                "description": "Get news related to specific tickers from Yahoo Finance",
+                "payload": {
+                    "tickers": "Stock symbol (optional, e.g., AAPL)",
+                    "type": "News type (optional): ALL, VIDEO, or PRESS_RELEASE",
+                }
+            },
+            "get_stock_module": {
+                "route": "/v1/markets/stock/modules",
+                "method": "GET",
+                "name": "Yahoo Finance Stock Module",
+                "description": "Get detailed information about a specific stock module",
+                "payload": {
+                    "ticker": "Company ticker symbol (required, e.g., AAPL)",
+                    "module": "Module to retrieve (required): asset-profile, financial-data, earnings, etc.",
+                }
+            },
+            "get_sma": {
+                "route": "/v1/markets/indicators/sma",
+                "method": "GET",
+                "name": "Yahoo Finance SMA Indicator",
+                "description": "Get Simple Moving Average (SMA) indicator data for a stock",
+                "payload": {
+                    "symbol": "Stock symbol (required, e.g., AAPL)",
+                    "interval": "Time interval (required): 5m, 15m, 30m, 1h, 1d, 1wk, 1mo, 3mo",
+                    "series_type": "Series type (required): open, close, high, low",
+                    "time_period": "Number of data points used for calculation (required)",
+                    "limit": "Limit the number of results (optional, default: 50)",
+                }
+            },
+            "get_rsi": {
+                "route": "/v1/markets/indicators/rsi",
+                "method": "GET",
+                "name": "Yahoo Finance RSI Indicator",
+                "description": "Get Relative Strength Index (RSI) indicator data for a stock",
+                "payload": {
+                    "symbol": "Stock symbol (required, e.g., AAPL)",
+                    "interval": "Time interval (required): 5m, 15m, 30m, 1h, 1d, 1wk, 1mo, 3mo",
+                    "series_type": "Series type (required): open, close, high, low",
+                    "time_period": "Number of data points used for calculation (required)",
+                    "limit": "Limit the number of results (optional, default: 50)",
+                }
+            },
+            "get_earnings_calendar": {
+                "route": "/v1/markets/calendar/earnings",
+                "method": "GET",
+                "name": "Yahoo Finance Earnings Calendar",
+                "description": "Get earnings calendar data for a specific date",
+                "payload": {
+                    "date": "Calendar date in yyyy-mm-dd format (optional, e.g., 2023-11-30)",
+                }
+            },
+            "get_insider_trades": {
+                "route": "/v1/markets/insider-trades",
+                "method": "GET",
+                "name": "Yahoo Finance Insider Trades",
+                "description": "Get recent insider trading activity",
+                "payload": {}
+            },
+        }
+        base_url = "https://yahoo-finance15.p.rapidapi.com/api"
+        super().__init__(base_url, endpoints)
+
+
+if __name__ == "__main__":
+    from dotenv import load_dotenv
+    load_dotenv()
+    tool = YahooFinanceProvider()
+
+    # Example for getting stock tickers
+    tickers_result = tool.call_endpoint(
+        route="get_tickers",
+        payload={
+            "page": 1,
+            "type": "STOCKS"
+        }
+    )
+    print("Tickers Result:", tickers_result)
+    
+    # Example for searching financial instruments
+    search_result = tool.call_endpoint(
+        route="search",
+        payload={
+            "search": "AA"
+        }
+    )
+    print("Search Result:", search_result)
+    
+    # Example for getting financial news
+    news_result = tool.call_endpoint(
+        route="get_news",
+        payload={
+            "tickers": "AAPL",
+            "type": "ALL"
+        }
+    )
+    print("News Result:", news_result)
+    
+    # Example for getting stock asset profile module
+    stock_module_result = tool.call_endpoint(
+        route="get_stock_module",
+        payload={
+            "ticker": "AAPL",
+            "module": "asset-profile"
+        }
+    )
+    print("Asset Profile Result:", stock_module_result)
+    
+    # Example for getting financial data module
+    financial_data_result = tool.call_endpoint(
+        route="get_stock_module",
+        payload={
+            "ticker": "AAPL",
+            "module": "financial-data"
+        }
+    )
+    print("Financial Data Result:", financial_data_result)
+    
+    # Example for getting SMA indicator data
+    sma_result = tool.call_endpoint(
+        route="get_sma",
+        payload={
+            "symbol": "AAPL",
+            "interval": "5m",
+            "series_type": "close",
+            "time_period": "50",
+            "limit": "50"
+        }
+    )
+    print("SMA Result:", sma_result)
+    
+    # Example for getting RSI indicator data
+    rsi_result = tool.call_endpoint(
+        route="get_rsi",
+        payload={
+            "symbol": "AAPL",
+            "interval": "5m",
+            "series_type": "close",
+            "time_period": "50",
+            "limit": "50"
+        }
+    )
+    print("RSI Result:", rsi_result)
+    
+    # Example for getting earnings calendar data
+    earnings_calendar_result = tool.call_endpoint(
+        route="get_earnings_calendar",
+        payload={
+            "date": "2023-11-30"
+        }
+    )
+    print("Earnings Calendar Result:", earnings_calendar_result)
+    
+    # Example for getting insider trades
+    insider_trades_result = tool.call_endpoint(
+        route="get_insider_trades",
+        payload={}
+    )
+    print("Insider Trades Result:", insider_trades_result)
+

app/tool/data_providers/ZillowProvider.py

@@ -0,0 +1,187 @@
+from typing import Dict
+import logging
+
+from agent.tools.data_providers.RapidDataProviderBase import RapidDataProviderBase, EndpointSchema
+
+logger = logging.getLogger(__name__)
+
+
+class ZillowProvider(RapidDataProviderBase):
+    def __init__(self):
+        endpoints: Dict[str, EndpointSchema] = {
+            "search": {
+                "route": "/search",
+                "method": "GET",
+                "name": "Zillow Property Search",
+                "description": "Search for properties by neighborhood, city, or ZIP code with various filters.",
+                "payload": {
+                    "location": "Location can be an address, neighborhood, city, or ZIP code (required)",
+                    "page": "Page number for pagination (optional, default: 0)",
+                    "output": "Output format: json, csv, xlsx (optional, default: json)",
+                    "status": "Status of properties: forSale, forRent, recentlySold (optional, default: forSale)",
+                    "sortSelection": "Sorting criteria (optional, default: priorityscore)",
+                    "listing_type": "Listing type: by_agent, by_owner_other (optional, default: by_agent)",
+                    "doz": "Days on Zillow: any, 1, 7, 14, 30, 90, 6m, 12m, 24m, 36m (optional, default: any)",
+                    "price_min": "Minimum price (optional)",
+                    "price_max": "Maximum price (optional)",
+                    "sqft_min": "Minimum square footage (optional)",
+                    "sqft_max": "Maximum square footage (optional)",
+                    "beds_min": "Minimum number of bedrooms (optional)",
+                    "beds_max": "Maximum number of bedrooms (optional)",
+                    "baths_min": "Minimum number of bathrooms (optional)",
+                    "baths_max": "Maximum number of bathrooms (optional)",
+                    "built_min": "Minimum year built (optional)",
+                    "built_max": "Maximum year built (optional)",
+                    "lotSize_min": "Minimum lot size in sqft (optional)",
+                    "lotSize_max": "Maximum lot size in sqft (optional)",
+                    "keywords": "Keywords to search for (optional)"
+                }
+            },
+            "search_address": {
+                "route": "/search_address",
+                "method": "GET",
+                "name": "Zillow Address Search",
+                "description": "Search for a specific property by its full address.",
+                "payload": {
+                    "address": "Full property address (required)"
+                }
+            },
+            "propertyV2": {
+                "route": "/propertyV2",
+                "method": "GET",
+                "name": "Zillow Property Details",
+                "description": "Get detailed information about a specific property by zpid or URL.",
+                "payload": {
+                    "zpid": "Zillow property ID (optional if URL is provided)",
+                    "url": "Property details URL (optional if zpid is provided)"
+                }
+            },
+            "zestimate_history": {
+                "route": "/zestimate_history",
+                "method": "GET",
+                "name": "Zillow Zestimate History",
+                "description": "Get historical Zestimate values for a specific property.",
+                "payload": {
+                    "zpid": "Zillow property ID (optional if URL is provided)",
+                    "url": "Property details URL (optional if zpid is provided)"
+                }
+            },
+            "similar_properties": {
+                "route": "/similar_properties",
+                "method": "GET",
+                "name": "Zillow Similar Properties",
+                "description": "Find properties similar to a specific property.",
+                "payload": {
+                    "zpid": "Zillow property ID (optional if URL or address is provided)",
+                    "url": "Property details URL (optional if zpid or address is provided)",
+                    "address": "Property address (optional if zpid or URL is provided)"
+                }
+            },
+            "mortgage_rates": {
+                "route": "/mortgage/rates",
+                "method": "GET",
+                "name": "Zillow Mortgage Rates",
+                "description": "Get current mortgage rates for different loan programs and conditions.",
+                "payload": {
+                    "program": "Loan program (required): Fixed30Year, Fixed20Year, Fixed15Year, Fixed10Year, ARM3, ARM5, ARM7, etc.",
+                    "state": "State abbreviation (optional, default: US)",
+                    "refinance": "Whether this is for refinancing (optional, default: false)",
+                    "loanType": "Type of loan: Conventional, etc. (optional)",
+                    "loanAmount": "Loan amount category: Micro, SmallConforming, Conforming, SuperConforming, Jumbo (optional)",
+                    "loanToValue": "Loan to value ratio: Normal, High, VeryHigh (optional)",
+                    "creditScore": "Credit score category: Low, High, VeryHigh (optional)",
+                    "duration": "Duration in days (optional, default: 30)"
+                }
+            },
+        }
+        base_url = "https://zillow56.p.rapidapi.com"
+        super().__init__(base_url, endpoints)
+
+
+if __name__ == "__main__":
+    from dotenv import load_dotenv
+    from time import sleep
+    load_dotenv()
+    tool = ZillowProvider()
+
+    # Example for searching properties in Houston
+    search_result = tool.call_endpoint(
+        route="search",
+        payload={
+            "location": "houston, tx",
+            "status": "forSale",
+            "sortSelection": "priorityscore",
+            "listing_type": "by_agent",
+            "doz": "any"
+        }
+    )
+    logger.debug("Search Result: %s", search_result)
+    logger.debug("***")
+    logger.debug("***")
+    logger.debug("***")
+    sleep(1)
+    # Example for searching by address
+    address_result = tool.call_endpoint(
+        route="search_address",
+        payload={
+            "address": "1161 Natchez Dr College Station Texas 77845"
+        }
+    )
+    logger.debug("Address Search Result: %s", address_result)
+    logger.debug("***")
+    logger.debug("***")
+    logger.debug("***")
+    sleep(1)
+    # Example for getting property details
+    property_result = tool.call_endpoint(
+        route="propertyV2",
+        payload={
+            "zpid": "7594920"
+        }
+    )
+    logger.debug("Property Details Result: %s", property_result)
+    sleep(1)
+    logger.debug("***")
+    logger.debug("***")
+    logger.debug("***")
+
+    # Example for getting zestimate history
+    zestimate_result = tool.call_endpoint(
+        route="zestimate_history",
+        payload={
+            "zpid": "20476226"
+        }
+    )
+    logger.debug("Zestimate History Result: %s", zestimate_result)
+    sleep(1)
+    logger.debug("***")
+    logger.debug("***")
+    logger.debug("***")
+    # Example for getting similar properties
+    similar_result = tool.call_endpoint(
+        route="similar_properties",
+        payload={
+            "zpid": "28253016"
+        }
+    )
+    logger.debug("Similar Properties Result: %s", similar_result)
+    sleep(1)
+    logger.debug("***")
+    logger.debug("***")
+    logger.debug("***")
+    # Example for getting mortgage rates
+    mortgage_result = tool.call_endpoint(
+        route="mortgage_rates",
+        payload={
+            "program": "Fixed30Year",
+            "state": "US",
+            "refinance": "false",
+            "loanType": "Conventional",
+            "loanAmount": "Conforming",
+            "loanToValue": "Normal",
+            "creditScore": "Low",
+            "duration": "30"
+        }
+    )
+    logger.debug("Mortgage Rates Result: %s", mortgage_result)
+  

app/tool/data_providers_tool.py

@@ -0,0 +1,188 @@
+import json
+from typing import Union, Dict, Any
+
+from agentpress.tool import Tool, ToolResult, openapi_schema, xml_schema
+from agent.tools.data_providers.LinkedinProvider import LinkedinProvider
+from agent.tools.data_providers.YahooFinanceProvider import YahooFinanceProvider
+from agent.tools.data_providers.AmazonProvider import AmazonProvider
+from agent.tools.data_providers.ZillowProvider import ZillowProvider
+from agent.tools.data_providers.TwitterProvider import TwitterProvider
+
+class DataProvidersTool(Tool):
+    """Tool for making requests to various data providers."""
+
+    def __init__(self):
+        super().__init__()
+
+        self.register_data_providers = {
+            "linkedin": LinkedinProvider(),
+            "yahoo_finance": YahooFinanceProvider(),
+            "amazon": AmazonProvider(),
+            "zillow": ZillowProvider(),
+            "twitter": TwitterProvider()
+        }
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "get_data_provider_endpoints",
+            "description": "Get available endpoints for a specific data provider",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "service_name": {
+                        "type": "string",
+                        "description": "The name of the data provider (e.g., 'linkedin', 'twitter', 'zillow', 'amazon', 'yahoo_finance')"
+                    }
+                },
+                "required": ["service_name"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="get-data-provider-endpoints",
+        mappings=[
+            {"param_name": "service_name", "node_type": "attribute", "path": "."}
+        ],
+        example='''
+<!-- 
+The get-data-provider-endpoints tool returns available endpoints for a specific data provider.
+Use this tool when you need to discover what endpoints are available.
+-->
+
+<!-- Example to get LinkedIn API endpoints -->
+<function_calls>
+<invoke name="get_data_provider_endpoints">
+<parameter name="service_name">linkedin</parameter>
+</invoke>
+</function_calls>
+        '''
+    )
+    async def get_data_provider_endpoints(
+        self,
+        service_name: str
+    ) -> ToolResult:
+        """
+        Get available endpoints for a specific data provider.
+        
+        Parameters:
+        - service_name: The name of the data provider (e.g., 'linkedin')
+        """
+        try:
+            if not service_name:
+                return self.fail_response("Data provider name is required.")
+                
+            if service_name not in self.register_data_providers:
+                return self.fail_response(f"Data provider '{service_name}' not found. Available data providers: {list(self.register_data_providers.keys())}")
+                
+            endpoints = self.register_data_providers[service_name].get_endpoints()
+            return self.success_response(endpoints)
+            
+        except Exception as e:
+            error_message = str(e)
+            simplified_message = f"Error getting data provider endpoints: {error_message[:200]}"
+            if len(error_message) > 200:
+                simplified_message += "..."
+            return self.fail_response(simplified_message)
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "execute_data_provider_call",
+            "description": "Execute a call to a specific data provider endpoint",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "service_name": {
+                        "type": "string",
+                        "description": "The name of the API service (e.g., 'linkedin')"
+                    },
+                    "route": {
+                        "type": "string",
+                        "description": "The key of the endpoint to call"
+                    },
+                    "payload": {
+                        "type": "object",
+                        "description": "The payload to send with the API call"
+                    }
+                },
+                "required": ["service_name", "route"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="execute-data-provider-call",
+        mappings=[
+            {"param_name": "service_name", "node_type": "attribute", "path": "service_name"},
+            {"param_name": "route", "node_type": "attribute", "path": "route"},
+            {"param_name": "payload", "node_type": "content", "path": "."}
+        ],
+        example='''
+        <!-- 
+        The execute-data-provider-call tool makes a request to a specific data provider endpoint.
+        Use this tool when you need to call an data provider endpoint with specific parameters.
+        The route must be a valid endpoint key obtained from get-data-provider-endpoints tool!!
+        -->
+        
+        <!-- Example to call linkedIn service with the specific route person -->
+        <function_calls>
+        <invoke name="execute_data_provider_call">
+        <parameter name="service_name">linkedin</parameter>
+        <parameter name="route">person</parameter>
+        <parameter name="payload">{"link": "https://www.linkedin.com/in/johndoe/"}</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def execute_data_provider_call(
+        self,
+        service_name: str,
+        route: str,
+        payload: Union[Dict[str, Any], str, None] = None
+    ) -> ToolResult:
+        """
+        Execute a call to a specific data provider endpoint.
+        
+        Parameters:
+        - service_name: The name of the data provider (e.g., 'linkedin')
+        - route: The key of the endpoint to call
+        - payload: The payload to send with the data provider call (dict or JSON string)
+        """
+        try:
+            # Handle payload - it can be either a dict or a JSON string
+            if isinstance(payload, str):
+                try:
+                    payload = json.loads(payload)
+                except json.JSONDecodeError as e:
+                    return self.fail_response(f"Invalid JSON in payload: {str(e)}")
+            elif payload is None:
+                payload = {}
+            # If payload is already a dict, use it as-is
+
+            if not service_name:
+                return self.fail_response("service_name is required.")
+
+            if not route:
+                return self.fail_response("route is required.")
+                
+            if service_name not in self.register_data_providers:
+                return self.fail_response(f"API '{service_name}' not found. Available APIs: {list(self.register_data_providers.keys())}")
+            
+            data_provider = self.register_data_providers[service_name]
+            if route == service_name:
+                return self.fail_response(f"route '{route}' is the same as service_name '{service_name}'. YOU FUCKING IDIOT!")
+            
+            if route not in data_provider.get_endpoints().keys():
+                return self.fail_response(f"Endpoint '{route}' not found in {service_name} data provider.")
+            
+            
+            result = data_provider.call_endpoint(route, payload)
+            return self.success_response(result)
+            
+        except Exception as e:
+            error_message = str(e)
+            print(error_message)
+            simplified_message = f"Error executing data provider call: {error_message[:200]}"
+            if len(error_message) > 200:
+                simplified_message += "..."
+            return self.fail_response(simplified_message)

app/tool/expand_msg_tool.py

@@ -0,0 +1,103 @@
+from agentpress.tool import Tool, ToolResult, openapi_schema, xml_schema
+from agentpress.thread_manager import ThreadManager
+import json
+
+class ExpandMessageTool(Tool):
+    """Tool for expanding a previous message to the user."""
+
+    def __init__(self, thread_id: str, thread_manager: ThreadManager):
+        super().__init__()
+        self.thread_manager = thread_manager
+        self.thread_id = thread_id
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "expand_message",
+            "description": "Expand a message from the previous conversation with the user. Use this tool to expand a message that was truncated in the earlier conversation.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "message_id": {
+                        "type": "string",
+                        "description": "The ID of the message to expand. Must be a UUID."
+                    }
+                },
+                "required": ["message_id"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="expand-message",
+        mappings=[
+            {"param_name": "message_id", "node_type": "attribute", "path": "."}
+        ],
+        example='''
+        <!-- Example 1: Expand a message that was truncated in the previous conversation -->
+        <function_calls>
+        <invoke name="expand_message">
+        <parameter name="message_id">ecde3a4c-c7dc-4776-ae5c-8209517c5576</parameter>
+        </invoke>
+        </function_calls>
+
+        <!-- Example 2: Expand a message to create reports or analyze truncated data -->
+        <function_calls>
+        <invoke name="expand_message">
+        <parameter name="message_id">f47ac10b-58cc-4372-a567-0e02b2c3d479</parameter>
+        </invoke>
+        </function_calls>
+
+        <!-- Example 3: Expand a message when you need the full content for analysis -->
+        <function_calls>
+        <invoke name="expand_message">
+        <parameter name="message_id">550e8400-e29b-41d4-a716-446655440000</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def expand_message(self, message_id: str) -> ToolResult:
+        """Expand a message from the previous conversation with the user.
+
+        Args:
+            message_id: The ID of the message to expand
+
+        Returns:
+            ToolResult indicating the message was successfully expanded
+        """
+        try:
+            client = await self.thread_manager.db.client
+            message = await client.table('messages').select('*').eq('message_id', message_id).eq('thread_id', self.thread_id).execute()
+
+            if not message.data or len(message.data) == 0:
+                return self.fail_response(f"Message with ID {message_id} not found in thread {self.thread_id}")
+
+            message_data = message.data[0]
+            message_content = message_data['content']
+            final_content = message_content
+            if isinstance(message_content, dict) and 'content' in message_content:
+                final_content = message_content['content']
+            elif isinstance(message_content, str):
+                try:
+                    parsed_content = json.loads(message_content)
+                    if isinstance(parsed_content, dict) and 'content' in parsed_content:
+                        final_content = parsed_content['content']
+                except json.JSONDecodeError:
+                    pass
+
+            return self.success_response({"status": "Message expanded successfully.", "message": final_content})
+        except Exception as e:
+            return self.fail_response(f"Error expanding message: {str(e)}")
+
+if __name__ == "__main__":
+    import asyncio
+
+    async def test_expand_message_tool():
+        expand_message_tool = ExpandMessageTool()
+
+        # Test expand message
+        expand_message_result = await expand_message_tool.expand_message(
+            message_id="004ab969-ef9a-4656-8aba-e392345227cd"
+        )
+        print("Expand message result:", expand_message_result)
+
+    asyncio.run(test_expand_message_tool())

app/tool/mcp_tool_wrapper.py

@@ -0,0 +1,715 @@
+"""
+MCP Tool Wrapper for AgentPress
+
+This module provides a generic tool wrapper that handles all MCP (Model Context Protocol) 
+server tool calls through dynamically generated individual function methods.
+"""
+
+import json
+from typing import Any, Dict, List, Optional
+from agentpress.tool import Tool, ToolResult, openapi_schema, xml_schema, ToolSchema, SchemaType
+from mcp_local.client import MCPManager
+from utils.logger import logger
+import inspect
+from mcp import ClientSession
+from mcp.client.sse import sse_client
+from mcp.client.stdio import stdio_client
+from mcp.client.streamable_http import streamablehttp_client
+from mcp import StdioServerParameters
+import asyncio
+
+
+class MCPToolWrapper(Tool):
+    """
+    A generic tool wrapper that dynamically creates individual methods for each MCP tool.
+    
+    This tool creates separate function calls for each MCP tool while routing them all
+    through the same underlying implementation.
+    """
+    
+    def __init__(self, mcp_configs: Optional[List[Dict[str, Any]]] = None):
+        """
+        Initialize the MCP tool wrapper.
+        
+        Args:
+            mcp_configs: List of MCP configurations from agent's configured_mcps
+        """
+        # Don't call super().__init__() yet - we need to set up dynamic methods first
+        self.mcp_manager = MCPManager()
+        self.mcp_configs = mcp_configs or []
+        self._initialized = False
+        self._dynamic_tools = {}
+        self._schemas: Dict[str, List[ToolSchema]] = {}
+        self._custom_tools = {}  # Store custom MCP tools separately
+        
+        # Now initialize the parent class which will call _register_schemas
+        super().__init__()
+        
+    async def _ensure_initialized(self):
+        """Ensure MCP servers are initialized."""
+        if not self._initialized:
+            # Initialize standard MCP servers from Smithery
+            standard_configs = [cfg for cfg in self.mcp_configs if not cfg.get('isCustom', False)]
+            custom_configs = [cfg for cfg in self.mcp_configs if cfg.get('isCustom', False)]
+            
+            # Initialize standard MCPs through MCPManager
+            if standard_configs:
+                for config in standard_configs:
+                    try:
+                        logger.info(f"Attempting to connect to MCP server: {config['qualifiedName']}")
+                        await self.mcp_manager.connect_server(config)
+                        logger.info(f"Successfully connected to MCP server: {config['qualifiedName']}")
+                    except Exception as e:
+                        logger.error(f"Failed to connect to MCP server {config['qualifiedName']}: {e}")
+                        import traceback
+                        logger.error(f"Full traceback: {traceback.format_exc()}")
+            
+            # Initialize custom MCPs directly
+            if custom_configs:
+                await self._initialize_custom_mcps(custom_configs)
+            
+            # Create dynamic tools for all connected servers
+            await self._create_dynamic_tools()
+            self._initialized = True
+    
+    async def _connect_sse_server(self, server_name, server_config, all_tools, timeout):
+        url = server_config["url"]
+        headers = server_config.get("headers", {})
+        
+        async with asyncio.timeout(timeout):
+            try:
+                async with sse_client(url, headers=headers) as (read, write):
+                    async with ClientSession(read, write) as session:
+                        await session.initialize()
+                        tools_result = await session.list_tools()
+                        tools_info = []
+                        for tool in tools_result.tools:
+                            tool_info = {
+                                "name": tool.name,
+                                "description": tool.description,
+                                "input_schema": tool.inputSchema
+                            }
+                            tools_info.append(tool_info)
+                        
+                        all_tools[server_name] = {
+                            "status": "connected",
+                            "transport": "sse",
+                            "url": url,
+                            "tools": tools_info
+                        }
+                        
+                        logger.info(f"  {server_name}: Connected via SSE ({len(tools_info)} tools)")
+            except TypeError as e:
+                if "unexpected keyword argument" in str(e):
+                    async with sse_client(url) as (read, write):
+                        async with ClientSession(read, write) as session:
+                            await session.initialize()
+                            tools_result = await session.list_tools()
+                            tools_info = []
+                            for tool in tools_result.tools:
+                                tool_info = {
+                                    "name": tool.name,
+                                    "description": tool.description,
+                                    "input_schema": tool.inputSchema
+                                }
+                                tools_info.append(tool_info)
+                            
+                            all_tools[server_name] = {
+                                "status": "connected",
+                                "transport": "sse",
+                                "url": url,
+                                "tools": tools_info
+                            }
+                            logger.info(f"  {server_name}: Connected via SSE ({len(tools_info)} tools)")
+                else:
+                    raise
+    
+    async def _connect_streamable_http_server(self, url):
+        async with streamablehttp_client(url) as (
+            read_stream,
+            write_stream,
+            _,
+        ):
+            async with ClientSession(read_stream, write_stream) as session:
+                await session.initialize()
+                tool_result = await session.list_tools()
+                print(f"Connected via HTTP ({len(tool_result.tools)} tools)")
+                
+                tools_info = []
+                for tool in tool_result.tools:
+                    tool_info = {
+                        "name": tool.name,
+                        "description": tool.description,
+                        "inputSchema": tool.inputSchema
+                    }
+                    tools_info.append(tool_info)
+                
+                return tools_info
+        
+    async def _connect_stdio_server(self, server_name, server_config, all_tools, timeout):
+        """Connect to a stdio-based MCP server."""
+        server_params = StdioServerParameters(
+            command=server_config["command"],
+            args=server_config.get("args", []),
+            env=server_config.get("env", {})
+        )
+        
+        async with asyncio.timeout(timeout):
+            async with stdio_client(server_params) as (read, write):
+                async with ClientSession(read, write) as session:
+                    await session.initialize()
+                    tools_result = await session.list_tools()
+                    tools_info = []
+                    for tool in tools_result.tools:
+                        tool_info = {
+                            "name": tool.name,
+                            "description": tool.description,
+                            "input_schema": tool.inputSchema
+                        }
+                        tools_info.append(tool_info)
+                    
+                    all_tools[server_name] = {
+                        "status": "connected",
+                        "transport": "stdio",
+                        "tools": tools_info
+                    }
+                    
+                    logger.info(f"  {server_name}: Connected via stdio ({len(tools_info)} tools)")
+
+    async def _initialize_custom_mcps(self, custom_configs):
+        """Initialize custom MCP servers."""
+        for config in custom_configs:
+            try:
+                logger.info(f"Initializing custom MCP: {config}")
+                custom_type = config.get('customType', 'sse')
+                server_config = config.get('config', {})
+                enabled_tools = config.get('enabledTools', [])
+                server_name = config.get('name', 'Unknown')
+                
+                logger.info(f"Initializing custom MCP: {server_name} (type: {custom_type})")
+                
+                if custom_type == 'sse':
+                    if 'url' not in server_config:
+                        logger.error(f"Custom MCP {server_name}: Missing 'url' in config")
+                        continue
+                        
+                    url = server_config['url']
+                    logger.info(f"Initializing custom MCP {url} with SSE type")
+                    
+                    try:
+                        # Use the working connect_sse_server method
+                        all_tools = {}
+                        await self._connect_sse_server(server_name, server_config, all_tools, 15)
+                        
+                        # Process the results
+                        if server_name in all_tools and all_tools[server_name].get('status') == 'connected':
+                            tools_info = all_tools[server_name].get('tools', [])
+                            tools_registered = 0
+                            
+                            for tool_info in tools_info:
+                                tool_name_from_server = tool_info['name']
+                                if not enabled_tools or tool_name_from_server in enabled_tools:
+                                    tool_name = f"custom_{server_name.replace(' ', '_').lower()}_{tool_name_from_server}"
+                                    self._custom_tools[tool_name] = {
+                                        'name': tool_name,
+                                        'description': tool_info['description'],
+                                        'parameters': tool_info['input_schema'],
+                                        'server': server_name,
+                                        'original_name': tool_name_from_server,
+                                        'is_custom': True,
+                                        'custom_type': custom_type,
+                                        'custom_config': server_config
+                                    }
+                                    tools_registered += 1
+                                    logger.debug(f"Registered custom tool: {tool_name}")
+                            
+                            logger.info(f"Successfully initialized custom MCP {server_name} with {tools_registered} tools")
+                        else:
+                            logger.error(f"Failed to connect to custom MCP {server_name}")
+                            
+                    except Exception as e:
+                        logger.error(f"Custom MCP {server_name}: Connection failed - {str(e)}")
+                        continue
+                
+                elif custom_type == 'http':
+                    if 'url' not in server_config:
+                        logger.error(f"Custom MCP {server_name}: Missing 'url' in config")
+                        continue
+                        
+                    url = server_config['url']
+                    logger.info(f"Initializing custom MCP {url} with HTTP type")
+                    
+                    try:
+
+                        tools_info = await self._connect_streamable_http_server(url)
+                        tools_registered = 0
+                        
+                        for tool_info in tools_info:
+                            tool_name_from_server = tool_info['name']
+                            if not enabled_tools or tool_name_from_server in enabled_tools:
+                                tool_name = f"custom_{server_name.replace(' ', '_').lower()}_{tool_name_from_server}"
+                                self._custom_tools[tool_name] = {
+                                    'name': tool_name,
+                                    'description': tool_info['description'],
+                                    'parameters': tool_info['inputSchema'],
+                                    'server': server_name,
+                                    'original_name': tool_name_from_server,
+                                    'is_custom': True,
+                                    'custom_type': custom_type,
+                                    'custom_config': server_config
+                                }
+                                tools_registered += 1
+                                logger.debug(f"Registered custom tool: {tool_name}")
+                        
+                        logger.info(f"Successfully initialized custom MCP {server_name} with {tools_registered} tools")
+                            
+                    except Exception as e:
+                        logger.error(f"Custom MCP {server_name}: Connection failed - {str(e)}")
+                        continue
+                        
+                elif custom_type == 'json':
+                    if 'command' not in server_config:
+                        logger.error(f"Custom MCP {server_name}: Missing 'command' in config")
+                        continue
+                        
+                    logger.info(f"Initializing custom MCP {server_name} with JSON/stdio type")
+                    
+                    try:
+                        # Use the stdio connection method
+                        all_tools = {}
+                        await self._connect_stdio_server(server_name, server_config, all_tools, 15)
+                        
+                        # Process the results
+                        if server_name in all_tools and all_tools[server_name].get('status') == 'connected':
+                            tools_info = all_tools[server_name].get('tools', [])
+                            tools_registered = 0
+                            
+                            for tool_info in tools_info:
+                                tool_name_from_server = tool_info['name']
+                                if not enabled_tools or tool_name_from_server in enabled_tools:
+                                    tool_name = f"custom_{server_name.replace(' ', '_').lower()}_{tool_name_from_server}"
+                                    self._custom_tools[tool_name] = {
+                                        'name': tool_name,
+                                        'description': tool_info['description'],
+                                        'parameters': tool_info['input_schema'],
+                                        'server': server_name,
+                                        'original_name': tool_name_from_server,
+                                        'is_custom': True,
+                                        'custom_type': custom_type,
+                                        'custom_config': server_config
+                                    }
+                                    tools_registered += 1
+                                    logger.debug(f"Registered custom tool: {tool_name}")
+                            
+                            logger.info(f"Successfully initialized custom MCP {server_name} with {tools_registered} tools")
+                        else:
+                            logger.error(f"Failed to connect to custom MCP {server_name}")
+                            
+                    except Exception as e:
+                        logger.error(f"Custom MCP {server_name}: Connection failed - {str(e)}")
+                        continue
+                        
+                else:
+                    logger.error(f"Custom MCP {server_name}: Unsupported type '{custom_type}', supported types are 'sse', 'http' and 'json'")
+                    continue
+                    
+            except Exception as e:
+                logger.error(f"Failed to initialize custom MCP {config.get('name', 'Unknown')}: {e}")
+                continue
+    
+    async def initialize_and_register_tools(self, tool_registry=None):
+        """Initialize MCP tools and optionally update the tool registry.
+        
+        This method should be called after the tool has been registered to dynamically
+        add the MCP tool schemas to the registry.
+        
+        Args:
+            tool_registry: Optional ToolRegistry instance to update with new schemas
+        """
+        await self._ensure_initialized()
+        
+        if tool_registry and self._dynamic_tools:
+            logger.info(f"Updating tool registry with {len(self._dynamic_tools)} MCP tools")
+            for method_name, schemas in self._schemas.items():
+                if method_name not in ['call_mcp_tool']:  # Skip the fallback method
+                    pass
+    
+    async def _create_dynamic_tools(self):
+        """Create dynamic tool methods for each available MCP tool."""
+        try:
+            # Get standard MCP tools
+            available_tools = self.mcp_manager.get_all_tools_openapi()
+            logger.info(f"MCPManager returned {len(available_tools)} tools")
+            
+            for tool_info in available_tools:
+                tool_name = tool_info.get('name', '')
+                logger.info(f"Processing tool: {tool_name}")
+                if tool_name:
+                    # Create a dynamic method for this tool with proper OpenAI schema
+                    self._create_dynamic_method(tool_name, tool_info)
+            
+            # Get custom MCP tools
+            logger.info(f"Processing {len(self._custom_tools)} custom MCP tools")
+            for tool_name, tool_info in self._custom_tools.items():
+                logger.info(f"Processing custom tool: {tool_name}")
+                # Convert custom tool info to the expected format
+                openapi_tool_info = {
+                    "name": tool_name,
+                    "description": tool_info['description'],
+                    "parameters": tool_info['parameters']
+                }
+                self._create_dynamic_method(tool_name, openapi_tool_info)
+                    
+            logger.info(f"Created {len(self._dynamic_tools)} dynamic MCP tool methods")
+            
+        except Exception as e:
+            logger.error(f"Error creating dynamic MCP tools: {e}")
+    
+    def _create_dynamic_method(self, tool_name: str, tool_info: Dict[str, Any]):
+        """Create a dynamic method for a specific MCP tool with proper OpenAI schema."""
+        if tool_name.startswith("custom_"):
+            if tool_name in self._custom_tools:
+                clean_tool_name = self._custom_tools[tool_name]['original_name']
+                server_name = self._custom_tools[tool_name]['server']
+            else:
+                parts = tool_name.split("_")
+                if len(parts) >= 3:
+                    clean_tool_name = "_".join(parts[2:])
+                    server_name = parts[1] if len(parts) > 1 else "unknown"
+                else:
+                    clean_tool_name = tool_name
+                    server_name = "unknown"
+        else:
+            parts = tool_name.split("_", 2)
+            clean_tool_name = parts[2] if len(parts) > 2 else tool_name
+            server_name = parts[1] if len(parts) > 1 else "unknown"
+
+        method_name = clean_tool_name.replace('-', '_')
+        
+        logger.info(f"Creating dynamic method for tool '{tool_name}': clean_tool_name='{clean_tool_name}', method_name='{method_name}', server='{server_name}'")
+
+        original_full_name = tool_name
+        
+        # Create the dynamic method
+        async def dynamic_tool_method(**kwargs) -> ToolResult:
+            """Dynamically created method for MCP tool."""
+            # Use the original full tool name for execution
+            return await self._execute_mcp_tool(original_full_name, kwargs)
+        
+        # Set the method name to match the tool name
+        dynamic_tool_method.__name__ = method_name
+        dynamic_tool_method.__qualname__ = f"{self.__class__.__name__}.{method_name}"
+        
+        # Build a more descriptive description
+        base_description = tool_info.get("description", f"MCP tool from {server_name}")
+        full_description = f"{base_description} (MCP Server: {server_name})"
+        
+        # Create the OpenAI schema for this tool
+        openapi_function_schema = {
+            "type": "function",
+            "function": {
+                "name": method_name,  # Use the clean method name for function calling
+                "description": full_description,
+                "parameters": tool_info.get("parameters", {
+                    "type": "object",
+                    "properties": {},
+                    "required": []
+                })
+            }
+        }
+        
+        # Create a ToolSchema object
+        tool_schema = ToolSchema(
+            schema_type=SchemaType.OPENAPI,
+            schema=openapi_function_schema
+        )
+        
+        # Add the schema to our schemas dict
+        self._schemas[method_name] = [tool_schema]
+        
+        # Also add the schema to the method itself (for compatibility)
+        dynamic_tool_method.tool_schemas = [tool_schema]
+        
+        # Store the method and its info
+        self._dynamic_tools[tool_name] = {
+            'method': dynamic_tool_method,
+            'method_name': method_name,
+            'original_tool_name': tool_name,
+            'clean_tool_name': clean_tool_name,
+            'server_name': server_name,
+            'info': tool_info,
+            'schema': tool_schema
+        }
+        
+        # Add the method to this instance
+        setattr(self, method_name, dynamic_tool_method)
+        
+        logger.debug(f"Created dynamic method '{method_name}' for MCP tool '{tool_name}' from server '{server_name}'")
+    
+    def _register_schemas(self):
+        """Register schemas from all decorated methods and dynamic tools."""
+        # First register static schemas from decorated methods
+        for name, method in inspect.getmembers(self, predicate=inspect.ismethod):
+            if hasattr(method, 'tool_schemas'):
+                self._schemas[name] = method.tool_schemas
+                logger.debug(f"Registered schemas for method '{name}' in {self.__class__.__name__}")
+        
+        # Note: Dynamic schemas will be added after async initialization
+        logger.debug(f"Initial registration complete for MCPToolWrapper")
+    
+    def get_schemas(self) -> Dict[str, List[ToolSchema]]:
+        """Get all registered tool schemas including dynamic ones."""
+        # Return all schemas including dynamically added ones
+        return self._schemas
+    
+    def __getattr__(self, name: str):
+        """Handle calls to dynamically created MCP tool methods."""
+        # Look for exact method name match first
+        for tool_data in self._dynamic_tools.values():
+            if tool_data['method_name'] == name:
+                return tool_data['method']
+        
+        # Try with underscore/hyphen conversion
+        name_with_hyphens = name.replace('_', '-')
+        for tool_name, tool_data in self._dynamic_tools.items():
+            if tool_data['method_name'] == name or tool_name == name_with_hyphens:
+                return tool_data['method']
+        
+        raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'")
+            
+    async def get_available_tools(self) -> List[Dict[str, Any]]:
+        """Get all available MCP tools in OpenAPI format."""
+        await self._ensure_initialized()
+        return self.mcp_manager.get_all_tools_openapi()
+    
+    async def _execute_mcp_tool(self, tool_name: str, arguments: Dict[str, Any]) -> ToolResult:
+        """Execute an MCP tool call."""
+        await self._ensure_initialized()
+        logger.info(f"Executing MCP tool {tool_name} with arguments {arguments}")
+        try:
+            # Check if it's a custom MCP tool first
+            if tool_name in self._custom_tools:
+                tool_info = self._custom_tools[tool_name]
+                return await self._execute_custom_mcp_tool(tool_name, arguments, tool_info)
+            else:
+                # Use standard MCP manager for Smithery servers
+                result = await self.mcp_manager.execute_tool(tool_name, arguments)
+                
+                if isinstance(result, dict):
+                    if result.get('isError', False):
+                        return self.fail_response(result.get('content', 'Tool execution failed'))
+                    else:
+                        return self.success_response(result.get('content', result))
+                else:
+                    return self.success_response(result)
+                    
+        except Exception as e:
+            logger.error(f"Error executing MCP tool {tool_name}: {str(e)}")
+            return self.fail_response(f"Error executing tool: {str(e)}")
+    
+    async def _execute_custom_mcp_tool(self, tool_name: str, arguments: Dict[str, Any], tool_info: Dict[str, Any]) -> ToolResult:
+        """Execute a custom MCP tool call."""
+        try:
+            custom_type = tool_info['custom_type']
+            custom_config = tool_info['custom_config']
+            original_tool_name = tool_info['original_name']
+            
+            if custom_type == 'sse':
+                # Execute SSE-based custom MCP using the same pattern as _connect_sse_server
+                url = custom_config['url']
+                headers = custom_config.get('headers', {})
+                
+                async with asyncio.timeout(30):  # 30 second timeout for tool execution
+                    try:
+                        # Try with headers first (same pattern as _connect_sse_server)
+                        async with sse_client(url, headers=headers) as (read, write):
+                            async with ClientSession(read, write) as session:
+                                await session.initialize()
+                                result = await session.call_tool(original_tool_name, arguments)
+                                
+                                # Handle the result properly
+                                if hasattr(result, 'content'):
+                                    content = result.content
+                                    if isinstance(content, list):
+                                        # Extract text from content list
+                                        text_parts = []
+                                        for item in content:
+                                            if hasattr(item, 'text'):
+                                                text_parts.append(item.text)
+                                            else:
+                                                text_parts.append(str(item))
+                                        content_str = "\n".join(text_parts)
+                                    elif hasattr(content, 'text'):
+                                        content_str = content.text
+                                    else:
+                                        content_str = str(content)
+                                    
+                                    return self.success_response(content_str)
+                                else:
+                                    return self.success_response(str(result))
+                                    
+                    except TypeError as e:
+                        if "unexpected keyword argument" in str(e):
+                            # Fallback: try without headers (exact pattern from _connect_sse_server)
+                            async with sse_client(url) as (read, write):
+                                async with ClientSession(read, write) as session:
+                                    await session.initialize()
+                                    result = await session.call_tool(original_tool_name, arguments)
+                                    
+                                    # Handle the result properly
+                                    if hasattr(result, 'content'):
+                                        content = result.content
+                                        if isinstance(content, list):
+                                            # Extract text from content list
+                                            text_parts = []
+                                            for item in content:
+                                                if hasattr(item, 'text'):
+                                                    text_parts.append(item.text)
+                                                else:
+                                                    text_parts.append(str(item))
+                                            content_str = "\n".join(text_parts)
+                                        elif hasattr(content, 'text'):
+                                            content_str = content.text
+                                        else:
+                                            content_str = str(content)
+                                        
+                                        return self.success_response(content_str)
+                                    else:
+                                        return self.success_response(str(result))
+                        else:
+                            raise
+            
+            elif custom_type == 'http':
+                # Execute HTTP-based custom MCP
+                url = custom_config['url']
+                
+                async with asyncio.timeout(30):  # 30 second timeout for tool execution
+                    async with streamablehttp_client(url) as (read, write, _):
+                        async with ClientSession(read, write) as session:
+                            await session.initialize()
+                            result = await session.call_tool(original_tool_name, arguments)
+                            
+                            # Handle the result properly
+                            if hasattr(result, 'content'):
+                                content = result.content
+                                if isinstance(content, list):
+                                    # Extract text from content list
+                                    text_parts = []
+                                    for item in content:
+                                        if hasattr(item, 'text'):
+                                            text_parts.append(item.text)
+                                        else:
+                                            text_parts.append(str(item))
+                                    content_str = "\n".join(text_parts)
+                                elif hasattr(content, 'text'):
+                                    content_str = content.text
+                                else:
+                                    content_str = str(content)
+                                
+                                return self.success_response(content_str)
+                            else:
+                                return self.success_response(str(result))
+                                
+            elif custom_type == 'json':
+                # Execute stdio-based custom MCP using the same pattern as _connect_stdio_server
+                server_params = StdioServerParameters(
+                    command=custom_config["command"],
+                    args=custom_config.get("args", []),
+                    env=custom_config.get("env", {})
+                )
+                
+                async with asyncio.timeout(30):  # 30 second timeout for tool execution
+                    async with stdio_client(server_params) as (read, write):
+                        async with ClientSession(read, write) as session:
+                            await session.initialize()
+                            result = await session.call_tool(original_tool_name, arguments)
+                            
+                            # Handle the result properly
+                            if hasattr(result, 'content'):
+                                content = result.content
+                                if isinstance(content, list):
+                                    # Extract text from content list
+                                    text_parts = []
+                                    for item in content:
+                                        if hasattr(item, 'text'):
+                                            text_parts.append(item.text)
+                                        else:
+                                            text_parts.append(str(item))
+                                    content_str = "\n".join(text_parts)
+                                elif hasattr(content, 'text'):
+                                    content_str = content.text
+                                else:
+                                    content_str = str(content)
+                                
+                                return self.success_response(content_str)
+                            else:
+                                return self.success_response(str(result))
+            else:
+                return self.fail_response(f"Unsupported custom MCP type: {custom_type}")
+                                
+        except asyncio.TimeoutError:
+            return self.fail_response(f"Tool execution timeout for {tool_name}")
+        except Exception as e:
+            logger.error(f"Error executing custom MCP tool {tool_name}: {str(e)}")
+            return self.fail_response(f"Error executing custom tool: {str(e)}")
+    
+    # Keep the original call_mcp_tool method as a fallback
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "call_mcp_tool",
+            "description": "Execute a tool from any connected MCP server. This is a fallback wrapper that forwards calls to MCP tools. The tool_name should be in the format 'mcp_{server}_{tool}' where {server} is the MCP server's qualified name and {tool} is the specific tool name.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "tool_name": {
+                        "type": "string",
+                        "description": "The full MCP tool name in format 'mcp_{server}_{tool}', e.g., 'mcp_exa_web_search_exa'"
+                    },
+                    "arguments": {
+                        "type": "object",
+                        "description": "The arguments to pass to the MCP tool, as a JSON object. The required arguments depend on the specific tool being called.",
+                        "additionalProperties": True
+                    }
+                },
+                "required": ["tool_name", "arguments"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="call-mcp-tool",
+        mappings=[
+            {"param_name": "tool_name", "node_type": "attribute", "path": "."},
+            {"param_name": "arguments", "node_type": "content", "path": "."}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="call_mcp_tool">
+        <parameter name="tool_name">mcp_exa_web_search_exa</parameter>
+        <parameter name="arguments">{"query": "latest developments in AI", "num_results": 10}</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def call_mcp_tool(self, tool_name: str, arguments: Dict[str, Any]) -> ToolResult:
+        """
+        Execute an MCP tool call (fallback method).
+        
+        Args:
+            tool_name: The full MCP tool name (e.g., "mcp_exa_web_search_exa")
+            arguments: The arguments to pass to the tool
+            
+        Returns:
+            ToolResult with the tool execution result
+        """
+        return await self._execute_mcp_tool(tool_name, arguments)
+            
+    async def cleanup(self):
+        """Disconnect all MCP servers."""
+        if self._initialized:
+            try:
+                await self.mcp_manager.disconnect_all()
+            except Exception as e:
+                logger.error(f"Error during MCP cleanup: {str(e)}")
+            finally:
+                self._initialized = False 

app/tool/message_tool.py

@@ -0,0 +1,270 @@
+from typing import List, Optional, Union
+from agentpress.tool import Tool, ToolResult, openapi_schema, xml_schema
+from utils.logger import logger
+
+class MessageTool(Tool):
+    """Tool for user communication and interaction.
+
+    This tool provides methods for asking questions, with support for
+    attachments and user takeover suggestions.
+    """
+
+    def __init__(self):
+        super().__init__()
+
+    # Commented out as we are just doing this via prompt as there is no need to call it as a tool
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "ask",
+            "description": "Ask user a question and wait for response. Use for: 1) Requesting clarification on ambiguous requirements, 2) Seeking confirmation before proceeding with high-impact changes, 3) Gathering additional information needed to complete a task, 4) Offering options and requesting user preference, 5) Validating assumptions when critical to task success. IMPORTANT: Use this tool only when user input is essential to proceed. Always provide clear context and options when applicable. Include relevant attachments when the question relates to specific files or resources.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "text": {
+                        "type": "string",
+                        "description": "Question text to present to user - should be specific and clearly indicate what information you need. Include: 1) Clear question or request, 2) Context about why the input is needed, 3) Available options if applicable, 4) Impact of different choices, 5) Any relevant constraints or considerations."
+                    },
+                    "attachments": {
+                        "anyOf": [
+                            {"type": "string"},
+                            {"items": {"type": "string"}, "type": "array"}
+                        ],
+                        "description": "(Optional) List of files or URLs to attach to the question. Include when: 1) Question relates to specific files or configurations, 2) User needs to review content before answering, 3) Options or choices are documented in files, 4) Supporting evidence or context is needed. Always use relative paths to /workspace directory."
+                    }
+                },
+                "required": ["text"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="ask",
+        mappings=[
+            {"param_name": "text", "node_type": "content", "path": "."},
+            {"param_name": "attachments", "node_type": "attribute", "path": ".", "required": False}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="ask">
+        <parameter name="text">I'm planning to bake the chocolate cake for your birthday party. The recipe mentions "rich frosting" but doesn't specify what type. Could you clarify your preferences? For example:
+1. Would you prefer buttercream or cream cheese frosting?
+2. Do you want any specific flavor added to the frosting (vanilla, coffee, etc.)?
+3. Should I add any decorative toppings like sprinkles or fruit?
+4. Do you have any dietary restrictions I should be aware of?
+
+This information will help me make sure the cake meets your expectations for the celebration.</parameter>
+        <parameter name="attachments">recipes/chocolate_cake.txt,photos/cake_examples.jpg</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def ask(self, text: str, attachments: Optional[Union[str, List[str]]] = None) -> ToolResult:
+        """Ask the user a question and wait for a response.
+
+        Args:
+            text: The question to present to the user
+            attachments: Optional file paths or URLs to attach to the question
+
+        Returns:
+            ToolResult indicating the question was successfully sent
+        """
+        try:            
+            # Convert single attachment to list for consistent handling
+            if attachments and isinstance(attachments, str):
+                attachments = [attachments]
+          
+            return self.success_response({"status": "Awaiting user response..."})
+        except Exception as e:
+            return self.fail_response(f"Error asking user: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "web_browser_takeover",
+            "description": "Request user takeover of browser interaction. Use this tool when: 1) The page requires complex human interaction that automated tools cannot handle, 2) Authentication or verification steps require human input, 3) The page has anti-bot measures that prevent automated access, 4) Complex form filling or navigation is needed, 5) The page requires human verification (CAPTCHA, etc.). IMPORTANT: This tool should be used as a last resort after web-search and crawl-webpage have failed, and when direct browser tools are insufficient. Always provide clear context about why takeover is needed and what actions the user should take.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "text": {
+                        "type": "string",
+                        "description": "Instructions for the user about what actions to take in the browser. Include: 1) Clear explanation of why takeover is needed, 2) Specific steps the user should take, 3) What information to look for or extract, 4) How to indicate when they're done, 5) Any important context about the current page state."
+                    },
+                    "attachments": {
+                        "anyOf": [
+                            {"type": "string"},
+                            {"items": {"type": "string"}, "type": "array"}
+                        ],
+                        "description": "(Optional) List of files or URLs to attach to the takeover request. Include when: 1) Screenshots or visual references are needed, 2) Previous search results or crawled content is relevant, 3) Supporting documentation is required. Always use relative paths to /workspace directory."
+                    }
+                },
+                "required": ["text"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="web-browser-takeover",
+        mappings=[
+            {"param_name": "text", "node_type": "content", "path": "."},
+            {"param_name": "attachments", "node_type": "attribute", "path": ".", "required": False}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="web_browser_takeover">
+        <parameter name="text">I've encountered a CAPTCHA verification on the page. Please:
+1. Solve the CAPTCHA puzzle
+2. Let me know once you've completed it
+3. I'll then continue with the automated process
+
+If you encounter any issues or need to take additional steps, please let me know.</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def web_browser_takeover(self, text: str, attachments: Optional[Union[str, List[str]]] = None) -> ToolResult:
+        """Request user takeover of browser interaction.
+
+        Args:
+            text: Instructions for the user about what actions to take
+            attachments: Optional file paths or URLs to attach to the request
+
+        Returns:
+            ToolResult indicating the takeover request was successfully sent
+        """
+        try:
+            # Convert single attachment to list for consistent handling
+            if attachments and isinstance(attachments, str):
+                attachments = [attachments]
+
+            return self.success_response({"status": "Awaiting user browser takeover..."})
+        except Exception as e:
+            return self.fail_response(f"Error requesting browser takeover: {str(e)}")
+
+#     @openapi_schema({
+#         "type": "function",
+#         "function": {
+#             "name": "inform",
+#             "description": "Inform the user about progress, completion of a major step, or important context. Use this tool: 1) To provide updates between major sections of work, 2) After accomplishing significant milestones, 3) When transitioning to a new phase of work, 4) To confirm actions were completed successfully, 5) To provide context about upcoming steps. IMPORTANT: Use FREQUENTLY throughout execution to provide UI context to the user. The user CANNOT respond to this tool - they can only respond to the 'ask' tool. Use this tool to keep the user informed without requiring their input.",
+#             "parameters": {
+#                 "type": "object",
+#                 "properties": {
+#                     "text": {
+#                         "type": "string",
+#                         "description": "Information to present to the user. Include: 1) Clear statement of what has been accomplished or what is happening, 2) Relevant context or impact, 3) Brief indication of next steps if applicable."
+#                     },
+#                     "attachments": {
+#                         "anyOf": [
+#                             {"type": "string"},
+#                             {"items": {"type": "string"}, "type": "array"}
+#                         ],
+#                         "description": "(Optional) List of files or URLs to attach to the information. Include when: 1) Information relates to specific files or resources, 2) Showing intermediate results or outputs, 3) Providing supporting documentation. Always use relative paths to /workspace directory."
+#                     }
+#                 },
+#                 "required": ["text"]
+#             }
+#         }
+#     })
+#     @xml_schema(
+#         tag_name="inform",
+#         mappings=[
+#             {"param_name": "text", "node_type": "content", "path": "."},
+#             {"param_name": "attachments", "node_type": "attribute", "path": ".", "required": False}
+#         ],
+#         example='''
+
+# Inform the user about progress, completion of a major step, or important context. Use this tool: 1) To provide updates between major sections of work, 2) After accomplishing significant milestones, 3) When transitioning to a new phase of work, 4) To confirm actions were completed successfully, 5) To provide context about upcoming steps. IMPORTANT: Use FREQUENTLY throughout execution to provide UI context to the user. The user CANNOT respond to this tool - they can only respond to the 'ask' tool. Use this tool to keep the user informed without requiring their input."
+
+#         <!-- Use inform FREQUENTLY to provide UI context and progress updates - THE USER CANNOT RESPOND to this tool -->
+#         <!-- The user can ONLY respond to the ask tool, not to inform -->
+#         <!-- Examples of when to use inform: -->
+#         <!-- 1. Completing major milestones -->
+#         <!-- 2. Transitioning between work phases -->
+#         <!-- 3. Confirming important actions -->
+#         <!-- 4. Providing context about upcoming steps -->
+#         <!-- 5. Sharing significant intermediate results -->
+#         <!-- 6. Providing regular UI updates throughout execution -->
+
+#         <inform attachments="analysis_results.csv,summary_chart.png">
+#             I've completed the data analysis of the sales figures. Key findings include:
+#             - Q4 sales were 28% higher than Q3
+#             - Product line A showed the strongest performance
+#             - Three regions missed their targets
+
+#             I'll now proceed with creating the executive summary report based on these findings.
+#         </inform>
+#         '''
+#     )
+#     async def inform(self, text: str, attachments: Optional[Union[str, List[str]]] = None) -> ToolResult:
+#         """Inform the user about progress or important updates without requiring a response.
+
+#         Args:
+#             text: The information to present to the user
+#             attachments: Optional file paths or URLs to attach
+
+#         Returns:
+#             ToolResult indicating the information was successfully sent
+#         """
+#         try:
+#             # Convert single attachment to list for consistent handling
+#             if attachments and isinstance(attachments, str):
+#                 attachments = [attachments]
+
+#             return self.success_response({"status": "Information sent"})
+#         except Exception as e:
+#             return self.fail_response(f"Error informing user: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "complete",
+            "description": "A special tool to indicate you have completed all tasks and are about to enter complete state. Use ONLY when: 1) All tasks in todo.md are marked complete [x], 2) The user's original request has been fully addressed, 3) There are no pending actions or follow-ups required, 4) You've delivered all final outputs and results to the user. IMPORTANT: This is the ONLY way to properly terminate execution. Never use this tool unless ALL tasks are complete and verified. Always ensure you've provided all necessary outputs and references before using this tool.",
+            "parameters": {
+                "type": "object",
+                "properties": {},
+                "required": []
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="complete",
+        mappings=[],
+        example='''
+        <function_calls>
+        <invoke name="complete">
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def complete(self) -> ToolResult:
+        """Indicate that the agent has completed all tasks and is entering complete state.
+
+        Returns:
+            ToolResult indicating successful transition to complete state
+        """
+        try:
+            return self.success_response({"status": "complete"})
+        except Exception as e:
+            return self.fail_response(f"Error entering complete state: {str(e)}")
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    async def test_message_tool():
+        message_tool = MessageTool()
+
+        # Test question
+        ask_result = await message_tool.ask(
+            text="Would you like to proceed with the next phase?",
+            attachments="summary.pdf"
+        )
+        print("Question result:", ask_result)
+
+        # Test inform
+        inform_result = await message_tool.inform(
+            text="Completed analysis of data. Processing results now.",
+            attachments="analysis.pdf"
+        )
+        print("Inform result:", inform_result)
+
+    asyncio.run(test_message_tool())

app/tool/sb_deploy_tool.py

@@ -0,0 +1,147 @@
+import os
+from dotenv import load_dotenv
+from agentpress.tool import ToolResult, openapi_schema, xml_schema
+from sandbox.tool_base import SandboxToolsBase
+from utils.files_utils import clean_path
+from agentpress.thread_manager import ThreadManager
+
+# Load environment variables
+load_dotenv()
+
+class SandboxDeployTool(SandboxToolsBase):
+    """Tool for deploying static websites from a Daytona sandbox to Cloudflare Pages."""
+
+    def __init__(self, project_id: str, thread_manager: ThreadManager):
+        super().__init__(project_id, thread_manager)
+        self.workspace_path = "/workspace"  # Ensure we're always operating in /workspace
+        self.cloudflare_api_token = os.getenv("CLOUDFLARE_API_TOKEN")
+
+    def clean_path(self, path: str) -> str:
+        """Clean and normalize a path to be relative to /workspace"""
+        return clean_path(path, self.workspace_path)
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "deploy",
+            "description": "Deploy a static website (HTML+CSS+JS) from a directory in the sandbox to Cloudflare Pages. Only use this tool when permanent deployment to a production environment is needed. The directory path must be relative to /workspace. The website will be deployed to {name}.kortix.cloud.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "name": {
+                        "type": "string",
+                        "description": "Name for the deployment, will be used in the URL as {name}.kortix.cloud"
+                    },
+                    "directory_path": {
+                        "type": "string",
+                        "description": "Path to the directory containing the static website files to deploy, relative to /workspace (e.g., 'build')"
+                    }
+                },
+                "required": ["name", "directory_path"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="deploy",
+        mappings=[
+            {"param_name": "name", "node_type": "attribute", "path": "name"},
+            {"param_name": "directory_path", "node_type": "attribute", "path": "directory_path"}
+        ],
+        example='''
+        <!-- 
+        IMPORTANT: Only use this tool when:
+        1. The user explicitly requests permanent deployment to production
+        2. You have a complete, ready-to-deploy directory 
+        
+        NOTE: If the same name is used, it will redeploy to the same project as before
+        -->
+
+        <function_calls>
+        <invoke name="deploy">
+        <parameter name="name">my-site</parameter>
+        <parameter name="directory_path">website</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def deploy(self, name: str, directory_path: str) -> ToolResult:
+        """
+        Deploy a static website (HTML+CSS+JS) from the sandbox to Cloudflare Pages.
+        Only use this tool when permanent deployment to a production environment is needed.
+        
+        Args:
+            name: Name for the deployment, will be used in the URL as {name}.kortix.cloud
+            directory_path: Path to the directory to deploy, relative to /workspace
+            
+        Returns:
+            ToolResult containing:
+            - Success: Deployment information including URL
+            - Failure: Error message if deployment fails
+        """
+        try:
+            # Ensure sandbox is initialized
+            await self._ensure_sandbox()
+            
+            directory_path = self.clean_path(directory_path)
+            full_path = f"{self.workspace_path}/{directory_path}"
+            
+            # Verify the directory exists
+            try:
+                dir_info = self.sandbox.fs.get_file_info(full_path)
+                if not dir_info.is_dir:
+                    return self.fail_response(f"'{directory_path}' is not a directory")
+            except Exception as e:
+                return self.fail_response(f"Directory '{directory_path}' does not exist: {str(e)}")
+            
+            # Deploy to Cloudflare Pages directly from the container
+            try:
+                # Get Cloudflare API token from environment
+                if not self.cloudflare_api_token:
+                    return self.fail_response("CLOUDFLARE_API_TOKEN environment variable not set")
+                    
+                # Single command that creates the project if it doesn't exist and then deploys
+                project_name = f"{self.sandbox_id}-{name}"
+                deploy_cmd = f'''cd {self.workspace_path} && export CLOUDFLARE_API_TOKEN={self.cloudflare_api_token} && 
+                    (npx wrangler pages deploy {full_path} --project-name {project_name} || 
+                    (npx wrangler pages project create {project_name} --production-branch production && 
+                    npx wrangler pages deploy {full_path} --project-name {project_name}))'''
+
+                # Execute the command directly using the sandbox's process.exec method
+                response = self.sandbox.process.exec(f"/bin/sh -c \"{deploy_cmd}\"",
+                                 timeout=300)
+                
+                print(f"Deployment command output: {response.result}")
+                
+                if response.exit_code == 0:
+                    return self.success_response({
+                        "message": f"Website deployed successfully",
+                        "output": response.result
+                    })
+                else:
+                    return self.fail_response(f"Deployment failed with exit code {response.exit_code}: {response.result}")
+            except Exception as e:
+                return self.fail_response(f"Error during deployment: {str(e)}")
+        except Exception as e:
+            return self.fail_response(f"Error deploying website: {str(e)}")
+
+if __name__ == "__main__":
+    import asyncio
+    import sys
+    
+    async def test_deploy():
+        # Replace these with actual values for testing
+        sandbox_id = "sandbox-ccb30b35"
+        password = "test-password"
+        
+        # Initialize the deploy tool
+        deploy_tool = SandboxDeployTool(sandbox_id, password)
+        
+        # Test deployment - replace with actual directory path and site name
+        result = await deploy_tool.deploy(
+            name="test-site-1x",
+            directory_path="website"  # Directory containing static site files
+        )
+        print(f"Deployment result: {result}")
+            
+    asyncio.run(test_deploy())
+

app/tool/sb_expose_tool.py

@@ -0,0 +1,126 @@
+from agentpress.tool import ToolResult, openapi_schema, xml_schema
+from sandbox.tool_base import SandboxToolsBase
+from agentpress.thread_manager import ThreadManager
+import asyncio
+import time
+
+class SandboxExposeTool(SandboxToolsBase):
+    """Tool for exposing and retrieving preview URLs for sandbox ports."""
+
+    def __init__(self, project_id: str, thread_manager: ThreadManager):
+        super().__init__(project_id, thread_manager)
+
+    async def _wait_for_sandbox_services(self, timeout: int = 30) -> bool:
+        """Wait for sandbox services to be fully started before exposing ports."""
+        start_time = time.time()
+        
+        while time.time() - start_time < timeout:
+            try:
+                # Check if supervisord is running and managing services
+                result = self.sandbox.process.exec("supervisorctl status", timeout=10)
+                
+                if result.exit_code == 0:
+                    # Check if key services are running
+                    status_output = result.output
+                    if "http_server" in status_output and "RUNNING" in status_output:
+                        return True
+                
+                # If services aren't ready, wait a bit
+                await asyncio.sleep(2)
+                
+            except Exception as e:
+                # If we can't check status, wait a bit and try again
+                await asyncio.sleep(2)
+        
+        return False
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "expose_port",
+            "description": "Expose a port from the agent's sandbox environment to the public internet and get its preview URL. This is essential for making services running in the sandbox accessible to users, such as web applications, APIs, or other network services. The exposed URL can be shared with users to allow them to interact with the sandbox environment.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "port": {
+                        "type": "integer",
+                        "description": "The port number to expose. Must be a valid port number between 1 and 65535.",
+                        "minimum": 1,
+                        "maximum": 65535
+                    }
+                },
+                "required": ["port"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="expose-port",
+        mappings=[
+            {"param_name": "port", "node_type": "content", "path": "."}
+        ],
+        example='''
+        <!-- Example 1: Expose a web server running on port 8000 -->
+        <function_calls>
+        <invoke name="expose_port">
+        <parameter name="port">8000</parameter>
+        </invoke>
+        </function_calls>
+
+        <!-- Example 2: Expose an API service running on port 3000 -->
+        <function_calls>
+        <invoke name="expose_port">
+        <parameter name="port">3000</parameter>
+        </invoke>
+        </function_calls>
+
+        <!-- Example 3: Expose a development server running on port 5173 -->
+        <function_calls>
+        <invoke name="expose_port">
+        <parameter name="port">5173</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def expose_port(self, port: int) -> ToolResult:
+        try:
+            # Ensure sandbox is initialized
+            await self._ensure_sandbox()
+            
+            # Convert port to integer if it's a string
+            port = int(port)
+            
+            # Validate port number
+            if not 1 <= port <= 65535:
+                return self.fail_response(f"Invalid port number: {port}. Must be between 1 and 65535.")
+
+            # Wait for sandbox services to be ready (especially important for workflows)
+            services_ready = await self._wait_for_sandbox_services()
+            if not services_ready:
+                return self.fail_response(f"Sandbox services are not fully started yet. Please wait a moment and try again, or ensure a service is running on port {port}.")
+
+            # Check if something is actually listening on the port (for custom ports)
+            if port not in [6080, 8080, 8003]:  # Skip check for known sandbox ports
+                try:
+                    port_check = self.sandbox.process.exec(f"netstat -tlnp | grep :{port}", timeout=5)
+                    if port_check.exit_code != 0:
+                        return self.fail_response(f"No service is currently listening on port {port}. Please start a service on this port first.")
+                except Exception:
+                    # If we can't check, proceed anyway - the user might be starting a service
+                    pass
+
+            # Get the preview link for the specified port
+            preview_link = self.sandbox.get_preview_link(port)
+            
+            # Extract the actual URL from the preview link object
+            url = preview_link.url if hasattr(preview_link, 'url') else str(preview_link)
+            
+            return self.success_response({
+                "url": url,
+                "port": port,
+                "message": f"Successfully exposed port {port} to the public. Users can now access this service at: {url}"
+            })
+                
+        except ValueError:
+            return self.fail_response(f"Invalid port number: {port}. Must be a valid integer between 1 and 65535.")
+        except Exception as e:
+            return self.fail_response(f"Error exposing port {port}: {str(e)}")

app/tool/sb_files_tool.py

@@ -0,0 +1,462 @@
+from agentpress.tool import ToolResult, openapi_schema, xml_schema
+from sandbox.tool_base import SandboxToolsBase    
+from utils.files_utils import should_exclude_file, clean_path
+from agentpress.thread_manager import ThreadManager
+from utils.logger import logger
+import os
+
+class SandboxFilesTool(SandboxToolsBase):
+    """Tool for executing file system operations in a Daytona sandbox. All operations are performed relative to the /workspace directory."""
+
+    def __init__(self, project_id: str, thread_manager: ThreadManager):
+        super().__init__(project_id, thread_manager)
+        self.SNIPPET_LINES = 4  # Number of context lines to show around edits
+        self.workspace_path = "/workspace"  # Ensure we're always operating in /workspace
+
+    def clean_path(self, path: str) -> str:
+        """Clean and normalize a path to be relative to /workspace"""
+        return clean_path(path, self.workspace_path)
+
+    def _should_exclude_file(self, rel_path: str) -> bool:
+        """Check if a file should be excluded based on path, name, or extension"""
+        return should_exclude_file(rel_path)
+
+    def _file_exists(self, path: str) -> bool:
+        """Check if a file exists in the sandbox"""
+        try:
+            self.sandbox.fs.get_file_info(path)
+            return True
+        except Exception:
+            return False
+
+    async def get_workspace_state(self) -> dict:
+        """Get the current workspace state by reading all files"""
+        files_state = {}
+        try:
+            # Ensure sandbox is initialized
+            await self._ensure_sandbox()
+            
+            files = self.sandbox.fs.list_files(self.workspace_path)
+            for file_info in files:
+                rel_path = file_info.name
+                
+                # Skip excluded files and directories
+                if self._should_exclude_file(rel_path) or file_info.is_dir:
+                    continue
+
+                try:
+                    full_path = f"{self.workspace_path}/{rel_path}"
+                    content = self.sandbox.fs.download_file(full_path).decode()
+                    files_state[rel_path] = {
+                        "content": content,
+                        "is_dir": file_info.is_dir,
+                        "size": file_info.size,
+                        "modified": file_info.mod_time
+                    }
+                except Exception as e:
+                    print(f"Error reading file {rel_path}: {e}")
+                except UnicodeDecodeError:
+                    print(f"Skipping binary file: {rel_path}")
+
+            return files_state
+        
+        except Exception as e:
+            print(f"Error getting workspace state: {str(e)}")
+            return {}
+
+
+    # def _get_preview_url(self, file_path: str) -> Optional[str]:
+    #     """Get the preview URL for a file if it's an HTML file."""
+    #     if file_path.lower().endswith('.html') and self._sandbox_url:
+    #         return f"{self._sandbox_url}/{(file_path.replace('/workspace/', ''))}"
+    #     return None
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "create_file",
+            "description": "Create a new file with the provided contents at a given path in the workspace. The path must be relative to /workspace (e.g., 'src/main.py' for /workspace/src/main.py)",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "file_path": {
+                        "type": "string",
+                        "description": "Path to the file to be created, relative to /workspace (e.g., 'src/main.py')"
+                    },
+                    "file_contents": {
+                        "type": "string",
+                        "description": "The content to write to the file"
+                    },
+                    "permissions": {
+                        "type": "string",
+                        "description": "File permissions in octal format (e.g., '644')",
+                        "default": "644"
+                    }
+                },
+                "required": ["file_path", "file_contents"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="create-file",
+        mappings=[
+            {"param_name": "file_path", "node_type": "attribute", "path": "."},
+            {"param_name": "file_contents", "node_type": "content", "path": "."}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="create_file">
+        <parameter name="file_path">src/main.py</parameter>
+        <parameter name="file_contents">
+        # This is the file content
+        def main():
+            print("Hello, World!")
+        
+        if __name__ == "__main__":
+            main()
+        </parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def create_file(self, file_path: str, file_contents: str, permissions: str = "644") -> ToolResult:
+        try:
+            # Ensure sandbox is initialized
+            await self._ensure_sandbox()
+            
+            file_path = self.clean_path(file_path)
+            full_path = f"{self.workspace_path}/{file_path}"
+            if self._file_exists(full_path):
+                return self.fail_response(f"File '{file_path}' already exists. Use update_file to modify existing files.")
+            
+            # Create parent directories if needed
+            parent_dir = '/'.join(full_path.split('/')[:-1])
+            if parent_dir:
+                self.sandbox.fs.create_folder(parent_dir, "755")
+            
+            # Write the file content
+            self.sandbox.fs.upload_file(file_contents.encode(), full_path)
+            self.sandbox.fs.set_file_permissions(full_path, permissions)
+            
+            message = f"File '{file_path}' created successfully."
+            
+            # Check if index.html was created and add 8080 server info (only in root workspace)
+            if file_path.lower() == 'index.html':
+                try:
+                    website_link = self.sandbox.get_preview_link(8080)
+                    website_url = website_link.url if hasattr(website_link, 'url') else str(website_link).split("url='")[1].split("'")[0]
+                    message += f"\n\n[Auto-detected index.html - HTTP server available at: {website_url}]"
+                    message += "\n[Note: Use the provided HTTP server URL above instead of starting a new server]"
+                except Exception as e:
+                    logger.warning(f"Failed to get website URL for index.html: {str(e)}")
+            
+            return self.success_response(message)
+        except Exception as e:
+            return self.fail_response(f"Error creating file: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "str_replace",
+            "description": "Replace specific text in a file. The file path must be relative to /workspace (e.g., 'src/main.py' for /workspace/src/main.py). Use this when you need to replace a unique string that appears exactly once in the file.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "file_path": {
+                        "type": "string",
+                        "description": "Path to the target file, relative to /workspace (e.g., 'src/main.py')"
+                    },
+                    "old_str": {
+                        "type": "string",
+                        "description": "Text to be replaced (must appear exactly once)"
+                    },
+                    "new_str": {
+                        "type": "string",
+                        "description": "Replacement text"
+                    }
+                },
+                "required": ["file_path", "old_str", "new_str"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="str-replace",
+        mappings=[
+            {"param_name": "file_path", "node_type": "attribute", "path": "."},
+            {"param_name": "old_str", "node_type": "element", "path": "old_str"},
+            {"param_name": "new_str", "node_type": "element", "path": "new_str"}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="str_replace">
+        <parameter name="file_path">src/main.py</parameter>
+        <parameter name="old_str">text to replace (must appear exactly once in the file)</parameter>
+        <parameter name="new_str">replacement text that will be inserted instead</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def str_replace(self, file_path: str, old_str: str, new_str: str) -> ToolResult:
+        try:
+            # Ensure sandbox is initialized
+            await self._ensure_sandbox()
+            
+            file_path = self.clean_path(file_path)
+            full_path = f"{self.workspace_path}/{file_path}"
+            if not self._file_exists(full_path):
+                return self.fail_response(f"File '{file_path}' does not exist")
+            
+            content = self.sandbox.fs.download_file(full_path).decode()
+            old_str = old_str.expandtabs()
+            new_str = new_str.expandtabs()
+            
+            occurrences = content.count(old_str)
+            if occurrences == 0:
+                return self.fail_response(f"String '{old_str}' not found in file")
+            if occurrences > 1:
+                lines = [i+1 for i, line in enumerate(content.split('\n')) if old_str in line]
+                return self.fail_response(f"Multiple occurrences found in lines {lines}. Please ensure string is unique")
+            
+            # Perform replacement
+            new_content = content.replace(old_str, new_str)
+            self.sandbox.fs.upload_file(new_content.encode(), full_path)
+            
+            # Show snippet around the edit
+            replacement_line = content.split(old_str)[0].count('\n')
+            start_line = max(0, replacement_line - self.SNIPPET_LINES)
+            end_line = replacement_line + self.SNIPPET_LINES + new_str.count('\n')
+            snippet = '\n'.join(new_content.split('\n')[start_line:end_line + 1])
+            
+            # Get preview URL if it's an HTML file
+            # preview_url = self._get_preview_url(file_path)
+            message = f"Replacement successful."
+            # if preview_url:
+            #     message += f"\n\nYou can preview this HTML file at: {preview_url}"
+            
+            return self.success_response(message)
+            
+        except Exception as e:
+            return self.fail_response(f"Error replacing string: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "full_file_rewrite",
+            "description": "Completely rewrite an existing file with new content. The file path must be relative to /workspace (e.g., 'src/main.py' for /workspace/src/main.py). Use this when you need to replace the entire file content or make extensive changes throughout the file.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "file_path": {
+                        "type": "string",
+                        "description": "Path to the file to be rewritten, relative to /workspace (e.g., 'src/main.py')"
+                    },
+                    "file_contents": {
+                        "type": "string",
+                        "description": "The new content to write to the file, replacing all existing content"
+                    },
+                    "permissions": {
+                        "type": "string",
+                        "description": "File permissions in octal format (e.g., '644')",
+                        "default": "644"
+                    }
+                },
+                "required": ["file_path", "file_contents"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="full-file-rewrite",
+        mappings=[
+            {"param_name": "file_path", "node_type": "attribute", "path": "."},
+            {"param_name": "file_contents", "node_type": "content", "path": "."}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="full_file_rewrite">
+        <parameter name="file_path">src/main.py</parameter>
+        <parameter name="file_contents">
+        This completely replaces the entire file content.
+        Use when making major changes to a file or when the changes
+        are too extensive for str-replace.
+        All previous content will be lost and replaced with this text.
+        </parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def full_file_rewrite(self, file_path: str, file_contents: str, permissions: str = "644") -> ToolResult:
+        try:
+            # Ensure sandbox is initialized
+            await self._ensure_sandbox()
+            
+            file_path = self.clean_path(file_path)
+            full_path = f"{self.workspace_path}/{file_path}"
+            if not self._file_exists(full_path):
+                return self.fail_response(f"File '{file_path}' does not exist. Use create_file to create a new file.")
+            
+            self.sandbox.fs.upload_file(file_contents.encode(), full_path)
+            self.sandbox.fs.set_file_permissions(full_path, permissions)
+            
+            message = f"File '{file_path}' completely rewritten successfully."
+            
+            # Check if index.html was rewritten and add 8080 server info (only in root workspace)
+            if file_path.lower() == 'index.html':
+                try:
+                    website_link = self.sandbox.get_preview_link(8080)
+                    website_url = website_link.url if hasattr(website_link, 'url') else str(website_link).split("url='")[1].split("'")[0]
+                    message += f"\n\n[Auto-detected index.html - HTTP server available at: {website_url}]"
+                    message += "\n[Note: Use the provided HTTP server URL above instead of starting a new server]"
+                except Exception as e:
+                    logger.warning(f"Failed to get website URL for index.html: {str(e)}")
+            
+            return self.success_response(message)
+        except Exception as e:
+            return self.fail_response(f"Error rewriting file: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "delete_file",
+            "description": "Delete a file at the given path. The path must be relative to /workspace (e.g., 'src/main.py' for /workspace/src/main.py)",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "file_path": {
+                        "type": "string",
+                        "description": "Path to the file to be deleted, relative to /workspace (e.g., 'src/main.py')"
+                    }
+                },
+                "required": ["file_path"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="delete-file",
+        mappings=[
+            {"param_name": "file_path", "node_type": "attribute", "path": "."}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="delete_file">
+        <parameter name="file_path">src/main.py</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def delete_file(self, file_path: str) -> ToolResult:
+        try:
+            # Ensure sandbox is initialized
+            await self._ensure_sandbox()
+            
+            file_path = self.clean_path(file_path)
+            full_path = f"{self.workspace_path}/{file_path}"
+            if not self._file_exists(full_path):
+                return self.fail_response(f"File '{file_path}' does not exist")
+            
+            self.sandbox.fs.delete_file(full_path)
+            return self.success_response(f"File '{file_path}' deleted successfully.")
+        except Exception as e:
+            return self.fail_response(f"Error deleting file: {str(e)}")
+
+    # @openapi_schema({
+    #     "type": "function",
+    #     "function": {
+    #         "name": "read_file",
+    #         "description": "Read and return the contents of a file. This tool is essential for verifying data, checking file contents, and analyzing information. Always use this tool to read file contents before processing or analyzing data. The file path must be relative to /workspace.",
+    #         "parameters": {
+    #             "type": "object",
+    #             "properties": {
+    #                 "file_path": {
+    #                     "type": "string",
+    #                     "description": "Path to the file to read, relative to /workspace (e.g., 'src/main.py' for /workspace/src/main.py). Must be a valid file path within the workspace."
+    #                 },
+    #                 "start_line": {
+    #                     "type": "integer",
+    #                     "description": "Optional starting line number (1-based). Use this to read specific sections of large files. If not specified, reads from the beginning of the file.",
+    #                     "default": 1
+    #                 },
+    #                 "end_line": {
+    #                     "type": "integer",
+    #                     "description": "Optional ending line number (inclusive). Use this to read specific sections of large files. If not specified, reads to the end of the file.",
+    #                     "default": None
+    #                 }
+    #             },
+    #             "required": ["file_path"]
+    #         }
+    #     }
+    # })
+    # @xml_schema(
+    #     tag_name="read-file",
+    #     mappings=[
+    #         {"param_name": "file_path", "node_type": "attribute", "path": "."},
+    #         {"param_name": "start_line", "node_type": "attribute", "path": ".", "required": False},
+    #         {"param_name": "end_line", "node_type": "attribute", "path": ".", "required": False}
+    #     ],
+    #     example='''
+    #     <!-- Example 1: Read entire file -->
+    #     <read-file file_path="src/main.py">
+    #     </read-file>
+
+    #     <!-- Example 2: Read specific lines (lines 10-20) -->
+    #     <read-file file_path="src/main.py" start_line="10" end_line="20">
+    #     </read-file>
+
+    #     <!-- Example 3: Read from line 5 to end -->
+    #     <read-file file_path="config.json" start_line="5">
+    #     </read-file>
+
+    #     <!-- Example 4: Read last 10 lines -->
+    #     <read-file file_path="logs/app.log" start_line="-10">
+    #     </read-file>
+    #     '''
+    # )
+    # async def read_file(self, file_path: str, start_line: int = 1, end_line: Optional[int] = None) -> ToolResult:
+    #     """Read file content with optional line range specification.
+        
+    #     Args:
+    #         file_path: Path to the file relative to /workspace
+    #         start_line: Starting line number (1-based), defaults to 1
+    #         end_line: Ending line number (inclusive), defaults to None (end of file)
+            
+    #     Returns:
+    #         ToolResult containing:
+    #         - Success: File content and metadata
+    #         - Failure: Error message if file doesn't exist or is binary
+    #     """
+    #     try:
+    #         file_path = self.clean_path(file_path)
+    #         full_path = f"{self.workspace_path}/{file_path}"
+            
+    #         if not self._file_exists(full_path):
+    #             return self.fail_response(f"File '{file_path}' does not exist")
+            
+    #         # Download and decode file content
+    #         content = self.sandbox.fs.download_file(full_path).decode()
+            
+    #         # Split content into lines
+    #         lines = content.split('\n')
+    #         total_lines = len(lines)
+            
+    #         # Handle line range if specified
+    #         if start_line > 1 or end_line is not None:
+    #             # Convert to 0-based indices
+    #             start_idx = max(0, start_line - 1)
+    #             end_idx = end_line if end_line is not None else total_lines
+    #             end_idx = min(end_idx, total_lines)  # Ensure we don't exceed file length
+                
+    #             # Extract the requested lines
+    #             content = '\n'.join(lines[start_idx:end_idx])
+            
+    #         return self.success_response({
+    #             "content": content,
+    #             "file_path": file_path,
+    #             "start_line": start_line,
+    #             "end_line": end_line if end_line is not None else total_lines,
+    #             "total_lines": total_lines
+    #         })
+            
+    #     except UnicodeDecodeError:
+    #         return self.fail_response(f"File '{file_path}' appears to be binary and cannot be read as text")
+    #     except Exception as e:
+    #         return self.fail_response(f"Error reading file: {str(e)}")
+

app/tool/sb_shell_tool.py

@@ -0,0 +1,423 @@
+from typing import Optional, Dict, Any
+import time
+from uuid import uuid4
+from agentpress.tool import ToolResult, openapi_schema, xml_schema
+from sandbox.tool_base import SandboxToolsBase
+from agentpress.thread_manager import ThreadManager
+
+class SandboxShellTool(SandboxToolsBase):
+    """Tool for executing tasks in a Daytona sandbox with browser-use capabilities. 
+    Uses sessions for maintaining state between commands and provides comprehensive process management."""
+
+    def __init__(self, project_id: str, thread_manager: ThreadManager):
+        super().__init__(project_id, thread_manager)
+        self._sessions: Dict[str, str] = {}  # Maps session names to session IDs
+        self.workspace_path = "/workspace"  # Ensure we're always operating in /workspace
+
+    async def _ensure_session(self, session_name: str = "default") -> str:
+        """Ensure a session exists and return its ID."""
+        if session_name not in self._sessions:
+            session_id = str(uuid4())
+            try:
+                await self._ensure_sandbox()  # Ensure sandbox is initialized
+                self.sandbox.process.create_session(session_id)
+                self._sessions[session_name] = session_id
+            except Exception as e:
+                raise RuntimeError(f"Failed to create session: {str(e)}")
+        return self._sessions[session_name]
+
+    async def _cleanup_session(self, session_name: str):
+        """Clean up a session if it exists."""
+        if session_name in self._sessions:
+            try:
+                await self._ensure_sandbox()  # Ensure sandbox is initialized
+                self.sandbox.process.delete_session(self._sessions[session_name])
+                del self._sessions[session_name]
+            except Exception as e:
+                print(f"Warning: Failed to cleanup session {session_name}: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "execute_command",
+            "description": "Execute a shell command in the workspace directory. IMPORTANT: Commands are non-blocking by default and run in a tmux session. This is ideal for long-running operations like starting servers or build processes. Uses sessions to maintain state between commands. This tool is essential for running CLI tools, installing packages, and managing system operations.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "command": {
+                        "type": "string",
+                        "description": "The shell command to execute. Use this for running CLI tools, installing packages, or system operations. Commands can be chained using &&, ||, and | operators."
+                    },
+                    "folder": {
+                        "type": "string",
+                        "description": "Optional relative path to a subdirectory of /workspace where the command should be executed. Example: 'data/pdfs'"
+                    },
+                    "session_name": {
+                        "type": "string",
+                        "description": "Optional name of the tmux session to use. Use named sessions for related commands that need to maintain state. Defaults to a random session name.",
+                    },
+                    "blocking": {
+                        "type": "boolean",
+                        "description": "Whether to wait for the command to complete. Defaults to false for non-blocking execution.",
+                        "default": False
+                    },
+                    "timeout": {
+                        "type": "integer",
+                        "description": "Optional timeout in seconds for blocking commands. Defaults to 60. Ignored for non-blocking commands.",
+                        "default": 60
+                    }
+                },
+                "required": ["command"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="execute-command",
+        mappings=[
+            {"param_name": "command", "node_type": "content", "path": "."},
+            {"param_name": "folder", "node_type": "attribute", "path": ".", "required": False},
+            {"param_name": "session_name", "node_type": "attribute", "path": ".", "required": False},
+            {"param_name": "blocking", "node_type": "attribute", "path": ".", "required": False},
+            {"param_name": "timeout", "node_type": "attribute", "path": ".", "required": False}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="execute_command">
+        <parameter name="command">npm run dev</parameter>
+        <parameter name="session_name">dev_server</parameter>
+        </invoke>
+        </function_calls>
+
+        <!-- Example 2: Running in Specific Directory -->
+        <function_calls>
+        <invoke name="execute_command">
+        <parameter name="command">npm run build</parameter>
+        <parameter name="folder">frontend</parameter>
+        <parameter name="session_name">build_process</parameter>
+        </invoke>
+        </function_calls>
+
+        <!-- Example 3: Blocking command (wait for completion) -->
+        <function_calls>
+        <invoke name="execute_command">
+        <parameter name="command">npm install</parameter>
+        <parameter name="blocking">true</parameter>
+        <parameter name="timeout">300</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def execute_command(
+        self, 
+        command: str, 
+        folder: Optional[str] = None,
+        session_name: Optional[str] = None,
+        blocking: bool = False,
+        timeout: int = 60
+    ) -> ToolResult:
+        try:
+            # Ensure sandbox is initialized
+            await self._ensure_sandbox()
+            
+            # Set up working directory
+            cwd = self.workspace_path
+            if folder:
+                folder = folder.strip('/')
+                cwd = f"{self.workspace_path}/{folder}"
+            
+            # Generate a session name if not provided
+            if not session_name:
+                session_name = f"session_{str(uuid4())[:8]}"
+            
+            # Check if tmux session already exists
+            check_session = await self._execute_raw_command(f"tmux has-session -t {session_name} 2>/dev/null || echo 'not_exists'")
+            session_exists = "not_exists" not in check_session.get("output", "")
+            
+            if not session_exists:
+                # Create a new tmux session
+                await self._execute_raw_command(f"tmux new-session -d -s {session_name}")
+                
+            # Ensure we're in the correct directory and send command to tmux
+            full_command = f"cd {cwd} && {command}"
+            wrapped_command = full_command.replace('"', '\\"')  # Escape double quotes
+            
+            # Send command to tmux session
+            await self._execute_raw_command(f'tmux send-keys -t {session_name} "{wrapped_command}" Enter')
+            
+            if blocking:
+                # For blocking execution, wait and capture output
+                start_time = time.time()
+                while (time.time() - start_time) < timeout:
+                    # Wait a bit before checking
+                    time.sleep(2)
+                    
+                    # Check if session still exists (command might have exited)
+                    check_result = await self._execute_raw_command(f"tmux has-session -t {session_name} 2>/dev/null || echo 'ended'")
+                    if "ended" in check_result.get("output", ""):
+                        break
+                        
+                    # Get current output and check for common completion indicators
+                    output_result = await self._execute_raw_command(f"tmux capture-pane -t {session_name} -p -S - -E -")
+                    current_output = output_result.get("output", "")
+                    
+                    # Check for prompt indicators that suggest command completion
+                    last_lines = current_output.split('\n')[-3:]
+                    completion_indicators = ['$', '#', '>', 'Done', 'Completed', 'Finished', '✓']
+                    if any(indicator in line for indicator in completion_indicators for line in last_lines):
+                        break
+                
+                # Capture final output
+                output_result = await self._execute_raw_command(f"tmux capture-pane -t {session_name} -p -S - -E -")
+                final_output = output_result.get("output", "")
+                
+                # Kill the session after capture
+                await self._execute_raw_command(f"tmux kill-session -t {session_name}")
+                
+                return self.success_response({
+                    "output": final_output,
+                    "session_name": session_name,
+                    "cwd": cwd,
+                    "completed": True
+                })
+            else:
+                # For non-blocking, just return immediately
+                return self.success_response({
+                    "session_name": session_name,
+                    "cwd": cwd,
+                    "message": f"Command sent to tmux session '{session_name}'. Use check_command_output to view results.",
+                    "completed": False
+                })
+                
+        except Exception as e:
+            # Attempt to clean up session in case of error
+            if session_name:
+                try:
+                    await self._execute_raw_command(f"tmux kill-session -t {session_name}")
+                except:
+                    pass
+            return self.fail_response(f"Error executing command: {str(e)}")
+
+    async def _execute_raw_command(self, command: str) -> Dict[str, Any]:
+        """Execute a raw command directly in the sandbox."""
+        # Ensure session exists for raw commands
+        session_id = await self._ensure_session("raw_commands")
+        
+        # Execute command in session
+        from sandbox.sandbox import SessionExecuteRequest
+        req = SessionExecuteRequest(
+            command=command,
+            var_async=False,
+            cwd=self.workspace_path
+        )
+        
+        response = self.sandbox.process.execute_session_command(
+            session_id=session_id,
+            req=req,
+            timeout=30  # Short timeout for utility commands
+        )
+        
+        logs = self.sandbox.process.get_session_command_logs(
+            session_id=session_id,
+            command_id=response.cmd_id
+        )
+        
+        return {
+            "output": logs,
+            "exit_code": response.exit_code
+        }
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "check_command_output",
+            "description": "Check the output of a previously executed command in a tmux session. Use this to monitor the progress or results of non-blocking commands.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "session_name": {
+                        "type": "string",
+                        "description": "The name of the tmux session to check."
+                    },
+                    "kill_session": {
+                        "type": "boolean",
+                        "description": "Whether to terminate the tmux session after checking. Set to true when you're done with the command.",
+                        "default": False
+                    }
+                },
+                "required": ["session_name"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="check-command-output",
+        mappings=[
+            {"param_name": "session_name", "node_type": "attribute", "path": ".", "required": True},
+            {"param_name": "kill_session", "node_type": "attribute", "path": ".", "required": False}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="check_command_output">
+        <parameter name="session_name">dev_server</parameter>
+        </invoke>
+        </function_calls>
+        
+        <!-- Example 2: Check final output and kill session -->
+        <function_calls>
+        <invoke name="check_command_output">
+        <parameter name="session_name">build_process</parameter>
+        <parameter name="kill_session">true</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def check_command_output(
+        self,
+        session_name: str,
+        kill_session: bool = False
+    ) -> ToolResult:
+        try:
+            # Ensure sandbox is initialized
+            await self._ensure_sandbox()
+            
+            # Check if session exists
+            check_result = await self._execute_raw_command(f"tmux has-session -t {session_name} 2>/dev/null || echo 'not_exists'")
+            if "not_exists" in check_result.get("output", ""):
+                return self.fail_response(f"Tmux session '{session_name}' does not exist.")
+            
+            # Get output from tmux pane
+            output_result = await self._execute_raw_command(f"tmux capture-pane -t {session_name} -p -S - -E -")
+            output = output_result.get("output", "")
+            
+            # Kill session if requested
+            if kill_session:
+                await self._execute_raw_command(f"tmux kill-session -t {session_name}")
+                termination_status = "Session terminated."
+            else:
+                termination_status = "Session still running."
+            
+            return self.success_response({
+                "output": output,
+                "session_name": session_name,
+                "status": termination_status
+            })
+                
+        except Exception as e:
+            return self.fail_response(f"Error checking command output: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "terminate_command",
+            "description": "Terminate a running command by killing its tmux session.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "session_name": {
+                        "type": "string",
+                        "description": "The name of the tmux session to terminate."
+                    }
+                },
+                "required": ["session_name"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="terminate-command",
+        mappings=[
+            {"param_name": "session_name", "node_type": "attribute", "path": ".", "required": True}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="terminate_command">
+        <parameter name="session_name">dev_server</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def terminate_command(
+        self,
+        session_name: str
+    ) -> ToolResult:
+        try:
+            # Ensure sandbox is initialized
+            await self._ensure_sandbox()
+            
+            # Check if session exists
+            check_result = await self._execute_raw_command(f"tmux has-session -t {session_name} 2>/dev/null || echo 'not_exists'")
+            if "not_exists" in check_result.get("output", ""):
+                return self.fail_response(f"Tmux session '{session_name}' does not exist.")
+            
+            # Kill the session
+            await self._execute_raw_command(f"tmux kill-session -t {session_name}")
+            
+            return self.success_response({
+                "message": f"Tmux session '{session_name}' terminated successfully."
+            })
+                
+        except Exception as e:
+            return self.fail_response(f"Error terminating command: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "list_commands",
+            "description": "List all running tmux sessions and their status.",
+            "parameters": {
+                "type": "object",
+                "properties": {}
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="list-commands",
+        mappings=[],
+        example='''
+        <function_calls>
+        <invoke name="list_commands">
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def list_commands(self) -> ToolResult:
+        try:
+            # Ensure sandbox is initialized
+            await self._ensure_sandbox()
+            
+            # List all tmux sessions
+            result = await self._execute_raw_command("tmux list-sessions 2>/dev/null || echo 'No sessions'")
+            output = result.get("output", "")
+            
+            if "No sessions" in output or not output.strip():
+                return self.success_response({
+                    "message": "No active tmux sessions found.",
+                    "sessions": []
+                })
+            
+            # Parse session list
+            sessions = []
+            for line in output.split('\n'):
+                if line.strip():
+                    parts = line.split(':')
+                    if parts:
+                        session_name = parts[0].strip()
+                        sessions.append(session_name)
+            
+            return self.success_response({
+                "message": f"Found {len(sessions)} active sessions.",
+                "sessions": sessions
+            })
+                
+        except Exception as e:
+            return self.fail_response(f"Error listing commands: {str(e)}")
+
+    async def cleanup(self):
+        """Clean up all sessions."""
+        for session_name in list(self._sessions.keys()):
+            await self._cleanup_session(session_name)
+        
+        # Also clean up any tmux sessions
+        try:
+            await self._ensure_sandbox()
+            await self._execute_raw_command("tmux kill-server 2>/dev/null || true")
+        except:
+            pass

app/tool/sb_vision_tool.py

@@ -0,0 +1,206 @@
+import os
+import base64
+import mimetypes
+from typing import Optional, Tuple
+from io import BytesIO
+from PIL import Image
+
+from agentpress.tool import ToolResult, openapi_schema, xml_schema
+from sandbox.tool_base import SandboxToolsBase
+from agentpress.thread_manager import ThreadManager
+import json
+
+# Add common image MIME types if mimetypes module is limited
+mimetypes.add_type("image/webp", ".webp")
+mimetypes.add_type("image/jpeg", ".jpg")
+mimetypes.add_type("image/jpeg", ".jpeg")
+mimetypes.add_type("image/png", ".png")
+mimetypes.add_type("image/gif", ".gif")
+
+# Maximum file size in bytes (e.g., 10MB for original, 5MB for compressed)
+MAX_IMAGE_SIZE = 10 * 1024 * 1024
+MAX_COMPRESSED_SIZE = 5 * 1024 * 1024
+
+# Compression settings
+DEFAULT_MAX_WIDTH = 1920
+DEFAULT_MAX_HEIGHT = 1080
+DEFAULT_JPEG_QUALITY = 85
+DEFAULT_PNG_COMPRESS_LEVEL = 6
+
+class SandboxVisionTool(SandboxToolsBase):
+    """Tool for allowing the agent to 'see' images within the sandbox."""
+
+    def __init__(self, project_id: str, thread_id: str, thread_manager: ThreadManager):
+        super().__init__(project_id, thread_manager)
+        self.thread_id = thread_id
+        # Make thread_manager accessible within the tool instance
+        self.thread_manager = thread_manager
+
+    def compress_image(self, image_bytes: bytes, mime_type: str, file_path: str) -> Tuple[bytes, str]:
+        """Compress an image to reduce its size while maintaining reasonable quality.
+        
+        Args:
+            image_bytes: Original image bytes
+            mime_type: MIME type of the image
+            file_path: Path to the image file (for logging)
+            
+        Returns:
+            Tuple of (compressed_bytes, new_mime_type)
+        """
+        try:
+            # Open image from bytes
+            img = Image.open(BytesIO(image_bytes))
+            
+            # Convert RGBA to RGB if necessary (for JPEG)
+            if img.mode in ('RGBA', 'LA', 'P'):
+                # Create a white background
+                background = Image.new('RGB', img.size, (255, 255, 255))
+                if img.mode == 'P':
+                    img = img.convert('RGBA')
+                background.paste(img, mask=img.split()[-1] if img.mode == 'RGBA' else None)
+                img = background
+            
+            # Calculate new dimensions while maintaining aspect ratio
+            width, height = img.size
+            if width > DEFAULT_MAX_WIDTH or height > DEFAULT_MAX_HEIGHT:
+                ratio = min(DEFAULT_MAX_WIDTH / width, DEFAULT_MAX_HEIGHT / height)
+                new_width = int(width * ratio)
+                new_height = int(height * ratio)
+                img = img.resize((new_width, new_height), Image.Resampling.LANCZOS)
+                print(f"[SeeImage] Resized image from {width}x{height} to {new_width}x{new_height}")
+            
+            # Save to bytes with compression
+            output = BytesIO()
+            
+            # Determine output format based on original mime type
+            if mime_type == 'image/gif':
+                # Keep GIFs as GIFs to preserve animation
+                img.save(output, format='GIF', optimize=True)
+                output_mime = 'image/gif'
+            elif mime_type == 'image/png':
+                # Compress PNG
+                img.save(output, format='PNG', optimize=True, compress_level=DEFAULT_PNG_COMPRESS_LEVEL)
+                output_mime = 'image/png'
+            else:
+                # Convert everything else to JPEG for better compression
+                img.save(output, format='JPEG', quality=DEFAULT_JPEG_QUALITY, optimize=True)
+                output_mime = 'image/jpeg'
+            
+            compressed_bytes = output.getvalue()
+            
+            # Log compression results
+            original_size = len(image_bytes)
+            compressed_size = len(compressed_bytes)
+            compression_ratio = (1 - compressed_size / original_size) * 100
+            print(f"[SeeImage] Compressed '{file_path}' from {original_size / 1024:.1f}KB to {compressed_size / 1024:.1f}KB ({compression_ratio:.1f}% reduction)")
+            
+            return compressed_bytes, output_mime
+            
+        except Exception as e:
+            print(f"[SeeImage] Failed to compress image: {str(e)}. Using original.")
+            return image_bytes, mime_type
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "see_image",
+            "description": "Allows the agent to 'see' an image file located in the /workspace directory. Provide the relative path to the image. The image will be compressed before sending to reduce token usage. The image content will be made available in the next turn's context.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "file_path": {
+                        "type": "string",
+                        "description": "The relative path to the image file within the /workspace directory (e.g., 'screenshots/image.png'). Supported formats: JPG, PNG, GIF, WEBP. Max size: 10MB."
+                    }
+                },
+                "required": ["file_path"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="see-image",
+        mappings=[
+            {"param_name": "file_path", "node_type": "attribute", "path": "."}
+        ],
+        example='''
+        <!-- Example: Request to see an image named 'diagram.png' inside the 'docs' folder -->
+        <function_calls>
+        <invoke name="see_image">
+        <parameter name="file_path">docs/diagram.png</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def see_image(self, file_path: str) -> ToolResult:
+        """Reads an image file, compresses it, converts it to base64, and adds it as a temporary message."""
+        try:
+            # Ensure sandbox is initialized
+            await self._ensure_sandbox()
+
+            # Clean and construct full path
+            cleaned_path = self.clean_path(file_path)
+            full_path = f"{self.workspace_path}/{cleaned_path}"
+
+            # Check if file exists and get info
+            try:
+                file_info = self.sandbox.fs.get_file_info(full_path)
+                if file_info.is_dir:
+                    return self.fail_response(f"Path '{cleaned_path}' is a directory, not an image file.")
+            except Exception as e:
+                return self.fail_response(f"Image file not found at path: '{cleaned_path}'")
+
+            # Check file size
+            if file_info.size > MAX_IMAGE_SIZE:
+                return self.fail_response(f"Image file '{cleaned_path}' is too large ({file_info.size / (1024*1024):.2f}MB). Maximum size is {MAX_IMAGE_SIZE / (1024*1024)}MB.")
+
+            # Read image file content
+            try:
+                image_bytes = self.sandbox.fs.download_file(full_path)
+            except Exception as e:
+                return self.fail_response(f"Could not read image file: {cleaned_path}")
+
+            # Determine MIME type
+            mime_type, _ = mimetypes.guess_type(full_path)
+            if not mime_type or not mime_type.startswith('image/'):
+                # Basic fallback based on extension if mimetypes fails
+                ext = os.path.splitext(cleaned_path)[1].lower()
+                if ext == '.jpg' or ext == '.jpeg': mime_type = 'image/jpeg'
+                elif ext == '.png': mime_type = 'image/png'
+                elif ext == '.gif': mime_type = 'image/gif'
+                elif ext == '.webp': mime_type = 'image/webp'
+                else:
+                    return self.fail_response(f"Unsupported or unknown image format for file: '{cleaned_path}'. Supported: JPG, PNG, GIF, WEBP.")
+
+            # Compress the image
+            compressed_bytes, compressed_mime_type = self.compress_image(image_bytes, mime_type, cleaned_path)
+            
+            # Check if compressed image is still too large
+            if len(compressed_bytes) > MAX_COMPRESSED_SIZE:
+                return self.fail_response(f"Image file '{cleaned_path}' is still too large after compression ({len(compressed_bytes) / (1024*1024):.2f}MB). Maximum compressed size is {MAX_COMPRESSED_SIZE / (1024*1024)}MB.")
+
+            # Convert to base64
+            base64_image = base64.b64encode(compressed_bytes).decode('utf-8')
+
+            # Prepare the temporary message content
+            image_context_data = {
+                "mime_type": compressed_mime_type,
+                "base64": base64_image,
+                "file_path": cleaned_path, # Include path for context
+                "original_size": file_info.size,
+                "compressed_size": len(compressed_bytes)
+            }
+
+            # Add the temporary message using the thread_manager callback
+            # Use a distinct type like 'image_context'
+            await self.thread_manager.add_message(
+                thread_id=self.thread_id,
+                type="image_context", # Use a specific type for this
+                content=image_context_data, # Store the dict directly
+                is_llm_message=False # This is context generated by a tool
+            )
+
+            # Inform the agent the image will be available next turn
+            return self.success_response(f"Successfully loaded and compressed the image '{cleaned_path}' (reduced from {file_info.size / 1024:.1f}KB to {len(compressed_bytes) / 1024:.1f}KB).")
+
+        except Exception as e:
+            return self.fail_response(f"An unexpected error occurred while trying to see the image: {str(e)}") 

app/tool/update_agent_tool.py

@@ -0,0 +1,889 @@
+import json
+import httpx
+from typing import Optional, Dict, Any, List
+from agentpress.tool import Tool, ToolResult, openapi_schema, xml_schema
+from agentpress.thread_manager import ThreadManager
+
+class UpdateAgentTool(Tool):
+    """Tool for updating agent configuration.
+    
+    This tool is used by the agent builder to update agent properties
+    based on user requirements.
+    """
+
+    def __init__(self, thread_manager: ThreadManager, db_connection, agent_id: str):
+        super().__init__()
+        self.thread_manager = thread_manager
+        self.db = db_connection
+        self.agent_id = agent_id
+        # Smithery API configuration
+        self.smithery_api_base_url = "https://registry.smithery.ai"
+        import os
+        self.smithery_api_key = os.getenv("SMITHERY_API_KEY")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "update_agent",
+            "description": "Update the agent's configuration including name, description, system prompt, tools, and MCP servers. Call this whenever the user wants to modify any aspect of the agent.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "name": {
+                        "type": "string",
+                        "description": "The name of the agent. Should be descriptive and indicate the agent's purpose."
+                    },
+                    "description": {
+                        "type": "string",
+                        "description": "A brief description of what the agent does and its capabilities."
+                    },
+                    "system_prompt": {
+                        "type": "string",
+                        "description": "The system instructions that define the agent's behavior, expertise, and approach. This should be comprehensive and well-structured."
+                    },
+                    "agentpress_tools": {
+                        "type": "object",
+                        "description": "Configuration for AgentPress tools. Each key is a tool name, and the value is an object with 'enabled' (boolean) and 'description' (string) properties.",
+                        "additionalProperties": {
+                            "type": "object",
+                            "properties": {
+                                "enabled": {"type": "boolean"},
+                                "description": {"type": "string"}
+                            }
+                        }
+                    },
+                    "configured_mcps": {
+                        "type": "array",
+                        "description": "List of configured MCP servers for external integrations.",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "name": {"type": "string"},
+                                "qualifiedName": {"type": "string"},
+                                "config": {"type": "object"},
+                                "enabledTools": {
+                                    "type": "array",
+                                    "items": {"type": "string"}
+                                }
+                            }
+                        }
+                    },
+                    "avatar": {
+                        "type": "string",
+                        "description": "Emoji to use as the agent's avatar."
+                    },
+                    "avatar_color": {
+                        "type": "string",
+                        "description": "Hex color code for the agent's avatar background."
+                    }
+                },
+                "required": []
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="update-agent",
+        mappings=[
+            {"param_name": "name", "node_type": "attribute", "path": ".", "required": False},
+            {"param_name": "description", "node_type": "element", "path": "description", "required": False},
+            {"param_name": "system_prompt", "node_type": "element", "path": "system_prompt", "required": False},
+            {"param_name": "agentpress_tools", "node_type": "element", "path": "agentpress_tools", "required": False},
+            {"param_name": "configured_mcps", "node_type": "element", "path": "configured_mcps", "required": False},
+            {"param_name": "avatar", "node_type": "attribute", "path": ".", "required": False},
+            {"param_name": "avatar_color", "node_type": "attribute", "path": ".", "required": False}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="update_agent">
+        <parameter name="name">Research Assistant</parameter>
+        <parameter name="description">An AI assistant specialized in conducting research and providing comprehensive analysis</parameter>
+        <parameter name="system_prompt">You are a research assistant with expertise in gathering, analyzing, and synthesizing information. Your approach is thorough and methodical...</parameter>
+        <parameter name="agentpress_tools">{"web_search": {"enabled": true, "description": "Search the web for information"}, "sb_files": {"enabled": true, "description": "Read and write files"}}</parameter>
+        <parameter name="avatar">🔬</parameter>
+        <parameter name="avatar_color">#4F46E5</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def update_agent(
+        self,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        system_prompt: Optional[str] = None,
+        agentpress_tools: Optional[Dict[str, Dict[str, Any]]] = None,
+        configured_mcps: Optional[list] = None,
+        avatar: Optional[str] = None,
+        avatar_color: Optional[str] = None
+    ) -> ToolResult:
+        """Update agent configuration with provided fields.
+        
+        Args:
+            name: Agent name
+            description: Agent description
+            system_prompt: System instructions for the agent
+            agentpress_tools: AgentPress tools configuration
+            configured_mcps: MCP servers configuration
+            avatar: Emoji avatar
+            avatar_color: Avatar background color
+            
+        Returns:
+            ToolResult with updated agent data or error
+        """
+        try:
+            client = await self.db.client
+            
+            update_data = {}
+            if name is not None:
+                update_data["name"] = name
+            if description is not None:
+                update_data["description"] = description
+            if system_prompt is not None:
+                update_data["system_prompt"] = system_prompt
+            if agentpress_tools is not None:
+                formatted_tools = {}
+                for tool_name, tool_config in agentpress_tools.items():
+                    if isinstance(tool_config, dict):
+                        formatted_tools[tool_name] = {
+                            "enabled": tool_config.get("enabled", False),
+                            "description": tool_config.get("description", "")
+                        }
+                update_data["agentpress_tools"] = formatted_tools
+            if configured_mcps is not None:
+                if isinstance(configured_mcps, str):
+                    configured_mcps = json.loads(configured_mcps)
+                update_data["configured_mcps"] = configured_mcps
+            if avatar is not None:
+                update_data["avatar"] = avatar
+            if avatar_color is not None:
+                update_data["avatar_color"] = avatar_color
+                
+            if not update_data:
+                return self.fail_response("No fields provided to update")
+                
+            result = await client.table('agents').update(update_data).eq('agent_id', self.agent_id).execute()
+            
+            if not result.data:
+                return self.fail_response("Failed to update agent")
+
+            return self.success_response({
+                "message": "Agent updated successfully",
+                "updated_fields": list(update_data.keys()),
+                "agent": result.data[0]
+            })
+            
+        except Exception as e:
+            return self.fail_response(f"Error updating agent: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "get_current_agent_config",
+            "description": "Get the current configuration of the agent being edited. Use this to check what's already configured before making updates.",
+            "parameters": {
+                "type": "object",
+                "properties": {},
+                "required": []
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="get-current-agent-config",
+        mappings=[],
+        example='''
+        <function_calls>
+        <invoke name="get_current_agent_config">
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def get_current_agent_config(self) -> ToolResult:
+        """Get the current agent configuration.
+        
+        Returns:
+            ToolResult with current agent configuration
+        """
+        try:
+            client = await self.db.client
+            
+            result = await client.table('agents').select('*').eq('agent_id', self.agent_id).execute()
+            
+            if not result.data:
+                return self.fail_response("Agent not found")
+                
+            agent = result.data[0]
+            
+            config_summary = {
+                "agent_id": agent["agent_id"],
+                "name": agent.get("name", "Untitled Agent"),
+                "description": agent.get("description", "No description set"),
+                "system_prompt": agent.get("system_prompt", "No system prompt set"),
+                "avatar": agent.get("avatar", "🤖"),
+                "avatar_color": agent.get("avatar_color", "#6B7280"),
+                "agentpress_tools": agent.get("agentpress_tools", {}),
+                "configured_mcps": agent.get("configured_mcps", []),
+                "created_at": agent.get("created_at"),
+                "updated_at": agent.get("updated_at")
+            }
+            
+            tools_count = len([t for t, cfg in config_summary["agentpress_tools"].items() if cfg.get("enabled")])
+            mcps_count = len(config_summary["configured_mcps"])
+            
+            return self.success_response({
+                "summary": f"Agent '{config_summary['name']}' has {tools_count} tools enabled and {mcps_count} MCP servers configured.",
+                "configuration": config_summary
+            })
+            
+        except Exception as e:
+            return self.fail_response(f"Error getting agent configuration: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "search_mcp_servers",
+            "description": "Search for MCP servers from the Smithery registry based on user requirements. Use this when the user wants to add MCP tools to their agent.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "query": {
+                        "type": "string",
+                        "description": "Search query for finding relevant MCP servers (e.g., 'linear', 'github', 'database', 'search')"
+                    },
+                    "category": {
+                        "type": "string",
+                        "description": "Optional category filter",
+                        "enum": ["AI & Search", "Development & Version Control", "Project Management", "Communication & Collaboration", "Data & Analytics", "Cloud & Infrastructure", "File Storage", "Marketing & Sales", "Customer Support", "Finance", "Automation & Productivity", "Utilities"]
+                    },
+                    "limit": {
+                        "type": "integer",
+                        "description": "Maximum number of servers to return (default: 10)",
+                        "default": 10
+                    }
+                },
+                "required": ["query"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="search-mcp-servers",
+        mappings=[
+            {"param_name": "query", "node_type": "attribute", "path": "."},
+            {"param_name": "category", "node_type": "attribute", "path": "."},
+            {"param_name": "limit", "node_type": "attribute", "path": "."}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="search_mcp_servers">
+        <parameter name="query">linear</parameter>
+        <parameter name="limit">5</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def search_mcp_servers(
+        self,
+        query: str,
+        category: Optional[str] = None,
+        limit: int = 10
+    ) -> ToolResult:
+        """Search for MCP servers based on user requirements.
+        
+        Args:
+            query: Search query for finding relevant MCP servers
+            category: Optional category filter
+            limit: Maximum number of servers to return
+            
+        Returns:
+            ToolResult with matching MCP servers
+        """
+        try:
+            async with httpx.AsyncClient() as client:
+                headers = {
+                    "Accept": "application/json",
+                    "User-Agent": "Suna-MCP-Integration/1.0"
+                }
+                
+                if self.smithery_api_key:
+                    headers["Authorization"] = f"Bearer {self.smithery_api_key}"
+                
+                params = {
+                    "q": query,
+                    "page": 1,
+                    "pageSize": min(limit * 2, 50)  # Get more results to filter
+                }
+                
+                response = await client.get(
+                    f"{self.smithery_api_base_url}/servers",
+                    headers=headers,
+                    params=params,
+                    timeout=30.0
+                )
+                
+                response.raise_for_status()
+                data = response.json()
+                servers = data.get("servers", [])
+                
+                # Filter by category if specified
+                if category:
+                    filtered_servers = []
+                    for server in servers:
+                        server_category = self._categorize_server(server)
+                        if server_category == category:
+                            filtered_servers.append(server)
+                    servers = filtered_servers
+                
+                # Sort by useCount and limit results
+                servers = sorted(servers, key=lambda x: x.get("useCount", 0), reverse=True)[:limit]
+                
+                # Format results for user-friendly display
+                formatted_servers = []
+                for server in servers:
+                    formatted_servers.append({
+                        "name": server.get("displayName", server.get("qualifiedName", "Unknown")),
+                        "qualifiedName": server.get("qualifiedName"),
+                        "description": server.get("description", "No description available"),
+                        "useCount": server.get("useCount", 0),
+                        "category": self._categorize_server(server),
+                        "homepage": server.get("homepage", ""),
+                        "isDeployed": server.get("isDeployed", False)
+                    })
+                
+                if not formatted_servers:
+                    return ToolResult(
+                        success=False,
+                        output=json.dumps([], ensure_ascii=False)
+                    )
+                
+                return ToolResult(
+                    success=True,
+                    output=json.dumps(formatted_servers, ensure_ascii=False)
+                )
+                
+        except Exception as e:
+            return self.fail_response(f"Error searching MCP servers: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "get_mcp_server_tools",
+            "description": "Get detailed information about a specific MCP server including its available tools. Use this after the user selects a server they want to connect to.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "qualified_name": {
+                        "type": "string",
+                        "description": "The qualified name of the MCP server (e.g., 'exa', '@smithery-ai/github')"
+                    }
+                },
+                "required": ["qualified_name"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="get-mcp-server-tools",
+        mappings=[
+            {"param_name": "qualified_name", "node_type": "attribute", "path": ".", "required": True}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="get_mcp_server_tools">
+        <parameter name="qualified_name">exa</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def get_mcp_server_tools(self, qualified_name: str) -> ToolResult:
+        """Get detailed information about a specific MCP server and its tools.
+        
+        Args:
+            qualified_name: The qualified name of the MCP server
+            
+        Returns:
+            ToolResult with server details and available tools
+        """
+        try:
+            # First get server metadata from registry
+            async with httpx.AsyncClient() as client:
+                headers = {
+                    "Accept": "application/json",
+                    "User-Agent": "Suna-MCP-Integration/1.0"
+                }
+                
+                if self.smithery_api_key:
+                    headers["Authorization"] = f"Bearer {self.smithery_api_key}"
+                
+                # URL encode the qualified name if it contains special characters
+                from urllib.parse import quote
+                if '@' in qualified_name or '/' in qualified_name:
+                    encoded_name = quote(qualified_name, safe='')
+                else:
+                    encoded_name = qualified_name
+                
+                url = f"{self.smithery_api_base_url}/servers/{encoded_name}"
+                
+                response = await client.get(
+                    url,
+                    headers=headers,
+                    timeout=30.0
+                )
+                
+                response.raise_for_status()
+                server_data = response.json()
+            
+            # Now connect to the MCP server to get actual tools using ClientSession
+            try:
+                # Import MCP components
+                from mcp import ClientSession
+                from mcp.client.streamable_http import streamablehttp_client
+                import base64
+                import os
+                
+                # Check if Smithery API key is available
+                smithery_api_key = os.getenv("SMITHERY_API_KEY")
+                if not smithery_api_key:
+                    raise ValueError("SMITHERY_API_KEY environment variable is not set")
+                
+                # Create server URL with empty config for testing
+                config_json = json.dumps({})
+                config_b64 = base64.b64encode(config_json.encode()).decode()
+                server_url = f"https://server.smithery.ai/{qualified_name}/mcp?config={config_b64}&api_key={smithery_api_key}"
+                
+                # Connect and get tools
+                async with streamablehttp_client(server_url) as (read_stream, write_stream, _):
+                    async with ClientSession(read_stream, write_stream) as session:
+                        # Initialize the connection
+                        await session.initialize()
+                        
+                        # List available tools
+                        tools_result = await session.list_tools()
+                        tools = tools_result.tools if hasattr(tools_result, 'tools') else tools_result
+                
+                # Format tools for user-friendly display
+                formatted_tools = []
+                for tool in tools:
+                    tool_info = {
+                        "name": tool.name,
+                        "description": getattr(tool, 'description', 'No description available'),
+                    }
+                    
+                    # Extract parameters from inputSchema if available
+                    if hasattr(tool, 'inputSchema') and tool.inputSchema:
+                        schema = tool.inputSchema
+                        if isinstance(schema, dict):
+                            tool_info["parameters"] = schema.get("properties", {})
+                            tool_info["required_params"] = schema.get("required", [])
+                        else:
+                            tool_info["parameters"] = {}
+                            tool_info["required_params"] = []
+                    else:
+                        tool_info["parameters"] = {}
+                        tool_info["required_params"] = []
+                    
+                    formatted_tools.append(tool_info)
+                
+                # Extract configuration requirements from server metadata
+                config_requirements = []
+                security = server_data.get("security", {})
+                if security:
+                    for key, value in security.items():
+                        if isinstance(value, dict):
+                            config_requirements.append({
+                                "name": key,
+                                "description": value.get("description", f"Configuration for {key}"),
+                                "required": value.get("required", False),
+                                "type": value.get("type", "string")
+                            })
+                
+                server_info = {
+                    "name": server_data.get("displayName", qualified_name),
+                    "qualifiedName": qualified_name,
+                    "description": server_data.get("description", "No description available"),
+                    "homepage": server_data.get("homepage", ""),
+                    "iconUrl": server_data.get("iconUrl", ""),
+                    "isDeployed": server_data.get("isDeployed", False),
+                    "tools": formatted_tools,
+                    "config_requirements": config_requirements,
+                    "total_tools": len(formatted_tools)
+                }
+                
+                return self.success_response({
+                    "message": f"Found {len(formatted_tools)} tools for {server_info['name']}",
+                    "server": server_info
+                })
+                
+            except Exception as mcp_error:
+                # If MCP connection fails, fall back to registry data
+                tools = server_data.get("tools", [])
+                formatted_tools = []
+                for tool in tools:
+                    formatted_tools.append({
+                        "name": tool.get("name", "Unknown"),
+                        "description": tool.get("description", "No description available"),
+                        "parameters": tool.get("inputSchema", {}).get("properties", {}),
+                        "required_params": tool.get("inputSchema", {}).get("required", [])
+                    })
+                
+                config_requirements = []
+                security = server_data.get("security", {})
+                if security:
+                    for key, value in security.items():
+                        if isinstance(value, dict):
+                            config_requirements.append({
+                                "name": key,
+                                "description": value.get("description", f"Configuration for {key}"),
+                                "required": value.get("required", False),
+                                "type": value.get("type", "string")
+                            })
+                
+                server_info = {
+                    "name": server_data.get("displayName", qualified_name),
+                    "qualifiedName": qualified_name,
+                    "description": server_data.get("description", "No description available"),
+                    "homepage": server_data.get("homepage", ""),
+                    "iconUrl": server_data.get("iconUrl", ""),
+                    "isDeployed": server_data.get("isDeployed", False),
+                    "tools": formatted_tools,
+                    "config_requirements": config_requirements,
+                    "total_tools": len(formatted_tools),
+                    "note": "Tools listed from registry metadata (MCP connection failed - may need configuration)"
+                }
+                
+                return self.success_response({
+                    "message": f"Found {len(formatted_tools)} tools for {server_info['name']} (from registry)",
+                    "server": server_info
+                })
+                
+        except Exception as e:
+            return self.fail_response(f"Error getting MCP server tools: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "configure_mcp_server",
+            "description": "Configure and add an MCP server to the agent with selected tools. Use this after the user has chosen which tools they want from a server.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "qualified_name": {
+                        "type": "string",
+                        "description": "The qualified name of the MCP server"
+                    },
+                    "display_name": {
+                        "type": "string",
+                        "description": "Display name for the server"
+                    },
+                    "enabled_tools": {
+                        "type": "array",
+                        "description": "List of tool names to enable for this server",
+                        "items": {"type": "string"}
+                    },
+                    "config": {
+                        "type": "object",
+                        "description": "Configuration object with API keys and other settings",
+                        "additionalProperties": True
+                    }
+                },
+                "required": ["qualified_name", "display_name", "enabled_tools"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="configure-mcp-server",
+        mappings=[
+            {"param_name": "qualified_name", "node_type": "attribute", "path": ".", "required": True},
+            {"param_name": "display_name", "node_type": "attribute", "path": ".", "required": True},
+            {"param_name": "enabled_tools", "node_type": "element", "path": "enabled_tools", "required": True},
+            {"param_name": "config", "node_type": "element", "path": "config", "required": False}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="configure_mcp_server">
+        <parameter name="qualified_name">exa</parameter>
+        <parameter name="display_name">Exa Search</parameter>
+        <parameter name="enabled_tools">["search", "find_similar"]</parameter>
+        <parameter name="config">{"exaApiKey": "user-api-key"}</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def configure_mcp_server(
+        self,
+        qualified_name: str,
+        display_name: str,
+        enabled_tools: List[str],
+        config: Optional[Dict[str, Any]] = None
+    ) -> ToolResult:
+        """Configure and add an MCP server to the agent.
+        
+        Args:
+            qualified_name: The qualified name of the MCP server
+            display_name: Display name for the server
+            enabled_tools: List of tool names to enable
+            config: Configuration object with API keys and settings
+            
+        Returns:
+            ToolResult with configuration status
+        """
+        try:
+            client = await self.db.client
+            
+            # Get current agent configuration
+            result = await client.table('agents').select('configured_mcps').eq('agent_id', self.agent_id).execute()
+            
+            if not result.data:
+                return self.fail_response("Agent not found")
+            
+            current_mcps = result.data[0].get('configured_mcps', [])
+            
+            # Check if server is already configured
+            existing_server_index = None
+            for i, mcp in enumerate(current_mcps):
+                if mcp.get('qualifiedName') == qualified_name:
+                    existing_server_index = i
+                    break
+            
+            # Create new MCP configuration
+            new_mcp_config = {
+                "name": display_name,
+                "qualifiedName": qualified_name,
+                "config": config or {},
+                "enabledTools": enabled_tools
+            }
+            
+            # Update or add the configuration
+            if existing_server_index is not None:
+                current_mcps[existing_server_index] = new_mcp_config
+                action = "updated"
+            else:
+                current_mcps.append(new_mcp_config)
+                action = "added"
+            
+            # Save to database
+            update_result = await client.table('agents').update({
+                'configured_mcps': current_mcps
+            }).eq('agent_id', self.agent_id).execute()
+            
+            if not update_result.data:
+                return self.fail_response("Failed to save MCP configuration")
+            
+            return self.success_response({
+                "message": f"Successfully {action} MCP server '{display_name}' with {len(enabled_tools)} tools",
+                "server": new_mcp_config,
+                "total_mcp_servers": len(current_mcps),
+                "action": action
+            })
+            
+        except Exception as e:
+            return self.fail_response(f"Error configuring MCP server: {str(e)}")
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "get_popular_mcp_servers",
+            "description": "Get a list of popular and recommended MCP servers organized by category. Use this to show users popular options when they want to add MCP tools.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "category": {
+                        "type": "string",
+                        "description": "Optional category filter to show only servers from a specific category",
+                        "enum": ["AI & Search", "Development & Version Control", "Project Management", "Communication & Collaboration", "Data & Analytics", "Cloud & Infrastructure", "File Storage", "Marketing & Sales", "Customer Support", "Finance", "Automation & Productivity", "Utilities"]
+                    }
+                },
+                "required": []
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="get-popular-mcp-servers",
+        mappings=[
+            {"param_name": "category", "node_type": "attribute", "path": ".", "required": False}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="get_popular_mcp_servers">
+        <parameter name="category">AI & Search</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def get_popular_mcp_servers(self, category: Optional[str] = None) -> ToolResult:
+        """Get popular MCP servers organized by category.
+        
+        Args:
+            category: Optional category filter
+            
+        Returns:
+            ToolResult with popular MCP servers
+        """
+        try:
+            async with httpx.AsyncClient() as client:
+                headers = {
+                    "Accept": "application/json",
+                    "User-Agent": "Suna-MCP-Integration/1.0"
+                }
+                
+                if self.smithery_api_key:
+                    headers["Authorization"] = f"Bearer {self.smithery_api_key}"
+                
+                response = await client.get(
+                    f"{self.smithery_api_base_url}/servers",
+                    headers=headers,
+                    params={"page": 1, "pageSize": 50},
+                    timeout=30.0
+                )
+                
+                response.raise_for_status()
+                data = response.json()
+                servers = data.get("servers", [])
+                
+                # Categorize servers
+                categorized = {}
+                for server in servers:
+                    server_category = self._categorize_server(server)
+                    if category and server_category != category:
+                        continue
+                        
+                    if server_category not in categorized:
+                        categorized[server_category] = []
+                    
+                    categorized[server_category].append({
+                        "name": server.get("displayName", server.get("qualifiedName", "Unknown")),
+                        "qualifiedName": server.get("qualifiedName"),
+                        "description": server.get("description", "No description available"),
+                        "useCount": server.get("useCount", 0),
+                        "homepage": server.get("homepage", ""),
+                        "isDeployed": server.get("isDeployed", False)
+                    })
+                
+                # Sort categories and servers within each category
+                for cat in categorized:
+                    categorized[cat] = sorted(categorized[cat], key=lambda x: x["useCount"], reverse=True)[:5]
+                
+                return self.success_response({
+                    "message": f"Found popular MCP servers" + (f" in category '{category}'" if category else ""),
+                    "categorized_servers": categorized,
+                    "total_categories": len(categorized)
+                })
+                
+        except Exception as e:
+            return self.fail_response(f"Error getting popular MCP servers: {str(e)}")
+
+    def _categorize_server(self, server: Dict[str, Any]) -> str:
+        """Categorize a server based on its qualified name and description."""
+        qualified_name = server.get("qualifiedName", "").lower()
+        description = server.get("description", "").lower()
+        
+        # Category mappings
+        category_mappings = {
+            "AI & Search": ["exa", "perplexity", "openai", "anthropic", "duckduckgo", "brave", "google", "search"],
+            "Development & Version Control": ["github", "gitlab", "bitbucket", "git"],
+            "Project Management": ["linear", "jira", "asana", "notion", "trello", "monday", "clickup"],
+            "Communication & Collaboration": ["slack", "discord", "teams", "zoom", "telegram"],
+            "Data & Analytics": ["postgres", "mysql", "mongodb", "bigquery", "snowflake", "sqlite", "redis", "database"],
+            "Cloud & Infrastructure": ["aws", "gcp", "azure", "vercel", "netlify", "cloudflare", "docker"],
+            "File Storage": ["gdrive", "google-drive", "dropbox", "box", "onedrive", "s3", "drive"],
+            "Marketing & Sales": ["hubspot", "salesforce", "mailchimp", "sendgrid"],
+            "Customer Support": ["zendesk", "intercom", "freshdesk", "helpscout"],
+            "Finance": ["stripe", "quickbooks", "xero", "plaid"],
+            "Automation & Productivity": ["playwright", "puppeteer", "selenium", "desktop-commander", "sequential-thinking", "automation"],
+            "Utilities": ["filesystem", "memory", "fetch", "time", "weather", "currency", "file"]
+        }
+        
+        # Check qualified name and description for category keywords
+        for category, keywords in category_mappings.items():
+            for keyword in keywords:
+                if keyword in qualified_name or keyword in description:
+                    return category
+        
+        return "Other"
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "test_mcp_server_connection",
+            "description": "Test connectivity to an MCP server with provided configuration. Use this to validate that a server can be connected to before adding it to the agent.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "qualified_name": {
+                        "type": "string",
+                        "description": "The qualified name of the MCP server"
+                    },
+                    "config": {
+                        "type": "object",
+                        "description": "Configuration object with API keys and other settings",
+                        "additionalProperties": True
+                    }
+                },
+                "required": ["qualified_name"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="test-mcp-server-connection",
+        mappings=[
+            {"param_name": "qualified_name", "node_type": "attribute", "path": ".", "required": True},
+            {"param_name": "config", "node_type": "element", "path": "config", "required": False}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="test_mcp_server_connection">
+        <parameter name="qualified_name">exa</parameter>
+        <parameter name="config">{"exaApiKey": "user-api-key"}</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def test_mcp_server_connection(
+        self,
+        qualified_name: str,
+        config: Optional[Dict[str, Any]] = None
+    ) -> ToolResult:
+        """Test connectivity to an MCP server with provided configuration.
+        
+        Args:
+            qualified_name: The qualified name of the MCP server
+            config: Configuration object with API keys and settings
+            
+        Returns:
+            ToolResult with connection test results
+        """
+        try:
+            # Import MCP components
+            from mcp import ClientSession
+            from mcp.client.streamable_http import streamablehttp_client
+            import base64
+            import os
+            
+            # Check if Smithery API key is available
+            smithery_api_key = os.getenv("SMITHERY_API_KEY")
+            if not smithery_api_key:
+                return self.fail_response("SMITHERY_API_KEY environment variable is not set")
+            
+            # Create server URL with provided config
+            config_json = json.dumps(config or {})
+            config_b64 = base64.b64encode(config_json.encode()).decode()
+            server_url = f"https://server.smithery.ai/{qualified_name}/mcp?config={config_b64}&api_key={smithery_api_key}"
+            
+            # Test connection
+            async with streamablehttp_client(server_url) as (read_stream, write_stream, _):
+                async with ClientSession(read_stream, write_stream) as session:
+                    # Initialize the connection
+                    await session.initialize()
+                    
+                    # List available tools to verify connection
+                    tools_result = await session.list_tools()
+                    tools = tools_result.tools if hasattr(tools_result, 'tools') else tools_result
+                    
+                    tool_names = [tool.name for tool in tools]
+                    
+                    return self.success_response({
+                        "message": f"Successfully connected to {qualified_name}",
+                        "qualified_name": qualified_name,
+                        "connection_status": "success",
+                        "available_tools": tool_names,
+                        "total_tools": len(tool_names)
+                    })
+            
+        except Exception as e:
+            return self.fail_response(f"Failed to connect to {qualified_name}: {str(e)}") 

app/tool/web_search_tool.py

@@ -0,0 +1,395 @@
+from tavily import AsyncTavilyClient
+import httpx
+from dotenv import load_dotenv
+from agentpress.tool import Tool, ToolResult, openapi_schema, xml_schema
+from utils.config import config
+from daytona.tool_base import SandboxToolsBase
+from agentpress.thread_manager import ThreadManager
+import json
+import os
+import datetime
+import asyncio
+import logging
+
+# TODO: add subpages, etc... in filters as sometimes its necessary
+
+class SandboxWebSearchTool(SandboxToolsBase):
+    """Tool for performing web searches using Tavily API and web scraping using Firecrawl."""
+
+    def __init__(self, project_id: str, thread_manager: ThreadManager):
+        super().__init__(project_id, thread_manager)
+        # Load environment variables
+        load_dotenv()
+        # Use API keys from config
+        self.tavily_api_key = config.TAVILY_API_KEY
+        self.firecrawl_api_key = config.FIRECRAWL_API_KEY
+        self.firecrawl_url = config.FIRECRAWL_URL
+
+        if not self.tavily_api_key:
+            raise ValueError("TAVILY_API_KEY not found in configuration")
+        if not self.firecrawl_api_key:
+            raise ValueError("FIRECRAWL_API_KEY not found in configuration")
+
+        # Tavily asynchronous search client
+        self.tavily_client = AsyncTavilyClient(api_key=self.tavily_api_key)
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "web_search",
+            "description": "Search the web for up-to-date information on a specific topic using the Tavily API. This tool allows you to gather real-time information from the internet to answer user queries, research topics, validate facts, and find recent developments. Results include titles, URLs, and publication dates. Use this tool for discovering relevant web pages before potentially crawling them for complete content.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "query": {
+                        "type": "string",
+                        "description": "The search query to find relevant web pages. Be specific and include key terms to improve search accuracy. For best results, use natural language questions or keyword combinations that precisely describe what you're looking for."
+                    },
+                    "num_results": {
+                        "type": "integer",
+                        "description": "The number of search results to return. Increase for more comprehensive research or decrease for focused, high-relevance results.",
+                        "default": 20
+                    }
+                },
+                "required": ["query"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="web-search",
+        mappings=[
+            {"param_name": "query", "node_type": "attribute", "path": "."},
+            {"param_name": "num_results", "node_type": "attribute", "path": "."}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="web_search">
+        <parameter name="query">what is Kortix AI and what are they building?</parameter>
+        <parameter name="num_results">20</parameter>
+        </invoke>
+        </function_calls>
+
+        <!-- Another search example -->
+        <function_calls>
+        <invoke name="web_search">
+        <parameter name="query">latest AI research on transformer models</parameter>
+        <parameter name="num_results">20</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def web_search(
+        self,
+        query: str,
+        num_results: int = 20
+    ) -> ToolResult:
+        """
+        Search the web using the Tavily API to find relevant and up-to-date information.
+        """
+        try:
+            # Ensure we have a valid query
+            if not query or not isinstance(query, str):
+                return self.fail_response("A valid search query is required.")
+
+            # Normalize num_results
+            if num_results is None:
+                num_results = 20
+            elif isinstance(num_results, int):
+                num_results = max(1, min(num_results, 50))
+            elif isinstance(num_results, str):
+                try:
+                    num_results = max(1, min(int(num_results), 50))
+                except ValueError:
+                    num_results = 20
+            else:
+                num_results = 20
+
+            # Execute the search with Tavily
+            logging.info(f"Executing web search for query: '{query}' with {num_results} results")
+            search_response = await self.tavily_client.search(
+                query=query,
+                max_results=num_results,
+                include_images=True,
+                include_answer="advanced",
+                search_depth="advanced",
+            )
+
+            # Check if we have actual results or an answer
+            results = search_response.get('results', [])
+            answer = search_response.get('answer', '')
+
+            # Return the complete Tavily response
+            # This includes the query, answer, results, images and more
+            logging.info(f"Retrieved search results for query: '{query}' with answer and {len(results)} results")
+
+            # Consider search successful if we have either results OR an answer
+            if len(results) > 0 or (answer and answer.strip()):
+                return ToolResult(
+                    success=True,
+                    output=json.dumps(search_response, ensure_ascii=False)
+                )
+            else:
+                # No results or answer found
+                logging.warning(f"No search results or answer found for query: '{query}'")
+                return ToolResult(
+                    success=False,
+                    output=json.dumps(search_response, ensure_ascii=False)
+                )
+
+        except Exception as e:
+            error_message = str(e)
+            logging.error(f"Error performing web search for '{query}': {error_message}")
+            simplified_message = f"Error performing web search: {error_message[:200]}"
+            if len(error_message) > 200:
+                simplified_message += "..."
+            return self.fail_response(simplified_message)
+
+    @openapi_schema({
+        "type": "function",
+        "function": {
+            "name": "scrape_webpage",
+            "description": "Extract full text content from multiple webpages in a single operation. IMPORTANT: You should ALWAYS collect multiple relevant URLs from web-search results and scrape them all in a single call for efficiency. This tool saves time by processing multiple pages simultaneously rather than one at a time. The extracted text includes the main content of each page without HTML markup.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "urls": {
+                        "type": "string",
+                        "description": "Multiple URLs to scrape, separated by commas. You should ALWAYS include several URLs when possible for efficiency. Example: 'https://example.com/page1,https://example.com/page2,https://example.com/page3'"
+                    }
+                },
+                "required": ["urls"]
+            }
+        }
+    })
+    @xml_schema(
+        tag_name="scrape-webpage",
+        mappings=[
+            {"param_name": "urls", "node_type": "attribute", "path": "."}
+        ],
+        example='''
+        <function_calls>
+        <invoke name="scrape_webpage">
+        <parameter name="urls">https://www.kortix.ai/,https://github.com/kortix-ai/suna</parameter>
+        </invoke>
+        </function_calls>
+        '''
+    )
+    async def scrape_webpage(
+        self,
+        urls: str
+    ) -> ToolResult:
+        """
+        Retrieve the complete text content of multiple webpages in a single efficient operation.
+
+        ALWAYS collect multiple relevant URLs from search results and scrape them all at once
+        rather than making separate calls for each URL. This is much more efficient.
+
+        Parameters:
+        - urls: Multiple URLs to scrape, separated by commas
+        """
+        try:
+            logging.info(f"Starting to scrape webpages: {urls}")
+
+            # Ensure sandbox is initialized
+            await self._ensure_sandbox()
+
+            # Parse the URLs parameter
+            if not urls:
+                logging.warning("Scrape attempt with empty URLs")
+                return self.fail_response("Valid URLs are required.")
+
+            # Split the URLs string into a list
+            url_list = [url.strip() for url in urls.split(',') if url.strip()]
+
+            if not url_list:
+                logging.warning("No valid URLs found in the input")
+                return self.fail_response("No valid URLs provided.")
+
+            if len(url_list) == 1:
+                logging.warning("Only a single URL provided - for efficiency you should scrape multiple URLs at once")
+
+            logging.info(f"Processing {len(url_list)} URLs: {url_list}")
+
+            # Process each URL and collect results
+            results = []
+            for url in url_list:
+                try:
+                    # Add protocol if missing
+                    if not (url.startswith('http://') or url.startswith('https://')):
+                        url = 'https://' + url
+                        logging.info(f"Added https:// protocol to URL: {url}")
+
+                    # Scrape this URL
+                    result = await self._scrape_single_url(url)
+                    results.append(result)
+
+                except Exception as e:
+                    logging.error(f"Error processing URL {url}: {str(e)}")
+                    results.append({
+                        "url": url,
+                        "success": False,
+                        "error": str(e)
+                    })
+
+            # Summarize results
+            successful = sum(1 for r in results if r.get("success", False))
+            failed = len(results) - successful
+
+            # Create success/failure message
+            if successful == len(results):
+                message = f"Successfully scraped all {len(results)} URLs. Results saved to:"
+                for r in results:
+                    if r.get("file_path"):
+                        message += f"\n- {r.get('file_path')}"
+            elif successful > 0:
+                message = f"Scraped {successful} URLs successfully and {failed} failed. Results saved to:"
+                for r in results:
+                    if r.get("success", False) and r.get("file_path"):
+                        message += f"\n- {r.get('file_path')}"
+                message += "\n\nFailed URLs:"
+                for r in results:
+                    if not r.get("success", False):
+                        message += f"\n- {r.get('url')}: {r.get('error', 'Unknown error')}"
+            else:
+                error_details = "; ".join([f"{r.get('url')}: {r.get('error', 'Unknown error')}" for r in results])
+                return self.fail_response(f"Failed to scrape all {len(results)} URLs. Errors: {error_details}")
+
+            return ToolResult(
+                success=True,
+                output=message
+            )
+
+        except Exception as e:
+            error_message = str(e)
+            logging.error(f"Error in scrape_webpage: {error_message}")
+            return self.fail_response(f"Error processing scrape request: {error_message[:200]}")
+
+    async def _scrape_single_url(self, url: str) -> dict:
+        """
+        Helper function to scrape a single URL and return the result information.
+        """
+        logging.info(f"Scraping single URL: {url}")
+
+        try:
+            # ---------- Firecrawl scrape endpoint ----------
+            logging.info(f"Sending request to Firecrawl for URL: {url}")
+            async with httpx.AsyncClient() as client:
+                headers = {
+                    "Authorization": f"Bearer {self.firecrawl_api_key}",
+                    "Content-Type": "application/json",
+                }
+                payload = {
+                    "url": url,
+                    "formats": ["markdown"]
+                }
+
+                # Use longer timeout and retry logic for more reliability
+                max_retries = 3
+                timeout_seconds = 120
+                retry_count = 0
+
+                while retry_count < max_retries:
+                    try:
+                        logging.info(f"Sending request to Firecrawl (attempt {retry_count + 1}/{max_retries})")
+                        response = await client.post(
+                            f"{self.firecrawl_url}/v1/scrape",
+                            json=payload,
+                            headers=headers,
+                            timeout=timeout_seconds,
+                        )
+                        response.raise_for_status()
+                        data = response.json()
+                        logging.info(f"Successfully received response from Firecrawl for {url}")
+                        break
+                    except (httpx.ReadTimeout, httpx.ConnectTimeout, httpx.ReadError) as timeout_err:
+                        retry_count += 1
+                        logging.warning(f"Request timed out (attempt {retry_count}/{max_retries}): {str(timeout_err)}")
+                        if retry_count >= max_retries:
+                            raise Exception(f"Request timed out after {max_retries} attempts with {timeout_seconds}s timeout")
+                        # Exponential backoff
+                        logging.info(f"Waiting {2 ** retry_count}s before retry")
+                        await asyncio.sleep(2 ** retry_count)
+                    except Exception as e:
+                        # Don't retry on non-timeout errors
+                        logging.error(f"Error during scraping: {str(e)}")
+                        raise e
+
+            # Format the response
+            title = data.get("data", {}).get("metadata", {}).get("title", "")
+            markdown_content = data.get("data", {}).get("markdown", "")
+            logging.info(f"Extracted content from {url}: title='{title}', content length={len(markdown_content)}")
+
+            formatted_result = {
+                "title": title,
+                "url": url,
+                "text": markdown_content
+            }
+
+            # Add metadata if available
+            if "metadata" in data.get("data", {}):
+                formatted_result["metadata"] = data["data"]["metadata"]
+                logging.info(f"Added metadata: {data['data']['metadata'].keys()}")
+
+            # Create a simple filename from the URL domain and date
+            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+
+            # Extract domain from URL for the filename
+            from urllib.parse import urlparse
+            parsed_url = urlparse(url)
+            domain = parsed_url.netloc.replace("www.", "")
+
+            # Clean up domain for filename
+            domain = "".join([c if c.isalnum() else "_" for c in domain])
+            safe_filename = f"{timestamp}_{domain}.json"
+
+            logging.info(f"Generated filename: {safe_filename}")
+
+            # Save results to a file in the /workspace/scrape directory
+            scrape_dir = f"{self.workspace_path}/scrape"
+            self.sandbox.fs.create_folder(scrape_dir, "755")
+
+            results_file_path = f"{scrape_dir}/{safe_filename}"
+            json_content = json.dumps(formatted_result, ensure_ascii=False, indent=2)
+            logging.info(f"Saving content to file: {results_file_path}, size: {len(json_content)} bytes")
+
+            self.sandbox.fs.upload_file(
+                json_content.encode(),
+                results_file_path,
+            )
+
+            return {
+                "url": url,
+                "success": True,
+                "title": title,
+                "file_path": results_file_path,
+                "content_length": len(markdown_content)
+            }
+
+        except Exception as e:
+            error_message = str(e)
+            logging.error(f"Error scraping URL '{url}': {error_message}")
+
+            # Create an error result
+            return {
+                "url": url,
+                "success": False,
+                "error": error_message
+            }
+
+if __name__ == "__main__":
+    async def test_web_search():
+        """Test function for the web search tool"""
+        # This test function is not compatible with the sandbox version
+        print("Test function needs to be updated for sandbox version")
+
+    async def test_scrape_webpage():
+        """Test function for the webpage scrape tool"""
+        # This test function is not compatible with the sandbox version
+        print("Test function needs to be updated for sandbox version")
+
+    async def run_tests():
+        """Run all test functions"""
+        await test_web_search()
+        await test_scrape_webpage()
+
+    asyncio.run(run_tests())