patchy631--ai-engineering-hub/pixeltable-mcp/image-index/tools.py

import pixeltable as pxt
import os
from mcp.server.fastmcp import FastMCP
from pixeltable.functions.openai import vision
from pixeltable.functions.huggingface import sentence_transformer

mcp = FastMCP("Pixeltable")

# Base directory for all indexes
DIRECTORY = 'image_search'

# Registry to hold all image indexes
image_indexes = {}


def _get_openai_api_key() -> str:
    """Get OpenAI API key from environment variables.
    
    Returns:
        The OpenAI API key
        
    Raises:
        ValueError: If the API key is not found
    """
    api_key = os.getenv('OPENAI_API_KEY')
    if not api_key:
        raise ValueError("OPENAI_API_KEY not found in environment variables")
    return api_key


@mcp.tool()
def setup_image_index(table_name: str) -> str:
    """Set up an image index with the provided name and OpenAI API key.

    Args:
        table_name: The name of the image index (e.g., 'photos', 'artwork').

    Returns:
        A message indicating whether the index was created, already exists, or failed.
    """
    global image_indexes

    # Construct full table name
    full_table_name = f'{DIRECTORY}.{table_name}'

    try:
        # Set the API key
        openai_api_key = _get_openai_api_key()
        os.environ['OPENAI_API_KEY'] = openai_api_key

        # Check if the table already exists
        existing_tables = pxt.list_tables()
        if full_table_name in existing_tables:
            image_index = pxt.get_table(full_table_name)
            image_indexes[full_table_name] = image_index
            return f"Image index '{full_table_name}' already exists and is ready for use."

        # Create directory and table
        pxt.create_dir(DIRECTORY, if_exists='ignore')
        image_index = pxt.create_table(
            full_table_name, 
            {'image_file': pxt.Image},
            if_exists='ignore'
        )

        # Add GPT-4 Vision analysis
        image_index.add_computed_column(
            image_description=vision(
                prompt="Describe the image. Be specific on the colors you see.",
                image=image_index.image_file,
                model="gpt-4o-mini"
            )
        )

        # Define the embedding model and create embedding index
        embed_model = sentence_transformer.using(model_id='intfloat/e5-large-v2')
        image_index.add_embedding_index(
            column='image_description', 
            string_embed=embed_model,
            if_exists='ignore'
        )

        # Store in the registry
        image_indexes[full_table_name] = image_index
        return f"Image index '{full_table_name}' created successfully."
    except Exception as e:
        return f"Error setting up image index '{full_table_name}': {str(e)}"

@mcp.tool()
def insert_image(table_name: str, image_location: str) -> str:
    """Insert an image file into the specified image index.

    Args:
        table_name: The name of the image index (e.g., 'photos', 'artwork').
        image_location: The URL or path to the image file to insert (e.g., local path or URL).

    Returns:
        A confirmation message indicating success or failure.
    """
    full_table_name = f'{DIRECTORY}.{table_name}'
    try:
        if full_table_name not in image_indexes:
            return f"Error: Image index '{full_table_name}' not set up. Please call setup_image_index first."
        image_index = image_indexes[full_table_name]
        image_index.insert([{'image_file': image_location}])
        return f"Image file '{image_location}' inserted successfully into index '{full_table_name}'."
    except Exception as e:
        return f"Error inserting image file into '{full_table_name}': {str(e)}"

@mcp.tool()
def query_image(table_name: str, query_text: str, top_n: int = 5) -> str:
    """Query the specified image index with a text description.

    Args:
        table_name: The name of the image index (e.g., 'photos', 'artwork').
        query_text: The text description to search for in the image descriptions.
        top_n: Number of top results to return (default is 5).

    Returns:
        A string containing the top matching images and their similarity scores.
    """
    full_table_name = f'{DIRECTORY}.{table_name}'
    try:
        if full_table_name not in image_indexes:
            return f"Error: Image index '{full_table_name}' not set up. Please call setup_image_index first."
        image_index = image_indexes[full_table_name]
        
        # Calculate similarity scores
        sim = image_index.image_description.similarity(query_text)

        # Get top results
        results = (image_index.order_by(sim, asc=False)
                  .select(image_index.image_file, image_index.image_description, sim=sim)
                  .limit(top_n)
                  .collect())

        # Format the results
        result_str = f"Query Results for '{query_text}' in '{full_table_name}':\n\n"
        for i, row in enumerate(results.to_pandas().itertuples(), 1):
            result_str += f"{i}. Score: {row.sim:.4f}\n"
            result_str += f"   Description: {row.image_description}\n"
            result_str += f"   Image: {row.image_file}\n\n"
        
        return result_str if result_str else "No results found."
    except Exception as e:
        return f"Error querying image index '{full_table_name}': {str(e)}"

@mcp.tool()
def list_image_tables() -> str:
    """List all image indexes currently available.

    Returns:
        A string listing the current image indexes.
    """
    tables = pxt.list_tables()
    image_tables = [t for t in tables if t.startswith(f'{DIRECTORY}.')]
    return f"Current image indexes: {', '.join(image_tables)}" if image_tables else "No image indexes exist."