{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "9ba5c49e", "metadata": {}, "outputs": [], "source": [ "import asyncio\n", "import os\n", "from typing import List, Dict, Any\n", "from dotenv import load_dotenv\n", "from code_ingestion import ingest_github_repo\n", "from model_service import get_model_response_async\n", "from code_evaluation import evaluate_code\n", "\n", "load_dotenv()\n", "\n", "async def get_full_response(model_name: str, prompt: str, context: Dict[str, Any]) -> str:\n", " \"\"\"\n", " Get a complete response from a model (non-streaming)\n", " \"\"\"\n", " response_text = \"\"\n", " async for chunk in get_model_response_async(model_name, prompt, context):\n", " response_text += chunk\n", " \n", " # Clean up the response by removing markdown code blocks\n", " cleaned_text = response_text.strip().removeprefix(\"```python\").removeprefix(\"```\").removesuffix(\"```\").strip()\n", " return cleaned_text\n", "\n", "async def process_single_query(query: str, context: Dict[str, Any]) -> Dict[str, Any]:\n", " \"\"\"\n", " Process a single query and return results for both models\n", " \"\"\"\n", " try:\n", " # Get responses from both models\n", " claude_response, qwen_response = await asyncio.gather(\n", " get_full_response(\"claude-4\", query, context),\n", " get_full_response(\"qwen3-coder\", query, context)\n", " )\n", " \n", " # Evaluate both responses\n", " claude_eval = evaluate_code(claude_response)\n", " qwen_eval = evaluate_code(qwen_response)\n", " \n", " # Return results in the specified format\n", " return {\n", " \"query\": query,\n", " \"Correctness_Qwen\": qwen_eval[\"detailed_metrics\"][\"correctness\"][\"score\"],\n", " \"Correctness_Sonnet\": claude_eval[\"detailed_metrics\"][\"correctness\"][\"score\"],\n", " \"readability_Qwen\": qwen_eval[\"detailed_metrics\"][\"readability\"][\"score\"],\n", " \"readability_Sonnet\": claude_eval[\"detailed_metrics\"][\"readability\"][\"score\"],\n", " \"bestpractices_Qwen\": qwen_eval[\"detailed_metrics\"][\"best_practices\"][\"score\"],\n", " \"bestpractices_Sonnet\": claude_eval[\"detailed_metrics\"][\"best_practices\"][\"score\"],\n", " # Additional fields for debugging/analysis\n", " \"claude_response\": claude_response,\n", " \"qwen_response\": qwen_response,\n", " \"claude_overall_score\": claude_eval[\"overall_score\"],\n", " \"qwen_overall_score\": qwen_eval[\"overall_score\"]\n", " }\n", " \n", " except Exception as e:\n", " print(f\"Error processing query '{query}': {str(e)}\")\n", " return {\n", " \"query\": query,\n", " \"Correctness_Qwen\": 0.0,\n", " \"Correctness_Sonnet\": 0.0,\n", " \"readability_Qwen\": 0.0,\n", " \"readability_Sonnet\": 0.0,\n", " \"bestpractices_Qwen\": 0.0,\n", " \"bestpractices_Sonnet\": 0.0,\n", " \"error\": str(e)\n", " }\n", "\n", "async def process_queries(queries: List[str], github_repo_url: str) -> List[Dict[str, Any]]:\n", " \"\"\"\n", " Process a list of queries against a GitHub repository\n", " \n", " Args:\n", " queries: List of query strings\n", " github_repo_url: URL of the GitHub repository to ingest\n", " \n", " Returns:\n", " List of dictionaries with evaluation results for each query\n", " \"\"\"\n", " # Ingest the repository to get context\n", " print(f\"Ingesting repository: {github_repo_url}\")\n", " context = ingest_github_repo(github_repo_url)\n", " print(\"Repository ingested successfully!\")\n", " \n", " # Process all queries\n", " results = []\n", " for i, query in enumerate(queries, 1):\n", " print(f\"Processing query {i}/{len(queries)}: {query[:50]}...\")\n", " result = await process_single_query(query, context)\n", " results.append(result)\n", " print(result)\n", " print(f\"Completed query {i}/{len(queries)}\")\n", " \n", " return results\n", "\n", "def process_queries_sync(queries: List[str], github_repo_url: str) -> List[Dict[str, Any]]:\n", " \"\"\"\n", " Synchronous wrapper for process_queries - works in Jupyter notebooks\n", " \"\"\"\n", " try:\n", " # Try to get the current event loop\n", " loop = asyncio.get_event_loop()\n", " if loop.is_running():\n", " # If we're in a running event loop (like Jupyter), use asyncio.create_task\n", " import nest_asyncio\n", " nest_asyncio.apply()\n", " return loop.run_until_complete(process_queries(queries, github_repo_url))\n", " else:\n", " # If no event loop is running, use asyncio.run\n", " return asyncio.run(process_queries(queries, github_repo_url))\n", " except RuntimeError:\n", " # Fallback for Jupyter notebooks\n", " import nest_asyncio\n", " nest_asyncio.apply()\n", " loop = asyncio.get_event_loop()\n", " return loop.run_until_complete(process_queries(queries, github_repo_url))\n", "\n", "# Example usage:\n", "if True:\n", " # Define your queries\n", " mcp_prompts = [\n", " \"Build an MCP server that fetches the latest tweets from a user and sends them as a daily digest email via Gmail.\",\n", " \"Build an MCP server that creates a new Notion page when someone drops a file into a specific Google Drive folder.\",\n", " \"Create a small MCP server that listens for incoming questions from Discord and uses OpenAI to generate answers.\",\n", " \"Build an MCP server that watches a GitHub repo for new issues and posts them into a Telegram group.\",\n", " \"Write an MCP agent that reads a URL, extracts metadata with BeautifulSoup, and logs it to a CSV file.\",\n", " \"Create an MCP endpoint that receives a YouTube link, downloads the audio, and sends it via email.\",\n", " \"Create an MCP server that takes a PDF uploaded via a web form, summarizes it using an LLM, and returns the summary.\",\n", " \"Write an MCP-compatible server that pulls job listings from LinkedIn and posts them to a Slack channel every hour.\",\n", " \"Build a server that monitors a subreddit for new posts, filters them by keyword, and logs them in a Notion database.\",\n", " \"Create a minimal MCP server that extracts tabular data from a URL and sends it to an Airtable base.\"\n", "]\n", " \n", " # Define the GitHub repository URL\n", " repo_url = \"https://github.com/jlowin/fastmcp\"\n", " \n", " # Process all queries\n", " results = process_queries_sync(mcp_prompts, repo_url)\n", " \n", " # Print results\n", " for result in results:\n", " print(f\"\\nQuery: {result['query']}\")\n", " print(f\"Qwen Correctness: {result['Correctness_Qwen']:.2f}\")\n", " print(f\"Sonnet Correctness: {result['Correctness_Sonnet']:.2f}\")\n", " print(f\"Qwen Readability: {result['readability_Qwen']:.2f}\")\n", " print(f\"Sonnet Readability: {result['readability_Sonnet']:.2f}\")\n", " print(f\"Qwen Best Practices: {result['bestpractices_Qwen']:.2f}\")\n", " print(f\"Sonnet Best Practices: {result['bestpractices_Sonnet']:.2f}\")\n", " if 'error' in result:\n", " print(f\"Error: {result['error']}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "4f14722e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
 QueryCorrectness (Qwen)Readability (Qwen)Best Practices (Qwen)Correctness (Claude)Readability (Claude)Best Practices (Claude)Overall Score (Qwen)Overall Score (Claude)Winner
0Build an MCP server that fetches the latest tweets from a user and sends them as a daily digest email via Gmail.0.9140.9030.7100.7380.9000.7580.8420.799Qwen
1Build an MCP server that creates a new Notion page when someone drops a file into a specific Google Drive folder.0.9350.9100.7790.8900.8990.7340.8740.841Qwen
2Create a small MCP server that listens for incoming questions from Discord and uses OpenAI to generate answers.0.9200.9040.7550.8940.9050.7730.8600.857Qwen
3Build an MCP server that watches a GitHub repo for new issues and posts them into a Telegram group.0.9170.9130.8130.7410.9050.7960.8810.814Qwen
4Write an MCP agent that reads a URL, extracts metadata with BeautifulSoup, and logs it to a CSV file.0.9010.9060.7700.7620.9040.7540.8590.807Qwen
5Create an MCP endpoint that receives a YouTube link, downloads the audio, and sends it via email.0.9320.9080.7310.9000.9100.7130.8570.841Qwen
6Create an MCP server that takes a PDF uploaded via a web form, summarizes it using an LLM, and returns the summary.0.9200.9050.7030.8450.9080.8020.8430.851Claude
7Write an MCP-compatible server that pulls job listings from LinkedIn and posts them to a Slack channel every hour.0.9200.7800.7930.6940.8980.7080.8310.767Qwen
8Build a server that monitors a subreddit for new posts, filters them by keyword, and logs them in a Notion database.0.9190.9050.7180.6320.9060.7910.8470.776Qwen
9Create a minimal MCP server that extracts tabular data from a URL and sends it to an Airtable base.0.9200.8900.8600.8970.9050.8270.8900.876Qwen
\n" ], "text/plain": [ "" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "import numpy as np\n", "from typing import List, Dict, Any\n", "\n", "def create_results_table(results: List[Dict[str, Any]]) -> pd.DataFrame:\n", " \"\"\"\n", " Create a formatted table from evaluation results\n", " \"\"\"\n", " # Create DataFrame from results\n", " df = pd.DataFrame(results)\n", " \n", " # Calculate overall scores (average of the three metrics)\n", " df['overall_score_qwen'] = (df['Correctness_Qwen'] + df['readability_Qwen'] + df['bestpractices_Qwen']) / 3\n", " df['overall_score_claude'] = (df['Correctness_Sonnet'] + df['readability_Sonnet'] + df['bestpractices_Sonnet']) / 3\n", " \n", " # Determine winner\n", " df['winner'] = df.apply(lambda row: 'Qwen' if row['overall_score_qwen'] > row['overall_score_claude'] else 'Claude', axis=1)\n", " \n", " # Round all numeric columns to 3 decimal places\n", " numeric_columns = [\n", " 'Correctness_Qwen', 'readability_Qwen', 'bestpractices_Qwen',\n", " 'Correctness_Sonnet', 'readability_Sonnet', 'bestpractices_Sonnet',\n", " 'overall_score_qwen', 'overall_score_claude'\n", " ]\n", " \n", " for col in numeric_columns:\n", " if col in df.columns:\n", " df[col] = df[col].round(3)\n", " \n", " # Select and rename columns for the final table\n", " final_df = df[[\n", " 'query',\n", " 'Correctness_Qwen', 'readability_Qwen', 'bestpractices_Qwen',\n", " 'Correctness_Sonnet', 'readability_Sonnet', 'bestpractices_Sonnet',\n", " 'overall_score_qwen', 'overall_score_claude', 'winner'\n", " ]].copy()\n", " \n", " # Rename columns for better readability\n", " final_df.columns = [\n", " 'Query',\n", " 'Correctness (Qwen)', 'Readability (Qwen)', 'Best Practices (Qwen)',\n", " 'Correctness (Claude)', 'Readability (Claude)', 'Best Practices (Claude)',\n", " 'Overall Score (Qwen)', 'Overall Score (Claude)', 'Winner'\n", " ]\n", " \n", " return final_df\n", "\n", "def display_results_table(results: List[Dict[str, Any]]):\n", " \"\"\"\n", " Display the results table with formatting\n", " \"\"\"\n", " df = create_results_table(results)\n", " \n", " # Display the table\n", " print(\"=== EVALUATION RESULTS TABLE ===\")\n", " print(df.to_string(index=False))\n", " \n", " # Print summary statistics\n", " print(\"\\n=== SUMMARY STATISTICS ===\")\n", " print(f\"Total queries: {len(df)}\")\n", " print(f\"Qwen wins: {len(df[df['Winner'] == 'Qwen'])}\")\n", " print(f\"Claude wins: {len(df[df['Winner'] == 'Claude'])}\")\n", " \n", " # Average scores\n", " print(f\"\\nAverage Overall Scores:\")\n", " print(f\"Qwen: {df['Overall Score (Qwen)'].mean():.3f}\")\n", " print(f\"Claude: {df['Overall Score (Claude)'].mean():.3f}\")\n", " \n", " return df\n", "\n", "# If you want to display in Jupyter with better formatting:\n", "def display_formatted_table(results: List[Dict[str, Any]]):\n", " \"\"\"\n", " Display formatted table with styling (for Jupyter notebooks)\n", " \"\"\"\n", " df = create_results_table(results)\n", " \n", " # Apply styling\n", " styled_df = df.style.set_properties(**{\n", " 'background-color': 'lightblue',\n", " 'color': 'black',\n", " 'border-color': 'white',\n", " 'border-style': 'solid',\n", " 'border-width': '1px'\n", " }).format({\n", " 'Correctness (Qwen)': '{:.3f}',\n", " 'Readability (Qwen)': '{:.3f}',\n", " 'Best Practices (Qwen)': '{:.3f}',\n", " 'Correctness (Claude)': '{:.3f}',\n", " 'Readability (Claude)': '{:.3f}',\n", " 'Best Practices (Claude)': '{:.3f}',\n", " 'Overall Score (Qwen)': '{:.3f}',\n", " 'Overall Score (Claude)': '{:.3f}'\n", " }).apply(lambda x: ['background-color: lightgreen' if x['Winner'] == 'Qwen' else 'background-color: lightcoral' for i in range(len(x))], axis=1)\n", " \n", " return styled_df\n", "\n", "styled_table = display_formatted_table(results)\n", "styled_table" ] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.10" } }, "nbformat": 4, "nbformat_minor": 5 }