wehub-resource-sync
471 lines
28 KiB
Plaintext
471 lines
28 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "9ba5c49e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import asyncio\n",
|
|
"import os\n",
|
|
"from typing import List, Dict, Any\n",
|
|
"from dotenv import load_dotenv\n",
|
|
"from code_ingestion import ingest_github_repo\n",
|
|
"from model_service import get_model_response_async\n",
|
|
"from code_evaluation import evaluate_code\n",
|
|
"\n",
|
|
"load_dotenv()\n",
|
|
"\n",
|
|
"async def get_full_response(model_name: str, prompt: str, context: Dict[str, Any]) -> str:\n",
|
|
" \"\"\"\n",
|
|
" Get a complete response from a model (non-streaming)\n",
|
|
" \"\"\"\n",
|
|
" response_text = \"\"\n",
|
|
" async for chunk in get_model_response_async(model_name, prompt, context):\n",
|
|
" response_text += chunk\n",
|
|
" \n",
|
|
" # Clean up the response by removing markdown code blocks\n",
|
|
" cleaned_text = response_text.strip().removeprefix(\"```python\").removeprefix(\"```\").removesuffix(\"```\").strip()\n",
|
|
" return cleaned_text\n",
|
|
"\n",
|
|
"async def process_single_query(query: str, context: Dict[str, Any]) -> Dict[str, Any]:\n",
|
|
" \"\"\"\n",
|
|
" Process a single query and return results for both models\n",
|
|
" \"\"\"\n",
|
|
" try:\n",
|
|
" # Get responses from both models\n",
|
|
" claude_response, qwen_response = await asyncio.gather(\n",
|
|
" get_full_response(\"claude-4\", query, context),\n",
|
|
" get_full_response(\"qwen3-coder\", query, context)\n",
|
|
" )\n",
|
|
" \n",
|
|
" # Evaluate both responses\n",
|
|
" claude_eval = evaluate_code(claude_response)\n",
|
|
" qwen_eval = evaluate_code(qwen_response)\n",
|
|
" \n",
|
|
" # Return results in the specified format\n",
|
|
" return {\n",
|
|
" \"query\": query,\n",
|
|
" \"Correctness_Qwen\": qwen_eval[\"detailed_metrics\"][\"correctness\"][\"score\"],\n",
|
|
" \"Correctness_Sonnet\": claude_eval[\"detailed_metrics\"][\"correctness\"][\"score\"],\n",
|
|
" \"readability_Qwen\": qwen_eval[\"detailed_metrics\"][\"readability\"][\"score\"],\n",
|
|
" \"readability_Sonnet\": claude_eval[\"detailed_metrics\"][\"readability\"][\"score\"],\n",
|
|
" \"bestpractices_Qwen\": qwen_eval[\"detailed_metrics\"][\"best_practices\"][\"score\"],\n",
|
|
" \"bestpractices_Sonnet\": claude_eval[\"detailed_metrics\"][\"best_practices\"][\"score\"],\n",
|
|
" # Additional fields for debugging/analysis\n",
|
|
" \"claude_response\": claude_response,\n",
|
|
" \"qwen_response\": qwen_response,\n",
|
|
" \"claude_overall_score\": claude_eval[\"overall_score\"],\n",
|
|
" \"qwen_overall_score\": qwen_eval[\"overall_score\"]\n",
|
|
" }\n",
|
|
" \n",
|
|
" except Exception as e:\n",
|
|
" print(f\"Error processing query '{query}': {str(e)}\")\n",
|
|
" return {\n",
|
|
" \"query\": query,\n",
|
|
" \"Correctness_Qwen\": 0.0,\n",
|
|
" \"Correctness_Sonnet\": 0.0,\n",
|
|
" \"readability_Qwen\": 0.0,\n",
|
|
" \"readability_Sonnet\": 0.0,\n",
|
|
" \"bestpractices_Qwen\": 0.0,\n",
|
|
" \"bestpractices_Sonnet\": 0.0,\n",
|
|
" \"error\": str(e)\n",
|
|
" }\n",
|
|
"\n",
|
|
"async def process_queries(queries: List[str], github_repo_url: str) -> List[Dict[str, Any]]:\n",
|
|
" \"\"\"\n",
|
|
" Process a list of queries against a GitHub repository\n",
|
|
" \n",
|
|
" Args:\n",
|
|
" queries: List of query strings\n",
|
|
" github_repo_url: URL of the GitHub repository to ingest\n",
|
|
" \n",
|
|
" Returns:\n",
|
|
" List of dictionaries with evaluation results for each query\n",
|
|
" \"\"\"\n",
|
|
" # Ingest the repository to get context\n",
|
|
" print(f\"Ingesting repository: {github_repo_url}\")\n",
|
|
" context = ingest_github_repo(github_repo_url)\n",
|
|
" print(\"Repository ingested successfully!\")\n",
|
|
" \n",
|
|
" # Process all queries\n",
|
|
" results = []\n",
|
|
" for i, query in enumerate(queries, 1):\n",
|
|
" print(f\"Processing query {i}/{len(queries)}: {query[:50]}...\")\n",
|
|
" result = await process_single_query(query, context)\n",
|
|
" results.append(result)\n",
|
|
" print(result)\n",
|
|
" print(f\"Completed query {i}/{len(queries)}\")\n",
|
|
" \n",
|
|
" return results\n",
|
|
"\n",
|
|
"def process_queries_sync(queries: List[str], github_repo_url: str) -> List[Dict[str, Any]]:\n",
|
|
" \"\"\"\n",
|
|
" Synchronous wrapper for process_queries - works in Jupyter notebooks\n",
|
|
" \"\"\"\n",
|
|
" try:\n",
|
|
" # Try to get the current event loop\n",
|
|
" loop = asyncio.get_event_loop()\n",
|
|
" if loop.is_running():\n",
|
|
" # If we're in a running event loop (like Jupyter), use asyncio.create_task\n",
|
|
" import nest_asyncio\n",
|
|
" nest_asyncio.apply()\n",
|
|
" return loop.run_until_complete(process_queries(queries, github_repo_url))\n",
|
|
" else:\n",
|
|
" # If no event loop is running, use asyncio.run\n",
|
|
" return asyncio.run(process_queries(queries, github_repo_url))\n",
|
|
" except RuntimeError:\n",
|
|
" # Fallback for Jupyter notebooks\n",
|
|
" import nest_asyncio\n",
|
|
" nest_asyncio.apply()\n",
|
|
" loop = asyncio.get_event_loop()\n",
|
|
" return loop.run_until_complete(process_queries(queries, github_repo_url))\n",
|
|
"\n",
|
|
"# Example usage:\n",
|
|
"if True:\n",
|
|
" # Define your queries\n",
|
|
" mcp_prompts = [\n",
|
|
" \"Build an MCP server that fetches the latest tweets from a user and sends them as a daily digest email via Gmail.\",\n",
|
|
" \"Build an MCP server that creates a new Notion page when someone drops a file into a specific Google Drive folder.\",\n",
|
|
" \"Create a small MCP server that listens for incoming questions from Discord and uses OpenAI to generate answers.\",\n",
|
|
" \"Build an MCP server that watches a GitHub repo for new issues and posts them into a Telegram group.\",\n",
|
|
" \"Write an MCP agent that reads a URL, extracts metadata with BeautifulSoup, and logs it to a CSV file.\",\n",
|
|
" \"Create an MCP endpoint that receives a YouTube link, downloads the audio, and sends it via email.\",\n",
|
|
" \"Create an MCP server that takes a PDF uploaded via a web form, summarizes it using an LLM, and returns the summary.\",\n",
|
|
" \"Write an MCP-compatible server that pulls job listings from LinkedIn and posts them to a Slack channel every hour.\",\n",
|
|
" \"Build a server that monitors a subreddit for new posts, filters them by keyword, and logs them in a Notion database.\",\n",
|
|
" \"Create a minimal MCP server that extracts tabular data from a URL and sends it to an Airtable base.\"\n",
|
|
"]\n",
|
|
" \n",
|
|
" # Define the GitHub repository URL\n",
|
|
" repo_url = \"https://github.com/jlowin/fastmcp\"\n",
|
|
" \n",
|
|
" # Process all queries\n",
|
|
" results = process_queries_sync(mcp_prompts, repo_url)\n",
|
|
" \n",
|
|
" # Print results\n",
|
|
" for result in results:\n",
|
|
" print(f\"\\nQuery: {result['query']}\")\n",
|
|
" print(f\"Qwen Correctness: {result['Correctness_Qwen']:.2f}\")\n",
|
|
" print(f\"Sonnet Correctness: {result['Correctness_Sonnet']:.2f}\")\n",
|
|
" print(f\"Qwen Readability: {result['readability_Qwen']:.2f}\")\n",
|
|
" print(f\"Sonnet Readability: {result['readability_Sonnet']:.2f}\")\n",
|
|
" print(f\"Qwen Best Practices: {result['bestpractices_Qwen']:.2f}\")\n",
|
|
" print(f\"Sonnet Best Practices: {result['bestpractices_Sonnet']:.2f}\")\n",
|
|
" if 'error' in result:\n",
|
|
" print(f\"Error: {result['error']}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "4f14722e",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<style type=\"text/css\">\n",
|
|
"#T_339c8_row0_col0, #T_339c8_row0_col1, #T_339c8_row0_col2, #T_339c8_row0_col3, #T_339c8_row0_col4, #T_339c8_row0_col5, #T_339c8_row0_col6, #T_339c8_row0_col7, #T_339c8_row0_col8, #T_339c8_row0_col9, #T_339c8_row1_col0, #T_339c8_row1_col1, #T_339c8_row1_col2, #T_339c8_row1_col3, #T_339c8_row1_col4, #T_339c8_row1_col5, #T_339c8_row1_col6, #T_339c8_row1_col7, #T_339c8_row1_col8, #T_339c8_row1_col9, #T_339c8_row2_col0, #T_339c8_row2_col1, #T_339c8_row2_col2, #T_339c8_row2_col3, #T_339c8_row2_col4, #T_339c8_row2_col5, #T_339c8_row2_col6, #T_339c8_row2_col7, #T_339c8_row2_col8, #T_339c8_row2_col9, #T_339c8_row3_col0, #T_339c8_row3_col1, #T_339c8_row3_col2, #T_339c8_row3_col3, #T_339c8_row3_col4, #T_339c8_row3_col5, #T_339c8_row3_col6, #T_339c8_row3_col7, #T_339c8_row3_col8, #T_339c8_row3_col9, #T_339c8_row4_col0, #T_339c8_row4_col1, #T_339c8_row4_col2, #T_339c8_row4_col3, #T_339c8_row4_col4, #T_339c8_row4_col5, #T_339c8_row4_col6, #T_339c8_row4_col7, #T_339c8_row4_col8, #T_339c8_row4_col9, #T_339c8_row5_col0, #T_339c8_row5_col1, #T_339c8_row5_col2, #T_339c8_row5_col3, #T_339c8_row5_col4, #T_339c8_row5_col5, #T_339c8_row5_col6, #T_339c8_row5_col7, #T_339c8_row5_col8, #T_339c8_row5_col9, #T_339c8_row7_col0, #T_339c8_row7_col1, #T_339c8_row7_col2, #T_339c8_row7_col3, #T_339c8_row7_col4, #T_339c8_row7_col5, #T_339c8_row7_col6, #T_339c8_row7_col7, #T_339c8_row7_col8, #T_339c8_row7_col9, #T_339c8_row8_col0, #T_339c8_row8_col1, #T_339c8_row8_col2, #T_339c8_row8_col3, #T_339c8_row8_col4, #T_339c8_row8_col5, #T_339c8_row8_col6, #T_339c8_row8_col7, #T_339c8_row8_col8, #T_339c8_row8_col9, #T_339c8_row9_col0, #T_339c8_row9_col1, #T_339c8_row9_col2, #T_339c8_row9_col3, #T_339c8_row9_col4, #T_339c8_row9_col5, #T_339c8_row9_col6, #T_339c8_row9_col7, #T_339c8_row9_col8, #T_339c8_row9_col9 {\n",
|
|
" background-color: lightblue;\n",
|
|
" color: black;\n",
|
|
" border-color: white;\n",
|
|
" border-style: solid;\n",
|
|
" border-width: 1px;\n",
|
|
" background-color: lightgreen;\n",
|
|
"}\n",
|
|
"#T_339c8_row6_col0, #T_339c8_row6_col1, #T_339c8_row6_col2, #T_339c8_row6_col3, #T_339c8_row6_col4, #T_339c8_row6_col5, #T_339c8_row6_col6, #T_339c8_row6_col7, #T_339c8_row6_col8, #T_339c8_row6_col9 {\n",
|
|
" background-color: lightblue;\n",
|
|
" color: black;\n",
|
|
" border-color: white;\n",
|
|
" border-style: solid;\n",
|
|
" border-width: 1px;\n",
|
|
" background-color: lightcoral;\n",
|
|
"}\n",
|
|
"</style>\n",
|
|
"<table id=\"T_339c8\">\n",
|
|
" <thead>\n",
|
|
" <tr>\n",
|
|
" <th class=\"blank level0\" > </th>\n",
|
|
" <th id=\"T_339c8_level0_col0\" class=\"col_heading level0 col0\" >Query</th>\n",
|
|
" <th id=\"T_339c8_level0_col1\" class=\"col_heading level0 col1\" >Correctness (Qwen)</th>\n",
|
|
" <th id=\"T_339c8_level0_col2\" class=\"col_heading level0 col2\" >Readability (Qwen)</th>\n",
|
|
" <th id=\"T_339c8_level0_col3\" class=\"col_heading level0 col3\" >Best Practices (Qwen)</th>\n",
|
|
" <th id=\"T_339c8_level0_col4\" class=\"col_heading level0 col4\" >Correctness (Claude)</th>\n",
|
|
" <th id=\"T_339c8_level0_col5\" class=\"col_heading level0 col5\" >Readability (Claude)</th>\n",
|
|
" <th id=\"T_339c8_level0_col6\" class=\"col_heading level0 col6\" >Best Practices (Claude)</th>\n",
|
|
" <th id=\"T_339c8_level0_col7\" class=\"col_heading level0 col7\" >Overall Score (Qwen)</th>\n",
|
|
" <th id=\"T_339c8_level0_col8\" class=\"col_heading level0 col8\" >Overall Score (Claude)</th>\n",
|
|
" <th id=\"T_339c8_level0_col9\" class=\"col_heading level0 col9\" >Winner</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th id=\"T_339c8_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
|
|
" <td id=\"T_339c8_row0_col0\" class=\"data row0 col0\" >Build an MCP server that fetches the latest tweets from a user and sends them as a daily digest email via Gmail.</td>\n",
|
|
" <td id=\"T_339c8_row0_col1\" class=\"data row0 col1\" >0.914</td>\n",
|
|
" <td id=\"T_339c8_row0_col2\" class=\"data row0 col2\" >0.903</td>\n",
|
|
" <td id=\"T_339c8_row0_col3\" class=\"data row0 col3\" >0.710</td>\n",
|
|
" <td id=\"T_339c8_row0_col4\" class=\"data row0 col4\" >0.738</td>\n",
|
|
" <td id=\"T_339c8_row0_col5\" class=\"data row0 col5\" >0.900</td>\n",
|
|
" <td id=\"T_339c8_row0_col6\" class=\"data row0 col6\" >0.758</td>\n",
|
|
" <td id=\"T_339c8_row0_col7\" class=\"data row0 col7\" >0.842</td>\n",
|
|
" <td id=\"T_339c8_row0_col8\" class=\"data row0 col8\" >0.799</td>\n",
|
|
" <td id=\"T_339c8_row0_col9\" class=\"data row0 col9\" >Qwen</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th id=\"T_339c8_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
|
|
" <td id=\"T_339c8_row1_col0\" class=\"data row1 col0\" >Build an MCP server that creates a new Notion page when someone drops a file into a specific Google Drive folder.</td>\n",
|
|
" <td id=\"T_339c8_row1_col1\" class=\"data row1 col1\" >0.935</td>\n",
|
|
" <td id=\"T_339c8_row1_col2\" class=\"data row1 col2\" >0.910</td>\n",
|
|
" <td id=\"T_339c8_row1_col3\" class=\"data row1 col3\" >0.779</td>\n",
|
|
" <td id=\"T_339c8_row1_col4\" class=\"data row1 col4\" >0.890</td>\n",
|
|
" <td id=\"T_339c8_row1_col5\" class=\"data row1 col5\" >0.899</td>\n",
|
|
" <td id=\"T_339c8_row1_col6\" class=\"data row1 col6\" >0.734</td>\n",
|
|
" <td id=\"T_339c8_row1_col7\" class=\"data row1 col7\" >0.874</td>\n",
|
|
" <td id=\"T_339c8_row1_col8\" class=\"data row1 col8\" >0.841</td>\n",
|
|
" <td id=\"T_339c8_row1_col9\" class=\"data row1 col9\" >Qwen</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th id=\"T_339c8_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
|
|
" <td id=\"T_339c8_row2_col0\" class=\"data row2 col0\" >Create a small MCP server that listens for incoming questions from Discord and uses OpenAI to generate answers.</td>\n",
|
|
" <td id=\"T_339c8_row2_col1\" class=\"data row2 col1\" >0.920</td>\n",
|
|
" <td id=\"T_339c8_row2_col2\" class=\"data row2 col2\" >0.904</td>\n",
|
|
" <td id=\"T_339c8_row2_col3\" class=\"data row2 col3\" >0.755</td>\n",
|
|
" <td id=\"T_339c8_row2_col4\" class=\"data row2 col4\" >0.894</td>\n",
|
|
" <td id=\"T_339c8_row2_col5\" class=\"data row2 col5\" >0.905</td>\n",
|
|
" <td id=\"T_339c8_row2_col6\" class=\"data row2 col6\" >0.773</td>\n",
|
|
" <td id=\"T_339c8_row2_col7\" class=\"data row2 col7\" >0.860</td>\n",
|
|
" <td id=\"T_339c8_row2_col8\" class=\"data row2 col8\" >0.857</td>\n",
|
|
" <td id=\"T_339c8_row2_col9\" class=\"data row2 col9\" >Qwen</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th id=\"T_339c8_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
|
|
" <td id=\"T_339c8_row3_col0\" class=\"data row3 col0\" >Build an MCP server that watches a GitHub repo for new issues and posts them into a Telegram group.</td>\n",
|
|
" <td id=\"T_339c8_row3_col1\" class=\"data row3 col1\" >0.917</td>\n",
|
|
" <td id=\"T_339c8_row3_col2\" class=\"data row3 col2\" >0.913</td>\n",
|
|
" <td id=\"T_339c8_row3_col3\" class=\"data row3 col3\" >0.813</td>\n",
|
|
" <td id=\"T_339c8_row3_col4\" class=\"data row3 col4\" >0.741</td>\n",
|
|
" <td id=\"T_339c8_row3_col5\" class=\"data row3 col5\" >0.905</td>\n",
|
|
" <td id=\"T_339c8_row3_col6\" class=\"data row3 col6\" >0.796</td>\n",
|
|
" <td id=\"T_339c8_row3_col7\" class=\"data row3 col7\" >0.881</td>\n",
|
|
" <td id=\"T_339c8_row3_col8\" class=\"data row3 col8\" >0.814</td>\n",
|
|
" <td id=\"T_339c8_row3_col9\" class=\"data row3 col9\" >Qwen</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th id=\"T_339c8_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
|
|
" <td id=\"T_339c8_row4_col0\" class=\"data row4 col0\" >Write an MCP agent that reads a URL, extracts metadata with BeautifulSoup, and logs it to a CSV file.</td>\n",
|
|
" <td id=\"T_339c8_row4_col1\" class=\"data row4 col1\" >0.901</td>\n",
|
|
" <td id=\"T_339c8_row4_col2\" class=\"data row4 col2\" >0.906</td>\n",
|
|
" <td id=\"T_339c8_row4_col3\" class=\"data row4 col3\" >0.770</td>\n",
|
|
" <td id=\"T_339c8_row4_col4\" class=\"data row4 col4\" >0.762</td>\n",
|
|
" <td id=\"T_339c8_row4_col5\" class=\"data row4 col5\" >0.904</td>\n",
|
|
" <td id=\"T_339c8_row4_col6\" class=\"data row4 col6\" >0.754</td>\n",
|
|
" <td id=\"T_339c8_row4_col7\" class=\"data row4 col7\" >0.859</td>\n",
|
|
" <td id=\"T_339c8_row4_col8\" class=\"data row4 col8\" >0.807</td>\n",
|
|
" <td id=\"T_339c8_row4_col9\" class=\"data row4 col9\" >Qwen</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th id=\"T_339c8_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
|
|
" <td id=\"T_339c8_row5_col0\" class=\"data row5 col0\" >Create an MCP endpoint that receives a YouTube link, downloads the audio, and sends it via email.</td>\n",
|
|
" <td id=\"T_339c8_row5_col1\" class=\"data row5 col1\" >0.932</td>\n",
|
|
" <td id=\"T_339c8_row5_col2\" class=\"data row5 col2\" >0.908</td>\n",
|
|
" <td id=\"T_339c8_row5_col3\" class=\"data row5 col3\" >0.731</td>\n",
|
|
" <td id=\"T_339c8_row5_col4\" class=\"data row5 col4\" >0.900</td>\n",
|
|
" <td id=\"T_339c8_row5_col5\" class=\"data row5 col5\" >0.910</td>\n",
|
|
" <td id=\"T_339c8_row5_col6\" class=\"data row5 col6\" >0.713</td>\n",
|
|
" <td id=\"T_339c8_row5_col7\" class=\"data row5 col7\" >0.857</td>\n",
|
|
" <td id=\"T_339c8_row5_col8\" class=\"data row5 col8\" >0.841</td>\n",
|
|
" <td id=\"T_339c8_row5_col9\" class=\"data row5 col9\" >Qwen</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th id=\"T_339c8_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
|
|
" <td id=\"T_339c8_row6_col0\" class=\"data row6 col0\" >Create an MCP server that takes a PDF uploaded via a web form, summarizes it using an LLM, and returns the summary.</td>\n",
|
|
" <td id=\"T_339c8_row6_col1\" class=\"data row6 col1\" >0.920</td>\n",
|
|
" <td id=\"T_339c8_row6_col2\" class=\"data row6 col2\" >0.905</td>\n",
|
|
" <td id=\"T_339c8_row6_col3\" class=\"data row6 col3\" >0.703</td>\n",
|
|
" <td id=\"T_339c8_row6_col4\" class=\"data row6 col4\" >0.845</td>\n",
|
|
" <td id=\"T_339c8_row6_col5\" class=\"data row6 col5\" >0.908</td>\n",
|
|
" <td id=\"T_339c8_row6_col6\" class=\"data row6 col6\" >0.802</td>\n",
|
|
" <td id=\"T_339c8_row6_col7\" class=\"data row6 col7\" >0.843</td>\n",
|
|
" <td id=\"T_339c8_row6_col8\" class=\"data row6 col8\" >0.851</td>\n",
|
|
" <td id=\"T_339c8_row6_col9\" class=\"data row6 col9\" >Claude</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th id=\"T_339c8_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
|
|
" <td id=\"T_339c8_row7_col0\" class=\"data row7 col0\" >Write an MCP-compatible server that pulls job listings from LinkedIn and posts them to a Slack channel every hour.</td>\n",
|
|
" <td id=\"T_339c8_row7_col1\" class=\"data row7 col1\" >0.920</td>\n",
|
|
" <td id=\"T_339c8_row7_col2\" class=\"data row7 col2\" >0.780</td>\n",
|
|
" <td id=\"T_339c8_row7_col3\" class=\"data row7 col3\" >0.793</td>\n",
|
|
" <td id=\"T_339c8_row7_col4\" class=\"data row7 col4\" >0.694</td>\n",
|
|
" <td id=\"T_339c8_row7_col5\" class=\"data row7 col5\" >0.898</td>\n",
|
|
" <td id=\"T_339c8_row7_col6\" class=\"data row7 col6\" >0.708</td>\n",
|
|
" <td id=\"T_339c8_row7_col7\" class=\"data row7 col7\" >0.831</td>\n",
|
|
" <td id=\"T_339c8_row7_col8\" class=\"data row7 col8\" >0.767</td>\n",
|
|
" <td id=\"T_339c8_row7_col9\" class=\"data row7 col9\" >Qwen</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th id=\"T_339c8_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
|
|
" <td id=\"T_339c8_row8_col0\" class=\"data row8 col0\" >Build a server that monitors a subreddit for new posts, filters them by keyword, and logs them in a Notion database.</td>\n",
|
|
" <td id=\"T_339c8_row8_col1\" class=\"data row8 col1\" >0.919</td>\n",
|
|
" <td id=\"T_339c8_row8_col2\" class=\"data row8 col2\" >0.905</td>\n",
|
|
" <td id=\"T_339c8_row8_col3\" class=\"data row8 col3\" >0.718</td>\n",
|
|
" <td id=\"T_339c8_row8_col4\" class=\"data row8 col4\" >0.632</td>\n",
|
|
" <td id=\"T_339c8_row8_col5\" class=\"data row8 col5\" >0.906</td>\n",
|
|
" <td id=\"T_339c8_row8_col6\" class=\"data row8 col6\" >0.791</td>\n",
|
|
" <td id=\"T_339c8_row8_col7\" class=\"data row8 col7\" >0.847</td>\n",
|
|
" <td id=\"T_339c8_row8_col8\" class=\"data row8 col8\" >0.776</td>\n",
|
|
" <td id=\"T_339c8_row8_col9\" class=\"data row8 col9\" >Qwen</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th id=\"T_339c8_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
|
|
" <td id=\"T_339c8_row9_col0\" class=\"data row9 col0\" >Create a minimal MCP server that extracts tabular data from a URL and sends it to an Airtable base.</td>\n",
|
|
" <td id=\"T_339c8_row9_col1\" class=\"data row9 col1\" >0.920</td>\n",
|
|
" <td id=\"T_339c8_row9_col2\" class=\"data row9 col2\" >0.890</td>\n",
|
|
" <td id=\"T_339c8_row9_col3\" class=\"data row9 col3\" >0.860</td>\n",
|
|
" <td id=\"T_339c8_row9_col4\" class=\"data row9 col4\" >0.897</td>\n",
|
|
" <td id=\"T_339c8_row9_col5\" class=\"data row9 col5\" >0.905</td>\n",
|
|
" <td id=\"T_339c8_row9_col6\" class=\"data row9 col6\" >0.827</td>\n",
|
|
" <td id=\"T_339c8_row9_col7\" class=\"data row9 col7\" >0.890</td>\n",
|
|
" <td id=\"T_339c8_row9_col8\" class=\"data row9 col8\" >0.876</td>\n",
|
|
" <td id=\"T_339c8_row9_col9\" class=\"data row9 col9\" >Qwen</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n"
|
|
],
|
|
"text/plain": [
|
|
"<pandas.io.formats.style.Styler at 0x1380724e0>"
|
|
]
|
|
},
|
|
"execution_count": 18,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"import numpy as np\n",
|
|
"from typing import List, Dict, Any\n",
|
|
"\n",
|
|
"def create_results_table(results: List[Dict[str, Any]]) -> pd.DataFrame:\n",
|
|
" \"\"\"\n",
|
|
" Create a formatted table from evaluation results\n",
|
|
" \"\"\"\n",
|
|
" # Create DataFrame from results\n",
|
|
" df = pd.DataFrame(results)\n",
|
|
" \n",
|
|
" # Calculate overall scores (average of the three metrics)\n",
|
|
" df['overall_score_qwen'] = (df['Correctness_Qwen'] + df['readability_Qwen'] + df['bestpractices_Qwen']) / 3\n",
|
|
" df['overall_score_claude'] = (df['Correctness_Sonnet'] + df['readability_Sonnet'] + df['bestpractices_Sonnet']) / 3\n",
|
|
" \n",
|
|
" # Determine winner\n",
|
|
" df['winner'] = df.apply(lambda row: 'Qwen' if row['overall_score_qwen'] > row['overall_score_claude'] else 'Claude', axis=1)\n",
|
|
" \n",
|
|
" # Round all numeric columns to 3 decimal places\n",
|
|
" numeric_columns = [\n",
|
|
" 'Correctness_Qwen', 'readability_Qwen', 'bestpractices_Qwen',\n",
|
|
" 'Correctness_Sonnet', 'readability_Sonnet', 'bestpractices_Sonnet',\n",
|
|
" 'overall_score_qwen', 'overall_score_claude'\n",
|
|
" ]\n",
|
|
" \n",
|
|
" for col in numeric_columns:\n",
|
|
" if col in df.columns:\n",
|
|
" df[col] = df[col].round(3)\n",
|
|
" \n",
|
|
" # Select and rename columns for the final table\n",
|
|
" final_df = df[[\n",
|
|
" 'query',\n",
|
|
" 'Correctness_Qwen', 'readability_Qwen', 'bestpractices_Qwen',\n",
|
|
" 'Correctness_Sonnet', 'readability_Sonnet', 'bestpractices_Sonnet',\n",
|
|
" 'overall_score_qwen', 'overall_score_claude', 'winner'\n",
|
|
" ]].copy()\n",
|
|
" \n",
|
|
" # Rename columns for better readability\n",
|
|
" final_df.columns = [\n",
|
|
" 'Query',\n",
|
|
" 'Correctness (Qwen)', 'Readability (Qwen)', 'Best Practices (Qwen)',\n",
|
|
" 'Correctness (Claude)', 'Readability (Claude)', 'Best Practices (Claude)',\n",
|
|
" 'Overall Score (Qwen)', 'Overall Score (Claude)', 'Winner'\n",
|
|
" ]\n",
|
|
" \n",
|
|
" return final_df\n",
|
|
"\n",
|
|
"def display_results_table(results: List[Dict[str, Any]]):\n",
|
|
" \"\"\"\n",
|
|
" Display the results table with formatting\n",
|
|
" \"\"\"\n",
|
|
" df = create_results_table(results)\n",
|
|
" \n",
|
|
" # Display the table\n",
|
|
" print(\"=== EVALUATION RESULTS TABLE ===\")\n",
|
|
" print(df.to_string(index=False))\n",
|
|
" \n",
|
|
" # Print summary statistics\n",
|
|
" print(\"\\n=== SUMMARY STATISTICS ===\")\n",
|
|
" print(f\"Total queries: {len(df)}\")\n",
|
|
" print(f\"Qwen wins: {len(df[df['Winner'] == 'Qwen'])}\")\n",
|
|
" print(f\"Claude wins: {len(df[df['Winner'] == 'Claude'])}\")\n",
|
|
" \n",
|
|
" # Average scores\n",
|
|
" print(f\"\\nAverage Overall Scores:\")\n",
|
|
" print(f\"Qwen: {df['Overall Score (Qwen)'].mean():.3f}\")\n",
|
|
" print(f\"Claude: {df['Overall Score (Claude)'].mean():.3f}\")\n",
|
|
" \n",
|
|
" return df\n",
|
|
"\n",
|
|
"# If you want to display in Jupyter with better formatting:\n",
|
|
"def display_formatted_table(results: List[Dict[str, Any]]):\n",
|
|
" \"\"\"\n",
|
|
" Display formatted table with styling (for Jupyter notebooks)\n",
|
|
" \"\"\"\n",
|
|
" df = create_results_table(results)\n",
|
|
" \n",
|
|
" # Apply styling\n",
|
|
" styled_df = df.style.set_properties(**{\n",
|
|
" 'background-color': 'lightblue',\n",
|
|
" 'color': 'black',\n",
|
|
" 'border-color': 'white',\n",
|
|
" 'border-style': 'solid',\n",
|
|
" 'border-width': '1px'\n",
|
|
" }).format({\n",
|
|
" 'Correctness (Qwen)': '{:.3f}',\n",
|
|
" 'Readability (Qwen)': '{:.3f}',\n",
|
|
" 'Best Practices (Qwen)': '{:.3f}',\n",
|
|
" 'Correctness (Claude)': '{:.3f}',\n",
|
|
" 'Readability (Claude)': '{:.3f}',\n",
|
|
" 'Best Practices (Claude)': '{:.3f}',\n",
|
|
" 'Overall Score (Qwen)': '{:.3f}',\n",
|
|
" 'Overall Score (Claude)': '{:.3f}'\n",
|
|
" }).apply(lambda x: ['background-color: lightgreen' if x['Winner'] == 'Qwen' else 'background-color: lightcoral' for i in range(len(x))], axis=1)\n",
|
|
" \n",
|
|
" return styled_df\n",
|
|
"\n",
|
|
"styled_table = display_formatted_table(results)\n",
|
|
"styled_table"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": ".venv",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.12.10"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|