import asyncio import streamlit as st import pandas as pd import plotly.express as px from dotenv import load_dotenv from model_service import get_parallel_responses from code_ingestion import ingest_github_repo from code_evaluation import evaluate_code load_dotenv() # Set page config st.set_page_config( page_title="Code Generation Model Comparison", layout="wide" ) # Custom CSS for responsive code containers st.markdown(""" """, unsafe_allow_html=True) # Initialize session state if 'chat_history' not in st.session_state: st.session_state.chat_history = [] if 'context' not in st.session_state: st.session_state.context = None if 'reference_code' not in st.session_state: st.session_state.reference_code = None if 'last_generated_code' not in st.session_state: st.session_state.last_generated_code = {"claude": None, "qwen3-coder": None} if 'evaluation_results' not in st.session_state: st.session_state.evaluation_results = {"claude": None, "qwen3-coder": None} with st.sidebar: st.title("Configuration") github_repo = st.text_input( "GitHub Repository URL", placeholder="https://github.com/username/repository" ) if st.button("Ingest Repository"): if github_repo: with st.spinner("Ingesting repository..."): st.session_state.context = ingest_github_repo(github_repo) st.success("Repository ingested successfully!") else: st.error("Please enter a valid repository URL") st.session_state.reference_code = st.text_area( "Reference Code (Optional)", help="Enter reference/ground truth code to compare against", height=200 ) # Evaluation section st.write("### Evaluation") if st.button("Evaluate Generated Code"): if st.session_state.last_generated_code["claude"] and st.session_state.last_generated_code["qwen3-coder"]: with st.spinner("Evaluating code..."): st.session_state.evaluation_results["claude"] = evaluate_code( st.session_state.last_generated_code["claude"], st.session_state.reference_code if st.session_state.reference_code else None ) st.session_state.evaluation_results["qwen3-coder"] = evaluate_code( st.session_state.last_generated_code["qwen3-coder"], st.session_state.reference_code if st.session_state.reference_code else None ) st.success("Evaluation complete!") else: st.error("Please generate code from both models first") async def handle_chat_input(prompt: str): st.session_state.chat_history.append({"role": "user", "content": prompt}) with st.chat_message("user"): st.markdown(prompt) # Get streaming responses from both models with st.chat_message("assistant"): col1, col2 = st.columns(2) with col1: st.write("##### Claude Sonnet 4") claude_container = st.empty() claude_container = claude_container.code("", language="python") with col2: st.write("##### Qwen3-Coder") qwen3_coder_container = st.empty() qwen3_coder_container = qwen3_coder_container.code("", language="python") claude_gen, qwen3_coder_gen = await get_parallel_responses(prompt, st.session_state.context) async def process_claude_stream(container): response_text = "" async for chunk in claude_gen: response_text += chunk cleaned_text = response_text.strip().removeprefix("```python").removeprefix("```").removesuffix("```").strip() container.code(cleaned_text, language="python") return cleaned_text async def process_qwen3_coder_stream(container): response_text = "" async for chunk in qwen3_coder_gen: response_text += chunk cleaned_text = response_text.strip().removeprefix("```python").removeprefix("```").removesuffix("```").strip() container.code(cleaned_text, language="python") return cleaned_text # Run both streams concurrently final_claude_response, final_qwen3_coder_response = await asyncio.gather( process_claude_stream(claude_container), process_qwen3_coder_stream(qwen3_coder_container) ) message = { "role": "assistant", "content": "", "claude_response": final_claude_response, "qwen3-coder_response": final_qwen3_coder_response } st.session_state.chat_history.append(message) st.session_state.last_generated_code["claude"] = final_claude_response st.session_state.last_generated_code["qwen3-coder"] = final_qwen3_coder_response # Main interface st.title("Claude Sonnet 4 vs Qwen3-Coder using DeepEval") # Display chat history for message in st.session_state.chat_history: with st.chat_message(message["role"]): st.markdown(message["content"]) if message["role"] == "assistant": col1, col2 = st.columns(2) with col1: st.write("##### Claude Sonnet 4") st.code(message["claude_response"], language="python") with col2: st.write("##### Qwen3-Coder") st.code(message["qwen3-coder_response"], language="python") if prompt := st.chat_input("What code would you like to generate?"): if not st.session_state.context: st.error("Please ingest a GitHub repository first!") else: asyncio.run(handle_chat_input(prompt)) # Display evaluation results if st.session_state.evaluation_results["claude"] and st.session_state.evaluation_results["qwen3-coder"]: st.write("---") st.header("Evaluation results generated with GPT-4o using DeepEval") plot_data = pd.DataFrame({ 'Metric': ["Correctness", "Readability", "Best Practices", "Overall Score"], 'Claude': [ st.session_state.evaluation_results['claude']['detailed_metrics']['correctness']['score'], st.session_state.evaluation_results['claude']['detailed_metrics']['readability']['score'], st.session_state.evaluation_results['claude']['detailed_metrics']['best_practices']['score'], st.session_state.evaluation_results['claude']['overall_score'] ], 'Qwen3-Coder': [ st.session_state.evaluation_results['qwen3-coder']['detailed_metrics']['correctness']['score'], st.session_state.evaluation_results['qwen3-coder']['detailed_metrics']['readability']['score'], st.session_state.evaluation_results['qwen3-coder']['detailed_metrics']['best_practices']['score'], st.session_state.evaluation_results['qwen3-coder']['overall_score'] ] }) fig = px.bar( plot_data.melt('Metric', var_name='Model', value_name='Score'), x='Metric', y='Score', color='Model', barmode='group', title='Model Performance Comparison', template='plotly_dark', color_discrete_sequence=['#00CED1', '#FF69B4'] ) fig.update_layout( xaxis_title="Evaluation Metrics", yaxis_title="Score", legend_title="Models", plot_bgcolor='rgba(32, 32, 32, 1)', paper_bgcolor='rgba(32, 32, 32, 1)', bargap=0.2, bargroupgap=0.1, font=dict(color='#E0E0E0'), title_font=dict(color='#E0E0E0'), showlegend=True, legend=dict( bgcolor='rgba(32, 32, 32, 0.8)', bordercolor='rgba(255, 255, 255, 0.3)', borderwidth=1 ) ) fig.update_xaxes( gridcolor='rgba(128, 128, 128, 0.2)', zerolinecolor='rgba(128, 128, 128, 0.2)' ) fig.update_yaxes( gridcolor='rgba(128, 128, 128, 0.2)', zerolinecolor='rgba(128, 128, 128, 0.2)' ) st.plotly_chart(fig, use_container_width=True) st.write("### Claude Sonnet 4 detailed metrics") claude_data = [] for metric in ["correctness", "readability", "best_practices"]: row = { "Metric": metric.title(), "Score": f"{st.session_state.evaluation_results['claude']['detailed_metrics'][metric]['score']:.2f}", "Reasoning": st.session_state.evaluation_results['claude']['detailed_metrics'][metric]['reason'] } claude_data.append(row) claude_data.append({ "Metric": "Overall Score", "Score": f"{st.session_state.evaluation_results['claude']['overall_score']:.2f}", "Reasoning": "Final weighted average" }) # Display Claude table claude_df = pd.DataFrame(claude_data) st.dataframe( claude_df, column_config={ "Metric": st.column_config.TextColumn("Metric", width="small"), "Score": st.column_config.TextColumn("Score", width="small"), "Reasoning": st.column_config.TextColumn("Reasoning", width="large") }, hide_index=True, use_container_width=True ) st.write("### Qwen3-Coder detailed metrics") qwen3_coder_data = [] for metric in ["correctness", "readability", "best_practices"]: row = { "Metric": metric.title(), "Score": f"{st.session_state.evaluation_results['qwen3-coder']['detailed_metrics'][metric]['score']:.2f}", "Reasoning": st.session_state.evaluation_results['qwen3-coder']['detailed_metrics'][metric]['reason'] } qwen3_coder_data.append(row) qwen3_coder_data.append({ "Metric": "Overall Score", "Score": f"{st.session_state.evaluation_results['qwen3-coder']['overall_score']:.2f}", "Reasoning": "Final weighted average" }) # Display Qwen3-Coder table qwen3_coder_df = pd.DataFrame(qwen3_coder_data) st.dataframe( qwen3_coder_df, column_config={ "Metric": st.column_config.TextColumn("Metric", width="small"), "Score": st.column_config.TextColumn("Score", width="small"), "Reasoning": st.column_config.TextColumn("Reasoning", width="large") }, hide_index=True, use_container_width=True )