patchy631--ai-engineering-hub/code-model-comparison/code_evaluation_opik.py

from opik.evaluation.metrics import GEval


def evaluate_code(generated_code: str, reference_code: str = None):
    """
    Evaluate generated Python code using Comet Opik's GEval metrics.

    1. Code Correctness - Assesses functional correctness, edge case handling,
       and completeness of implementation
    2. Code Readability - Evaluates naming conventions, formatting, documentation,
       and overall code structure
    3. Code Best Practices - Checks error handling, security practices, efficiency,
       and modularity

    Args:
        generated_code (str): The Python code to evaluate
        reference_code (str, optional): Reference code for comparison. If provided,
            the correctness evaluation will compare against this reference.

    Returns:
        dict: A dictionary containing evaluation results with the following structure:
            {
                "overall_score": float,  # Average score across all metrics (0-10 scale)
                "detailed_metrics": {
                    "correctness": {"score": float, "reason": str},
                    "readability": {"score": float, "reason": str},
                    "best_practices": {"score": float, "reason": str}
                },
                "passed": bool,  # Whether overall_score >= 7.0 (70% threshold)
                "error": str, optional  # Error message if evaluation fails
            }
    """
    try:
        # Validate input
        if not generated_code or not generated_code.strip():
            raise ValueError("Generated code cannot be empty")
     
        # Build the context string that includes both actual and expected code
        context = f"ACTUAL_CODE:\n```\n{generated_code}\n```"
        if reference_code:
            context += f"\nEXPECTED_CODE:\n```\n{reference_code}\n```"

        # Define rubric scoring criteria
        correctness_rubric_text = (
            "Score 0-2: Code is non-functional or has critical errors\n"
            "Score 3-5: Code works but misses key functionality\n"
            "Score 6-8: Code is mostly correct with minor issues\n"
            "Score 9-10: Code is completely correct"
        )

        readability_rubric_text = (
            "Score 0-2: Code is poorly formatted and hard to read\n"
            "Score 3-5: Code has basic formatting but lacks clarity\n"
            "Score 6-8: Code is well formatted with minor issues\n"
            "Score 9-10: Code is exceptionally readable and well documented"
        )

        best_practices_rubric_text = (
            "Score 0-2: Code ignores best practices\n"
            "Score 3-5: Code follows basic practices with gaps\n"
            "Score 6-8: Code mostly follows best practices\n"
            "Score 9-10: Code perfectly follows all best practices"
        )

        # 1. Code Correctness Metric
        correctness_metric = GEval(
            task_introduction=(
                "You are an expert judge evaluating Python code correctness. "
                "The expected implementation is under EXPECTED_CODE and the submitted code is under ACTUAL_CODE. "
                "Assess if the code is functionally correct, handles edge cases, and fully implements the required functionality. "
                "Use the following rubric to assign scores:"
            ),
            evaluation_criteria=(
                "EVALUATION STEPS:\n"
                "1. Check if all required functionality is implemented.\n"
                "2. Verify proper handling of edge cases.\n"
                "3. Identify potential runtime errors.\n"
                "4. Confirm the code produces the expected outputs.\n\n"
                "SCORING RUBRIC:\n"
                f"{correctness_rubric_text}\n\n"
                "Return only a score between 0 and 10, and a concise reason that references the rubric."
            ),
            name="Code Correctness",
        )

        # 2. Code Readability Metric
        readability_metric = GEval(
            task_introduction=(
                "You are an expert judge evaluating Python code readability. "
                "The code to review is under ACTUAL_CODE. Focus on naming, formatting, and documentation. "
                "Use the following rubric to assign scores:"
            ),
            evaluation_criteria=(
                "EVALUATION STEPS:\n"
                "1. Are naming conventions clear and consistent?\n"
                "2. Is formatting and indentation correct?\n"
                "3. Are comments and docstrings complete and helpful?\n"
                "4. Is the code organized logically?\n\n"
                "SCORING RUBRIC:\n"
                f"{readability_rubric_text}\n\n"
                "Return only a score between 0 and 10, and a concise reason that references the rubric."
            ),
            name="Code Readability",
        )

        # 3. Code Best Practices Metric
        best_practices_metric = GEval(
            task_introduction=(
                "You are an expert judge evaluating adherence to Python best practices. "
                "The code to review is under ACTUAL_CODE. Focus on error handling, security, efficiency, and modularity. "
                "Use the following rubric to assign scores:"
            ),
            evaluation_criteria=(
                "EVALUATION STEPS:\n"
                "1. Does it handle exceptions and errors properly?\n"
                "2. Are security best practices followed?\n"
                "3. Is the code efficient in performance?\n"
                "4. Is functionality split into reusable, modular components?\n\n"
                "SCORING RUBRIC:\n"
                f"{best_practices_rubric_text}\n\n"
                "Return only a score between 0 and 10, and a concise reason that references the rubric."
            ),
            name="Code Best Practices",
        )

        # Run evaluation for each metric using Opik's GEval
        correctness_result = correctness_metric.score(output=context)
        readability_result = readability_metric.score(output=context)
        best_practices_result = best_practices_metric.score(output=context)

        # Convert scores from Opik's 0-1 scale to 0-10 scale
        # Opik returns scores as 0-1, we multiply by 10 for consistency
        correctness_score = correctness_result.value * 10
        readability_score = readability_result.value * 10
        best_practices_score = best_practices_result.value * 10

        # Calculate overall score as average of all three metrics
        overall_score = (
            correctness_score + readability_score + best_practices_score
        ) / 3

        # Prepare detailed metrics structure
        detailed_metrics = {
            "correctness": {
                "score": correctness_score,
                "reason": correctness_result.reason,
            },
            "readability": {
                "score": readability_score,
                "reason": readability_result.reason,
            },
            "best_practices": {
                "score": best_practices_score,
                "reason": best_practices_result.reason,
            },
        }

        # Return results
        return {
            "overall_score": overall_score,
            "detailed_metrics": detailed_metrics,
            "passed": overall_score >= 7.0,  # 70% threshold
        }

    except Exception as e:
        # Error handling
        return {
            "error": f"Error evaluating code: {str(e)}",
            "overall_score": 0.0,
            "detailed_metrics": {},
            "passed": False,
        }