wehub-resource-sync
112 lines
4.9 KiB
Python
112 lines
4.9 KiB
Python
from deepeval import evaluate
|
|
from deepeval.metrics import GEval
|
|
from deepeval.test_case import LLMTestCase, LLMTestCaseParams
|
|
from deepeval.metrics.g_eval import Rubric
|
|
from typing import Dict, Any
|
|
|
|
def evaluate_code(generated_code: str, reference_code: str = None):
|
|
try:
|
|
# Initialize test case
|
|
test_case = LLMTestCase(
|
|
input="Code Generation Task",
|
|
actual_output=generated_code,
|
|
expected_output=reference_code if reference_code else ""
|
|
)
|
|
|
|
# Code Correctness Metric
|
|
correctness_metric = GEval(
|
|
name="Code Correctness",
|
|
criteria="Evaluate if the code is functionally correct, properly handles edge cases, and implements the required functionality completely.",
|
|
evaluation_steps=[
|
|
"Check if the code implements all required functionality",
|
|
"Verify proper handling of edge cases",
|
|
"Check for potential runtime errors",
|
|
"Assess if the code produces expected outputs"
|
|
],
|
|
evaluation_params=[LLMTestCaseParams.ACTUAL_OUTPUT, LLMTestCaseParams.EXPECTED_OUTPUT],
|
|
rubric=[
|
|
Rubric(score_range=(0,2), expected_outcome="Code is non-functional or has critical errors"),
|
|
Rubric(score_range=(3,5), expected_outcome="Code works but misses key functionality"),
|
|
Rubric(score_range=(6,8), expected_outcome="Code is mostly correct with minor issues"),
|
|
Rubric(score_range=(9,10), expected_outcome="Code is completely correct")
|
|
],
|
|
threshold=0.7
|
|
)
|
|
|
|
# Code Readability Metric
|
|
readability_metric = GEval(
|
|
name="Code Readability",
|
|
criteria="Evaluate code readability including proper naming, formatting, and documentation.",
|
|
evaluation_steps=[
|
|
"Check for clear and consistent naming conventions",
|
|
"Verify proper code formatting and indentation",
|
|
"Assess quality and completeness of comments and docstrings",
|
|
"Check for code organization and logical structure"
|
|
],
|
|
evaluation_params=[LLMTestCaseParams.ACTUAL_OUTPUT],
|
|
rubric=[
|
|
Rubric(score_range=(0,2), expected_outcome="Code is poorly formatted and hard to read"),
|
|
Rubric(score_range=(3,5), expected_outcome="Code has basic formatting but lacks clarity"),
|
|
Rubric(score_range=(6,8), expected_outcome="Code is well formatted with minor issues"),
|
|
Rubric(score_range=(9,10), expected_outcome="Code is exceptionally readable and well documented")
|
|
],
|
|
threshold=0.7
|
|
)
|
|
|
|
# Code Best Practices Metric
|
|
best_practices_metric = GEval(
|
|
name="Code Best Practices",
|
|
criteria="Evaluate adherence to coding best practices, including error handling, security, and efficiency.",
|
|
evaluation_steps=[
|
|
"Check for proper error handling and exceptions",
|
|
"Verify security best practices",
|
|
"Assess code efficiency and performance considerations",
|
|
"Check for code reusability and modularity"
|
|
],
|
|
evaluation_params=[LLMTestCaseParams.ACTUAL_OUTPUT],
|
|
rubric=[
|
|
Rubric(score_range=(0,2), expected_outcome="Code ignores best practices"),
|
|
Rubric(score_range=(3,5), expected_outcome="Code follows basic practices with gaps"),
|
|
Rubric(score_range=(6,8), expected_outcome="Code mostly follows best practices"),
|
|
Rubric(score_range=(9,10), expected_outcome="Code perfectly follows all best practices")
|
|
],
|
|
threshold=0.7
|
|
)
|
|
|
|
# Run evaluation
|
|
metrics = [correctness_metric, readability_metric, best_practices_metric]
|
|
for metric in metrics:
|
|
metric.measure(test_case)
|
|
|
|
# Calculate overall score
|
|
overall_score = (correctness_metric.score + readability_metric.score + best_practices_metric.score) / 3
|
|
|
|
# Prepare detailed metrics
|
|
detailed_metrics = {
|
|
"correctness": {
|
|
"score": correctness_metric.score,
|
|
"reason": correctness_metric.reason
|
|
},
|
|
"readability": {
|
|
"score": readability_metric.score,
|
|
"reason": readability_metric.reason
|
|
},
|
|
"best_practices": {
|
|
"score": best_practices_metric.score,
|
|
"reason": best_practices_metric.reason
|
|
}
|
|
}
|
|
|
|
return {
|
|
"overall_score": overall_score,
|
|
"detailed_metrics": detailed_metrics,
|
|
"passed": overall_score >= 0.7
|
|
}
|
|
|
|
except Exception as e:
|
|
return {
|
|
"error": f"Error evaluating code: {str(e)}",
|
|
"overall_score": 0.0,
|
|
"detailed_metrics": {},
|
|
"passed": False
|
|
} |