wehub-resource-sync
e768098d0e
Flake8 Lint / flake8 (push) Waiting to run
Spell check CI / Spell_Check (push) Waiting to run
tools_continuous_delivery / Private PyPI non-main branch release (push) Has been skipped
tools_continuous_delivery / Private PyPI main branch release (push) Failing after 2m42s
Publish Promptflow Doc / Build (push) Has been cancelled
Publish Promptflow Doc / Deploy (push) Has been cancelled
70 lines
2.1 KiB
Python
70 lines
2.1 KiB
Python
import json
|
|
|
|
from typing import TypedDict
|
|
from pathlib import Path
|
|
|
|
from jinja2 import Template
|
|
|
|
from promptflow.tracing import trace
|
|
from promptflow.core import AzureOpenAIModelConfiguration
|
|
from promptflow.core._flow import Prompty
|
|
|
|
BASE_DIR = Path(__file__).absolute().parent
|
|
|
|
|
|
@trace
|
|
def load_prompt(jinja2_template: str, code: str, examples: list) -> str:
|
|
"""Load prompt function."""
|
|
with open(BASE_DIR / jinja2_template, "r", encoding="utf-8") as f:
|
|
tmpl = Template(f.read(), trim_blocks=True, keep_trailing_newline=True)
|
|
prompt = tmpl.render(code=code, examples=examples)
|
|
return prompt
|
|
|
|
|
|
class Result(TypedDict):
|
|
correctness: float
|
|
readability: float
|
|
explanation: str
|
|
|
|
|
|
class CodeEvaluator:
|
|
def __init__(self, model_config: AzureOpenAIModelConfiguration):
|
|
self.model_config = model_config
|
|
|
|
def __call__(self, code: str) -> Result:
|
|
"""Evaluate the code based on correctness, readability."""
|
|
prompty = Prompty.load(
|
|
source=BASE_DIR / "eval_code_quality.prompty",
|
|
model={"configuration": self.model_config},
|
|
)
|
|
output = prompty(code=code)
|
|
output = json.loads(output)
|
|
output = Result(**output)
|
|
return output
|
|
|
|
def __aggregate__(self, line_results: list) -> dict:
|
|
"""Aggregate the results."""
|
|
total = len(line_results)
|
|
avg_correctness = sum(int(r["correctness"]) for r in line_results) / total
|
|
avg_readability = sum(int(r["readability"]) for r in line_results) / total
|
|
return {
|
|
"average_correctness": avg_correctness,
|
|
"average_readability": avg_readability,
|
|
"total": total,
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
from promptflow.tracing import start_trace
|
|
|
|
start_trace()
|
|
model_config = AzureOpenAIModelConfiguration(
|
|
connection="open_ai_connection",
|
|
azure_deployment="gpt-4o",
|
|
)
|
|
evaluator = CodeEvaluator(model_config)
|
|
result = evaluator('print("Hello, world!")')
|
|
print(result)
|
|
aggregate_result = evaluator.__aggregate__([result])
|
|
print(aggregate_result)
|