wehub-resource-sync
338 lines
12 KiB
Python
338 lines
12 KiB
Python
import os
|
|
from typing import (
|
|
List,
|
|
Optional,
|
|
Union,
|
|
Dict,
|
|
)
|
|
from rich.console import Console
|
|
import time
|
|
|
|
from deepeval.confident.api import Api, Endpoints, HttpMethods
|
|
from deepeval.evaluate.api import APIEvaluate
|
|
from deepeval.evaluate.configs import (
|
|
AsyncConfig,
|
|
DisplayConfig,
|
|
CacheConfig,
|
|
ErrorConfig,
|
|
)
|
|
from deepeval.evaluate.utils import (
|
|
validate_assert_test_inputs,
|
|
validate_evaluate_inputs,
|
|
)
|
|
from deepeval.evaluate.console_report import EvaluationConsoleReport
|
|
from deepeval.dataset import Golden
|
|
from deepeval.prompt import Prompt
|
|
from deepeval.test_case.utils import check_valid_test_cases_type
|
|
from deepeval.test_run.hyperparameters import (
|
|
process_hyperparameters,
|
|
process_prompts,
|
|
)
|
|
from deepeval.test_run.test_run import TEMP_FILE_PATH
|
|
from deepeval.utils import (
|
|
get_or_create_event_loop,
|
|
open_browser,
|
|
set_test_run_official,
|
|
should_ignore_errors,
|
|
should_skip_on_missing_params,
|
|
should_use_cache,
|
|
should_verbose_print,
|
|
get_identifier,
|
|
)
|
|
from deepeval.telemetry import capture_evaluation_run
|
|
from deepeval.metrics import (
|
|
BaseMetric,
|
|
BaseConversationalMetric,
|
|
)
|
|
from deepeval.metrics.indicator import (
|
|
format_metric_description,
|
|
)
|
|
from deepeval.test_case import (
|
|
LLMTestCase,
|
|
ConversationalTestCase,
|
|
)
|
|
from deepeval.test_run import (
|
|
global_test_run_manager,
|
|
MetricData,
|
|
)
|
|
from deepeval.utils import get_is_running_deepeval
|
|
from deepeval.evaluate.types import EvaluationResult
|
|
from deepeval.evaluate.execute import (
|
|
a_execute_test_cases,
|
|
_assert_test_from_current_trace,
|
|
execute_test_cases,
|
|
)
|
|
|
|
|
|
def assert_test(
|
|
test_case: Optional[Union[LLMTestCase, ConversationalTestCase]] = None,
|
|
metrics: Optional[
|
|
Union[
|
|
List[BaseMetric],
|
|
List[BaseConversationalMetric],
|
|
]
|
|
] = None,
|
|
golden: Optional[Golden] = None,
|
|
run_async: bool = True,
|
|
):
|
|
validate_assert_test_inputs(
|
|
golden=golden,
|
|
test_case=test_case,
|
|
metrics=metrics,
|
|
)
|
|
|
|
async_config = AsyncConfig(throttle_value=0, max_concurrent=100)
|
|
display_config = DisplayConfig(
|
|
verbose_mode=should_verbose_print(), show_indicator=True
|
|
)
|
|
error_config = ErrorConfig(
|
|
ignore_errors=should_ignore_errors(),
|
|
skip_on_missing_params=should_skip_on_missing_params(),
|
|
)
|
|
cache_config = CacheConfig(
|
|
write_cache=get_is_running_deepeval(), use_cache=should_use_cache()
|
|
)
|
|
|
|
if golden and not test_case:
|
|
# Trace-scoped assert_test: read the active trace set by the plugin.
|
|
test_result = _assert_test_from_current_trace(
|
|
golden=golden,
|
|
metrics=metrics,
|
|
error_config=error_config,
|
|
display_config=display_config,
|
|
)
|
|
|
|
elif test_case and metrics:
|
|
if run_async:
|
|
loop = get_or_create_event_loop()
|
|
test_result = loop.run_until_complete(
|
|
a_execute_test_cases(
|
|
[test_case],
|
|
metrics,
|
|
error_config=error_config,
|
|
display_config=display_config,
|
|
async_config=async_config,
|
|
cache_config=cache_config,
|
|
identifier=get_identifier(),
|
|
_use_bar_indicator=True,
|
|
_is_assert_test=True,
|
|
)
|
|
)[0]
|
|
else:
|
|
test_result = execute_test_cases(
|
|
[test_case],
|
|
metrics,
|
|
error_config=error_config,
|
|
display_config=display_config,
|
|
cache_config=cache_config,
|
|
identifier=get_identifier(),
|
|
_use_bar_indicator=False,
|
|
_is_assert_test=True,
|
|
)[0]
|
|
|
|
if not test_result.success:
|
|
failed_metrics_data: List[MetricData] = []
|
|
# even for conversations, test_result right now is just the
|
|
# result for the last message
|
|
for metric_data in test_result.metrics_data:
|
|
if metric_data.error is not None:
|
|
failed_metrics_data.append(metric_data)
|
|
else:
|
|
# This try block is for user defined custom metrics,
|
|
# which might not handle the score == undefined case elegantly
|
|
try:
|
|
if not metric_data.success:
|
|
failed_metrics_data.append(metric_data)
|
|
except Exception:
|
|
failed_metrics_data.append(metric_data)
|
|
|
|
failed_metrics_str = ", ".join(
|
|
[
|
|
f"{metrics_data.name} (score: {metrics_data.score}, threshold: {metrics_data.threshold}, strict: {metrics_data.strict_mode}, error: {metrics_data.error}, reason: {metrics_data.reason})"
|
|
for metrics_data in failed_metrics_data
|
|
]
|
|
)
|
|
raise AssertionError(f"Metrics: {failed_metrics_str} failed.")
|
|
|
|
|
|
def evaluate(
|
|
test_cases: Union[List[LLMTestCase], List[ConversationalTestCase]],
|
|
metrics: Optional[
|
|
Union[
|
|
List[BaseMetric],
|
|
List[BaseConversationalMetric],
|
|
]
|
|
] = None,
|
|
# Evals on Confident AI
|
|
metric_collection: Optional[str] = None,
|
|
hyperparameters: Optional[Dict[str, Union[str, int, float, Prompt]]] = None,
|
|
# agnostic
|
|
identifier: Optional[str] = None,
|
|
official: bool = False,
|
|
_skip_reset: bool = False,
|
|
# Configs
|
|
async_config: Optional[AsyncConfig] = AsyncConfig(),
|
|
display_config: Optional[DisplayConfig] = DisplayConfig(),
|
|
cache_config: Optional[CacheConfig] = CacheConfig(),
|
|
error_config: Optional[ErrorConfig] = ErrorConfig(),
|
|
) -> EvaluationResult:
|
|
validate_evaluate_inputs(
|
|
test_cases=test_cases,
|
|
metrics=metrics,
|
|
metric_collection=metric_collection,
|
|
)
|
|
check_valid_test_cases_type(test_cases)
|
|
|
|
if metrics:
|
|
|
|
if not _skip_reset and not get_is_running_deepeval():
|
|
global_test_run_manager.reset()
|
|
set_test_run_official(official)
|
|
start_time = time.perf_counter()
|
|
|
|
if display_config.show_indicator:
|
|
console = Console()
|
|
for metric in metrics:
|
|
console.print(
|
|
format_metric_description(
|
|
metric, async_mode=async_config.run_async
|
|
)
|
|
)
|
|
|
|
with capture_evaluation_run("evaluate()"):
|
|
if async_config.run_async:
|
|
loop = get_or_create_event_loop()
|
|
test_results = loop.run_until_complete(
|
|
a_execute_test_cases(
|
|
test_cases,
|
|
metrics,
|
|
identifier=identifier,
|
|
error_config=error_config,
|
|
display_config=display_config,
|
|
cache_config=cache_config,
|
|
async_config=async_config,
|
|
)
|
|
)
|
|
else:
|
|
test_results = execute_test_cases(
|
|
test_cases,
|
|
metrics,
|
|
identifier=identifier,
|
|
error_config=error_config,
|
|
display_config=display_config,
|
|
cache_config=cache_config,
|
|
)
|
|
|
|
end_time = time.perf_counter()
|
|
run_duration = end_time - start_time
|
|
if display_config.print_results:
|
|
console_report = EvaluationConsoleReport(test_results)
|
|
console_report.render_to_terminal(
|
|
truncate_passing_cases=display_config.truncate_passing_cases,
|
|
display_option=display_config.display_option,
|
|
)
|
|
|
|
# Handle full, un-truncated file exports
|
|
if display_config.file_output_dir is not None:
|
|
if display_config.file_type == "html":
|
|
console_report.export_to_html(
|
|
output_dir=display_config.file_output_dir,
|
|
evaluation_name=identifier,
|
|
theme_mode="dark",
|
|
)
|
|
elif display_config.file_type == "md":
|
|
console_report.export_to_markdown(
|
|
output_dir=display_config.file_output_dir,
|
|
evaluation_name=identifier,
|
|
)
|
|
else:
|
|
raise ValueError(
|
|
f"Invalid file type: {display_config.file_type}"
|
|
)
|
|
|
|
test_run = global_test_run_manager.get_test_run()
|
|
if hyperparameters is not None or test_run.hyperparameters is None:
|
|
test_run.hyperparameters = process_hyperparameters(hyperparameters)
|
|
test_run.prompts = process_prompts(hyperparameters)
|
|
|
|
global_test_run_manager.configure_local_store(
|
|
results_folder=display_config.results_folder,
|
|
results_subfolder=display_config.results_subfolder,
|
|
)
|
|
|
|
if _skip_reset:
|
|
test_run.run_duration += run_duration
|
|
global_test_run_manager.save_test_run(TEMP_FILE_PATH)
|
|
return EvaluationResult(
|
|
test_results=test_results,
|
|
confident_link=None,
|
|
test_run_id=None,
|
|
)
|
|
|
|
global_test_run_manager.save_test_run(TEMP_FILE_PATH)
|
|
|
|
# In CLI mode (`deepeval test run`), the CLI owns finalization and will
|
|
# call `wrap_up_test_run()` once after pytest finishes. Finalizing here
|
|
# as well would double finalize the run and consequently result in
|
|
# duplicate uploads / local saves and temp file races, so only
|
|
# do it when we're NOT in CLI mode.
|
|
if get_is_running_deepeval():
|
|
return EvaluationResult(
|
|
test_results=test_results,
|
|
confident_link=None,
|
|
test_run_id=None,
|
|
)
|
|
|
|
res = global_test_run_manager.wrap_up_test_run(
|
|
run_duration, display_table=False
|
|
)
|
|
if isinstance(res, tuple):
|
|
confident_link, test_run_id = res
|
|
else:
|
|
confident_link = test_run_id = None
|
|
|
|
# All other side-effects (saving locally, posting to Confident AI,
|
|
# rendering the table) have already happened inside wrap_up_test_run.
|
|
# Offer to open the inspect TUI as the very last thing the user sees,
|
|
# so it never competes with the run output for attention.
|
|
from deepeval.evaluate.inspect_prompt import maybe_offer_inspect_tui
|
|
|
|
maybe_offer_inspect_tui(global_test_run_manager, display_config)
|
|
|
|
return EvaluationResult(
|
|
test_results=test_results,
|
|
confident_link=confident_link,
|
|
test_run_id=test_run_id,
|
|
)
|
|
elif metric_collection:
|
|
api = Api()
|
|
api_evaluate = APIEvaluate(
|
|
metricCollection=metric_collection,
|
|
llmTestCases=(
|
|
test_cases if isinstance(test_cases[0], LLMTestCase) else None
|
|
),
|
|
conversationalTestCases=(
|
|
test_cases
|
|
if isinstance(test_cases[0], ConversationalTestCase)
|
|
else None
|
|
),
|
|
)
|
|
try:
|
|
body = api_evaluate.model_dump(by_alias=True, exclude_none=True)
|
|
except AttributeError:
|
|
# Pydantic version below 2.0
|
|
body = api_evaluate.dict(by_alias=True, exclude_none=True)
|
|
|
|
_, link = api.send_request(
|
|
method=HttpMethods.POST,
|
|
endpoint=Endpoints.EVALUATE_ENDPOINT,
|
|
body=body,
|
|
)
|
|
if link:
|
|
console = Console()
|
|
console.print(
|
|
"✅ Evaluation successfully pushed to Confident AI! View at "
|
|
f"[link={link}]{link}[/link]"
|
|
)
|
|
open_browser(link)
|