From 3db52600654737205ab0460eb46e6037c0f57079 Mon Sep 17 00:00:00 2001
From: wehub-resource-sync
Date: Mon, 13 Jul 2026 13:32:05 +0800
Subject: [PATCH] chore: import upstream snapshot with attribution
---
.claude-plugin/marketplace.json | 24 +
.claude-plugin/plugin.json | 22 +
.cursor-plugin/plugin.json | 23 +
.env.example | 113 +
.github/ISSUE_TEMPLATE/bug_report.md | 41 +
.github/ISSUE_TEMPLATE/feature_request.md | 23 +
.github/workflows/black.yml | 20 +
.github/workflows/changelog.yml | 76 +
.github/workflows/full_test_core_for_pr.yml | 106 +
.github/workflows/pr-title-check.yml | 27 +
.github/workflows/release.yml | 115 +
.github/workflows/test_confident.yml | 72 +
.github/workflows/test_core.yml | 107 +
.github/workflows/test_integrations.yml | 344 +
.github/workflows/test_metric_templates.yml | 42 +
.github/workflows/test_metrics.yml | 75 +
.github/workflows/typescript_lint.yml | 35 +
.github/workflows/typescript_test.yml | 36 +
.gitignore | 178 +
.pre-commit-config.yaml | 10 +
.scripts/changelog/generate.py | 1548 ++
.scripts/changelog/release_notes.py | 333 +
.vscode/settings.json | 13 +
CITATION.cff | 15 +
CONTRIBUTING.md | 57 +
LICENSE.md | 202 +
MAINTAINERS.md | 8 +
MANIFEST.in | 1 +
README.md | 611 +
README.wehub.md | 7 +
assets/confident-mcp-architecture.png | Bin 0 -> 544437 bytes
assets/demo.gif | Bin 0 -> 5004519 bytes
assets/hero/wordmark-dark.svg | 10 +
assets/hero/wordmark-light.svg | 10 +
deepeval/__init__.py | 140 +
deepeval/_version.py | 1 +
deepeval/annotation/__init__.py | 3 +
deepeval/annotation/annotation.py | 76 +
deepeval/annotation/api.py | 54 +
deepeval/anthropic/__init__.py | 19 +
deepeval/anthropic/extractors.py | 94 +
deepeval/anthropic/patch.py | 169 +
deepeval/anthropic/utils.py | 225 +
deepeval/benchmarks/__init__.py | 37 +
deepeval/benchmarks/arc/__init__.py | 0
deepeval/benchmarks/arc/arc.py | 182 +
deepeval/benchmarks/arc/mode.py | 6 +
deepeval/benchmarks/arc/template.py | 94 +
deepeval/benchmarks/base_benchmark.py | 32 +
deepeval/benchmarks/bbq/__init__.py | 0
deepeval/benchmarks/bbq/bbq.py | 218 +
deepeval/benchmarks/bbq/task.py | 15 +
deepeval/benchmarks/bbq/template.py | 163 +
.../benchmarks/big_bench_hard/__init__.py | 0
.../big_bench_hard/big_bench_hard.py | 329 +
.../big_bench_hard/cot_prompts/__init__.py | 0
.../cot_prompts/boolean_expressions.txt | 23 +
.../cot_prompts/causal_judgement.txt | 25 +
.../cot_prompts/date_understanding.txt | 33 +
.../cot_prompts/disambiguation_qa.txt | 37 +
.../cot_prompts/dyck_languages.txt | 72 +
.../cot_prompts/formal_fallacies.txt | 44 +
.../cot_prompts/geometric_shapes.txt | 78 +
.../big_bench_hard/cot_prompts/hyperbaton.txt | 28 +
.../logical_deduction_five_objects.txt | 37 +
.../logical_deduction_seven_objects.txt | 37 +
.../logical_deduction_three_objects.txt | 37 +
.../cot_prompts/movie_recommendation.txt | 42 +
.../cot_prompts/multistep_arithmetic_two.txt | 25 +
.../big_bench_hard/cot_prompts/navigate.txt | 43 +
.../cot_prompts/object_counting.txt | 37 +
.../cot_prompts/penguins_in_a_table.txt | 41 +
.../reasoning_about_colored_objects.txt | 63 +
.../big_bench_hard/cot_prompts/ruin_names.txt | 44 +
.../salient_translation_error_detection.txt | 40 +
.../big_bench_hard/cot_prompts/snarks.txt | 30 +
.../cot_prompts/sports_understanding.txt | 10 +
.../cot_prompts/temporal_sequences.txt | 77 +
...tracking_shuffled_objects_five_objects.txt | 40 +
...racking_shuffled_objects_seven_objects.txt | 40 +
...racking_shuffled_objects_three_objects.txt | 40 +
.../cot_prompts/web_of_lies.txt | 28 +
.../cot_prompts/word_sorting.txt | 17 +
.../big_bench_hard/shot_prompts/__init__.py | 0
.../shot_prompts/boolean_expressions.txt | 10 +
.../shot_prompts/causal_judgement.txt | 22 +
.../shot_prompts/date_understanding.txt | 30 +
.../shot_prompts/disambiguation_qa.txt | 25 +
.../shot_prompts/dyck_languages.txt | 10 +
.../shot_prompts/formal_fallacies.txt | 22 +
.../shot_prompts/geometric_shapes.txt | 43 +
.../shot_prompts/hyperbaton.txt | 19 +
.../logical_deduction_five_objects.txt | 22 +
.../logical_deduction_seven_objects.txt | 22 +
.../logical_deduction_three_objects.txt | 22 +
.../shot_prompts/movie_recommendation.txt | 27 +
.../shot_prompts/multistep_arithmetic_two.txt | 10 +
.../big_bench_hard/shot_prompts/navigate.txt | 19 +
.../shot_prompts/object_counting.txt | 10 +
.../shot_prompts/penguins_in_a_table.txt | 30 +
.../reasoning_about_colored_objects.txt | 56 +
.../shot_prompts/ruin_names.txt | 25 +
.../salient_translation_error_detection.txt | 37 +
.../big_bench_hard/shot_prompts/snarks.txt | 21 +
.../shot_prompts/sports_understanding.txt | 10 +
.../shot_prompts/temporal_sequences.txt | 51 +
...tracking_shuffled_objects_five_objects.txt | 25 +
...racking_shuffled_objects_seven_objects.txt | 25 +
...racking_shuffled_objects_three_objects.txt | 25 +
.../shot_prompts/web_of_lies.txt | 10 +
.../shot_prompts/word_sorting.txt | 10 +
deepeval/benchmarks/big_bench_hard/task.py | 37 +
.../benchmarks/big_bench_hard/template.py | 42 +
deepeval/benchmarks/bool_q/__init__.py | 0
deepeval/benchmarks/bool_q/bool_q.py | 165 +
deepeval/benchmarks/bool_q/template.py | 32 +
deepeval/benchmarks/drop/__init__.py | 0
deepeval/benchmarks/drop/drop.py | 362 +
deepeval/benchmarks/drop/task.py | 586 +
deepeval/benchmarks/drop/template.py | 40 +
deepeval/benchmarks/equity_med_qa/__init__.py | 0
.../benchmarks/equity_med_qa/equity_med_qa.py | 166 +
deepeval/benchmarks/equity_med_qa/task.py | 13 +
deepeval/benchmarks/equity_med_qa/template.py | 7 +
deepeval/benchmarks/gsm8k/__init__.py | 0
deepeval/benchmarks/gsm8k/gsm8k.py | 210 +
deepeval/benchmarks/gsm8k/template.py | 56 +
deepeval/benchmarks/hellaswag/__init__.py | 0
deepeval/benchmarks/hellaswag/hellaswag.py | 324 +
deepeval/benchmarks/hellaswag/task.py | 196 +
deepeval/benchmarks/hellaswag/template.py | 32 +
deepeval/benchmarks/human_eval/__init__.py | 0
deepeval/benchmarks/human_eval/human_eval.py | 263 +
deepeval/benchmarks/human_eval/task.py | 161 +
deepeval/benchmarks/human_eval/template.py | 19 +
deepeval/benchmarks/ifeval/__init__.py | 1 +
deepeval/benchmarks/ifeval/ifeval.py | 619 +
deepeval/benchmarks/ifeval/template.py | 131 +
deepeval/benchmarks/lambada/__init__.py | 0
deepeval/benchmarks/lambada/lambada.py | 165 +
deepeval/benchmarks/lambada/template.py | 64 +
deepeval/benchmarks/logi_qa/__init__.py | 0
deepeval/benchmarks/logi_qa/logi_qa.py | 303 +
deepeval/benchmarks/logi_qa/task.py | 9 +
deepeval/benchmarks/logi_qa/template.py | 42 +
deepeval/benchmarks/math_qa/__init__.py | 0
deepeval/benchmarks/math_qa/math_qa.py | 291 +
deepeval/benchmarks/math_qa/task.py | 10 +
deepeval/benchmarks/math_qa/template.py | 75 +
deepeval/benchmarks/mmlu/__init__.py | 0
deepeval/benchmarks/mmlu/mmlu.py | 317 +
deepeval/benchmarks/mmlu/task.py | 61 +
deepeval/benchmarks/mmlu/template.py | 39 +
deepeval/benchmarks/modes/__init__.py | 7 +
deepeval/benchmarks/results.py | 2 +
deepeval/benchmarks/schema.py | 171 +
deepeval/benchmarks/squad/__init__.py | 0
deepeval/benchmarks/squad/squad.py | 303 +
deepeval/benchmarks/squad/task.py | 54 +
deepeval/benchmarks/squad/template.py | 70 +
deepeval/benchmarks/tasks/__init__.py | 10 +
deepeval/benchmarks/truthful_qa/__init__.py | 0
deepeval/benchmarks/truthful_qa/mode.py | 6 +
deepeval/benchmarks/truthful_qa/task.py | 42 +
deepeval/benchmarks/truthful_qa/template.py | 110 +
.../benchmarks/truthful_qa/truthful_qa.py | 362 +
deepeval/benchmarks/utils.py | 13 +
deepeval/benchmarks/winogrande/__init__.py | 0
deepeval/benchmarks/winogrande/template.py | 63 +
deepeval/benchmarks/winogrande/winogrande.py | 167 +
deepeval/cli/__init__.py | 0
deepeval/cli/auth/__init__.py | 19 +
deepeval/cli/auth/api.py | 96 +
deepeval/cli/auth/command.py | 383 +
deepeval/cli/auth/flow.py | 455 +
deepeval/cli/diagnose/__init__.py | 6 +
deepeval/cli/diagnose/diagnose.py | 555 +
deepeval/cli/dotenv_handler.py | 71 +
deepeval/cli/generate/__init__.py | 0
deepeval/cli/generate/command.py | 332 +
deepeval/cli/generate/utils.py | 204 +
deepeval/cli/inspect.py | 126 +
deepeval/cli/main.py | 2957 ++
deepeval/cli/test/__init__.py | 0
deepeval/cli/test/command.py | 199 +
deepeval/cli/types.py | 13 +
deepeval/cli/utils.py | 451 +
deepeval/confident/__init__.py | 1 +
deepeval/confident/api.py | 360 +
deepeval/confident/types.py | 22 +
deepeval/config/__init__.py | 0
deepeval/config/dotenv_handler.py | 19 +
deepeval/config/logging.py | 33 +
deepeval/config/settings.py | 1628 ++
deepeval/config/settings_manager.py | 137 +
deepeval/config/utils.py | 151 +
deepeval/constants.py | 50 +
deepeval/contextvars.py | 24 +
deepeval/dataset/__init__.py | 10 +
deepeval/dataset/api.py | 95 +
deepeval/dataset/dataset.py | 1694 ++
deepeval/dataset/golden.py | 228 +
deepeval/dataset/test_run_tracer.py | 80 +
deepeval/dataset/types.py | 25 +
deepeval/dataset/utils.py | 303 +
deepeval/errors.py | 43 +
deepeval/evaluate/__init__.py | 13 +
deepeval/evaluate/api.py | 11 +
deepeval/evaluate/compare.py | 534 +
deepeval/evaluate/configs.py | 47 +
deepeval/evaluate/console_report.py | 396 +
deepeval/evaluate/evaluate.py | 337 +
deepeval/evaluate/execute/__init__.py | 39 +
deepeval/evaluate/execute/_common.py | 328 +
deepeval/evaluate/execute/agentic.py | 539 +
deepeval/evaluate/execute/e2e.py | 817 +
deepeval/evaluate/execute/loop.py | 1022 +
deepeval/evaluate/execute/trace_scope.py | 310 +
deepeval/evaluate/inspect_prompt.py | 133 +
deepeval/evaluate/local_store.py | 141 +
deepeval/evaluate/types.py | 38 +
deepeval/evaluate/utils.py | 581 +
deepeval/inspect/__init__.py | 11 +
deepeval/inspect/__main__.py | 42 +
deepeval/inspect/app.py | 243 +
.../inspect/fixtures/test_run_sample.json | 356 +
deepeval/inspect/loader.py | 159 +
deepeval/inspect/styles.tcss | 122 +
deepeval/inspect/types.py | 102 +
deepeval/inspect/widgets/__init__.py | 1 +
deepeval/inspect/widgets/_styling.py | 56 +
deepeval/inspect/widgets/details.py | 464 +
deepeval/inspect/widgets/header_bar.py | 48 +
deepeval/inspect/widgets/help_modal.py | 66 +
deepeval/inspect/widgets/search_bar.py | 39 +
deepeval/inspect/widgets/span_tree.py | 209 +
deepeval/integrations/README.md | 68 +
deepeval/integrations/__init__.py | 0
deepeval/integrations/agentcore/__init__.py | 4 +
.../integrations/agentcore/instrumentator.py | 962 +
deepeval/integrations/agentcore/otel.py | 167 +
deepeval/integrations/crewai/__init__.py | 16 +
deepeval/integrations/crewai/handler.py | 493 +
deepeval/integrations/crewai/subs.py | 64 +
deepeval/integrations/crewai/tool.py | 91 +
deepeval/integrations/crewai/wrapper.py | 196 +
deepeval/integrations/google_adk/__init__.py | 3 +
deepeval/integrations/google_adk/otel.py | 98 +
.../integrations/hugging_face/__init__.py | 3 +
.../integrations/hugging_face/callback.py | 231 +
.../integrations/hugging_face/rich_manager.py | 109 +
.../hugging_face/tests/test_callbacks.py | 158 +
deepeval/integrations/hugging_face/utils.py | 56 +
deepeval/integrations/langchain/__init__.py | 3 +
deepeval/integrations/langchain/callback.py | 929 +
deepeval/integrations/langchain/patch.py | 43 +
deepeval/integrations/langchain/utils.py | 499 +
deepeval/integrations/llama_index/__init__.py | 5 +
deepeval/integrations/llama_index/handler.py | 414 +
deepeval/integrations/llama_index/utils.py | 107 +
.../integrations/openinference/__init__.py | 5 +
.../openinference/instrumentator.py | 917 +
deepeval/integrations/openinference/otel.py | 173 +
deepeval/integrations/pydantic_ai/README.md | 1036 +
deepeval/integrations/pydantic_ai/__init__.py | 13 +
.../pydantic_ai/instrumentator.py | 817 +
deepeval/integrations/pydantic_ai/otel.py | 79 +
deepeval/integrations/strands/__init__.py | 4 +
.../integrations/strands/instrumentator.py | 976 +
deepeval/integrations/strands/otel.py | 177 +
deepeval/key_handler.py | 279 +
deepeval/metrics/README.md | 236 +
deepeval/metrics/__init__.py | 136 +
.../metrics/agent_loop_detection/__init__.py | 1 +
.../agent_loop_detection.py | 564 +
deepeval/metrics/answer_relevancy/__init__.py | 3 +
.../answer_relevancy/answer_relevancy.py | 321 +
deepeval/metrics/answer_relevancy/schema.py | 19 +
.../answer_relevancy/templates/class.txt | 1 +
.../templates/generate_reason.txt | 28 +
.../templates/generate_statements.txt | 27 +
.../templates/generate_verdicts.txt | 43 +
deepeval/metrics/arena_g_eval/__init__.py | 1 +
deepeval/metrics/arena_g_eval/arena_g_eval.py | 325 +
deepeval/metrics/arena_g_eval/schema.py | 20 +
.../metrics/arena_g_eval/templates/class.txt | 1 +
.../templates/generate_arena_winner.txt | 41 +
.../templates/generate_evaluation_steps.txt | 16 +
.../arena_g_eval/templates/rewrite_reason.txt | 32 +
deepeval/metrics/arena_g_eval/utils.py | 161 +
.../metrics/argument_correctness/__init__.py | 0
.../argument_correctness.py | 278 +
.../metrics/argument_correctness/schema.py | 15 +
.../argument_correctness/templates/class.txt | 1 +
.../templates/generate_reason.txt | 27 +
.../templates/generate_verdicts.txt | 79 +
deepeval/metrics/base_metric.py | 227 +
deepeval/metrics/bias/__init__.py | 3 +
deepeval/metrics/bias/bias.py | 297 +
deepeval/metrics/bias/schema.py | 20 +
deepeval/metrics/bias/templates/class.txt | 1 +
.../bias/templates/generate_opinions.txt | 25 +
.../bias/templates/generate_reason.txt | 25 +
.../bias/templates/generate_verdicts.txt | 54 +
deepeval/metrics/community/__init__.py | 7 +
.../citation_faithfulness/__init__.py | 3 +
.../citation_faithfulness.py | 216 +
.../community/citation_faithfulness/schema.py | 7 +
.../citation_faithfulness/template.py | 53 +
.../metrics/contextual_precision/__init__.py | 3 +
.../contextual_precision.py | 368 +
.../metrics/contextual_precision/schema.py | 15 +
.../contextual_precision/templates/class.txt | 1 +
.../templates/generate_reason.txt | 29 +
.../templates/generate_verdicts.txt | 38 +
.../metrics/contextual_recall/__init__.py | 3 +
.../contextual_recall/contextual_recall.py | 336 +
deepeval/metrics/contextual_recall/schema.py | 21 +
.../contextual_recall/templates/class.txt | 1 +
.../templates/generate_reason.txt | 29 +
.../templates/generate_verdicts.txt | 27 +
.../metrics/contextual_relevancy/__init__.py | 3 +
.../contextual_relevancy.py | 317 +
.../metrics/contextual_relevancy/schema.py | 16 +
.../contextual_relevancy/templates/class.txt | 1 +
.../templates/generate_reason.txt | 27 +
.../templates/generate_verdicts.txt | 32 +
.../conversation_completeness/__init__.py | 0
.../conversation_completeness.py | 315 +
.../conversation_completeness/schema.py | 15 +
.../templates/class.txt | 1 +
.../templates/extract_user_intentions.txt | 45 +
.../templates/generate_reason.txt | 30 +
.../templates/generate_verdicts.txt | 56 +
.../metrics/conversational_dag/__init__.py | 7 +
.../conversational_dag/conversational_dag.py | 150 +
deepeval/metrics/conversational_dag/nodes.py | 917 +
.../generate_binary_verdict.txt | 21 +
.../generate_non_binary_verdict.txt | 23 +
.../generate_task_output.txt | 19 +
.../generate_reason.txt | 24 +
.../metrics/conversational_g_eval/__init__.py | 3 +
.../conversational_g_eval.py | 496 +
.../metrics/conversational_g_eval/schema.py | 11 +
.../conversational_g_eval/templates/class.txt | 1 +
.../templates/generate_evaluation_results.txt | 36 +
.../templates/generate_evaluation_steps.txt | 18 +
deepeval/metrics/dag/__init__.py | 16 +
deepeval/metrics/dag/dag.py | 156 +
deepeval/metrics/dag/graph.py | 109 +
deepeval/metrics/dag/nodes.py | 744 +
deepeval/metrics/dag/schema.py | 20 +
.../metrics/dag/serialization/__init__.py | 16 +
.../metrics/dag/serialization/registry.py | 39 +
.../dag/serialization/serialization.py | 531 +
deepeval/metrics/dag/serialization/types.py | 14 +
.../generate_binary_verdict.txt | 16 +
.../generate_non_binary_verdict.txt | 16 +
.../TaskNode/generate_task_output.txt | 19 +
.../templates/VerdictNode/generate_reason.txt | 22 +
deepeval/metrics/dag/utils.py | 188 +
deepeval/metrics/exact_match/__init__.py | 0
deepeval/metrics/exact_match/exact_match.py | 96 +
deepeval/metrics/faithfulness/__init__.py | 3 +
deepeval/metrics/faithfulness/faithfulness.py | 406 +
deepeval/metrics/faithfulness/schema.py | 23 +
.../metrics/faithfulness/templates/class.txt | 1 +
.../templates/generate_claims.txt | 27 +
.../templates/generate_reason.txt | 23 +
.../templates/generate_truths.txt | 24 +
.../templates/generate_verdicts.txt | 19 +
deepeval/metrics/g_eval/__init__.py | 4 +
deepeval/metrics/g_eval/g_eval.py | 509 +
deepeval/metrics/g_eval/schema.py | 17 +
deepeval/metrics/g_eval/templates/class.txt | 1 +
.../templates/generate_evaluation_results.txt | 40 +
.../templates/generate_evaluation_steps.txt | 16 +
.../generate_strict_evaluation_results.txt | 24 +
deepeval/metrics/g_eval/utils.py | 404 +
deepeval/metrics/goal_accuracy/__init__.py | 1 +
.../metrics/goal_accuracy/goal_accuracy.py | 371 +
deepeval/metrics/goal_accuracy/schema.py | 17 +
.../metrics/goal_accuracy/templates/class.txt | 1 +
.../templates/get_accuracy_score.txt | 94 +
.../templates/get_final_reason.txt | 42 +
.../templates/get_plan_evaluation_score.txt | 81 +
deepeval/metrics/hallucination/__init__.py | 3 +
.../metrics/hallucination/hallucination.py | 265 +
deepeval/metrics/hallucination/schema.py | 15 +
.../metrics/hallucination/templates/class.txt | 1 +
.../templates/generate_reason.txt | 22 +
.../templates/generate_verdicts.txt | 37 +
deepeval/metrics/indicator.py | 305 +
deepeval/metrics/json_correctness/__init__.py | 0
.../json_correctness/json_correctness.py | 218 +
deepeval/metrics/json_correctness/schema.py | 5 +
.../json_correctness/templates/class.txt | 1 +
.../templates/generate_reason.txt | 24 +
.../metrics/knowledge_retention/__init__.py | 0
.../knowledge_retention.py | 332 +
.../metrics/knowledge_retention/schema.py | 21 +
.../knowledge_retention/templates/class.txt | 1 +
.../templates/extract_data.txt | 87 +
.../templates/generate_reason.txt | 21 +
.../templates/generate_verdict.txt | 72 +
deepeval/metrics/mcp/__init__.py | 0
deepeval/metrics/mcp/mcp_task_completion.py | 286 +
.../metrics/mcp/multi_turn_mcp_use_metric.py | 406 +
deepeval/metrics/mcp/schema.py | 26 +
deepeval/metrics/mcp/templates/class.txt | 1 +
.../mcp/templates/generate_final_reason.txt | 38 +
.../templates/get_args_correctness_score.txt | 42 +
.../templates/get_task_completion_score.txt | 34 +
.../templates/get_tool_correctness_score.txt | 35 +
deepeval/metrics/mcp/utils.py | 98 +
deepeval/metrics/mcp_use_metric/__init__.py | 0
.../metrics/mcp_use_metric/mcp_use_metric.py | 416 +
deepeval/metrics/mcp_use_metric/schema.py | 11 +
.../mcp_use_metric/templates/class.txt | 1 +
.../get_mcp_argument_correctness_prompt.txt | 48 +
.../get_primitive_correctness_prompt.txt | 46 +
deepeval/metrics/misuse/__init__.py | 3 +
deepeval/metrics/misuse/misuse.py | 292 +
deepeval/metrics/misuse/schema.py | 19 +
deepeval/metrics/misuse/templates/class.txt | 1 +
.../misuse/templates/generate_misuses.txt | 27 +
.../misuse/templates/generate_reason.txt | 14 +
.../misuse/templates/generate_verdicts.txt | 30 +
.../metrics/multimodal_metrics/__init__.py | 5 +
.../image_coherence/__init__.py | 0
.../image_coherence/image_coherence.py | 365 +
.../image_coherence/schema.py | 7 +
.../image_coherence/templates/class.txt | 1 +
.../templates/evaluate_image_coherence.txt | 34 +
.../image_editing/__init__.py | 0
.../image_editing/image_editing.py | 302 +
.../image_editing/schema.py | 7 +
.../image_editing/templates/class.txt | 1 +
..._perceptual_quality_evaluation_results.txt | 20 +
...emantic_consistency_evaluation_results.txt | 14 +
.../image_helpfulness/__init__.py | 0
.../image_helpfulness/image_helpfulness.py | 366 +
.../image_helpfulness/schema.py | 7 +
.../image_helpfulness/templates/class.txt | 1 +
.../templates/evaluate_image_helpfulness.txt | 34 +
.../image_reference/__init__.py | 0
.../image_reference/image_reference.py | 366 +
.../image_reference/schema.py | 7 +
.../image_reference/templates/class.txt | 1 +
.../templates/evaluate_image_reference.txt | 33 +
.../text_to_image/__init__.py | 0
.../text_to_image/schema.py | 7 +
.../text_to_image/templates/class.txt | 1 +
..._perceptual_quality_evaluation_results.txt | 20 +
...emantic_consistency_evaluation_results.txt | 15 +
.../text_to_image/text_to_image.py | 304 +
deepeval/metrics/non_advice/__init__.py | 1 +
deepeval/metrics/non_advice/non_advice.py | 314 +
deepeval/metrics/non_advice/schema.py | 19 +
.../metrics/non_advice/templates/class.txt | 1 +
.../non_advice/templates/generate_advices.txt | 29 +
.../non_advice/templates/generate_reason.txt | 12 +
.../templates/generate_verdicts.txt | 29 +
deepeval/metrics/pattern_match/__init__.py | 0
.../metrics/pattern_match/pattern_match.py | 105 +
deepeval/metrics/pii_leakage/__init__.py | 1 +
deepeval/metrics/pii_leakage/pii_leakage.py | 289 +
deepeval/metrics/pii_leakage/schema.py | 19 +
.../metrics/pii_leakage/templates/class.txt | 1 +
.../pii_leakage/templates/extract_pii.txt | 23 +
.../pii_leakage/templates/generate_reason.txt | 12 +
.../templates/generate_verdicts.txt | 29 +
deepeval/metrics/plan_adherence/__init__.py | 1 +
.../metrics/plan_adherence/plan_adherence.py | 289 +
deepeval/metrics/plan_adherence/schema.py | 11 +
.../plan_adherence/templates/class.txt | 1 +
.../templates/evaluate_adherence.txt | 101 +
.../templates/extract_plan_from_trace.txt | 53 +
deepeval/metrics/plan_quality/__init__.py | 1 +
deepeval/metrics/plan_quality/plan_quality.py | 280 +
deepeval/metrics/plan_quality/schema.py | 11 +
.../metrics/plan_quality/templates/class.txt | 1 +
.../templates/evaluate_plan_quality.txt | 91 +
deepeval/metrics/prompt_alignment/__init__.py | 0
.../prompt_alignment/prompt_alignment.py | 280 +
deepeval/metrics/prompt_alignment/schema.py | 15 +
.../prompt_alignment/templates/class.txt | 1 +
.../templates/generate_reason.txt | 28 +
.../templates/generate_verdicts.txt | 45 +
deepeval/metrics/ragas.py | 576 +
deepeval/metrics/retrieval_context_display.py | 23 +
deepeval/metrics/role_adherence/__init__.py | 0
.../metrics/role_adherence/role_adherence.py | 259 +
deepeval/metrics/role_adherence/schema.py | 16 +
.../role_adherence/templates/class.txt | 1 +
...act_out_of_character_response_verdicts.txt | 60 +
.../templates/generate_reason.txt | 29 +
deepeval/metrics/role_violation/__init__.py | 1 +
.../metrics/role_violation/role_violation.py | 302 +
deepeval/metrics/role_violation/schema.py | 19 +
.../role_violation/templates/class.txt | 1 +
.../templates/detect_role_violations.txt | 25 +
.../templates/generate_reason.txt | 12 +
.../templates/generate_verdicts.txt | 29 +
deepeval/metrics/step_efficiency/__init__.py | 1 +
deepeval/metrics/step_efficiency/schema.py | 11 +
.../step_efficiency/step_efficiency.py | 226 +
.../step_efficiency/templates/class.txt | 1 +
.../templates/extract_task_from_trace.txt | 124 +
.../templates/get_execution_efficiency.txt | 118 +
deepeval/metrics/summarization/__init__.py | 3 +
deepeval/metrics/summarization/schema.py | 36 +
.../metrics/summarization/summarization.py | 574 +
.../metrics/summarization/templates/class.txt | 1 +
.../templates/generate_alignment_verdicts.txt | 53 +
.../templates/generate_answers.txt | 27 +
.../templates/generate_questions.txt | 14 +
.../templates/generate_reason.txt | 24 +
deepeval/metrics/task_completion/__init__.py | 0
deepeval/metrics/task_completion/schema.py | 12 +
.../task_completion/task_completion.py | 256 +
.../task_completion/templates/class.txt | 1 +
.../templates/extract_goal_and_outcome.txt | 64 +
.../extract_task_and_outcome_from_trace.txt | 115 +
.../templates/generate_verdict.txt | 25 +
deepeval/metrics/tool_correctness/__init__.py | 0
deepeval/metrics/tool_correctness/schema.py | 6 +
.../tool_correctness/templates/class.txt | 1 +
.../templates/get_tool_selection_score.txt | 75 +
.../tool_correctness/tool_correctness.py | 529 +
deepeval/metrics/tool_permission/__init__.py | 3 +
.../tool_permission/tool_permission.py | 166 +
deepeval/metrics/tool_use/__init__.py | 1 +
deepeval/metrics/tool_use/schema.py | 23 +
deepeval/metrics/tool_use/templates/class.txt | 1 +
.../get_argument_correctness_score.txt | 63 +
.../get_tool_argument_final_reason.txt | 36 +
.../get_tool_selection_final_reason.txt | 37 +
.../templates/get_tool_selection_score.txt | 55 +
deepeval/metrics/tool_use/tool_use.py | 475 +
deepeval/metrics/topic_adherence/__init__.py | 1 +
deepeval/metrics/topic_adherence/schema.py | 20 +
.../topic_adherence/templates/class.txt | 1 +
.../templates/generate_reason.txt | 34 +
.../templates/get_qa_pair_verdict.txt | 58 +
.../templates/get_qa_pairs.txt | 54 +
.../topic_adherence/topic_adherence.py | 393 +
deepeval/metrics/toxicity/__init__.py | 3 +
deepeval/metrics/toxicity/schema.py | 20 +
deepeval/metrics/toxicity/templates/class.txt | 1 +
.../toxicity/templates/generate_opinions.txt | 26 +
.../toxicity/templates/generate_reason.txt | 22 +
.../toxicity/templates/generate_verdicts.txt | 62 +
deepeval/metrics/toxicity/toxicity.py | 294 +
.../turn_contextual_precision/__init__.py | 0
.../turn_contextual_precision/schema.py | 21 +
.../templates/class.txt | 1 +
.../templates/generate_final_reason.txt | 39 +
.../templates/generate_reason.txt | 31 +
.../templates/generate_verdicts.txt | 40 +
.../turn_contextual_precision.py | 576 +
.../turn_contextual_recall/__init__.py | 0
.../metrics/turn_contextual_recall/schema.py | 21 +
.../templates/class.txt | 1 +
.../templates/generate_final_reason.txt | 39 +
.../templates/generate_reason.txt | 31 +
.../templates/generate_verdicts.txt | 29 +
.../turn_contextual_recall.py | 552 +
.../turn_contextual_relevancy/__init__.py | 0
.../turn_contextual_relevancy/schema.py | 22 +
.../templates/class.txt | 1 +
.../templates/generate_final_reason.txt | 39 +
.../templates/generate_reason.txt | 28 +
.../templates/generate_verdicts.txt | 35 +
.../turn_contextual_relevancy.py | 562 +
.../metrics/turn_faithfulness/__init__.py | 0
deepeval/metrics/turn_faithfulness/schema.py | 31 +
.../turn_faithfulness/templates/class.txt | 1 +
.../templates/generate_claims.txt | 28 +
.../templates/generate_final_reason.txt | 39 +
.../templates/generate_reason.txt | 28 +
.../templates/generate_truths.txt | 23 +
.../templates/generate_verdicts.txt | 44 +
.../turn_faithfulness/turn_faithfulness.py | 590 +
deepeval/metrics/turn_relevancy/__init__.py | 0
deepeval/metrics/turn_relevancy/schema.py | 12 +
.../turn_relevancy/templates/class.txt | 1 +
.../templates/generate_reason.txt | 24 +
.../templates/generate_verdicts.txt | 46 +
.../metrics/turn_relevancy/turn_relevancy.py | 292 +
deepeval/metrics/utils.py | 775 +
deepeval/model_integrations/__init__.py | 0
deepeval/model_integrations/types.py | 20 +
deepeval/model_integrations/utils.py | 120 +
deepeval/models/__init__.py | 49 +
deepeval/models/_summac_model.py | 592 +
deepeval/models/answer_relevancy_model.py | 75 +
deepeval/models/base_model.py | 194 +
deepeval/models/detoxify_model.py | 26 +
deepeval/models/embedding_models/__init__.py | 11 +
.../embedding_models/azure_embedding_model.py | 165 +
.../embedding_models/local_embedding_model.py | 132 +
.../ollama_embedding_model.py | 113 +
.../openai_embedding_model.py | 134 +
deepeval/models/hallucination_model.py | 27 +
deepeval/models/llms/__init__.py | 29 +
deepeval/models/llms/amazon_bedrock_model.py | 368 +
deepeval/models/llms/anthropic_model.py | 313 +
deepeval/models/llms/azure_model.py | 551 +
deepeval/models/llms/constants.py | 2224 ++
deepeval/models/llms/deepseek_model.py | 246 +
deepeval/models/llms/gateway_model.py | 422 +
deepeval/models/llms/gemini_model.py | 483 +
deepeval/models/llms/grok_model.py | 315 +
deepeval/models/llms/kimi_model.py | 297 +
deepeval/models/llms/litellm_model.py | 276 +
deepeval/models/llms/local_model.py | 241 +
deepeval/models/llms/ollama_model.py | 237 +
deepeval/models/llms/openai_model.py | 559 +
deepeval/models/llms/openrouter_model.py | 82 +
deepeval/models/llms/portkey_model.py | 109 +
deepeval/models/llms/utils.py | 48 +
deepeval/models/retry_policy.py | 1053 +
deepeval/models/summac_model.py | 62 +
deepeval/models/unbias_model.py | 18 +
deepeval/models/utils.py | 205 +
deepeval/openai/__init__.py | 21 +
deepeval/openai/extractors.py | 198 +
deepeval/openai/patch.py | 309 +
deepeval/openai/utils.py | 211 +
deepeval/openai_agents/__init__.py | 7 +
deepeval/openai_agents/agent.py | 36 +
deepeval/openai_agents/callback_handler.py | 173 +
deepeval/openai_agents/extractors.py | 455 +
deepeval/openai_agents/patch.py | 402 +
deepeval/openai_agents/runner.py | 348 +
deepeval/optimizer/__init__.py | 5 +
deepeval/optimizer/algorithms/__init__.py | 6 +
deepeval/optimizer/algorithms/base.py | 29 +
deepeval/optimizer/algorithms/configs.py | 18 +
.../optimizer/algorithms/copro/__init__.py | 5 +
deepeval/optimizer/algorithms/copro/copro.py | 535 +
.../optimizer/algorithms/copro/proposer.py | 266 +
deepeval/optimizer/algorithms/copro/schema.py | 11 +
.../optimizer/algorithms/copro/template.py | 110 +
.../optimizer/algorithms/gepa/__init__.py | 5 +
deepeval/optimizer/algorithms/gepa/gepa.py | 968 +
.../optimizer/algorithms/miprov2/__init__.py | 1 +
.../algorithms/miprov2/bootstrapper.py | 385 +
.../optimizer/algorithms/miprov2/miprov2.py | 533 +
.../algorithms/miprov2/proposer/__init__.py | 1 +
.../algorithms/miprov2/proposer/proposer.py | 312 +
.../algorithms/miprov2/proposer/schema.py | 11 +
.../algorithms/miprov2/proposer/template.py | 91 +
.../optimizer/algorithms/simba/__init__.py | 5 +
.../optimizer/algorithms/simba/proposer.py | 168 +
deepeval/optimizer/algorithms/simba/schema.py | 7 +
deepeval/optimizer/algorithms/simba/simba.py | 756 +
.../optimizer/algorithms/simba/template.py | 59 +
deepeval/optimizer/configs.py | 12 +
deepeval/optimizer/policies.py | 229 +
deepeval/optimizer/prompt_optimizer.py | 297 +
deepeval/optimizer/rewriter/__init__.py | 5 +
deepeval/optimizer/rewriter/rewriter.py | 122 +
deepeval/optimizer/rewriter/schema.py | 7 +
deepeval/optimizer/rewriter/template.py | 65 +
deepeval/optimizer/scorer/__init__.py | 5 +
deepeval/optimizer/scorer/base.py | 77 +
deepeval/optimizer/scorer/schema.py | 15 +
deepeval/optimizer/scorer/scorer.py | 401 +
deepeval/optimizer/scorer/template.py | 40 +
deepeval/optimizer/scorer/utils.py | 30 +
deepeval/optimizer/types.py | 111 +
deepeval/optimizer/utils.py | 471 +
deepeval/plugins/__init__.py | 0
deepeval/plugins/plugin.py | 78 +
deepeval/progress_context.py | 108 +
deepeval/prompt/__init__.py | 23 +
deepeval/prompt/api.py | 291 +
deepeval/prompt/prompt.py | 1163 +
deepeval/prompt/utils.py | 390 +
deepeval/py.typed | 0
deepeval/red_teaming/README.md | 3 +
deepeval/scorer/__init__.py | 1 +
deepeval/scorer/scorer.py | 478 +
deepeval/simulator/__init__.py | 13 +
deepeval/simulator/controller/__init__.py | 3 +
deepeval/simulator/controller/controller.py | 251 +
deepeval/simulator/controller/template.py | 39 +
deepeval/simulator/controller/types.py | 22 +
deepeval/simulator/conversation_simulator.py | 611 +
deepeval/simulator/schema.py | 24 +
.../simulator/simulation_graph/__init__.py | 7 +
.../simulator/simulation_graph/default.py | 57 +
deepeval/simulator/simulation_graph/node.py | 70 +
deepeval/simulator/simulation_graph/runner.py | 241 +
.../simulator/simulation_graph/template.py | 47 +
deepeval/simulator/template.py | 100 +
deepeval/simulator/utils.py | 63 +
deepeval/singleton.py | 17 +
deepeval/synthesizer/__init__.py | 24 +
deepeval/synthesizer/base_synthesizer.py | 29 +
deepeval/synthesizer/chunking/__init__.py | 0
.../synthesizer/chunking/context_generator.py | 1041 +
deepeval/synthesizer/chunking/doc_chunker.py | 273 +
deepeval/synthesizer/config.py | 73 +
deepeval/synthesizer/schema.py | 85 +
deepeval/synthesizer/synthesizer.py | 3274 +++
deepeval/synthesizer/templates/__init__.py | 12 +
deepeval/synthesizer/templates/template.py | 1345 +
.../templates/template_extraction.py | 78 +
.../synthesizer/templates/template_prompt.py | 546 +
deepeval/synthesizer/types.py | 20 +
deepeval/synthesizer/utils.py | 36 +
deepeval/telemetry.py | 518 +
deepeval/templates/__init__.py | 17 +
...ithfulness_verdicts_example_multimodal.txt | 29 +
...ithfulness_verdicts_format_instruction.txt | 17 +
...fulness_verdicts_guidelines_multimodal.txt | 7 +
...hfulness_verdicts_guidelines_text_only.txt | 6 +
...ltimodal_image_generation_eval_context.txt | 9 +
.../fragments/multimodal_input_rules.txt | 6 +
.../multimodal_input_rules_turn_metric.txt | 8 +
deepeval/templates/metrics/templates.json | 225 +
deepeval/templates/resolver.py | 216 +
deepeval/test_case/__init__.py | 62 +
deepeval/test_case/api.py | 124 +
deepeval/test_case/arena_test_case.py | 48 +
.../test_case/conversational_test_case.py | 353 +
deepeval/test_case/llm_test_case.py | 627 +
deepeval/test_case/mcp.py | 57 +
deepeval/test_case/utils.py | 20 +
deepeval/test_run/__init__.py | 33 +
deepeval/test_run/api.py | 150 +
deepeval/test_run/cache.py | 404 +
deepeval/test_run/hooks.py | 18 +
deepeval/test_run/hyperparameters.py | 109 +
deepeval/test_run/test_run.py | 1218 +
deepeval/tracing/__init__.py | 49 +
deepeval/tracing/api.py | 175 +
deepeval/tracing/context.py | 700 +
deepeval/tracing/integrations.py | 35 +
deepeval/tracing/internal.py | 36 +
deepeval/tracing/offline_evals/__init__.py | 3 +
deepeval/tracing/offline_evals/api.py | 18 +
deepeval/tracing/offline_evals/span.py | 61 +
deepeval/tracing/offline_evals/thread.py | 75 +
deepeval/tracing/offline_evals/trace.py | 65 +
deepeval/tracing/otel/__init__.py | 7 +
.../tracing/otel/context_aware_processor.py | 196 +
deepeval/tracing/otel/exporter.py | 879 +
deepeval/tracing/otel/test_exporter.py | 35 +
deepeval/tracing/otel/utils.py | 648 +
deepeval/tracing/patchers.py | 221 +
deepeval/tracing/perf_epoch_bridge.py | 53 +
deepeval/tracing/trace_context.py | 114 +
deepeval/tracing/trace_test_manager.py | 19 +
deepeval/tracing/tracing.py | 1539 ++
deepeval/tracing/types.py | 301 +
deepeval/tracing/utils.py | 304 +
deepeval/utils.py | 968 +
demo_trace_scope/__init__.py | 0
demo_trace_scope/test_observed_app.py | 49 +
docs/.gitignore | 31 +
docs/README.md | 45 +
docs/app/(home)/layout.tsx | 5 +
docs/app/(home)/page.tsx | 33 +
docs/app/api/search/route.ts | 7 +
docs/app/apple-icon.png | Bin 0 -> 7031 bytes
docs/app/blog/[[...slug]]/page.tsx | 5 +
docs/app/blog/layout.tsx | 3 +
docs/app/changelog/[[...slug]]/page.tsx | 5 +
docs/app/changelog/layout.tsx | 3 +
docs/app/docs/[[...slug]]/page.tsx | 5 +
docs/app/docs/layout.tsx | 3 +
docs/app/enterprise/page.tsx | 31 +
docs/app/favicon.ico | Bin 0 -> 7406 bytes
docs/app/global.css | 791 +
docs/app/guides/[[...slug]]/page.tsx | 5 +
docs/app/guides/layout.tsx | 3 +
docs/app/icon.png | Bin 0 -> 1557 bytes
docs/app/icon.svg | 32 +
docs/app/integrations/[[...slug]]/page.tsx | 5 +
docs/app/integrations/layout.tsx | 3 +
docs/app/layout.tsx | 218 +
docs/app/llms-full.txt/route.ts | 10 +
docs/app/llms.mdx/blog/[[...slug]]/route.ts | 8 +
.../llms.mdx/changelog/[[...slug]]/route.ts | 8 +
docs/app/llms.mdx/docs/[[...slug]]/route.ts | 8 +
docs/app/llms.mdx/guides/[[...slug]]/route.ts | 8 +
.../integrations/[[...slug]]/route.ts | 8 +
.../llms.mdx/tutorials/[[...slug]]/route.ts | 8 +
docs/app/llms.txt/route.ts | 8 +
docs/app/og/blog/[...slug]/route.tsx | 30 +
docs/app/og/docs/[...slug]/route.tsx | 28 +
docs/app/robots.ts | 13 +
docs/app/sitemap.ts | 48 +
docs/app/tutorials/[[...slug]]/page.tsx | 5 +
docs/app/tutorials/layout.tsx | 3 +
docs/components/lang/code-term.tsx | 19 +
docs/components/lang/language-provider.tsx | 38 +
.../language-selector.module.scss | 119 +
.../language-selector/language-selector.tsx | 114 +
docs/components/mdx-anchor.tsx | 34 +
docs/components/mdx.tsx | 76 +
.../blog/deepeval-alternatives-compared.mdx | 167 +
docs/content/blog/deepeval-got-a-new-look.mdx | 37 +
docs/content/blog/deepeval-vs-arize.mdx | 156 +
docs/content/blog/deepeval-vs-langfuse.mdx | 134 +
docs/content/blog/deepeval-vs-ragas.mdx | 147 +
docs/content/blog/deepeval-vs-trulens.mdx | 121 +
docs/content/blog/eval-driven-development.mdx | 312 +
docs/content/blog/index.mdx | 8 +
docs/content/blog/introducing-deepeval-4.mdx | 131 +
docs/content/blog/llm-as-a-judge.mdx | 622 +
.../blog/medical-chatbot-deepeval-guide.mdx | 568 +
docs/content/blog/meta.json | 27 +
.../rag-contract-assistant-deepeval-guide.mdx | 600 +
docs/content/blog/top-5-geval-use-cases.mdx | 376 +
.../blog/typescript-in-deepeval-monorepo.mdx | 192 +
.../blog/use-case-cognee-ai-memory.mdx | 97 +
docs/content/blog/what-is-an-eval-harness.mdx | 261 +
docs/content/changelog/changelog-2024.mdx | 1488 +
docs/content/changelog/changelog-2025.mdx | 1270 +
docs/content/changelog/changelog-2026.mdx | 301 +
docs/content/changelog/index.mdx | 20 +
docs/content/changelog/meta.json | 10 +
docs/content/docs/(agentic)/meta.json | 11 +
.../metrics-argument-correctness.mdx | 201 +
.../docs/(agentic)/metrics-plan-adherence.mdx | 146 +
.../docs/(agentic)/metrics-plan-quality.mdx | 144 +
.../(agentic)/metrics-step-efficiency.mdx | 144 +
.../(agentic)/metrics-task-completion.mdx | 136 +
.../(agentic)/metrics-tool-correctness.mdx | 229 +
docs/content/docs/(algorithms)/meta.json | 9 +
.../prompt-optimization-copro.mdx | 331 +
.../(algorithms)/prompt-optimization-gepa.mdx | 364 +
.../prompt-optimization-miprov2.mdx | 337 +
.../prompt-optimization-simba.mdx | 387 +
.../docs/(benchmarks)/benchmarks-arc.mdx | 124 +
.../docs/(benchmarks)/benchmarks-bbq.mdx | 146 +
.../benchmarks-big-bench-hard.mdx | 169 +
.../docs/(benchmarks)/benchmarks-bool-q.mdx | 114 +
.../docs/(benchmarks)/benchmarks-drop.mdx | 718 +
.../docs/(benchmarks)/benchmarks-gsm8k.mdx | 123 +
.../(benchmarks)/benchmarks-hellaswag.mdx | 330 +
.../(benchmarks)/benchmarks-human-eval.mdx | 325 +
.../docs/(benchmarks)/benchmarks-ifeval.mdx | 115 +
.../docs/(benchmarks)/benchmarks-lambada.mdx | 115 +
.../docs/(benchmarks)/benchmarks-logi-qa.mdx | 138 +
.../docs/(benchmarks)/benchmarks-math-qa.mdx | 136 +
.../docs/(benchmarks)/benchmarks-mmlu.mdx | 191 +
.../docs/(benchmarks)/benchmarks-squad.mdx | 184 +
.../(benchmarks)/benchmarks-truthful-qa.mdx | 178 +
.../(benchmarks)/benchmarks-winogrande.mdx | 114 +
docs/content/docs/(benchmarks)/meta.json | 21 +
.../community-metrics-overview.mdx | 24 +
docs/content/docs/(community)/meta.json | 9 +
.../metrics-agent-loop-detection.mdx | 277 +
.../metrics-citation-faithfulness.mdx | 79 +
.../(community)/metrics-tool-permission.mdx | 115 +
.../evaluation-arena-test-cases.mdx | 303 +
.../evaluation-multiturn-test-cases.mdx | 366 +
.../(test-cases)/evaluation-test-cases.mdx | 644 +
.../docs/(concepts)/(test-cases)/meta.json | 8 +
.../docs/(concepts)/evaluation-datasets.mdx | 709 +
.../(concepts)/evaluation-llm-tracing.mdx | 670 +
.../docs/(concepts)/evaluation-mcp.mdx | 331 +
.../docs/(concepts)/evaluation-prompts.mdx | 508 +
docs/content/docs/(concepts)/meta.json | 10 +
docs/content/docs/(custom)/meta.json | 11 +
.../docs/(custom)/metrics-arena-g-eval.mdx | 147 +
.../(custom)/metrics-conversational-dag.mdx | 406 +
.../metrics-conversational-g-eval.mdx | 199 +
docs/content/docs/(custom)/metrics-custom.mdx | 590 +
docs/content/docs/(custom)/metrics-dag.mdx | 408 +
.../docs/(custom)/metrics-llm-evals.mdx | 498 +
.../content/docs/(generate-goldens)/meta.json | 9 +
.../synthesizer-generate-from-contexts.mdx | 141 +
.../synthesizer-generate-from-docs.mdx | 243 +
.../synthesizer-generate-from-goldens.mdx | 131 +
.../synthesizer-generate-from-scratch.mdx | 149 +
docs/content/docs/(images)/meta.json | 10 +
.../multimodal-metrics-image-coherence.mdx | 132 +
.../multimodal-metrics-image-editing.mdx | 137 +
.../multimodal-metrics-image-helpfulness.mdx | 143 +
.../multimodal-metrics-image-reference.mdx | 139 +
.../multimodal-metrics-text-to-image.mdx | 141 +
docs/content/docs/(mcp)/meta.json | 8 +
.../(mcp)/metrics-mcp-task-completion.mdx | 128 +
docs/content/docs/(mcp)/metrics-mcp-use.mdx | 142 +
.../docs/(mcp)/metrics-multi-turn-mcp-use.mdx | 130 +
docs/content/docs/(metrics-others)/meta.json | 9 +
.../metrics-hallucination.mdx | 168 +
.../metrics-prompt-alignment.mdx | 171 +
.../docs/(metrics-others)/metrics-ragas.mdx | 152 +
.../metrics-summarization.mdx | 215 +
docs/content/docs/(multi-turn)/meta.json | 16 +
.../metrics-conversation-completeness.mdx | 125 +
.../(multi-turn)/metrics-goal-accuracy.mdx | 129 +
.../metrics-knowledge-retention.mdx | 123 +
.../(multi-turn)/metrics-role-adherence.mdx | 127 +
.../docs/(multi-turn)/metrics-tool-use.mdx | 131 +
.../(multi-turn)/metrics-topic-adherence.mdx | 132 +
.../metrics-turn-contextual-precision.mdx | 162 +
.../metrics-turn-contextual-recall.mdx | 153 +
.../metrics-turn-contextual-relevancy.mdx | 152 +
.../metrics-turn-faithfulness.mdx | 146 +
.../(multi-turn)/metrics-turn-relevancy.mdx | 126 +
docs/content/docs/(non-llm)/meta.json | 8 +
.../docs/(non-llm)/metrics-exact-match.mdx | 128 +
.../(non-llm)/metrics-json-correctness.mdx | 194 +
.../docs/(non-llm)/metrics-pattern-match.mdx | 123 +
docs/content/docs/(rag)/meta.json | 10 +
.../docs/(rag)/metrics-answer-relevancy.mdx | 250 +
.../(rag)/metrics-contextual-precision.mdx | 282 +
.../docs/(rag)/metrics-contextual-recall.mdx | 270 +
.../(rag)/metrics-contextual-relevancy.mdx | 264 +
.../docs/(rag)/metrics-faithfulness.mdx | 261 +
docs/content/docs/(safety)/meta.json | 11 +
docs/content/docs/(safety)/metrics-bias.mdx | 193 +
docs/content/docs/(safety)/metrics-misuse.mdx | 191 +
.../docs/(safety)/metrics-non-advice.mdx | 188 +
.../docs/(safety)/metrics-pii-leakage.mdx | 193 +
.../docs/(safety)/metrics-role-violation.mdx | 192 +
.../docs/(safety)/metrics-toxicity.mdx | 193 +
.../(use-cases)/getting-started-agents.mdx | 515 +
.../(use-cases)/getting-started-chatbots.mdx | 506 +
.../(use-cases)/getting-started-llm-arena.mdx | 402 +
.../docs/(use-cases)/getting-started-mcp.mdx | 451 +
.../docs/(use-cases)/getting-started-rag.mdx | 405 +
docs/content/docs/(use-cases)/meta.json | 10 +
docs/content/docs/benchmarks-introduction.mdx | 326 +
docs/content/docs/command-line-interface.mdx | 381 +
...onversation-simulator-custom-templates.mdx | 156 +
...conversation-simulator-lifecycle-hooks.mdx | 106 +
.../conversation-simulator-model-callback.mdx | 120 +
...onversation-simulator-simulation-graph.mdx | 235 +
.../conversation-simulator-stopping-logic.mdx | 210 +
.../docs/conversation-simulator/index.mdx | 226 +
.../docs/conversation-simulator/meta.json | 10 +
docs/content/docs/data-privacy.mdx | 42 +
docs/content/docs/environment-variables.mdx | 318 +
.../evaluation-component-level-llm-evals.mdx | 484 +
.../evaluation-end-to-end-llm-evals/index.mdx | 122 +
.../evaluation-end-to-end-llm-evals/meta.json | 7 +
.../docs/evaluation-end-to-end-multi-turn.mdx | 477 +
.../evaluation-end-to-end-single-turn.mdx | 498 +
.../docs/evaluation-flags-and-configs.mdx | 331 +
docs/content/docs/evaluation-introduction.mdx | 347 +
.../docs/evaluation-unit-testing-in-ci-cd.mdx | 499 +
docs/content/docs/faq.mdx | 390 +
docs/content/docs/getting-started.mdx | 428 +
.../content/docs/golden-synthesizer/index.mdx | 519 +
.../content/docs/golden-synthesizer/meta.json | 9 +
.../content/docs/introduction-comparisons.mdx | 106 +
.../docs/introduction-design-philosophy.mdx | 161 +
docs/content/docs/introduction.mdx | 246 +
docs/content/docs/meta.json | 56 +
docs/content/docs/metrics-introduction.mdx | 859 +
docs/content/docs/miscellaneous.mdx | 15 +
.../docs/prompt-optimization-introduction.mdx | 342 +
...synthetic-data-generation-introduction.mdx | 191 +
docs/content/docs/troubleshooting.mdx | 173 +
docs/content/docs/vibe-coder-quickstart.mdx | 197 +
docs/content/docs/vibe-coding.mdx | 141 +
.../guides-ai-agent-evaluation-metrics.mdx | 529 +
.../guides/guides-ai-agent-evaluation.mdx | 602 +
.../guides-answer-correctness-metric.mdx | 221 +
.../guides/guides-building-custom-metrics.mdx | 378 +
.../guides/guides-llm-observability.mdx | 62 +
.../guides-multi-turn-evaluation-metrics.mdx | 548 +
.../guides/guides-multi-turn-evaluation.mdx | 483 +
.../guides/guides-multi-turn-simulation.mdx | 510 +
.../guides-optimizing-hyperparameters.mdx | 154 +
docs/content/guides/guides-rag-evaluation.mdx | 561 +
docs/content/guides/guides-rag-triad.mdx | 164 +
docs/content/guides/guides-red-teaming.mdx | 280 +
.../guides-regression-testing-in-cicd.mdx | 92 +
.../guides/guides-tracing-ai-agents.mdx | 662 +
.../guides/guides-tracing-multi-turn.mdx | 484 +
docs/content/guides/guides-tracing-rag.mdx | 609 +
.../guides-using-custom-embedding-models.mdx | 178 +
.../guides/guides-using-custom-llms.mdx | 739 +
.../guides/guides-using-synthesizer.mdx | 316 +
docs/content/guides/meta.json | 35 +
.../integrations/frameworks/agentcore.mdx | 404 +
.../integrations/frameworks/anthropic.mdx | 424 +
.../integrations/frameworks/crewai.mdx | 446 +
.../integrations/frameworks/google-adk.mdx | 415 +
.../integrations/frameworks/huggingface.mdx | 158 +
.../integrations/frameworks/langchain.mdx | 450 +
.../integrations/frameworks/langgraph.mdx | 522 +
.../integrations/frameworks/llamaindex.mdx | 434 +
.../content/integrations/frameworks/meta.json | 16 +
.../integrations/frameworks/openai-agents.mdx | 464 +
.../integrations/frameworks/openai.mdx | 421 +
.../integrations/frameworks/pydanticai.mdx | 485 +
.../integrations/frameworks/strands.mdx | 409 +
docs/content/integrations/index.mdx | 185 +
docs/content/integrations/meta.json | 10 +
.../integrations/models/amazon-bedrock.mdx | 116 +
.../content/integrations/models/anthropic.mdx | 98 +
.../integrations/models/azure-openai.mdx | 120 +
docs/content/integrations/models/deepseek.mdx | 97 +
docs/content/integrations/models/gemini.mdx | 104 +
docs/content/integrations/models/grok.mdx | 101 +
docs/content/integrations/models/litellm.mdx | 157 +
docs/content/integrations/models/lmstudio.mdx | 93 +
docs/content/integrations/models/meta.json | 20 +
docs/content/integrations/models/moonshot.mdx | 98 +
docs/content/integrations/models/ollama.mdx | 119 +
docs/content/integrations/models/openai.mdx | 147 +
.../integrations/models/openrouter.mdx | 103 +
docs/content/integrations/models/portkey.mdx | 87 +
.../content/integrations/models/vertex-ai.mdx | 99 +
docs/content/integrations/models/vllm.mdx | 91 +
docs/content/integrations/others/meta.json | 6 +
.../integrations/vector-databases/chroma.mdx | 256 +
.../integrations/vector-databases/cognee.mdx | 104 +
.../vector-databases/elasticsearch.mdx | 218 +
.../integrations/vector-databases/meta.json | 11 +
.../vector-databases/pgvector.mdx | 225 +
.../integrations/vector-databases/qdrant.mdx | 252 +
.../vector-databases/weaviate.mdx | 265 +
.../tutorials/medical-chatbot/development.mdx | 351 +
.../medical-chatbot/evals-in-prod.mdx | 145 +
.../tutorials/medical-chatbot/evaluation.mdx | 245 +
.../tutorials/medical-chatbot/improvement.mdx | 233 +
.../medical-chatbot/introduction.mdx | 104 +
docs/content/tutorials/meta.json | 29 +
.../tutorials/rag-qa-agent/development.mdx | 411 +
.../tutorials/rag-qa-agent/evals-in-prod.mdx | 171 +
.../tutorials/rag-qa-agent/evaluation.mdx | 240 +
.../tutorials/rag-qa-agent/improvement.mdx | 368 +
.../tutorials/rag-qa-agent/introduction.mdx | 103 +
.../summarization-agent/development.mdx | 490 +
.../summarization-agent/evals-in-prod.mdx | 190 +
.../summarization-agent/evaluation.mdx | 248 +
.../summarization-agent/improvement.mdx | 283 +
.../summarization-agent/introduction.mdx | 98 +
.../tutorials/tutorial-introduction.mdx | 104 +
docs/content/tutorials/tutorial-setup.mdx | 129 +
docs/enterprise/read-me.mdx | 177 +
docs/home/read-me.mdx | 267 +
docs/lib/authors.ts | 53 +
docs/lib/blog-categories.ts | 39 +
docs/lib/cn.ts | 1 +
docs/lib/contributors.ts | 36 +
docs/lib/defaults.ts | 1 +
.../lib/generated/changelog-contributors.json | 1912 ++
docs/lib/generated/contributors.json | 4630 ++++
docs/lib/generated/repo-contributors.json | 1640 ++
docs/lib/lang/terms.ts | 92 +
docs/lib/layout.shared.tsx | 86 +
docs/lib/llms-route.ts | 34 +
docs/lib/remark-admonitions.ts | 65 +
docs/lib/section.tsx | 270 +
docs/lib/sections.tsx | 186 +
docs/lib/shared.ts | 98 +
docs/lib/source.ts | 286 +
docs/next.config.mjs | 28 +
docs/package.json | 46 +
docs/postcss.config.mjs | 7 +
docs/proxy.ts | 29 +
docs/public/icons/DeepEval.svg | 4 +
docs/public/icons/brand-icons/amdocs.svg | 40 +
docs/public/icons/brand-icons/bcg.svg | 17 +
docs/public/icons/brand-icons/epic-games.svg | 57 +
docs/public/icons/brand-icons/finom.svg | 4 +
docs/public/icons/brand-icons/humach.svg | 93 +
docs/public/icons/brand-icons/panasonic.svg | 2 +
docs/public/icons/brand-icons/phreesia.svg | 55 +
docs/public/icons/brand-icons/siemens.svg | 23 +
.../icons/brand-icons/syngenta-group.svg | 35 +
docs/public/icons/brand-icons/toshiba.svg | 4 +
docs/public/icons/claudecode.svg | 1 +
docs/public/icons/companies/adobe.svg | 19 +
docs/public/icons/companies/aws.svg | 38 +
docs/public/icons/companies/axa.svg | 87 +
docs/public/icons/companies/bcg.svg | 17 +
docs/public/icons/companies/benz.svg | 2865 ++
docs/public/icons/companies/bosch.svg | 1 +
docs/public/icons/companies/cvs-health.svg | 23 +
docs/public/icons/companies/ey.svg | 4 +
docs/public/icons/companies/google.svg | 2 +
docs/public/icons/companies/lego.svg | 7 +
docs/public/icons/companies/mastercard.svg | 39 +
docs/public/icons/companies/microsoft.svg | 1 +
docs/public/icons/companies/nvidia.svg | 9 +
docs/public/icons/companies/openai.svg | 1 +
docs/public/icons/companies/pfizer.svg | 33 +
docs/public/icons/companies/samsung.svg | 97 +
docs/public/icons/companies/siemens.svg | 23 +
docs/public/icons/companies/toyota.svg | 3 +
docs/public/icons/companies/uber.svg | 1 +
docs/public/icons/companies/visa.svg | 76 +
docs/public/icons/companies/walmart.svg | 104 +
docs/public/icons/deepeval-icon.svg | 29 +
docs/public/icons/frameworks/anthropic.png | Bin 0 -> 27123 bytes
docs/public/icons/frameworks/crewai.png | Bin 0 -> 40767 bytes
docs/public/icons/frameworks/langchain.png | Bin 0 -> 31385 bytes
docs/public/icons/frameworks/langgraph.png | Bin 0 -> 28989 bytes
docs/public/icons/frameworks/llamaindex.png | Bin 0 -> 29488 bytes
docs/public/icons/frameworks/pydanticai.png | Bin 0 -> 3128 bytes
docs/public/icons/integrations/agentcore.svg | 46 +
docs/public/icons/integrations/ai-sdk.svg | 8 +
.../icons/integrations/azure-pipelines.svg | 1 +
docs/public/icons/integrations/azure.svg | 23 +
docs/public/icons/integrations/bedrock.svg | 1 +
docs/public/icons/integrations/buildkite.svg | 1 +
docs/public/icons/integrations/circleci.svg | 7 +
docs/public/icons/integrations/claude.svg | 7 +
docs/public/icons/integrations/crewai.svg | 11 +
docs/public/icons/integrations/gemini.svg | 1 +
docs/public/icons/integrations/github.svg | 3 +
docs/public/icons/integrations/gitlab.svg | 22 +
docs/public/icons/integrations/google-adk.png | Bin 0 -> 11976 bytes
docs/public/icons/integrations/jenkins.svg | 283 +
docs/public/icons/integrations/langchain.svg | 1 +
docs/public/icons/integrations/langgraph.svg | 1 +
docs/public/icons/integrations/litellm.svg | 15 +
docs/public/icons/integrations/llamaindex.svg | 11 +
docs/public/icons/integrations/mistral.svg | 19 +
docs/public/icons/integrations/openai.svg | 15 +
docs/public/icons/integrations/otel.svg | 4 +
docs/public/icons/integrations/portkey.svg | 10 +
docs/public/icons/integrations/pydanticai.svg | 1 +
docs/public/icons/integrations/strands.svg | 4 +
docs/public/icons/integrations/vertext_ai.svg | 1 +
docs/public/icons/python.svg | 1 +
docs/public/icons/typescript.svg | 1 +
docs/public/img/2025-dashboard-screenshot.png | Bin 0 -> 500022 bytes
docs/public/img/confident-alerts.png | Bin 0 -> 619531 bytes
.../img/confident-centralized-metrics.png | Bin 0 -> 387093 bytes
.../img/confident-dataset-management.png | Bin 0 -> 561195 bytes
docs/public/img/confident-experimentation.png | Bin 0 -> 437898 bytes
.../public/img/confident-human-annotation.png | Bin 0 -> 554884 bytes
.../img/confident-production-monitoring.png | Bin 0 -> 500931 bytes
.../img/confident-prompt-versioning.png | Bin 0 -> 417953 bytes
.../img/confident-regression-testing.png | Bin 0 -> 361082 bytes
.../public/img/confident-trace-to-dataset.png | Bin 0 -> 421134 bytes
.../img/confident-tracing-observability.png | Bin 0 -> 1010741 bytes
docs/public/img/dashboard-screenshot.png | Bin 0 -> 615231 bytes
docs/public/img/deepeval.png | Bin 0 -> 86028 bytes
docs/public/img/favicon.ico | Bin 0 -> 15406 bytes
docs/public/img/social_card.png | Bin 0 -> 40625 bytes
docs/public/img/white-2.png | Bin 0 -> 925727 bytes
docs/public/img/white-3.png | Bin 0 -> 493915 bytes
docs/public/img/white.png | Bin 0 -> 430588 bytes
docs/public/llms-full.txt | 22961 ++++++++++++++++
docs/public/llms.txt | 103 +
docs/scripts/build-readme-hero.mjs | 153 +
docs/scripts/fonts/Quicksand-SemiBold.ttf | Bin 0 -> 34124 bytes
.../generate-changelog-contributors.mjs | 66 +
docs/scripts/generate-contributors.mjs | 408 +
docs/scripts/generate-repo-contributors.mjs | 103 +
docs/scripts/normalize-admonition-titles.mjs | 69 +
.../replace-img-with-image-displayer.mjs | 93 +
docs/scripts/strip-redundant-mdx-imports.mjs | 64 +
docs/scripts/timeline-to-steps.mjs | 77 +
.../evaluation/cicd-agent-framework-tabs.mdx | 411 +
.../component-level-agent-framework-tabs.mdx | 822 +
.../end-to-end-agent-framework-tabs.mdx | 768 +
docs/source.config.ts | 116 +
docs/src/assets.ts | 109 +
.../AgentTraceTerminal.module.scss | 350 +
.../components/AgentTraceTerminal/index.tsx | 211 +
docs/src/components/AskAIButton/index.tsx | 43 +
.../BlogPostMeta/BlogPostMeta.module.scss | 147 +
docs/src/components/BlogPostMeta/index.tsx | 79 +
docs/src/components/BrandMarks/index.tsx | 46 +
.../components/Buttons/Buttons.module.scss | 78 +
docs/src/components/Buttons/index.tsx | 148 +
.../components/Callout/Callout.module.scss | 74 +
docs/src/components/Callout/index.tsx | 71 +
.../ChangelogContributors.module.scss | 60 +
.../ChangelogContributors/index.tsx | 77 +
.../CloudPlatformCallout.module.scss | 51 +
.../components/CloudPlatformCallout/index.tsx | 34 +
.../CommunityCta/CommunityCta.module.scss | 81 +
docs/src/components/CommunityCta/index.tsx | 49 +
.../ContributorDisplay.module.scss | 86 +
.../components/ContributorDisplay/index.tsx | 59 +
.../DiscordButton/DiscordButton.module.scss | 62 +
docs/src/components/DiscordButton/index.tsx | 59 +
.../components/Equation/Equation.module.scss | 4 +
docs/src/components/Equation/index.tsx | 22 +
docs/src/components/FAQ/index.tsx | 137 +
.../FeatureComparisonTable.module.scss | 68 +
.../FeatureComparisonTable/index.tsx | 2357 ++
.../GithubCtaButton.module.scss | 138 +
docs/src/components/GithubCtaButton/index.tsx | 69 +
.../GithubCtaButton/useGithubStarCount.ts | 75 +
docs/src/components/Hotkey/Hotkey.module.scss | 39 +
docs/src/components/Hotkey/index.tsx | 67 +
.../ImageDisplayer/ImageDisplayer.module.scss | 24 +
docs/src/components/ImageDisplayer/index.tsx | 20 +
.../IntegrationGrid.module.scss | 263 +
docs/src/components/IntegrationGrid/index.tsx | 168 +
.../IntegrationTagsDisplayer.module.scss | 47 +
.../IntegrationTagsDisplayer/index.tsx | 43 +
.../LinkCards/LinkCards.module.scss | 98 +
docs/src/components/LinkCards/index.tsx | 61 +
docs/src/components/Mermaid/index.tsx | 86 +
.../MetricTagsDisplayer.module.scss | 70 +
.../components/MetricTagsDisplayer/index.tsx | 81 +
.../PageContributors/ContributorsOverflow.tsx | 76 +
.../PageContributors.module.scss | 116 +
.../src/components/PageContributors/index.tsx | 86 +
docs/src/components/PauseOffscreen/index.tsx | 75 +
.../RedditButton/RedditButton.module.scss | 70 +
docs/src/components/RedditButton/index.tsx | 48 +
.../SchemaInjector/SchemaInjector.tsx | 31 +
.../SectionLabel/SectionLabel.module.scss | 13 +
docs/src/components/SectionLabel/index.tsx | 12 +
.../SiteThemeSwitch.module.scss | 27 +
docs/src/components/SiteThemeSwitch/index.tsx | 11 +
.../TechStackCards/TechStackCards.module.scss | 46 +
docs/src/components/TechStackCards/index.tsx | 37 +
.../TocFooter/TocFooter.module.scss | 35 +
docs/src/components/TocFooter/index.tsx | 51 +
.../VideoDisplayer/VideoDisplayer.module.scss | 81 +
docs/src/components/VideoDisplayer/index.tsx | 71 +
docs/src/components/Wordmark/index.tsx | 40 +
docs/src/components/index.ts | 9 +
docs/src/global.d.ts | 22 +
docs/src/layouts/Footer/Footer.module.scss | 133 +
docs/src/layouts/Footer/index.tsx | 151 +
.../layouts/HomeLayout/HomeLayout.module.scss | 86 +
docs/src/layouts/HomeLayout/HomeLayout.tsx | 128 +
docs/src/layouts/HomeLayout/index.ts | 1 +
.../HomeOverflowNav.module.scss | 136 +
docs/src/layouts/HomeOverflowNav/index.tsx | 204 +
.../HomePageShell/HomePageShell.module.scss | 22 +
docs/src/layouts/HomePageShell/index.tsx | 22 +
.../layouts/NavHeader/NavHeader.module.scss | 143 +
docs/src/layouts/NavHeader/index.tsx | 70 +
docs/src/layouts/NavLinks/index.tsx | 66 +
docs/src/layouts/NavMenu/NavMenu.module.scss | 76 +
docs/src/layouts/NavMenu/index.tsx | 64 +
docs/src/layouts/SidebarSearch/index.tsx | 40 +
.../layouts/SiteTopNav/SiteTopNav.module.scss | 299 +
docs/src/layouts/SiteTopNav/index.tsx | 153 +
docs/src/layouts/UtmCapture/UtmCapture.tsx | 203 +
docs/src/layouts/UtmCapture/index.ts | 1 +
.../EnterpriseComparisonTable.module.scss | 96 +
.../enterprise/EnterpriseComparisonTable.tsx | 72 +
.../EnterpriseHeroSection.module.scss | 36 +
.../enterprise/EnterpriseHeroSection.tsx | 92 +
.../EnterprisePlatformMockup.module.scss | 325 +
.../enterprise/EnterprisePlatformMockup.tsx | 147 +
.../ClaudeCodeTerminal.module.scss | 296 +
.../home/ClaudeCodeTerminal/index.tsx | 145 +
docs/src/sections/home/CompanyLogos/Adobe.tsx | 5 +
docs/src/sections/home/CompanyLogos/Aws.tsx | 31 +
docs/src/sections/home/CompanyLogos/Axa.tsx | 5 +
docs/src/sections/home/CompanyLogos/Benz.tsx | 40 +
docs/src/sections/home/CompanyLogos/Bosch.tsx | 24 +
.../CompanyLogos/CompanyLogos.module.scss | 53 +
.../sections/home/CompanyLogos/CvsHealth.tsx | 22 +
docs/src/sections/home/CompanyLogos/Ey.tsx | 20 +
.../src/sections/home/CompanyLogos/Google.tsx | 5 +
docs/src/sections/home/CompanyLogos/Lego.tsx | 5 +
.../sections/home/CompanyLogos/Mastercard.tsx | 30 +
.../sections/home/CompanyLogos/Microsoft.tsx | 5 +
.../src/sections/home/CompanyLogos/Nvidia.tsx | 34 +
.../src/sections/home/CompanyLogos/OpenAI.tsx | 18 +
.../src/sections/home/CompanyLogos/Pfizer.tsx | 5 +
.../sections/home/CompanyLogos/Samsung.tsx | 5 +
.../sections/home/CompanyLogos/Siemens.tsx | 5 +
.../src/sections/home/CompanyLogos/Toyota.tsx | 19 +
docs/src/sections/home/CompanyLogos/Uber.tsx | 20 +
docs/src/sections/home/CompanyLogos/Visa.tsx | 5 +
.../sections/home/CompanyLogos/Walmart.tsx | 5 +
.../home/CompanyLogos/_createInlineLogo.tsx | 26 +
.../home/CompanyLogos/_loadInlineSvg.ts | 88 +
docs/src/sections/home/CompanyLogos/index.ts | 46 +
docs/src/sections/home/CompanyLogos/types.ts | 3 +
.../DatasetDemos/DatasetDemos.module.scss | 677 +
.../DatasetDemos/GoldenGenerationDemo.tsx | 185 +
.../DatasetDemos/MultiTurnSimulationDemo.tsx | 197 +
docs/src/sections/home/DatasetDemos/index.tsx | 2 +
docs/src/sections/home/HomeHeroSection.tsx | 115 +
.../HomePytestDemo/HomePytestDemo.module.scss | 633 +
.../sections/home/HomePytestDemo/index.tsx | 493 +
.../src/sections/home/HomeSection.module.scss | 180 +
.../home/JudgeCards/JudgeCards.module.scss | 212 +
docs/src/sections/home/JudgeCards/index.tsx | 151 +
.../RepoContributors.module.scss | 73 +
.../sections/home/RepoContributors/index.tsx | 77 +
.../home/SOTACards/SOTACards.module.scss | 325 +
docs/src/sections/home/SOTACards/index.tsx | 238 +
.../TraceLoopConnector.module.scss | 76 +
.../home/TraceLoopConnector/index.tsx | 25 +
.../VibeCodingLoop/VibeCodingLoop.module.scss | 332 +
.../sections/home/VibeCodingLoop/index.tsx | 213 +
docs/src/utils/html-to-markdown.ts | 158 +
docs/src/utils/outbound-link-rel.ts | 9 +
docs/src/utils/schema-helpers.ts | 178 +
docs/src/utils/utm.ts | 124 +
docs/src/utils/visitor-attribution.ts | 130 +
docs/tsconfig.json | 36 +
docs/vercel.json | 120 +
docs/yarn.lock | 4029 +++
.../community/chatbot_evaluation/dataset.json | 14 +
.../community/chatbot_evaluation/evaluate.py | 32 +
.../chatbot_evaluation/generate_outputs.js | 27 +
.../community/chatbot_evaluation/outputs.json | 14 +
examples/create_tests.py | 12 +
.../dag-examples/conversational_dag.ipynb | 294 +
examples/getting_started/test_example.py | 39 +
.../mcp_evaluation/mcp_eval_multi_turn.py | 176 +
.../mcp_evaluation/mcp_eval_single_turn.py | 131 +
examples/notebooks/crewai.ipynb | 261 +
examples/notebooks/langgraph.ipynb | 598 +
examples/notebooks/openai.ipynb | 235 +
examples/notebooks/pydantic_ai.ipynb | 384 +
examples/notebooks/static/manual.txt | 107 +
examples/notebooks/static/output.png | Bin 0 -> 14809 bytes
.../rag_evaluation_with_qdrant.py | 163 +
examples/sample.txt | 56 +
examples/tracing/crewai_tracing.ipynb | 154 +
examples/tracing/test_chatbot.py | 92 +
manual_after_evals_iterator.py | 222 +
poetry.lock | 11260 ++++++++
pyproject.toml | 128 +
scripts/check_openai_model_capabilities.py | 177 +
scripts/compile_metric_templates.py | 134 +
skills/README.md | 63 +
skills/deepeval-otel/LICENSE | 4 +
skills/deepeval-otel/SKILL.md | 131 +
.../references/endpoint-and-exporter.md | 187 +
.../references/gen-ai-fallbacks.md | 41 +
.../references/span-attributes.md | 137 +
.../references/trace-attributes.md | 52 +
.../templates/confident_otel_setup.py | 115 +
skills/deepeval-tracing/LICENSE | 4 +
skills/deepeval-tracing/SKILL.md | 99 +
.../references/integrations.md | 75 +
skills/deepeval-tracing/references/tracing.md | 209 +
skills/deepeval/LICENSE | 4 +
skills/deepeval/SKILL.md | 178 +
.../deepeval/references/artifact-contracts.md | 77 +
skills/deepeval/references/choose-use-case.md | 46 +
skills/deepeval/references/confident-ai.md | 133 +
skills/deepeval/references/datasets.md | 84 +
skills/deepeval/references/intake.md | 131 +
skills/deepeval/references/iteration-loop.md | 117 +
skills/deepeval/references/metrics.md | 175 +
.../deepeval/references/pytest-e2e-evals.md | 187 +
skills/deepeval/references/synthetic-data.md | 281 +
skills/deepeval/references/traced-evals.md | 80 +
skills/deepeval/templates/metrics.py | 38 +
.../deepeval/templates/test_multi_turn_e2e.py | 28 +
.../templates/test_single_turn_no_tracing.py | 32 +
.../templates/test_single_turn_tracing.py | 21 +
test_agentcore_agent.py | 75 +
test_pydantic_agent.py | 61 +
tests/__init__.py | 0
tests/test_agent_loop_detection.py | 372 +
tests/test_confident/goldens.json | 135 +
tests/test_confident/goldens_multi_turn.json | 235 +
.../simulator/example_simulator.py | 59 +
tests/test_confident/test_annotation.py | 227 +
tests/test_confident/test_compare.py | 120 +
.../test_conversational_g_eval_upload.py | 109 +
tests/test_confident/test_dataset.py | 239 +
tests/test_confident/test_evaluate.py | 33 +
tests/test_confident/test_g_eval_upload.py | 109 +
tests/test_confident/test_prompt.py | 1485 +
.../test_region_autodetect_request_routing.py | 135 +
tests/test_core/__init__.py | 0
tests/test_core/conftest.py | 211 +
tests/test_core/helpers.py | 77 +
tests/test_core/stubs.py | 719 +
tests/test_core/test_cli/__init__.py | 0
tests/test_core/test_cli/test_auth_command.py | 44 +
.../test_cli/test_check_if_valid_file.py | 21 +
tests/test_core/test_cli/test_cli.py | 1055 +
tests/test_core/test_config/__init__.py | 0
.../test_deprecated_computed_aliases.py | 123 +
tests/test_core/test_config/test_settings.py | 518 +
tests/test_core/test_core.py | 25 +
.../test_core/test_dataset_test_run_tracer.py | 59 +
.../test_core/test_datasets/convo_goldens.csv | 16 +
.../test_datasets/convo_goldens.json | 182 +
tests/test_core/test_datasets/goldens.csv | 16 +
tests/test_core/test_datasets/goldens.json | 229 +
tests/test_core/test_datasets/test_dataset.py | 589 +
tests/test_core/test_drop_trace_and_span.py | 152 +
.../test_async_trace_metric_isolation.py | 149 +
.../test_evaluation/test_console_report.py | 255 +
.../test_end_to_end/__init__.py | 0
.../test_end_to_end/test_configs.py | 164 +
.../test_end_to_end/test_skip_reset.py | 339 +
.../test_execute/test_error_boundary.py | 862 +
.../test_execute_conversational_test_case.py | 247 +
.../test_execute_llm_test_case.py | 216 +
.../test_execute_mllm_test_case.py | 216 +
.../test_execute/test_execute_timeouts.py | 265 +
.../test_evaluation/test_local_store.py | 285 +
.../test_evaluation/test_printing.py | 89 +
.../test_results_extraction.py | 31 +
.../test_trace_results_extraction.py | 121 +
.../test_trace_scope_assert_test.py | 181 +
tests/test_core/test_imports.py | 393 +
.../test_models/test_amazon_bedrock_model.py | 247 +
.../test_models/test_anthropic_model.py | 313 +
.../test_core/test_models/test_azure_model.py | 668 +
.../test_models/test_azure_retry_config.py | 97 +
.../test_models/test_bedrock_retry_config.py | 135 +
.../test_models/test_deepseek_model.py | 233 +
.../test_embedding_models/__init__.py | 0
.../test_azure_embedding_model.py | 180 +
.../test_local_embedding_model.py | 99 +
.../test_ollama_embedding_model.py | 125 +
.../test_openai_embedding_model.py | 92 +
.../test_models/test_gemini_model.py | 458 +
.../test_core/test_models/test_grok_model.py | 138 +
.../test_core/test_models/test_kimi_model.py | 176 +
.../test_models/test_litellm_model.py | 177 +
.../test_core/test_models/test_local_model.py | 171 +
.../test_models/test_models_utils.py | 238 +
.../test_models/test_ollama_model.py | 110 +
.../test_models/test_openai_extractors.py | 32 +
.../test_models/test_openai_model.py | 896 +
.../test_models/test_openai_retry_policy.py | 116 +
.../test_models/test_openrouter_model.py | 401 +
.../test_models/test_portkey_model.py | 234 +
.../test_optimization/test_copro/__init__.py | 0
.../test_copro/test_configs.py | 34 +
.../test_optimization/test_copro/test_loop.py | 145 +
.../test_optimization/test_gepa/__init__.py | 0
.../test_gepa/test_gepa_config.py | 91 +
.../test_optimization/test_gepa/test_loop.py | 441 +
.../test_miprov2/test_report_contract.py | 124 +
.../test_mutations/__init__.py | 0
.../test_mutations/test_prompt_rewriter.py | 73 +
.../test_optimization/test_pareto.py | 73 +
.../test_policies/test_tie_breaker.py | 166 +
.../test_prompt_optimizer.py | 168 +
.../test_optimization/test_simba/__init__.py | 0
.../test_simba/test_configs.py | 42 +
.../test_optimization/test_simba/test_loop.py | 189 +
.../test_core/test_optimization/test_utils.py | 290 +
.../test_prompts/test_interpolation.py | 713 +
tests/test_core/test_prompts/test_load.py | 249 +
tests/test_core/test_prompts/test_prompt.py | 63 +
tests/test_core/test_retry_policy.py | 642 +
tests/test_core/test_run/__init__.py | 0
tests/test_core/test_run/test_file_sync.py | 88 +
tests/test_core/test_run/test_run_manager.py | 152 +
tests/test_core/test_run/test_turns_table.py | 67 +
tests/test_core/test_sanitize_nan.py | 227 +
tests/test_core/test_simulator/__init__.py | 1 +
tests/test_core/test_simulator/helpers.py | 96 +
.../test_conversation_simulator.py | 301 +
.../test_conversation_simulator_controller.py | 191 +
...t_conversation_simulator_decision_graph.py | 409 +
.../test_conversation_simulator_json_mode.py | 56 +
tests/test_core/test_stubs_contract.py | 167 +
tests/test_core/test_synthesizer/__init__.py | 0
.../test_synthesizer/example_simulator.py | 59 +
.../synthesizer_data/docx_example.docx | Bin 0 -> 10032 bytes
.../synthesizer_data/pdf_example.pdf | Bin 0 -> 2692290 bytes
.../synthesizer_data/txt_example.txt | 32 +
.../test_context_generator.py | 629 +
.../test_synthesizer/test_doc_chunker.py | 327 +
.../test_generate_from_goldens.py | 64 +
.../test_synthesizer/test_synthesizer.py | 479 +
tests/test_core/test_synthesizer_bugs.py | 590 +
tests/test_core/test_telemetry.py | 28 +
tests/test_core/test_test_case/__init__.py | 0
.../test_test_case/test_deprecated_params.py | 51 +
.../test_multi_turn/__init__.py | 0
.../test_conversational_test_case.py | 441 +
.../test_multi_turn/test_turn.py | 471 +
.../test_multi_turn/test_utils.py | 73 +
.../test_test_case/test_single_turn.py | 835 +
tests/test_core/test_threadpool_tracing.py | 74 +
tests/test_core/test_trace_memory_leak.py | 81 +
tests/test_core/test_tracing/__init__.py | 0
tests/test_core/test_tracing/apps/__init__.py | 1 +
.../test_core/test_tracing/apps/async_app.py | 117 +
tests/test_core/test_tracing/apps/sync_app.py | 284 +
tests/test_core/test_tracing/conftest.py | 113 +
.../test_tracing/example_e2e_trace_evals.py | 43 +
.../async_streaming_concurrent_schema.json | 27 +
.../async_streaming_llm_schema.json | 30 +
.../async_streaming_nested_schema.json | 40 +
.../async_streaming_processor_schema.json | 27 +
.../async_streaming_updates_schema.json | 30 +
..._safety_async_full_consumption_schema.json | 27 +
..._safety_async_gen_yields_inner_schema.json | 39 +
...xt_safety_async_nested_consume_schema.json | 79 +
...ntext_safety_async_three_level_schema.json | 51 +
...t_safety_sync_full_consumption_schema.json | 39 +
...afety_sync_gen_observe_between_schema.json | 78 +
...t_safety_sync_gen_yields_inner_schema.json | 53 +
...ext_safety_sync_nested_consume_schema.json | 85 +
...ty_sync_observe_between_yields_schema.json | 74 +
.../context_safety_sync_siblings_schema.json | 110 +
...ontext_safety_sync_three_level_schema.json | 64 +
.../fastapi_basic_threadpool_schema.json | 29 +
...fastapi_child_spans_threadpool_schema.json | 42 +
...astapi_deep_nesting_threadpool_schema.json | 99 +
...astapi_rag_pipeline_threadpool_schema.json | 66 +
.../fastapi_same_thread_sanity_schema.json | 66 +
.../generators/sync_streaming_llm_schema.json | 32 +
.../sync_streaming_nested_schema.json | 42 +
.../sync_streaming_processor_schema.json | 29 +
.../sync_streaming_updates_schema.json | 32 +
.../masking/comprehensive_masked_schema.json | 51 +
.../masking/credit_card_masked_schema.json | 29 +
.../schemas/masking/email_masked_schema.json | 29 +
.../schemas/masking/no_masking_schema.json | 29 +
.../metadata/agent_with_metadata_schema.json | 36 +
.../metadata/llm_with_metadata_schema.json | 34 +
.../metadata/span_basic_metadata_schema.json | 34 +
.../span_complex_metadata_schema.json | 44 +
.../metadata/trace_basic_metadata_schema.json | 34 +
.../metadata/trace_full_context_schema.json | 40 +
.../metadata/trace_nested_spans_schema.json | 45 +
.../metadata/trace_user_info_schema.json | 35 +
.../nested_spans/agent_workflow_schema.json | 61 +
.../nested_spans/async_nesting_schema.json | 60 +
.../nested_spans/deep_nesting_schema.json | 55 +
.../multiple_children_schema.json | 68 +
.../nested_spans/simple_nesting_schema.json | 42 +
.../span_types/agent_custom_name_schema.json | 33 +
.../agent_full_attributes_schema.json | 38 +
.../span_types/agent_minimal_schema.json | 31 +
.../agent_multiple_handoffs_schema.json | 35 +
.../schemas/span_types/agent_span_schema.json | 34 +
.../agent_with_handoffs_schema.json | 37 +
.../span_types/custom_processor_schema.json | 29 +
.../span_types/custom_validator_schema.json | 29 +
.../span_types/custom_with_name_schema.json | 41 +
.../span_types/default_span_schema.json | 29 +
.../span_types/llm_custom_name_schema.json | 30 +
.../llm_full_attributes_schema.json | 34 +
.../span_types/llm_minimal_schema.json | 29 +
.../span_types/llm_runtime_model_schema.json | 32 +
.../schemas/span_types/llm_span_schema.json | 32 +
.../span_types/llm_with_costs_schema.json | 34 +
.../retriever_custom_embedder_schema.json | 36 +
.../retriever_custom_name_schema.json | 34 +
.../retriever_full_attributes_schema.json | 40 +
.../span_types/retriever_minimal_schema.json | 33 +
.../retriever_override_embedder_schema.json | 34 +
.../span_types/retriever_span_schema.json | 42 +
.../span_with_only_name_schema.json | 29 +
.../span_types/tool_calculator_schema.json | 30 +
.../span_types/tool_custom_name_schema.json | 29 +
.../tool_description_and_name_schema.json | 36 +
.../tool_long_description_schema.json | 30 +
.../span_types/tool_minimal_schema.json | 29 +
.../schemas/span_types/tool_span_schema.json | 30 +
.../schemas/tags/basic_tags_schema.json | 34 +
.../schemas/tags/env_tags_schema.json | 36 +
.../schemas/tags/feature_tags_schema.json | 42 +
.../schemas/tags/name_and_tags_schema.json | 35 +
.../update_functions/span_context_schema.json | 37 +
.../span_expected_output_schema.json | 30 +
.../span_from_test_case_schema.json | 31 +
.../span_input_output_schema.json | 27 +
.../update_functions/span_name_schema.json | 29 +
.../span_override_test_case_schema.json | 28 +
.../update_functions/span_tools_schema.json | 48 +
.../trace_context_info_schema.json | 37 +
.../trace_from_test_case_schema.json | 31 +
.../trace_full_context_schema.json | 38 +
.../trace_identifiers_schema.json | 31 +
.../update_functions/trace_name_schema.json | 30 +
.../trace_nested_updates_schema.json | 47 +
.../update_functions/trace_tools_schema.json | 43 +
.../test_configure_multiple.py | 93 +
.../test_configuration/test_environment.py | 138 +
.../test_configuration/test_masking_config.py | 189 +
.../test_configuration/test_sampling_rate.py | 120 +
.../test_generators/test_async_generator.py | 121 +
.../test_fastapi_streaming_repro.py | 350 +
.../test_generator_context_safety.py | 544 +
.../test_generators/test_sync_generator.py | 83 +
.../test_current_golden_context.py | 133 +
.../test_integration/test_dataset_iterator.py | 372 +
.../test_execute_integration.py | 285 +
.../test_integration/test_tools_called.py | 37 +
.../test_tracing/test_masking/test_masking.py | 102 +
.../test_metadata/test_span_metadata.py | 75 +
.../test_metadata/test_trace_metadata.py | 72 +
.../test_nested_spans/test_nested_spans.py | 113 +
.../test_span_types/test_agent_span.py | 67 +
.../test_span_types/test_custom_span.py | 50 +
.../test_span_types/test_llm_span.py | 88 +
.../test_span_types/test_retriever_span.py | 77 +
.../test_span_types/test_tool_span.py | 62 +
.../test_tracing/test_tags/test_trace_tags.py | 50 +
.../test_update_current_span.py | 100 +
.../test_update_current_trace.py | 106 +
tests/test_core/test_trim_and_load_json.py | 39 +
tests/test_core/test_utils.py | 222 +
.../test_integrations/test_anthropic.py | 55 +
.../test_integrations/test_crewai.py | 59 +
.../test_integrations/test_langchain.py | 27 +
.../test_integrations/test_langgraph.py | 55 +
.../test_integrations/test_litellm.py | 53 +
.../test_integrations/test_openai.py | 51 +
.../test_integrations/test_openai_agents.py | 8 +
.../test_integrations/test_opentelemetry.py | 1 +
.../test_tracing_features/test_attributes.py | 34 +
.../test_tracing_features/test_environment.py | 19 +
.../test_input_output.py | 45 +
.../test_tracing_features/test_masking.py | 20 +
.../test_tracing_features/test_metadata.py | 20 +
.../test_tracing_features/test_name.py | 41 +
.../test_tracing_features/test_sampling.py | 20 +
.../test_tracing_features/test_span_types.py | 129 +
.../test_tracing_features/test_tags.py | 20 +
.../test_tracing_features/test_test_case.py | 49 +
.../test_tracing_features/test_threads.py | 51 +
.../test_tracing_features/test_users.py | 21 +
.../test_crewai/test_crewai_e2e.py | 78 +
.../test_ai_agent_evals/test_e2e_crewai.py | 28 +
.../test_ai_agent_evals/test_e2e_langchain.py | 21 +
.../test_ai_agent_evals/test_e2e_langraph.py | 25 +
.../test_langchain_e2e_async.py | 56 +
.../test_langchain/test_langchain_e2e_sync.py | 60 +
.../test_langchain_online_evals.py | 41 +
.../test_langchain/test_langchain_setup.py | 43 +
.../test_e2e_langgraph_async.py | 41 +
.../test_langgraph/test_e2e_langgraph_sync.py | 37 +
.../test_langgraph_component.py | 103 +
.../test_langgraph/test_langgraph_online.py | 30 +
.../test_langgraph/test_langgraph_setup.py | 28 +
.../test_llamaindex_e2e_async.py | 45 +
.../test_openai_agents/test_agent.py | 12 +
.../test_ai_agent_evals/test_overall.py | 72 +
.../test_pydanticai_component_level.py | 106 +
.../test_pydanticai_e2e_async.py | 27 +
.../test_pydanticai/test_pydanticai_online.py | 18 +
.../test_pydanticai/test_pydanticai_setup.py | 26 +
.../test_pydanticai_trace_attr.py | 22 +
.../test_ai_agent_evals/test_setup_crewai.py | 22 +
.../test_setup_end_to_end_python.py | 26 +
.../test_setup_langchain.py | 14 +
.../test_setup_langraph.py | 21 +
.../test_setup_llm_tracing.py | 15 +
.../test_component_level_evals.py | 77 +
.../test_llm_evals/test_setup_tracing.py | 95 +
tests/test_integrations/__init__.py | 0
.../test_agentcore/apps/agentcore_eval_app.py | 94 +
.../apps/agentcore_multiple_tools_app.py | 92 +
.../apps/agentcore_simple_app.py | 62 +
.../test_agentcore/apps/agentcore_tool_app.py | 75 +
.../test_agentcore/conftest.py | 61 +
...agentcore_async_parallel_tools_schema.json | 132 +
.../agentcore_async_simple_schema.json | 68 +
.../schemas/agentcore_async_tool_schema.json | 115 +
.../schemas/agentcore_features_async.json | 120 +
.../schemas/agentcore_features_sync.json | 120 +
.../agentcore_multiple_tools_time_schema.json | 115 +
...entcore_multiple_tools_weather_schema.json | 115 +
.../agentcore_parallel_tools_schema.json | 131 +
.../schemas/agentcore_simple_schema.json | 67 +
...entcore_tool_metric_collection_schema.json | 116 +
.../schemas/agentcore_tool_schema.json | 114 +
.../test_agentcore/test_async.py | 143 +
.../test_agentcore/test_evaluate_agent.py | 84 +
.../test_agentcore/test_span_interceptor.py | 719 +
.../test_agentcore/test_sync.py | 208 +
.../test_anthropic/conftest.py | 14 +
.../test_anthropic/simple_anthropic.py | 38 +
.../test_anthropic/test_async_anthropic.py | 54 +
...test_async_messages_create_with_trace.json | 44 +
.../test_anthropic/test_sync_anthropic.py | 54 +
...st_sync_messages_create_without_trace.json | 31 +
.../test_crewai/apps/async_app.py | 32 +
.../test_crewai/apps/evals_app.py | 58 +
.../test_crewai/apps/hierarchical_app.py | 34 +
.../apps/knowledge_retriever_app.py | 39 +
.../test_crewai/apps/multi_agent_app.py | 47 +
.../test_crewai/apps/simple_app.py | 40 +
.../test_crewai/apps/tool_usage_app.py | 42 +
.../test_integrations/test_crewai/conftest.py | 32 +
.../test_integrations/test_crewai/crewai.json | 149 +
.../test_crewai/crewai_component.json | 164 +
.../test_crewai/knowledge_retriever.json | 116 +
.../test_crewai/schemas/crewai_akickoff.json | 74 +
.../schemas/crewai_async_kickoff.json | 83 +
.../schemas/crewai_async_tool_usage.json | 161 +
.../schemas/crewai_features_async.json | 189 +
.../schemas/crewai_features_sync.json | 196 +
.../schemas/crewai_hierarchical.json | 188 +
.../schemas/crewai_kickoff_for_each.json | 139 +
.../crewai_kickoff_for_each_async.json | 163 +
.../schemas/crewai_knowledge_retrieval.json | 112 +
.../crewai_multi_agent_sequential.json | 106 +
.../schemas/crewai_simple_kickoff.json | 74 +
.../schemas/crewai_tool_usage.json | 149 +
.../test_crewai/test_async.py | 134 +
.../test_crewai/test_crewai.py | 102 +
.../test_crewai/test_crewai_component.py | 118 +
.../test_crewai/test_knowledge_retriever.py | 55 +
.../test_crewai/test_stress.py | 53 +
.../test_crewai/test_sync.py | 151 +
.../test_exporter/readable_spans.py | 271 +
.../test_exporter/test_pydantic_ai.py | 104 +
.../test_googleadk/__init__.py | 0
.../test_googleadk/apps/__init__.py | 0
.../test_googleadk/apps/googleadk_eval_app.py | 121 +
.../apps/googleadk_multiple_tools_app.py | 136 +
.../apps/googleadk_simple_app.py | 93 +
.../test_googleadk/apps/googleadk_tool_app.py | 115 +
.../test_googleadk/conftest.py | 43 +
.../test_googleadk/schemas/README.md | 54 +
...googleadk_async_parallel_tools_schema.json | 151 +
.../googleadk_async_simple_schema.json | 70 +
.../schemas/googleadk_async_tool_schema.json | 119 +
.../schemas/googleadk_features_async.json | 119 +
.../schemas/googleadk_features_sync.json | 118 +
.../googleadk_multiple_tools_time_schema.json | 115 +
...ogleadk_multiple_tools_weather_schema.json | 115 +
.../googleadk_parallel_tools_schema.json | 150 +
.../schemas/googleadk_simple_schema.json | 69 +
...ogleadk_tool_metric_collection_schema.json | 119 +
.../schemas/googleadk_tool_schema.json | 118 +
.../test_googleadk/test_async.py | 143 +
.../test_googleadk/test_evaluate_agent.py | 84 +
.../test_googleadk/test_span_interceptor.py | 1017 +
.../test_googleadk/test_sync.py | 211 +
.../test_langchain/__init__.py | 0
.../test_langchain/apps/__init__.py | 1 +
.../apps/langchain_agent_app.py | 252 +
.../apps/langchain_conditional_app.py | 277 +
.../apps/langchain_metric_collection_app.py | 66 +
.../apps/langchain_multiple_tools_app.py | 233 +
.../apps/langchain_next_span_app.py | 86 +
.../apps/langchain_parallel_tools_app.py | 264 +
.../apps/langchain_retriever_app.py | 240 +
.../apps/langchain_simple_app.py | 43 +
.../apps/langchain_single_tool_app.py | 144 +
.../apps/langchain_streaming_app.py | 327 +
.../test_langchain/conftest.py | 456 +
.../test_langchain/langchain.json | 723 +
.../langchain_agent_complex_schema.json | 428 +
.../langchain_agent_multi_step_schema.json | 432 +
.../langchain_agent_simple_schema.json | 427 +
.../langchain_async_agent_complex_schema.json | 429 +
...ngchain_async_agent_multi_step_schema.json | 433 +
.../langchain_async_agent_simple_schema.json | 428 +
...n_async_conditional_fact_check_schema.json | 425 +
...hain_async_conditional_general_schema.json | 189 +
...ain_async_conditional_research_schema.json | 428 +
...in_async_conditional_summarize_schema.json | 425 +
.../langchain_async_mixed_tools_schema.json | 432 +
...langchain_async_multiple_tools_schema.json | 445 +
.../langchain_async_next_llm_span_schema.json | 718 +
...langchain_async_parallel_mixed_schema.json | 449 +
...angchain_async_parallel_stocks_schema.json | 425 +
...ngchain_async_parallel_weather_schema.json | 622 +
...hain_async_retriever_langchain_schema.json | 248 +
...ngchain_async_retriever_python_schema.json | 251 +
.../langchain_async_simple_schema.json | 189 +
.../langchain_async_single_tool_schema.json | 429 +
...angchain_async_streaming_multi_schema.json | 388 +
.../langchain_async_streaming_schema.json | 383 +
...ngchain_conditional_fact_check_schema.json | 424 +
.../langchain_conditional_general_schema.json | 188 +
...langchain_conditional_research_schema.json | 427 +
...angchain_conditional_summarize_schema.json | 424 +
.../langchain_metric_collection_schema.json | 722 +
...langchain_multiple_tools_mixed_schema.json | 434 +
.../langchain_multiple_tools_schema.json | 444 +
.../langchain_next_llm_span_schema.json | 717 +
.../langchain_parallel_mixed_schema.json | 448 +
.../langchain_parallel_stocks_schema.json | 424 +
.../langchain_parallel_weather_schema.json | 621 +
.../langchain_retriever_langchain_schema.json | 247 +
...in_retriever_metric_collection_schema.json | 251 +
.../langchain_retriever_python_schema.json | 250 +
.../schemas/langchain_simple_schema.json | 188 +
.../schemas/langchain_single_tool_schema.json | 428 +
.../langchain_streaming_multi_schema.json | 387 +
.../schemas/langchain_streaming_schema.json | 382 +
.../test_langchain/test_async.py | 604 +
.../test_langchain/test_langchain.py | 51 +
.../test_langchain/test_next_span.py | 460 +
.../test_langchain/test_sync.py | 653 +
.../test_langgraph/__init__.py | 0
.../test_langgraph/apps/__init__.py | 0
.../apps/langgraph_async_app.py | 84 +
.../apps/langgraph_conditional_app.py | 191 +
.../apps/langgraph_metric_collection_app.py | 81 +
.../apps/langgraph_multi_turn_app.py | 172 +
.../apps/langgraph_multiple_tools_app.py | 112 +
.../apps/langgraph_next_span_app.py | 100 +
.../apps/langgraph_parallel_tools_app.py | 164 +
.../apps/langgraph_retriever_app.py | 179 +
.../apps/langgraph_simple_app.py | 68 +
.../apps/langgraph_streaming_app.py | 105 +
.../test_langgraph/apps/main.py | 236 +
.../test_langgraph/conftest.py | 692 +
.../langgraph_async_conditional_schema.json | 1132 +
.../langgraph_async_multi_turn_schema.json | 1884 ++
...langgraph_async_multiple_tools_schema.json | 1806 ++
.../langgraph_async_next_llm_span_schema.json | 966 +
.../langgraph_async_no_tools_schema.json | 341 +
...langgraph_async_parallel_heavy_schema.json | 2717 ++
.../langgraph_async_parallel_schema.json | 1340 +
.../langgraph_async_single_tool_schema.json | 969 +
...anggraph_async_streaming_multi_schema.json | 999 +
.../langgraph_async_streaming_schema.json | 786 +
...nggraph_conditional_fact_check_schema.json | 1132 +
.../langgraph_conditional_general_schema.json | 405 +
...langgraph_conditional_research_schema.json | 1135 +
...anggraph_conditional_summarize_schema.json | 1132 +
.../schemas/langgraph_full_flow_schema.json | 3591 +++
.../langgraph_metric_collection_schema.json | 970 +
.../schemas/langgraph_multi_turn_schema.json | 2743 ++
...langgraph_multiple_tools_mixed_schema.json | 1154 +
.../langgraph_multiple_tools_schema.json | 1328 +
.../langgraph_next_llm_span_schema.json | 965 +
.../langgraph_parallel_mixed_schema.json | 1523 +
.../langgraph_parallel_stocks_schema.json | 1684 ++
.../langgraph_parallel_weather_schema.json | 1343 +
.../langgraph_retriever_langchain_schema.json | 380 +
...ph_retriever_metric_collection_schema.json | 384 +
.../langgraph_retriever_python_schema.json | 383 +
.../schemas/langgraph_simple_schema.json | 960 +
.../schemas/langgraph_stateless_schema.json | 1014 +
.../langgraph_streaming_multi_schema.json | 1004 +
.../schemas/langgraph_streaming_schema.json | 783 +
.../test_langgraph/test_async.py | 390 +
.../test_langgraph/test_create_task.py | 545 +
.../test_langgraph/test_next_span.py | 282 +
.../test_langgraph/test_sync.py | 694 +
.../test_llamaindex/apps/agent_app.py | 45 +
.../test_llamaindex/apps/eval_app.py | 25 +
.../test_llamaindex/apps/rag_app.py | 58 +
.../test_llamaindex/apps/router_app.py | 46 +
.../test_llamaindex/apps/simple_app.py | 26 +
.../test_llamaindex/conftest.py | 30 +
.../llama_index_async_agent_math_schema.json | 762 +
.../llama_index_async_agent_schema.json | 747 +
.../schemas/llama_index_async_rag_schema.json | 416 +
.../llama_index_async_router_schema.json | 253 +
.../llama_index_async_simple_schema.json | 3518 +++
.../schemas/llama_index_features_async.json | 1156 +
.../schemas/llama_index_rag_llama_schema.json | 435 +
.../llama_index_rag_python_schema.json | 435 +
.../llama_index_router_math_schema.json | 252 +
.../schemas/llama_index_simple_schema.json | 9684 +++++++
.../test_llamaindex/test_async.py | 224 +
.../test_llamaindex/test_sync.py | 127 +
.../test_integrations/test_openai/conftest.py | 13 +
.../test_openai/simple_openai.py | 37 +
.../test_openai/test_async_openai.py | 86 +
.../test_async_openai_without_trace.json | 35 +
...test_async_response_create_with_trace.json | 53 +
.../test_openai/test_sync_openai.py | 85 +
.../test_sync_openai_with_trace.json | 49 +
...st_sync_response_create_without_trace.json | 39 +
.../test_tool_call_flow_completion.json | 155 +
.../test_tool_call_flow_completion.py | 141 +
.../test_tool_call_flow_response.json | 137 +
.../test_tool_call_flow_response.py | 126 +
.../test_openai_agents/apps/eval_agent.py | 17 +
.../test_openai_agents/apps/handoff_agent.py | 28 +
.../test_openai_agents/apps/session_agent.py | 20 +
.../test_openai_agents/apps/simple_agent.py | 13 +
.../apps/streaming_agent.py | 28 +
.../test_openai_agents/apps/tool_agent.py | 45 +
.../test_openai_agents/conftest.py | 23 +
.../openai_agents_async_handoff_schema.json | 233 +
.../openai_agents_async_simple_schema.json | 97 +
.../openai_agents_async_tool_schema.json | 265 +
.../schemas/openai_agents_eval_schema.json | 99 +
.../openai_agents_handoff_spanish_schema.json | 232 +
.../schemas/openai_agents_session_schema.json | 142 +
.../schemas/openai_agents_simple_schema.json | 98 +
.../openai_agents_tool_math_schema.json | 265 +
.../openai_agents_tool_weather_schema.json | 264 +
.../test_openai_agents/test_async.py | 72 +
.../test_scenerios/multi_agents.json | 1 +
.../test_scenerios/run.json | 1 +
.../test_scenerios/run_streamed.json | 1 +
.../test_scenerios/test_multi_agents.py | 70 +
.../test_scenerios/test_run.py | 134 +
.../test_scenerios/test_run_streamed.py | 137 +
.../test_scenerios/test_run_sync.py | 129 +
.../test_weather_agent_patched.py | 152 +
.../test_scenerios/test_with_trace.py | 143 +
.../test_with_trace_and_wrapped.py | 163 +
.../test_scenerios/weather_agent_patched.json | 1 +
.../test_scenerios/with_trace.json | 1 +
.../with_trace_and_wrapped.json | 1 +
.../test_openai_agents/test_sync.py | 128 +
.../test_openrouter_generation.py | 84 +
.../test_pydanticai/apps/__init__.py | 0
.../test_pydanticai/apps/eval_app.py | 87 +
.../apps/pydanticai_isolation_app.py | 167 +
.../apps/pydanticai_metric_collection_app.py | 58 +
.../apps/pydanticai_modes_app.py | 186 +
.../apps/pydanticai_multiple_tools_app.py | 98 +
.../apps/pydanticai_next_span_app.py | 92 +
.../apps/pydanticai_simple_app.py | 49 +
.../apps/pydanticai_streaming_app.py | 46 +
.../apps/pydanticai_tool_app.py | 80 +
.../test_pydanticai/conftest.py | 61 +
...ydanticai_async_parallel_tools_schema.json | 337 +
.../pydanticai_async_simple_schema.json | 95 +
.../schemas/pydanticai_async_tool_schema.json | 260 +
...ydanticai_bare_tool_enrichment_schema.json | 241 +
.../schemas/pydanticai_features_async.json | 239 +
.../schemas/pydanticai_features_sync.json | 239 +
...pydanticai_multiple_tools_time_schema.json | 296 +
...anticai_multiple_tools_weather_schema.json | 296 +
.../pydanticai_next_llm_only_schema.json | 99 +
.../pydanticai_next_stacked_schema.json | 104 +
.../pydanticai_observe_mode_schema.json | 112 +
.../pydanticai_parallel_tools_schema.json | 336 +
.../schemas/pydanticai_simple_schema.json | 94 +
.../schemas/pydanticai_streaming_schema.json | 94 +
.../schemas/pydanticai_tool_schema.json | 259 +
...nticai_trace_metric_collection_schema.json | 95 +
.../pydanticai_with_trace_mode_schema.json | 96 +
.../test_pydanticai/test_async.py | 275 +
.../test_pydanticai/test_evaluate_agent.py | 58 +
.../test_pydanticai/test_span_interceptor.py | 1269 +
.../test_pydanticai/test_sync.py | 495 +
.../test_strands/__init__.py | 0
.../test_strands/apps/__init__.py | 0
.../test_strands/apps/strands_eval_app.py | 104 +
.../apps/strands_multiple_tools_app.py | 103 +
.../test_strands/apps/strands_simple_app.py | 77 +
.../test_strands/apps/strands_tool_app.py | 92 +
.../test_strands/conftest.py | 61 +
.../test_strands/schemas/README.md | 54 +
.../strands_async_parallel_tools_schema.json | 132 +
.../schemas/strands_async_simple_schema.json | 68 +
.../schemas/strands_async_tool_schema.json | 115 +
.../schemas/strands_features_async.json | 120 +
.../schemas/strands_features_sync.json | 120 +
.../strands_multiple_tools_time_schema.json | 115 +
...strands_multiple_tools_weather_schema.json | 115 +
.../strands_parallel_tools_schema.json | 131 +
.../schemas/strands_simple_schema.json | 67 +
...strands_tool_metric_collection_schema.json | 116 +
.../schemas/strands_tool_schema.json | 114 +
.../test_strands/test_async.py | 144 +
.../test_strands/test_evaluate_agent.py | 84 +
.../test_strands/test_span_interceptor.py | 761 +
.../test_strands/test_sync.py | 205 +
tests/test_integrations/utils.py | 605 +
tests/test_metrics/images/car.png | Bin 0 -> 1468920 bytes
.../test_answer_relevancy_metric.py | 173 +
...st_answer_relevancy_metric_empty_output.py | 78 +
tests/test_metrics/test_arena_geval_metric.py | 336 +
tests/test_metrics/test_bias_metric.py | 171 +
.../test_citation_faithfulness_metric.py | 110 +
.../test_contextual_precision_metric.py | 173 +
.../test_contextual_recall_metric.py | 173 +
.../test_contextual_relevancy_metric.py | 173 +
.../test_conversation_completeness_metric.py | 175 +
tests/test_metrics/test_conversational_dag.py | 274 +
.../test_conversational_g_eval.py | 211 +
tests/test_metrics/test_dag.py | 271 +
tests/test_metrics/test_dag_serialization.py | 405 +
tests/test_metrics/test_exact_match_metric.py | 66 +
.../test_metrics/test_faithfulness_metric.py | 171 +
tests/test_metrics/test_g_eval_metric.py | 224 +
tests/test_metrics/test_g_eval_utils.py | 135 +
.../test_metrics/test_goal_accuracy_metric.py | 175 +
.../test_metrics/test_hallucination_metric.py | 173 +
.../test_image_coherence_metric.py | 101 +
.../test_metrics/test_image_editing_metric.py | 99 +
.../test_image_helpfulness_metric.py | 101 +
.../test_image_reference_metric.py | 101 +
.../test_json_correctness_metric.py | 193 +
.../test_knowledge_retention_metric.py | 244 +
.../test_mcp_task_completetion_metric.py | 187 +
tests/test_metrics/test_mcp_use_metric.py | 144 +
tests/test_metrics/test_misuse_metric.py | 181 +
.../test_multi_turn_mcp_use_metric.py | 187 +
tests/test_metrics/test_non_advice_metric.py | 187 +
.../test_metrics/test_pattern_match_metric.py | 66 +
tests/test_metrics/test_pii_lekage_metric.py | 171 +
.../test_plan_adherence_metric.py | 175 +
.../test_metrics/test_plan_quality_metric.py | 175 +
.../test_prompt_alignment_metric.py | 187 +
.../test_role_adherence_metric.py | 175 +
.../test_role_violation_metric.py | 183 +
.../test_step_efficiency_metric.py | 175 +
.../test_metrics/test_summarization_metric.py | 173 +
.../test_task_completetion_metric.py | 175 +
.../test_task_completion_tools_called.py | 154 +
.../test_metrics/test_text_to_image_metric.py | 99 +
.../test_tool_permission_metric.py | 74 +
tests/test_metrics/test_tool_use_metric.py | 213 +
.../test_topic_adherence_metric.py | 181 +
tests/test_metrics/test_toxicity_metric.py | 171 +
.../test_turn_contextual_precision_metric.py | 175 +
...ontextual_recall_async_expected_outcome.py | 74 +
.../test_turn_contextual_recall_metric.py | 180 +
.../test_turn_faithfulness_metric.py | 175 +
...turn_faithfulness_metric_empty_verdicts.py | 74 +
.../test_turn_relevancy_aggregation.py | 253 +
.../test_turn_relevancy_metric.py | 175 +
.../turn_contextual_relevancy_metric.py | 180 +
tests/test_templates/test_metric_templates.py | 59 +
typescript/.gitignore | 146 +
typescript/.prettierignore | 30 +
typescript/.prettierrc | 1 +
typescript/README.md | 30 +
typescript/eslint.config.mts | 29 +
.../annotation/example-send-annotation.ts | 7 +
.../dataset/example-custom-dataset-columns.ts | 15 +
.../examples/dataset/example-dataset.ts | 254 +
.../examples/dataset/example-e2e-multiturn.ts | 16 +
.../dataset/example-e2e-singleturn.ts | 44 +
.../dataset/example-push-multiturn.ts | 7 +
.../dataset/example-push-singleturn.ts | 5 +
.../dataset/example-queue-multiturn.ts | 7 +
.../dataset/example-queue-singleturn.ts | 5 +
.../examples/evaluate/example-evaluate.ts | 53 +
.../integrations/langchain/prompt-template.ts | 36 +
.../integrations/langchain/streaming.ts | 35 +
.../examples/integrations/langgraph/agent.ts | 134 +
.../integrations/openai/completions-parse.ts | 58 +
.../integrations/openai/completions-tools.ts | 62 +
.../integrations/openai/completions.ts | 43 +
.../integrations/openai/responses-parse.ts | 58 +
.../integrations/openai/responses-tools.ts | 60 +
.../examples/integrations/openai/responses.ts | 30 +
.../examples/prompt/example-prompt-pull.ts | 28 +
.../examples/simulate/example-simulate.ts | 77 +
.../examples/tracing/example-api-key.ts | 30 +
.../examples/tracing/example-attributes.ts | 69 +
.../examples/tracing/example-environment.ts | 21 +
.../tracing/example-input-output-span.ts | 19 +
.../tracing/example-input-output-trace.ts | 19 +
.../examples/tracing/example-masking.ts | 21 +
.../examples/tracing/example-metadata.ts | 27 +
.../tracing/example-name-span-types.ts | 61 +
.../examples/tracing/example-name-span.ts | 20 +
.../examples/tracing/example-name-trace.ts | 19 +
.../examples/tracing/example-observe.ts | 20 +
.../tracing/example-online-evaluations.ts | 27 +
.../examples/tracing/example-quickstart.ts | 37 +
.../examples/tracing/example-sampling.ts | 20 +
typescript/examples/tracing/example-tags.ts | 19 +
.../examples/tracing/example-test-case-llm.ts | 78 +
.../tracing/example-test-case-span.ts | 57 +
.../tracing/example-test-case-trace.ts | 20 +
typescript/examples/tracing/example-thread.ts | 22 +
.../examples/tracing/example-tracing.ts | 192 +
typescript/examples/tracing/example-user.ts | 18 +
typescript/jest.config.js | 6 +
typescript/package-lock.json | 10356 +++++++
typescript/package.json | 203 +
typescript/src/annotation/api.ts | 16 +
typescript/src/annotation/index.ts | 41 +
typescript/src/annotation/utils.ts | 47 +
typescript/src/cli/index.ts | 78 +
typescript/src/confident/api.ts | 284 +
typescript/src/confident/index.ts | 5 +
typescript/src/confident/types.ts | 13 +
typescript/src/config/settings.ts | 48 +
typescript/src/constants.ts | 4 +
typescript/src/dataset/api.ts | 34 +
typescript/src/dataset/dataset.ts | 676 +
typescript/src/dataset/golden.ts | 93 +
typescript/src/dataset/index.ts | 7 +
typescript/src/dataset/utils.ts | 160 +
typescript/src/errors.ts | 19 +
typescript/src/evaluate/compare.ts | 190 +
typescript/src/evaluate/confident.ts | 292 +
typescript/src/evaluate/configs.ts | 57 +
typescript/src/evaluate/console-report.ts | 470 +
typescript/src/evaluate/evaluate.ts | 288 +
typescript/src/evaluate/index.ts | 14 +
typescript/src/evaluate/trace-eval.ts | 109 +
typescript/src/evaluate/types.ts | 115 +
typescript/src/governance/index.ts | 22 +
typescript/src/index.ts | 40 +
typescript/src/integrations/ai-sdk/index.ts | 256 +
.../src/integrations/ai-sdk/processor.ts | 785 +
typescript/src/integrations/index.ts | 0
.../langchain/callback-handler.ts | 515 +
.../src/integrations/langchain/index.ts | 1 +
.../integrations/langchain/langgraph-utils.ts | 135 +
.../src/integrations/langchain/patch-tool.ts | 69 +
.../src/integrations/langchain/utils.ts | 415 +
.../integrations/openai-agents/callback.ts | 192 +
.../integrations/openai-agents/extractors.ts | 549 +
.../src/integrations/openai-agents/index.ts | 3 +
.../src/integrations/openinference/index.ts | 266 +
.../integrations/openinference/processor.ts | 539 +
typescript/src/metrics/README.md | 215 +
.../answer-relevancy/answer-relevancy.ts | 139 +
.../src/metrics/answer-relevancy/index.ts | 5 +
.../src/metrics/answer-relevancy/schema.ts | 20 +
.../src/metrics/arena-g-eval/arena-g-eval.ts | 185 +
typescript/src/metrics/arena-g-eval/index.ts | 2 +
typescript/src/metrics/arena-g-eval/schema.ts | 12 +
typescript/src/metrics/arena-g-eval/utils.ts | 99 +
.../argument-correctness.ts | 134 +
.../src/metrics/argument-correctness/index.ts | 5 +
.../metrics/argument-correctness/schema.ts | 20 +
typescript/src/metrics/base-arena-metric.ts | 21 +
.../src/metrics/base-conversational-metric.ts | 15 +
typescript/src/metrics/base-metrics.ts | 127 +
typescript/src/metrics/bias/bias.ts | 135 +
typescript/src/metrics/bias/index.ts | 2 +
typescript/src/metrics/bias/schema.ts | 18 +
.../contextual-precision.ts | 188 +
.../src/metrics/contextual-precision/index.ts | 5 +
.../metrics/contextual-precision/schema.ts | 20 +
.../contextual-recall/contextual-recall.ts | 141 +
.../src/metrics/contextual-recall/index.ts | 5 +
.../src/metrics/contextual-recall/schema.ts | 20 +
.../contextual-relevancy.ts | 159 +
.../src/metrics/contextual-relevancy/index.ts | 5 +
.../metrics/contextual-relevancy/schema.ts | 24 +
.../conversation-completeness.ts | 150 +
.../conversation-completeness/index.ts | 5 +
.../conversation-completeness/schema.ts | 20 +
.../conversational-g-eval.ts | 176 +
.../metrics/conversational-g-eval/index.ts | 4 +
.../metrics/conversational-g-eval/utils.ts | 73 +
.../src/metrics/conversational-utils.ts | 161 +
.../src/metrics/exact-match/exact-match.ts | 69 +
typescript/src/metrics/exact-match/index.ts | 1 +
.../src/metrics/faithfulness/faithfulness.ts | 177 +
typescript/src/metrics/faithfulness/index.ts | 5 +
typescript/src/metrics/faithfulness/schema.ts | 19 +
typescript/src/metrics/g-eval/g-eval.ts | 168 +
typescript/src/metrics/g-eval/index.ts | 3 +
typescript/src/metrics/g-eval/schema.ts | 10 +
typescript/src/metrics/g-eval/utils.ts | 137 +
.../metrics/goal-accuracy/goal-accuracy.ts | 175 +
typescript/src/metrics/goal-accuracy/index.ts | 5 +
.../src/metrics/goal-accuracy/schema.ts | 22 +
.../metrics/hallucination/hallucination.ts | 135 +
typescript/src/metrics/hallucination/index.ts | 5 +
.../src/metrics/hallucination/schema.ts | 16 +
typescript/src/metrics/index.ts | 159 +
.../src/metrics/json-correctness/index.ts | 5 +
.../json-correctness/json-correctness.ts | 117 +
.../src/metrics/json-correctness/schema.ts | 7 +
.../src/metrics/knowledge-retention/index.ts | 5 +
.../knowledge-retention.ts | 165 +
.../src/metrics/knowledge-retention/schema.ts | 23 +
.../src/metrics/mcp-use-metric/index.ts | 2 +
.../metrics/mcp-use-metric/mcp-use-metric.ts | 159 +
.../src/metrics/mcp-use-metric/schema.ts | 13 +
typescript/src/metrics/mcp/index.ts | 9 +
.../src/metrics/mcp/mcp-task-completion.ts | 120 +
.../src/metrics/mcp/multi-turn-mcp-use.ts | 159 +
typescript/src/metrics/mcp/schema.ts | 18 +
typescript/src/metrics/mcp/utils.ts | 165 +
typescript/src/metrics/misuse/index.ts | 2 +
typescript/src/metrics/misuse/misuse.ts | 140 +
typescript/src/metrics/misuse/schema.ts | 18 +
.../image-coherence/image-coherence.ts | 123 +
.../image-coherence/index.ts | 1 +
.../image-editing/image-editing.ts | 151 +
.../multimodal-metrics/image-editing/index.ts | 1 +
.../image-helpfulness/image-helpfulness.ts | 123 +
.../image-helpfulness/index.ts | 1 +
.../image-reference/image-reference.ts | 123 +
.../image-reference/index.ts | 1 +
.../src/metrics/multimodal-metrics/index.ts | 21 +
.../src/metrics/multimodal-metrics/schema.ts | 15 +
.../multimodal-metrics/text-to-image/index.ts | 1 +
.../text-to-image/text-to-image.ts | 146 +
.../src/metrics/multimodal-metrics/utils.ts | 51 +
typescript/src/metrics/non-advice/index.ts | 2 +
.../src/metrics/non-advice/non-advice.ts | 140 +
typescript/src/metrics/non-advice/schema.ts | 18 +
typescript/src/metrics/pattern-match/index.ts | 4 +
.../metrics/pattern-match/pattern-match.ts | 79 +
typescript/src/metrics/pii-leakage/index.ts | 2 +
.../src/metrics/pii-leakage/pii-leakage.ts | 138 +
typescript/src/metrics/pii-leakage/schema.ts | 20 +
.../src/metrics/plan-adherence/index.ts | 2 +
.../metrics/plan-adherence/plan-adherence.ts | 121 +
.../src/metrics/plan-adherence/schema.ts | 10 +
typescript/src/metrics/plan-quality/index.ts | 2 +
.../src/metrics/plan-quality/plan-quality.ts | 122 +
typescript/src/metrics/plan-quality/schema.ts | 10 +
.../src/metrics/prompt-alignment/index.ts | 5 +
.../prompt-alignment/prompt-alignment.ts | 145 +
.../src/metrics/prompt-alignment/schema.ts | 20 +
.../src/metrics/role-adherence/index.ts | 5 +
.../metrics/role-adherence/role-adherence.ts | 147 +
.../src/metrics/role-adherence/schema.ts | 19 +
.../src/metrics/role-violation/index.ts | 5 +
.../metrics/role-violation/role-violation.ts | 144 +
.../src/metrics/role-violation/schema.ts | 20 +
.../src/metrics/step-efficiency/index.ts | 2 +
.../src/metrics/step-efficiency/schema.ts | 10 +
.../step-efficiency/step-efficiency.ts | 102 +
typescript/src/metrics/summarization/index.ts | 5 +
.../src/metrics/summarization/schema.ts | 28 +
.../metrics/summarization/summarization.ts | 303 +
.../src/metrics/task-completion/index.ts | 2 +
.../src/metrics/task-completion/schema.ts | 13 +
.../task-completion/task-completion.ts | 117 +
.../src/metrics/tool-correctness/index.ts | 5 +
.../src/metrics/tool-correctness/schema.ts | 10 +
.../tool-correctness/tool-correctness.ts | 403 +
typescript/src/metrics/tool-use/index.ts | 2 +
typescript/src/metrics/tool-use/schema.ts | 29 +
typescript/src/metrics/tool-use/tool-use.ts | 209 +
.../src/metrics/topic-adherence/index.ts | 5 +
.../src/metrics/topic-adherence/schema.ts | 20 +
.../topic-adherence/topic-adherence.ts | 164 +
typescript/src/metrics/toxicity/index.ts | 2 +
typescript/src/metrics/toxicity/schema.ts | 18 +
typescript/src/metrics/toxicity/toxicity.ts | 134 +
.../turn-contextual-precision/index.ts | 5 +
.../turn-contextual-precision/schema.ts | 26 +
.../turn-contextual-precision.ts | 222 +
.../metrics/turn-contextual-recall/index.ts | 5 +
.../metrics/turn-contextual-recall/schema.ts | 26 +
.../turn-contextual-recall.ts | 221 +
.../turn-contextual-relevancy/index.ts | 5 +
.../turn-contextual-relevancy/schema.ts | 27 +
.../turn-contextual-relevancy.ts | 230 +
.../src/metrics/turn-faithfulness/index.ts | 5 +
.../src/metrics/turn-faithfulness/schema.ts | 28 +
.../turn-faithfulness/turn-faithfulness.ts | 255 +
.../src/metrics/turn-relevancy/index.ts | 5 +
.../src/metrics/turn-relevancy/schema.ts | 14 +
.../metrics/turn-relevancy/turn-relevancy.ts | 143 +
typescript/src/metrics/utils.ts | 196 +
typescript/src/models/README.md | 149 +
typescript/src/models/base-model.ts | 47 +
.../src/models/gateways/openrouter-model.ts | 32 +
.../src/models/gateways/portkey-model.ts | 38 +
typescript/src/models/index.ts | 43 +
typescript/src/models/multimodal.ts | 85 +
.../src/models/openai-compatible-model.ts | 114 +
.../src/models/providers/ai-sdk-model.ts | 97 +
.../src/models/providers/anthropic-model.ts | 96 +
.../src/models/providers/azure-model.ts | 59 +
.../src/models/providers/bedrock-model.ts | 121 +
.../src/models/providers/deepseek-model.ts | 28 +
.../src/models/providers/gemini-model.ts | 109 +
typescript/src/models/providers/grok-model.ts | 28 +
typescript/src/models/providers/kimi-model.ts | 28 +
.../src/models/providers/local-model.ts | 40 +
.../src/models/providers/ollama-model.ts | 87 +
.../src/models/providers/openai-model.ts | 26 +
typescript/src/models/utils.ts | 64 +
typescript/src/openai/extractor.ts | 200 +
typescript/src/openai/index.ts | 14 +
typescript/src/openai/patch.ts | 231 +
typescript/src/openai/types.ts | 17 +
typescript/src/openai/utils.ts | 198 +
typescript/src/prompt/index.ts | 624 +
typescript/src/prompt/types.ts | 227 +
typescript/src/prompt/utils.ts | 283 +
typescript/src/simulate/index.ts | 221 +
typescript/src/telemetry.ts | 336 +
typescript/src/telemetry.txt | 4 +
typescript/src/templates/index.ts | 8 +
.../src/templates/metrics/templates.json | 225 +
typescript/src/templates/resolver.ts | 198 +
typescript/src/test-case/arena-test-case.ts | 55 +
.../src/test-case/conversational-test-case.ts | 209 +
typescript/src/test-case/index.ts | 34 +
typescript/src/test-case/llm-test-case.ts | 248 +
typescript/src/test-case/mcp.ts | 111 +
typescript/src/test-case/mllm-image.ts | 204 +
typescript/src/test-case/utils.ts | 34 +
typescript/src/tracing/api.ts | 123 +
typescript/src/tracing/index.ts | 17 +
typescript/src/tracing/logging.ts | 58 +
typescript/src/tracing/offline-evals/api.ts | 27 +
typescript/src/tracing/offline-evals/index.ts | 3 +
typescript/src/tracing/offline-evals/span.ts | 35 +
.../src/tracing/offline-evals/thread.ts | 40 +
typescript/src/tracing/offline-evals/trace.ts | 38 +
typescript/src/tracing/trace-context.ts | 97 +
typescript/src/tracing/tracing.ts | 1650 ++
typescript/src/tracing/utils.ts | 51 +
typescript/src/utils.ts | 84 +
typescript/test/test-core/annotation.test.ts | 17 +
typescript/test/test-core/api.test.ts | 45 +
typescript/test/test-core/dataset.test.ts | 183 +
typescript/test/test-core/evaluate.test.ts | 37 +
typescript/test/test-core/prompt.test.ts | 502 +
typescript/test/test-core/tracing.test.ts | 288 +
.../test-ai-sdk/ai-sdk-tools.test.ts | 80 +
.../test-ai-sdk/ai-sdk.test.ts | 190 +
.../expected_ai_sdk_confident_attributes.json | 87 +
.../expected_ai_sdk_generate_text.json | 71 +
.../fixtures/expected_ai_sdk_stream_text.json | 73 +
.../expected_ai_sdk_structured_output.json | 142 +
.../fixtures/expected_ai_sdk_tools.json | 201 +
.../apps/langchain-agent-app.ts | 133 +
.../apps/langchain-conditional-app.ts | 171 +
.../apps/langchain-metric-collection-app.ts | 88 +
.../apps/langchain-multiple-tools-app.ts | 159 +
.../apps/langchain-parallel-tools-app.ts | 159 +
.../apps/langchain-retriever-app.ts | 121 +
.../apps/langchain-simple-app.ts | 27 +
.../apps/langchain-single-tool-app.ts | 76 +
.../apps/langchain-streaming-app.ts | 111 +
.../langchain_agent_multi_step_schema.json | 486 +
...langchain_conditional_research_schema.json | 486 +
.../langchain_metric_collection_schema.json | 512 +
.../langchain_multiple_tools_schema.json | 486 +
.../langchain_parallel_weather_schema.json | 620 +
.../langchain_retriever_python_schema.json | 249 +
.../fixtures/langchain_simple_schema.json | 203 +
.../langchain_single_tool_schema.json | 487 +
.../fixtures/langchain_streaming_schema.json | 516 +
.../test-langchain-apps.test.ts | 226 +
.../apps/openai-agents-eval-app.ts | 7 +
.../apps/openai-agents-handoff-app.ts | 21 +
.../apps/openai-agents-simple-app.ts | 8 +
.../apps/openai-agents-streaming-app.ts | 29 +
.../apps/openai-agents-tool-app.ts | 49 +
.../openai_agents_evals_app_schema.json | 112 +
.../openai_agents_handoff_schema.json | 226 +
.../fixtures/openai_agents_simple_schema.json | 103 +
.../openai_agents_streaming_schema.json | 125 +
.../fixtures/openai_agents_tool_schema.json | 262 +
.../test-openai-agents/openai-agents.test.ts | 163 +
.../expected_openai_confident_attributes.json | 53 +
.../fixtures/expected_openai_with_trace.json | 50 +
.../expected_openai_without_trace.json | 71 +
.../expected_response_create_with_trace.json | 55 +
.../expected_tool_call_flow_completion.json | 263 +
.../expected_tool_call_flow_response.json | 294 +
.../test-openai/openai-tool-calls.test.ts | 254 +
.../test-openai/openai.test.ts | 177 +
typescript/test/test-integrations/utils.ts | 250 +
typescript/tsconfig.json | 17 +
2344 files changed, 440315 insertions(+)
create mode 100644 .claude-plugin/marketplace.json
create mode 100644 .claude-plugin/plugin.json
create mode 100644 .cursor-plugin/plugin.json
create mode 100644 .env.example
create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md
create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md
create mode 100644 .github/workflows/black.yml
create mode 100644 .github/workflows/changelog.yml
create mode 100644 .github/workflows/full_test_core_for_pr.yml
create mode 100644 .github/workflows/pr-title-check.yml
create mode 100644 .github/workflows/release.yml
create mode 100644 .github/workflows/test_confident.yml
create mode 100644 .github/workflows/test_core.yml
create mode 100644 .github/workflows/test_integrations.yml
create mode 100644 .github/workflows/test_metric_templates.yml
create mode 100644 .github/workflows/test_metrics.yml
create mode 100644 .github/workflows/typescript_lint.yml
create mode 100644 .github/workflows/typescript_test.yml
create mode 100644 .gitignore
create mode 100644 .pre-commit-config.yaml
create mode 100644 .scripts/changelog/generate.py
create mode 100644 .scripts/changelog/release_notes.py
create mode 100644 .vscode/settings.json
create mode 100644 CITATION.cff
create mode 100644 CONTRIBUTING.md
create mode 100644 LICENSE.md
create mode 100644 MAINTAINERS.md
create mode 100644 MANIFEST.in
create mode 100644 README.md
create mode 100644 README.wehub.md
create mode 100644 assets/confident-mcp-architecture.png
create mode 100644 assets/demo.gif
create mode 100644 assets/hero/wordmark-dark.svg
create mode 100644 assets/hero/wordmark-light.svg
create mode 100644 deepeval/__init__.py
create mode 100644 deepeval/_version.py
create mode 100644 deepeval/annotation/__init__.py
create mode 100644 deepeval/annotation/annotation.py
create mode 100644 deepeval/annotation/api.py
create mode 100644 deepeval/anthropic/__init__.py
create mode 100644 deepeval/anthropic/extractors.py
create mode 100644 deepeval/anthropic/patch.py
create mode 100644 deepeval/anthropic/utils.py
create mode 100644 deepeval/benchmarks/__init__.py
create mode 100644 deepeval/benchmarks/arc/__init__.py
create mode 100644 deepeval/benchmarks/arc/arc.py
create mode 100644 deepeval/benchmarks/arc/mode.py
create mode 100644 deepeval/benchmarks/arc/template.py
create mode 100644 deepeval/benchmarks/base_benchmark.py
create mode 100644 deepeval/benchmarks/bbq/__init__.py
create mode 100644 deepeval/benchmarks/bbq/bbq.py
create mode 100644 deepeval/benchmarks/bbq/task.py
create mode 100644 deepeval/benchmarks/bbq/template.py
create mode 100644 deepeval/benchmarks/big_bench_hard/__init__.py
create mode 100644 deepeval/benchmarks/big_bench_hard/big_bench_hard.py
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/__init__.py
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/boolean_expressions.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/causal_judgement.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/date_understanding.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/disambiguation_qa.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/dyck_languages.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/formal_fallacies.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/geometric_shapes.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/hyperbaton.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/logical_deduction_five_objects.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/logical_deduction_seven_objects.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/logical_deduction_three_objects.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/movie_recommendation.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/multistep_arithmetic_two.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/navigate.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/object_counting.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/penguins_in_a_table.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/reasoning_about_colored_objects.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/ruin_names.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/salient_translation_error_detection.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/snarks.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/sports_understanding.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/temporal_sequences.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/tracking_shuffled_objects_five_objects.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/tracking_shuffled_objects_seven_objects.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/tracking_shuffled_objects_three_objects.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/web_of_lies.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/cot_prompts/word_sorting.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/__init__.py
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/boolean_expressions.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/causal_judgement.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/date_understanding.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/disambiguation_qa.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/dyck_languages.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/formal_fallacies.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/geometric_shapes.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/hyperbaton.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/logical_deduction_five_objects.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/logical_deduction_seven_objects.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/logical_deduction_three_objects.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/movie_recommendation.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/multistep_arithmetic_two.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/navigate.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/object_counting.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/penguins_in_a_table.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/reasoning_about_colored_objects.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/ruin_names.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/salient_translation_error_detection.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/snarks.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/sports_understanding.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/temporal_sequences.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/tracking_shuffled_objects_five_objects.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/tracking_shuffled_objects_seven_objects.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/tracking_shuffled_objects_three_objects.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/web_of_lies.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/shot_prompts/word_sorting.txt
create mode 100644 deepeval/benchmarks/big_bench_hard/task.py
create mode 100644 deepeval/benchmarks/big_bench_hard/template.py
create mode 100644 deepeval/benchmarks/bool_q/__init__.py
create mode 100644 deepeval/benchmarks/bool_q/bool_q.py
create mode 100644 deepeval/benchmarks/bool_q/template.py
create mode 100644 deepeval/benchmarks/drop/__init__.py
create mode 100644 deepeval/benchmarks/drop/drop.py
create mode 100644 deepeval/benchmarks/drop/task.py
create mode 100644 deepeval/benchmarks/drop/template.py
create mode 100644 deepeval/benchmarks/equity_med_qa/__init__.py
create mode 100644 deepeval/benchmarks/equity_med_qa/equity_med_qa.py
create mode 100644 deepeval/benchmarks/equity_med_qa/task.py
create mode 100644 deepeval/benchmarks/equity_med_qa/template.py
create mode 100644 deepeval/benchmarks/gsm8k/__init__.py
create mode 100644 deepeval/benchmarks/gsm8k/gsm8k.py
create mode 100644 deepeval/benchmarks/gsm8k/template.py
create mode 100644 deepeval/benchmarks/hellaswag/__init__.py
create mode 100644 deepeval/benchmarks/hellaswag/hellaswag.py
create mode 100644 deepeval/benchmarks/hellaswag/task.py
create mode 100644 deepeval/benchmarks/hellaswag/template.py
create mode 100644 deepeval/benchmarks/human_eval/__init__.py
create mode 100644 deepeval/benchmarks/human_eval/human_eval.py
create mode 100644 deepeval/benchmarks/human_eval/task.py
create mode 100644 deepeval/benchmarks/human_eval/template.py
create mode 100644 deepeval/benchmarks/ifeval/__init__.py
create mode 100644 deepeval/benchmarks/ifeval/ifeval.py
create mode 100644 deepeval/benchmarks/ifeval/template.py
create mode 100644 deepeval/benchmarks/lambada/__init__.py
create mode 100644 deepeval/benchmarks/lambada/lambada.py
create mode 100644 deepeval/benchmarks/lambada/template.py
create mode 100644 deepeval/benchmarks/logi_qa/__init__.py
create mode 100644 deepeval/benchmarks/logi_qa/logi_qa.py
create mode 100644 deepeval/benchmarks/logi_qa/task.py
create mode 100644 deepeval/benchmarks/logi_qa/template.py
create mode 100644 deepeval/benchmarks/math_qa/__init__.py
create mode 100644 deepeval/benchmarks/math_qa/math_qa.py
create mode 100644 deepeval/benchmarks/math_qa/task.py
create mode 100644 deepeval/benchmarks/math_qa/template.py
create mode 100644 deepeval/benchmarks/mmlu/__init__.py
create mode 100644 deepeval/benchmarks/mmlu/mmlu.py
create mode 100644 deepeval/benchmarks/mmlu/task.py
create mode 100644 deepeval/benchmarks/mmlu/template.py
create mode 100644 deepeval/benchmarks/modes/__init__.py
create mode 100644 deepeval/benchmarks/results.py
create mode 100644 deepeval/benchmarks/schema.py
create mode 100644 deepeval/benchmarks/squad/__init__.py
create mode 100644 deepeval/benchmarks/squad/squad.py
create mode 100644 deepeval/benchmarks/squad/task.py
create mode 100644 deepeval/benchmarks/squad/template.py
create mode 100644 deepeval/benchmarks/tasks/__init__.py
create mode 100644 deepeval/benchmarks/truthful_qa/__init__.py
create mode 100644 deepeval/benchmarks/truthful_qa/mode.py
create mode 100644 deepeval/benchmarks/truthful_qa/task.py
create mode 100644 deepeval/benchmarks/truthful_qa/template.py
create mode 100644 deepeval/benchmarks/truthful_qa/truthful_qa.py
create mode 100644 deepeval/benchmarks/utils.py
create mode 100644 deepeval/benchmarks/winogrande/__init__.py
create mode 100644 deepeval/benchmarks/winogrande/template.py
create mode 100644 deepeval/benchmarks/winogrande/winogrande.py
create mode 100644 deepeval/cli/__init__.py
create mode 100644 deepeval/cli/auth/__init__.py
create mode 100644 deepeval/cli/auth/api.py
create mode 100644 deepeval/cli/auth/command.py
create mode 100644 deepeval/cli/auth/flow.py
create mode 100644 deepeval/cli/diagnose/__init__.py
create mode 100644 deepeval/cli/diagnose/diagnose.py
create mode 100644 deepeval/cli/dotenv_handler.py
create mode 100644 deepeval/cli/generate/__init__.py
create mode 100644 deepeval/cli/generate/command.py
create mode 100644 deepeval/cli/generate/utils.py
create mode 100644 deepeval/cli/inspect.py
create mode 100644 deepeval/cli/main.py
create mode 100644 deepeval/cli/test/__init__.py
create mode 100644 deepeval/cli/test/command.py
create mode 100644 deepeval/cli/types.py
create mode 100644 deepeval/cli/utils.py
create mode 100644 deepeval/confident/__init__.py
create mode 100644 deepeval/confident/api.py
create mode 100644 deepeval/confident/types.py
create mode 100644 deepeval/config/__init__.py
create mode 100644 deepeval/config/dotenv_handler.py
create mode 100644 deepeval/config/logging.py
create mode 100644 deepeval/config/settings.py
create mode 100644 deepeval/config/settings_manager.py
create mode 100644 deepeval/config/utils.py
create mode 100644 deepeval/constants.py
create mode 100644 deepeval/contextvars.py
create mode 100644 deepeval/dataset/__init__.py
create mode 100644 deepeval/dataset/api.py
create mode 100644 deepeval/dataset/dataset.py
create mode 100644 deepeval/dataset/golden.py
create mode 100644 deepeval/dataset/test_run_tracer.py
create mode 100644 deepeval/dataset/types.py
create mode 100644 deepeval/dataset/utils.py
create mode 100644 deepeval/errors.py
create mode 100644 deepeval/evaluate/__init__.py
create mode 100644 deepeval/evaluate/api.py
create mode 100644 deepeval/evaluate/compare.py
create mode 100644 deepeval/evaluate/configs.py
create mode 100644 deepeval/evaluate/console_report.py
create mode 100644 deepeval/evaluate/evaluate.py
create mode 100644 deepeval/evaluate/execute/__init__.py
create mode 100644 deepeval/evaluate/execute/_common.py
create mode 100644 deepeval/evaluate/execute/agentic.py
create mode 100644 deepeval/evaluate/execute/e2e.py
create mode 100644 deepeval/evaluate/execute/loop.py
create mode 100644 deepeval/evaluate/execute/trace_scope.py
create mode 100644 deepeval/evaluate/inspect_prompt.py
create mode 100644 deepeval/evaluate/local_store.py
create mode 100644 deepeval/evaluate/types.py
create mode 100644 deepeval/evaluate/utils.py
create mode 100644 deepeval/inspect/__init__.py
create mode 100644 deepeval/inspect/__main__.py
create mode 100644 deepeval/inspect/app.py
create mode 100644 deepeval/inspect/fixtures/test_run_sample.json
create mode 100644 deepeval/inspect/loader.py
create mode 100644 deepeval/inspect/styles.tcss
create mode 100644 deepeval/inspect/types.py
create mode 100644 deepeval/inspect/widgets/__init__.py
create mode 100644 deepeval/inspect/widgets/_styling.py
create mode 100644 deepeval/inspect/widgets/details.py
create mode 100644 deepeval/inspect/widgets/header_bar.py
create mode 100644 deepeval/inspect/widgets/help_modal.py
create mode 100644 deepeval/inspect/widgets/search_bar.py
create mode 100644 deepeval/inspect/widgets/span_tree.py
create mode 100644 deepeval/integrations/README.md
create mode 100644 deepeval/integrations/__init__.py
create mode 100644 deepeval/integrations/agentcore/__init__.py
create mode 100644 deepeval/integrations/agentcore/instrumentator.py
create mode 100644 deepeval/integrations/agentcore/otel.py
create mode 100644 deepeval/integrations/crewai/__init__.py
create mode 100644 deepeval/integrations/crewai/handler.py
create mode 100644 deepeval/integrations/crewai/subs.py
create mode 100644 deepeval/integrations/crewai/tool.py
create mode 100644 deepeval/integrations/crewai/wrapper.py
create mode 100644 deepeval/integrations/google_adk/__init__.py
create mode 100644 deepeval/integrations/google_adk/otel.py
create mode 100644 deepeval/integrations/hugging_face/__init__.py
create mode 100644 deepeval/integrations/hugging_face/callback.py
create mode 100644 deepeval/integrations/hugging_face/rich_manager.py
create mode 100644 deepeval/integrations/hugging_face/tests/test_callbacks.py
create mode 100644 deepeval/integrations/hugging_face/utils.py
create mode 100644 deepeval/integrations/langchain/__init__.py
create mode 100644 deepeval/integrations/langchain/callback.py
create mode 100644 deepeval/integrations/langchain/patch.py
create mode 100644 deepeval/integrations/langchain/utils.py
create mode 100644 deepeval/integrations/llama_index/__init__.py
create mode 100644 deepeval/integrations/llama_index/handler.py
create mode 100644 deepeval/integrations/llama_index/utils.py
create mode 100644 deepeval/integrations/openinference/__init__.py
create mode 100644 deepeval/integrations/openinference/instrumentator.py
create mode 100644 deepeval/integrations/openinference/otel.py
create mode 100644 deepeval/integrations/pydantic_ai/README.md
create mode 100644 deepeval/integrations/pydantic_ai/__init__.py
create mode 100644 deepeval/integrations/pydantic_ai/instrumentator.py
create mode 100644 deepeval/integrations/pydantic_ai/otel.py
create mode 100644 deepeval/integrations/strands/__init__.py
create mode 100644 deepeval/integrations/strands/instrumentator.py
create mode 100644 deepeval/integrations/strands/otel.py
create mode 100644 deepeval/key_handler.py
create mode 100644 deepeval/metrics/README.md
create mode 100644 deepeval/metrics/__init__.py
create mode 100644 deepeval/metrics/agent_loop_detection/__init__.py
create mode 100644 deepeval/metrics/agent_loop_detection/agent_loop_detection.py
create mode 100644 deepeval/metrics/answer_relevancy/__init__.py
create mode 100644 deepeval/metrics/answer_relevancy/answer_relevancy.py
create mode 100644 deepeval/metrics/answer_relevancy/schema.py
create mode 100644 deepeval/metrics/answer_relevancy/templates/class.txt
create mode 100644 deepeval/metrics/answer_relevancy/templates/generate_reason.txt
create mode 100644 deepeval/metrics/answer_relevancy/templates/generate_statements.txt
create mode 100644 deepeval/metrics/answer_relevancy/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/arena_g_eval/__init__.py
create mode 100644 deepeval/metrics/arena_g_eval/arena_g_eval.py
create mode 100644 deepeval/metrics/arena_g_eval/schema.py
create mode 100644 deepeval/metrics/arena_g_eval/templates/class.txt
create mode 100644 deepeval/metrics/arena_g_eval/templates/generate_arena_winner.txt
create mode 100644 deepeval/metrics/arena_g_eval/templates/generate_evaluation_steps.txt
create mode 100644 deepeval/metrics/arena_g_eval/templates/rewrite_reason.txt
create mode 100644 deepeval/metrics/arena_g_eval/utils.py
create mode 100644 deepeval/metrics/argument_correctness/__init__.py
create mode 100644 deepeval/metrics/argument_correctness/argument_correctness.py
create mode 100644 deepeval/metrics/argument_correctness/schema.py
create mode 100644 deepeval/metrics/argument_correctness/templates/class.txt
create mode 100644 deepeval/metrics/argument_correctness/templates/generate_reason.txt
create mode 100644 deepeval/metrics/argument_correctness/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/base_metric.py
create mode 100644 deepeval/metrics/bias/__init__.py
create mode 100644 deepeval/metrics/bias/bias.py
create mode 100644 deepeval/metrics/bias/schema.py
create mode 100644 deepeval/metrics/bias/templates/class.txt
create mode 100644 deepeval/metrics/bias/templates/generate_opinions.txt
create mode 100644 deepeval/metrics/bias/templates/generate_reason.txt
create mode 100644 deepeval/metrics/bias/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/community/__init__.py
create mode 100644 deepeval/metrics/community/citation_faithfulness/__init__.py
create mode 100644 deepeval/metrics/community/citation_faithfulness/citation_faithfulness.py
create mode 100644 deepeval/metrics/community/citation_faithfulness/schema.py
create mode 100644 deepeval/metrics/community/citation_faithfulness/template.py
create mode 100644 deepeval/metrics/contextual_precision/__init__.py
create mode 100644 deepeval/metrics/contextual_precision/contextual_precision.py
create mode 100644 deepeval/metrics/contextual_precision/schema.py
create mode 100644 deepeval/metrics/contextual_precision/templates/class.txt
create mode 100644 deepeval/metrics/contextual_precision/templates/generate_reason.txt
create mode 100644 deepeval/metrics/contextual_precision/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/contextual_recall/__init__.py
create mode 100644 deepeval/metrics/contextual_recall/contextual_recall.py
create mode 100644 deepeval/metrics/contextual_recall/schema.py
create mode 100644 deepeval/metrics/contextual_recall/templates/class.txt
create mode 100644 deepeval/metrics/contextual_recall/templates/generate_reason.txt
create mode 100644 deepeval/metrics/contextual_recall/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/contextual_relevancy/__init__.py
create mode 100644 deepeval/metrics/contextual_relevancy/contextual_relevancy.py
create mode 100644 deepeval/metrics/contextual_relevancy/schema.py
create mode 100644 deepeval/metrics/contextual_relevancy/templates/class.txt
create mode 100644 deepeval/metrics/contextual_relevancy/templates/generate_reason.txt
create mode 100644 deepeval/metrics/contextual_relevancy/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/conversation_completeness/__init__.py
create mode 100644 deepeval/metrics/conversation_completeness/conversation_completeness.py
create mode 100644 deepeval/metrics/conversation_completeness/schema.py
create mode 100644 deepeval/metrics/conversation_completeness/templates/class.txt
create mode 100644 deepeval/metrics/conversation_completeness/templates/extract_user_intentions.txt
create mode 100644 deepeval/metrics/conversation_completeness/templates/generate_reason.txt
create mode 100644 deepeval/metrics/conversation_completeness/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/conversational_dag/__init__.py
create mode 100644 deepeval/metrics/conversational_dag/conversational_dag.py
create mode 100644 deepeval/metrics/conversational_dag/nodes.py
create mode 100644 deepeval/metrics/conversational_dag/templates/ConversationalBinaryJudgement/generate_binary_verdict.txt
create mode 100644 deepeval/metrics/conversational_dag/templates/ConversationalNonBinaryJudgement/generate_non_binary_verdict.txt
create mode 100644 deepeval/metrics/conversational_dag/templates/ConversationalTaskNode/generate_task_output.txt
create mode 100644 deepeval/metrics/conversational_dag/templates/ConversationalVerdictNode/generate_reason.txt
create mode 100644 deepeval/metrics/conversational_g_eval/__init__.py
create mode 100644 deepeval/metrics/conversational_g_eval/conversational_g_eval.py
create mode 100644 deepeval/metrics/conversational_g_eval/schema.py
create mode 100644 deepeval/metrics/conversational_g_eval/templates/class.txt
create mode 100644 deepeval/metrics/conversational_g_eval/templates/generate_evaluation_results.txt
create mode 100644 deepeval/metrics/conversational_g_eval/templates/generate_evaluation_steps.txt
create mode 100644 deepeval/metrics/dag/__init__.py
create mode 100644 deepeval/metrics/dag/dag.py
create mode 100644 deepeval/metrics/dag/graph.py
create mode 100644 deepeval/metrics/dag/nodes.py
create mode 100644 deepeval/metrics/dag/schema.py
create mode 100644 deepeval/metrics/dag/serialization/__init__.py
create mode 100644 deepeval/metrics/dag/serialization/registry.py
create mode 100644 deepeval/metrics/dag/serialization/serialization.py
create mode 100644 deepeval/metrics/dag/serialization/types.py
create mode 100644 deepeval/metrics/dag/templates/BinaryJudgement/generate_binary_verdict.txt
create mode 100644 deepeval/metrics/dag/templates/NonBinaryJudgement/generate_non_binary_verdict.txt
create mode 100644 deepeval/metrics/dag/templates/TaskNode/generate_task_output.txt
create mode 100644 deepeval/metrics/dag/templates/VerdictNode/generate_reason.txt
create mode 100644 deepeval/metrics/dag/utils.py
create mode 100644 deepeval/metrics/exact_match/__init__.py
create mode 100644 deepeval/metrics/exact_match/exact_match.py
create mode 100644 deepeval/metrics/faithfulness/__init__.py
create mode 100644 deepeval/metrics/faithfulness/faithfulness.py
create mode 100644 deepeval/metrics/faithfulness/schema.py
create mode 100644 deepeval/metrics/faithfulness/templates/class.txt
create mode 100644 deepeval/metrics/faithfulness/templates/generate_claims.txt
create mode 100644 deepeval/metrics/faithfulness/templates/generate_reason.txt
create mode 100644 deepeval/metrics/faithfulness/templates/generate_truths.txt
create mode 100644 deepeval/metrics/faithfulness/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/g_eval/__init__.py
create mode 100644 deepeval/metrics/g_eval/g_eval.py
create mode 100644 deepeval/metrics/g_eval/schema.py
create mode 100644 deepeval/metrics/g_eval/templates/class.txt
create mode 100644 deepeval/metrics/g_eval/templates/generate_evaluation_results.txt
create mode 100644 deepeval/metrics/g_eval/templates/generate_evaluation_steps.txt
create mode 100644 deepeval/metrics/g_eval/templates/generate_strict_evaluation_results.txt
create mode 100644 deepeval/metrics/g_eval/utils.py
create mode 100644 deepeval/metrics/goal_accuracy/__init__.py
create mode 100644 deepeval/metrics/goal_accuracy/goal_accuracy.py
create mode 100644 deepeval/metrics/goal_accuracy/schema.py
create mode 100644 deepeval/metrics/goal_accuracy/templates/class.txt
create mode 100644 deepeval/metrics/goal_accuracy/templates/get_accuracy_score.txt
create mode 100644 deepeval/metrics/goal_accuracy/templates/get_final_reason.txt
create mode 100644 deepeval/metrics/goal_accuracy/templates/get_plan_evaluation_score.txt
create mode 100644 deepeval/metrics/hallucination/__init__.py
create mode 100644 deepeval/metrics/hallucination/hallucination.py
create mode 100644 deepeval/metrics/hallucination/schema.py
create mode 100644 deepeval/metrics/hallucination/templates/class.txt
create mode 100644 deepeval/metrics/hallucination/templates/generate_reason.txt
create mode 100644 deepeval/metrics/hallucination/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/indicator.py
create mode 100644 deepeval/metrics/json_correctness/__init__.py
create mode 100644 deepeval/metrics/json_correctness/json_correctness.py
create mode 100644 deepeval/metrics/json_correctness/schema.py
create mode 100644 deepeval/metrics/json_correctness/templates/class.txt
create mode 100644 deepeval/metrics/json_correctness/templates/generate_reason.txt
create mode 100644 deepeval/metrics/knowledge_retention/__init__.py
create mode 100644 deepeval/metrics/knowledge_retention/knowledge_retention.py
create mode 100644 deepeval/metrics/knowledge_retention/schema.py
create mode 100644 deepeval/metrics/knowledge_retention/templates/class.txt
create mode 100644 deepeval/metrics/knowledge_retention/templates/extract_data.txt
create mode 100644 deepeval/metrics/knowledge_retention/templates/generate_reason.txt
create mode 100644 deepeval/metrics/knowledge_retention/templates/generate_verdict.txt
create mode 100644 deepeval/metrics/mcp/__init__.py
create mode 100644 deepeval/metrics/mcp/mcp_task_completion.py
create mode 100644 deepeval/metrics/mcp/multi_turn_mcp_use_metric.py
create mode 100644 deepeval/metrics/mcp/schema.py
create mode 100644 deepeval/metrics/mcp/templates/class.txt
create mode 100644 deepeval/metrics/mcp/templates/generate_final_reason.txt
create mode 100644 deepeval/metrics/mcp/templates/get_args_correctness_score.txt
create mode 100644 deepeval/metrics/mcp/templates/get_task_completion_score.txt
create mode 100644 deepeval/metrics/mcp/templates/get_tool_correctness_score.txt
create mode 100644 deepeval/metrics/mcp/utils.py
create mode 100644 deepeval/metrics/mcp_use_metric/__init__.py
create mode 100644 deepeval/metrics/mcp_use_metric/mcp_use_metric.py
create mode 100644 deepeval/metrics/mcp_use_metric/schema.py
create mode 100644 deepeval/metrics/mcp_use_metric/templates/class.txt
create mode 100644 deepeval/metrics/mcp_use_metric/templates/get_mcp_argument_correctness_prompt.txt
create mode 100644 deepeval/metrics/mcp_use_metric/templates/get_primitive_correctness_prompt.txt
create mode 100644 deepeval/metrics/misuse/__init__.py
create mode 100644 deepeval/metrics/misuse/misuse.py
create mode 100644 deepeval/metrics/misuse/schema.py
create mode 100644 deepeval/metrics/misuse/templates/class.txt
create mode 100644 deepeval/metrics/misuse/templates/generate_misuses.txt
create mode 100644 deepeval/metrics/misuse/templates/generate_reason.txt
create mode 100644 deepeval/metrics/misuse/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/multimodal_metrics/__init__.py
create mode 100644 deepeval/metrics/multimodal_metrics/image_coherence/__init__.py
create mode 100644 deepeval/metrics/multimodal_metrics/image_coherence/image_coherence.py
create mode 100644 deepeval/metrics/multimodal_metrics/image_coherence/schema.py
create mode 100644 deepeval/metrics/multimodal_metrics/image_coherence/templates/class.txt
create mode 100644 deepeval/metrics/multimodal_metrics/image_coherence/templates/evaluate_image_coherence.txt
create mode 100644 deepeval/metrics/multimodal_metrics/image_editing/__init__.py
create mode 100644 deepeval/metrics/multimodal_metrics/image_editing/image_editing.py
create mode 100644 deepeval/metrics/multimodal_metrics/image_editing/schema.py
create mode 100644 deepeval/metrics/multimodal_metrics/image_editing/templates/class.txt
create mode 100644 deepeval/metrics/multimodal_metrics/image_editing/templates/generate_perceptual_quality_evaluation_results.txt
create mode 100644 deepeval/metrics/multimodal_metrics/image_editing/templates/generate_semantic_consistency_evaluation_results.txt
create mode 100644 deepeval/metrics/multimodal_metrics/image_helpfulness/__init__.py
create mode 100644 deepeval/metrics/multimodal_metrics/image_helpfulness/image_helpfulness.py
create mode 100644 deepeval/metrics/multimodal_metrics/image_helpfulness/schema.py
create mode 100644 deepeval/metrics/multimodal_metrics/image_helpfulness/templates/class.txt
create mode 100644 deepeval/metrics/multimodal_metrics/image_helpfulness/templates/evaluate_image_helpfulness.txt
create mode 100644 deepeval/metrics/multimodal_metrics/image_reference/__init__.py
create mode 100644 deepeval/metrics/multimodal_metrics/image_reference/image_reference.py
create mode 100644 deepeval/metrics/multimodal_metrics/image_reference/schema.py
create mode 100644 deepeval/metrics/multimodal_metrics/image_reference/templates/class.txt
create mode 100644 deepeval/metrics/multimodal_metrics/image_reference/templates/evaluate_image_reference.txt
create mode 100644 deepeval/metrics/multimodal_metrics/text_to_image/__init__.py
create mode 100644 deepeval/metrics/multimodal_metrics/text_to_image/schema.py
create mode 100644 deepeval/metrics/multimodal_metrics/text_to_image/templates/class.txt
create mode 100644 deepeval/metrics/multimodal_metrics/text_to_image/templates/generate_perceptual_quality_evaluation_results.txt
create mode 100644 deepeval/metrics/multimodal_metrics/text_to_image/templates/generate_semantic_consistency_evaluation_results.txt
create mode 100644 deepeval/metrics/multimodal_metrics/text_to_image/text_to_image.py
create mode 100644 deepeval/metrics/non_advice/__init__.py
create mode 100644 deepeval/metrics/non_advice/non_advice.py
create mode 100644 deepeval/metrics/non_advice/schema.py
create mode 100644 deepeval/metrics/non_advice/templates/class.txt
create mode 100644 deepeval/metrics/non_advice/templates/generate_advices.txt
create mode 100644 deepeval/metrics/non_advice/templates/generate_reason.txt
create mode 100644 deepeval/metrics/non_advice/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/pattern_match/__init__.py
create mode 100644 deepeval/metrics/pattern_match/pattern_match.py
create mode 100644 deepeval/metrics/pii_leakage/__init__.py
create mode 100644 deepeval/metrics/pii_leakage/pii_leakage.py
create mode 100644 deepeval/metrics/pii_leakage/schema.py
create mode 100644 deepeval/metrics/pii_leakage/templates/class.txt
create mode 100644 deepeval/metrics/pii_leakage/templates/extract_pii.txt
create mode 100644 deepeval/metrics/pii_leakage/templates/generate_reason.txt
create mode 100644 deepeval/metrics/pii_leakage/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/plan_adherence/__init__.py
create mode 100644 deepeval/metrics/plan_adherence/plan_adherence.py
create mode 100644 deepeval/metrics/plan_adherence/schema.py
create mode 100644 deepeval/metrics/plan_adherence/templates/class.txt
create mode 100644 deepeval/metrics/plan_adherence/templates/evaluate_adherence.txt
create mode 100644 deepeval/metrics/plan_adherence/templates/extract_plan_from_trace.txt
create mode 100644 deepeval/metrics/plan_quality/__init__.py
create mode 100644 deepeval/metrics/plan_quality/plan_quality.py
create mode 100644 deepeval/metrics/plan_quality/schema.py
create mode 100644 deepeval/metrics/plan_quality/templates/class.txt
create mode 100644 deepeval/metrics/plan_quality/templates/evaluate_plan_quality.txt
create mode 100644 deepeval/metrics/prompt_alignment/__init__.py
create mode 100644 deepeval/metrics/prompt_alignment/prompt_alignment.py
create mode 100644 deepeval/metrics/prompt_alignment/schema.py
create mode 100644 deepeval/metrics/prompt_alignment/templates/class.txt
create mode 100644 deepeval/metrics/prompt_alignment/templates/generate_reason.txt
create mode 100644 deepeval/metrics/prompt_alignment/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/ragas.py
create mode 100644 deepeval/metrics/retrieval_context_display.py
create mode 100644 deepeval/metrics/role_adherence/__init__.py
create mode 100644 deepeval/metrics/role_adherence/role_adherence.py
create mode 100644 deepeval/metrics/role_adherence/schema.py
create mode 100644 deepeval/metrics/role_adherence/templates/class.txt
create mode 100644 deepeval/metrics/role_adherence/templates/extract_out_of_character_response_verdicts.txt
create mode 100644 deepeval/metrics/role_adherence/templates/generate_reason.txt
create mode 100644 deepeval/metrics/role_violation/__init__.py
create mode 100644 deepeval/metrics/role_violation/role_violation.py
create mode 100644 deepeval/metrics/role_violation/schema.py
create mode 100644 deepeval/metrics/role_violation/templates/class.txt
create mode 100644 deepeval/metrics/role_violation/templates/detect_role_violations.txt
create mode 100644 deepeval/metrics/role_violation/templates/generate_reason.txt
create mode 100644 deepeval/metrics/role_violation/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/step_efficiency/__init__.py
create mode 100644 deepeval/metrics/step_efficiency/schema.py
create mode 100644 deepeval/metrics/step_efficiency/step_efficiency.py
create mode 100644 deepeval/metrics/step_efficiency/templates/class.txt
create mode 100644 deepeval/metrics/step_efficiency/templates/extract_task_from_trace.txt
create mode 100644 deepeval/metrics/step_efficiency/templates/get_execution_efficiency.txt
create mode 100644 deepeval/metrics/summarization/__init__.py
create mode 100644 deepeval/metrics/summarization/schema.py
create mode 100644 deepeval/metrics/summarization/summarization.py
create mode 100644 deepeval/metrics/summarization/templates/class.txt
create mode 100644 deepeval/metrics/summarization/templates/generate_alignment_verdicts.txt
create mode 100644 deepeval/metrics/summarization/templates/generate_answers.txt
create mode 100644 deepeval/metrics/summarization/templates/generate_questions.txt
create mode 100644 deepeval/metrics/summarization/templates/generate_reason.txt
create mode 100644 deepeval/metrics/task_completion/__init__.py
create mode 100644 deepeval/metrics/task_completion/schema.py
create mode 100644 deepeval/metrics/task_completion/task_completion.py
create mode 100644 deepeval/metrics/task_completion/templates/class.txt
create mode 100644 deepeval/metrics/task_completion/templates/extract_goal_and_outcome.txt
create mode 100644 deepeval/metrics/task_completion/templates/extract_task_and_outcome_from_trace.txt
create mode 100644 deepeval/metrics/task_completion/templates/generate_verdict.txt
create mode 100644 deepeval/metrics/tool_correctness/__init__.py
create mode 100644 deepeval/metrics/tool_correctness/schema.py
create mode 100644 deepeval/metrics/tool_correctness/templates/class.txt
create mode 100644 deepeval/metrics/tool_correctness/templates/get_tool_selection_score.txt
create mode 100644 deepeval/metrics/tool_correctness/tool_correctness.py
create mode 100644 deepeval/metrics/tool_permission/__init__.py
create mode 100644 deepeval/metrics/tool_permission/tool_permission.py
create mode 100644 deepeval/metrics/tool_use/__init__.py
create mode 100644 deepeval/metrics/tool_use/schema.py
create mode 100644 deepeval/metrics/tool_use/templates/class.txt
create mode 100644 deepeval/metrics/tool_use/templates/get_argument_correctness_score.txt
create mode 100644 deepeval/metrics/tool_use/templates/get_tool_argument_final_reason.txt
create mode 100644 deepeval/metrics/tool_use/templates/get_tool_selection_final_reason.txt
create mode 100644 deepeval/metrics/tool_use/templates/get_tool_selection_score.txt
create mode 100644 deepeval/metrics/tool_use/tool_use.py
create mode 100644 deepeval/metrics/topic_adherence/__init__.py
create mode 100644 deepeval/metrics/topic_adherence/schema.py
create mode 100644 deepeval/metrics/topic_adherence/templates/class.txt
create mode 100644 deepeval/metrics/topic_adherence/templates/generate_reason.txt
create mode 100644 deepeval/metrics/topic_adherence/templates/get_qa_pair_verdict.txt
create mode 100644 deepeval/metrics/topic_adherence/templates/get_qa_pairs.txt
create mode 100644 deepeval/metrics/topic_adherence/topic_adherence.py
create mode 100644 deepeval/metrics/toxicity/__init__.py
create mode 100644 deepeval/metrics/toxicity/schema.py
create mode 100644 deepeval/metrics/toxicity/templates/class.txt
create mode 100644 deepeval/metrics/toxicity/templates/generate_opinions.txt
create mode 100644 deepeval/metrics/toxicity/templates/generate_reason.txt
create mode 100644 deepeval/metrics/toxicity/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/toxicity/toxicity.py
create mode 100644 deepeval/metrics/turn_contextual_precision/__init__.py
create mode 100644 deepeval/metrics/turn_contextual_precision/schema.py
create mode 100644 deepeval/metrics/turn_contextual_precision/templates/class.txt
create mode 100644 deepeval/metrics/turn_contextual_precision/templates/generate_final_reason.txt
create mode 100644 deepeval/metrics/turn_contextual_precision/templates/generate_reason.txt
create mode 100644 deepeval/metrics/turn_contextual_precision/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/turn_contextual_precision/turn_contextual_precision.py
create mode 100644 deepeval/metrics/turn_contextual_recall/__init__.py
create mode 100644 deepeval/metrics/turn_contextual_recall/schema.py
create mode 100644 deepeval/metrics/turn_contextual_recall/templates/class.txt
create mode 100644 deepeval/metrics/turn_contextual_recall/templates/generate_final_reason.txt
create mode 100644 deepeval/metrics/turn_contextual_recall/templates/generate_reason.txt
create mode 100644 deepeval/metrics/turn_contextual_recall/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/turn_contextual_recall/turn_contextual_recall.py
create mode 100644 deepeval/metrics/turn_contextual_relevancy/__init__.py
create mode 100644 deepeval/metrics/turn_contextual_relevancy/schema.py
create mode 100644 deepeval/metrics/turn_contextual_relevancy/templates/class.txt
create mode 100644 deepeval/metrics/turn_contextual_relevancy/templates/generate_final_reason.txt
create mode 100644 deepeval/metrics/turn_contextual_relevancy/templates/generate_reason.txt
create mode 100644 deepeval/metrics/turn_contextual_relevancy/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/turn_contextual_relevancy/turn_contextual_relevancy.py
create mode 100644 deepeval/metrics/turn_faithfulness/__init__.py
create mode 100644 deepeval/metrics/turn_faithfulness/schema.py
create mode 100644 deepeval/metrics/turn_faithfulness/templates/class.txt
create mode 100644 deepeval/metrics/turn_faithfulness/templates/generate_claims.txt
create mode 100644 deepeval/metrics/turn_faithfulness/templates/generate_final_reason.txt
create mode 100644 deepeval/metrics/turn_faithfulness/templates/generate_reason.txt
create mode 100644 deepeval/metrics/turn_faithfulness/templates/generate_truths.txt
create mode 100644 deepeval/metrics/turn_faithfulness/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/turn_faithfulness/turn_faithfulness.py
create mode 100644 deepeval/metrics/turn_relevancy/__init__.py
create mode 100644 deepeval/metrics/turn_relevancy/schema.py
create mode 100644 deepeval/metrics/turn_relevancy/templates/class.txt
create mode 100644 deepeval/metrics/turn_relevancy/templates/generate_reason.txt
create mode 100644 deepeval/metrics/turn_relevancy/templates/generate_verdicts.txt
create mode 100644 deepeval/metrics/turn_relevancy/turn_relevancy.py
create mode 100644 deepeval/metrics/utils.py
create mode 100644 deepeval/model_integrations/__init__.py
create mode 100644 deepeval/model_integrations/types.py
create mode 100644 deepeval/model_integrations/utils.py
create mode 100644 deepeval/models/__init__.py
create mode 100644 deepeval/models/_summac_model.py
create mode 100644 deepeval/models/answer_relevancy_model.py
create mode 100644 deepeval/models/base_model.py
create mode 100644 deepeval/models/detoxify_model.py
create mode 100644 deepeval/models/embedding_models/__init__.py
create mode 100644 deepeval/models/embedding_models/azure_embedding_model.py
create mode 100644 deepeval/models/embedding_models/local_embedding_model.py
create mode 100644 deepeval/models/embedding_models/ollama_embedding_model.py
create mode 100644 deepeval/models/embedding_models/openai_embedding_model.py
create mode 100644 deepeval/models/hallucination_model.py
create mode 100644 deepeval/models/llms/__init__.py
create mode 100644 deepeval/models/llms/amazon_bedrock_model.py
create mode 100644 deepeval/models/llms/anthropic_model.py
create mode 100644 deepeval/models/llms/azure_model.py
create mode 100644 deepeval/models/llms/constants.py
create mode 100644 deepeval/models/llms/deepseek_model.py
create mode 100644 deepeval/models/llms/gateway_model.py
create mode 100644 deepeval/models/llms/gemini_model.py
create mode 100644 deepeval/models/llms/grok_model.py
create mode 100644 deepeval/models/llms/kimi_model.py
create mode 100644 deepeval/models/llms/litellm_model.py
create mode 100644 deepeval/models/llms/local_model.py
create mode 100644 deepeval/models/llms/ollama_model.py
create mode 100644 deepeval/models/llms/openai_model.py
create mode 100644 deepeval/models/llms/openrouter_model.py
create mode 100644 deepeval/models/llms/portkey_model.py
create mode 100644 deepeval/models/llms/utils.py
create mode 100644 deepeval/models/retry_policy.py
create mode 100644 deepeval/models/summac_model.py
create mode 100644 deepeval/models/unbias_model.py
create mode 100644 deepeval/models/utils.py
create mode 100644 deepeval/openai/__init__.py
create mode 100644 deepeval/openai/extractors.py
create mode 100644 deepeval/openai/patch.py
create mode 100644 deepeval/openai/utils.py
create mode 100644 deepeval/openai_agents/__init__.py
create mode 100644 deepeval/openai_agents/agent.py
create mode 100644 deepeval/openai_agents/callback_handler.py
create mode 100644 deepeval/openai_agents/extractors.py
create mode 100644 deepeval/openai_agents/patch.py
create mode 100644 deepeval/openai_agents/runner.py
create mode 100644 deepeval/optimizer/__init__.py
create mode 100644 deepeval/optimizer/algorithms/__init__.py
create mode 100644 deepeval/optimizer/algorithms/base.py
create mode 100644 deepeval/optimizer/algorithms/configs.py
create mode 100644 deepeval/optimizer/algorithms/copro/__init__.py
create mode 100644 deepeval/optimizer/algorithms/copro/copro.py
create mode 100644 deepeval/optimizer/algorithms/copro/proposer.py
create mode 100644 deepeval/optimizer/algorithms/copro/schema.py
create mode 100644 deepeval/optimizer/algorithms/copro/template.py
create mode 100644 deepeval/optimizer/algorithms/gepa/__init__.py
create mode 100644 deepeval/optimizer/algorithms/gepa/gepa.py
create mode 100644 deepeval/optimizer/algorithms/miprov2/__init__.py
create mode 100644 deepeval/optimizer/algorithms/miprov2/bootstrapper.py
create mode 100644 deepeval/optimizer/algorithms/miprov2/miprov2.py
create mode 100644 deepeval/optimizer/algorithms/miprov2/proposer/__init__.py
create mode 100644 deepeval/optimizer/algorithms/miprov2/proposer/proposer.py
create mode 100644 deepeval/optimizer/algorithms/miprov2/proposer/schema.py
create mode 100644 deepeval/optimizer/algorithms/miprov2/proposer/template.py
create mode 100644 deepeval/optimizer/algorithms/simba/__init__.py
create mode 100644 deepeval/optimizer/algorithms/simba/proposer.py
create mode 100644 deepeval/optimizer/algorithms/simba/schema.py
create mode 100644 deepeval/optimizer/algorithms/simba/simba.py
create mode 100644 deepeval/optimizer/algorithms/simba/template.py
create mode 100644 deepeval/optimizer/configs.py
create mode 100644 deepeval/optimizer/policies.py
create mode 100644 deepeval/optimizer/prompt_optimizer.py
create mode 100644 deepeval/optimizer/rewriter/__init__.py
create mode 100644 deepeval/optimizer/rewriter/rewriter.py
create mode 100644 deepeval/optimizer/rewriter/schema.py
create mode 100644 deepeval/optimizer/rewriter/template.py
create mode 100644 deepeval/optimizer/scorer/__init__.py
create mode 100644 deepeval/optimizer/scorer/base.py
create mode 100644 deepeval/optimizer/scorer/schema.py
create mode 100644 deepeval/optimizer/scorer/scorer.py
create mode 100644 deepeval/optimizer/scorer/template.py
create mode 100644 deepeval/optimizer/scorer/utils.py
create mode 100644 deepeval/optimizer/types.py
create mode 100644 deepeval/optimizer/utils.py
create mode 100644 deepeval/plugins/__init__.py
create mode 100644 deepeval/plugins/plugin.py
create mode 100644 deepeval/progress_context.py
create mode 100644 deepeval/prompt/__init__.py
create mode 100644 deepeval/prompt/api.py
create mode 100644 deepeval/prompt/prompt.py
create mode 100644 deepeval/prompt/utils.py
create mode 100644 deepeval/py.typed
create mode 100644 deepeval/red_teaming/README.md
create mode 100644 deepeval/scorer/__init__.py
create mode 100644 deepeval/scorer/scorer.py
create mode 100644 deepeval/simulator/__init__.py
create mode 100644 deepeval/simulator/controller/__init__.py
create mode 100644 deepeval/simulator/controller/controller.py
create mode 100644 deepeval/simulator/controller/template.py
create mode 100644 deepeval/simulator/controller/types.py
create mode 100644 deepeval/simulator/conversation_simulator.py
create mode 100644 deepeval/simulator/schema.py
create mode 100644 deepeval/simulator/simulation_graph/__init__.py
create mode 100644 deepeval/simulator/simulation_graph/default.py
create mode 100644 deepeval/simulator/simulation_graph/node.py
create mode 100644 deepeval/simulator/simulation_graph/runner.py
create mode 100644 deepeval/simulator/simulation_graph/template.py
create mode 100644 deepeval/simulator/template.py
create mode 100644 deepeval/simulator/utils.py
create mode 100644 deepeval/singleton.py
create mode 100644 deepeval/synthesizer/__init__.py
create mode 100644 deepeval/synthesizer/base_synthesizer.py
create mode 100644 deepeval/synthesizer/chunking/__init__.py
create mode 100644 deepeval/synthesizer/chunking/context_generator.py
create mode 100644 deepeval/synthesizer/chunking/doc_chunker.py
create mode 100644 deepeval/synthesizer/config.py
create mode 100644 deepeval/synthesizer/schema.py
create mode 100644 deepeval/synthesizer/synthesizer.py
create mode 100644 deepeval/synthesizer/templates/__init__.py
create mode 100644 deepeval/synthesizer/templates/template.py
create mode 100644 deepeval/synthesizer/templates/template_extraction.py
create mode 100644 deepeval/synthesizer/templates/template_prompt.py
create mode 100644 deepeval/synthesizer/types.py
create mode 100644 deepeval/synthesizer/utils.py
create mode 100644 deepeval/telemetry.py
create mode 100644 deepeval/templates/__init__.py
create mode 100644 deepeval/templates/metrics/fragments/faithfulness_verdicts_example_multimodal.txt
create mode 100644 deepeval/templates/metrics/fragments/faithfulness_verdicts_format_instruction.txt
create mode 100644 deepeval/templates/metrics/fragments/faithfulness_verdicts_guidelines_multimodal.txt
create mode 100644 deepeval/templates/metrics/fragments/faithfulness_verdicts_guidelines_text_only.txt
create mode 100644 deepeval/templates/metrics/fragments/multimodal_image_generation_eval_context.txt
create mode 100644 deepeval/templates/metrics/fragments/multimodal_input_rules.txt
create mode 100644 deepeval/templates/metrics/fragments/multimodal_input_rules_turn_metric.txt
create mode 100644 deepeval/templates/metrics/templates.json
create mode 100644 deepeval/templates/resolver.py
create mode 100644 deepeval/test_case/__init__.py
create mode 100644 deepeval/test_case/api.py
create mode 100644 deepeval/test_case/arena_test_case.py
create mode 100644 deepeval/test_case/conversational_test_case.py
create mode 100644 deepeval/test_case/llm_test_case.py
create mode 100644 deepeval/test_case/mcp.py
create mode 100644 deepeval/test_case/utils.py
create mode 100644 deepeval/test_run/__init__.py
create mode 100644 deepeval/test_run/api.py
create mode 100644 deepeval/test_run/cache.py
create mode 100644 deepeval/test_run/hooks.py
create mode 100644 deepeval/test_run/hyperparameters.py
create mode 100644 deepeval/test_run/test_run.py
create mode 100644 deepeval/tracing/__init__.py
create mode 100644 deepeval/tracing/api.py
create mode 100644 deepeval/tracing/context.py
create mode 100644 deepeval/tracing/integrations.py
create mode 100644 deepeval/tracing/internal.py
create mode 100644 deepeval/tracing/offline_evals/__init__.py
create mode 100644 deepeval/tracing/offline_evals/api.py
create mode 100644 deepeval/tracing/offline_evals/span.py
create mode 100644 deepeval/tracing/offline_evals/thread.py
create mode 100644 deepeval/tracing/offline_evals/trace.py
create mode 100644 deepeval/tracing/otel/__init__.py
create mode 100644 deepeval/tracing/otel/context_aware_processor.py
create mode 100644 deepeval/tracing/otel/exporter.py
create mode 100644 deepeval/tracing/otel/test_exporter.py
create mode 100644 deepeval/tracing/otel/utils.py
create mode 100644 deepeval/tracing/patchers.py
create mode 100644 deepeval/tracing/perf_epoch_bridge.py
create mode 100644 deepeval/tracing/trace_context.py
create mode 100644 deepeval/tracing/trace_test_manager.py
create mode 100644 deepeval/tracing/tracing.py
create mode 100644 deepeval/tracing/types.py
create mode 100644 deepeval/tracing/utils.py
create mode 100644 deepeval/utils.py
create mode 100644 demo_trace_scope/__init__.py
create mode 100644 demo_trace_scope/test_observed_app.py
create mode 100644 docs/.gitignore
create mode 100644 docs/README.md
create mode 100644 docs/app/(home)/layout.tsx
create mode 100644 docs/app/(home)/page.tsx
create mode 100644 docs/app/api/search/route.ts
create mode 100644 docs/app/apple-icon.png
create mode 100644 docs/app/blog/[[...slug]]/page.tsx
create mode 100644 docs/app/blog/layout.tsx
create mode 100644 docs/app/changelog/[[...slug]]/page.tsx
create mode 100644 docs/app/changelog/layout.tsx
create mode 100644 docs/app/docs/[[...slug]]/page.tsx
create mode 100644 docs/app/docs/layout.tsx
create mode 100644 docs/app/enterprise/page.tsx
create mode 100644 docs/app/favicon.ico
create mode 100644 docs/app/global.css
create mode 100644 docs/app/guides/[[...slug]]/page.tsx
create mode 100644 docs/app/guides/layout.tsx
create mode 100644 docs/app/icon.png
create mode 100644 docs/app/icon.svg
create mode 100644 docs/app/integrations/[[...slug]]/page.tsx
create mode 100644 docs/app/integrations/layout.tsx
create mode 100644 docs/app/layout.tsx
create mode 100644 docs/app/llms-full.txt/route.ts
create mode 100644 docs/app/llms.mdx/blog/[[...slug]]/route.ts
create mode 100644 docs/app/llms.mdx/changelog/[[...slug]]/route.ts
create mode 100644 docs/app/llms.mdx/docs/[[...slug]]/route.ts
create mode 100644 docs/app/llms.mdx/guides/[[...slug]]/route.ts
create mode 100644 docs/app/llms.mdx/integrations/[[...slug]]/route.ts
create mode 100644 docs/app/llms.mdx/tutorials/[[...slug]]/route.ts
create mode 100644 docs/app/llms.txt/route.ts
create mode 100644 docs/app/og/blog/[...slug]/route.tsx
create mode 100644 docs/app/og/docs/[...slug]/route.tsx
create mode 100644 docs/app/robots.ts
create mode 100644 docs/app/sitemap.ts
create mode 100644 docs/app/tutorials/[[...slug]]/page.tsx
create mode 100644 docs/app/tutorials/layout.tsx
create mode 100644 docs/components/lang/code-term.tsx
create mode 100644 docs/components/lang/language-provider.tsx
create mode 100644 docs/components/language-selector/language-selector.module.scss
create mode 100644 docs/components/language-selector/language-selector.tsx
create mode 100644 docs/components/mdx-anchor.tsx
create mode 100644 docs/components/mdx.tsx
create mode 100644 docs/content/blog/deepeval-alternatives-compared.mdx
create mode 100644 docs/content/blog/deepeval-got-a-new-look.mdx
create mode 100644 docs/content/blog/deepeval-vs-arize.mdx
create mode 100644 docs/content/blog/deepeval-vs-langfuse.mdx
create mode 100644 docs/content/blog/deepeval-vs-ragas.mdx
create mode 100644 docs/content/blog/deepeval-vs-trulens.mdx
create mode 100644 docs/content/blog/eval-driven-development.mdx
create mode 100644 docs/content/blog/index.mdx
create mode 100644 docs/content/blog/introducing-deepeval-4.mdx
create mode 100644 docs/content/blog/llm-as-a-judge.mdx
create mode 100644 docs/content/blog/medical-chatbot-deepeval-guide.mdx
create mode 100644 docs/content/blog/meta.json
create mode 100644 docs/content/blog/rag-contract-assistant-deepeval-guide.mdx
create mode 100644 docs/content/blog/top-5-geval-use-cases.mdx
create mode 100644 docs/content/blog/typescript-in-deepeval-monorepo.mdx
create mode 100644 docs/content/blog/use-case-cognee-ai-memory.mdx
create mode 100644 docs/content/blog/what-is-an-eval-harness.mdx
create mode 100644 docs/content/changelog/changelog-2024.mdx
create mode 100644 docs/content/changelog/changelog-2025.mdx
create mode 100644 docs/content/changelog/changelog-2026.mdx
create mode 100644 docs/content/changelog/index.mdx
create mode 100644 docs/content/changelog/meta.json
create mode 100644 docs/content/docs/(agentic)/meta.json
create mode 100644 docs/content/docs/(agentic)/metrics-argument-correctness.mdx
create mode 100644 docs/content/docs/(agentic)/metrics-plan-adherence.mdx
create mode 100644 docs/content/docs/(agentic)/metrics-plan-quality.mdx
create mode 100644 docs/content/docs/(agentic)/metrics-step-efficiency.mdx
create mode 100644 docs/content/docs/(agentic)/metrics-task-completion.mdx
create mode 100644 docs/content/docs/(agentic)/metrics-tool-correctness.mdx
create mode 100644 docs/content/docs/(algorithms)/meta.json
create mode 100644 docs/content/docs/(algorithms)/prompt-optimization-copro.mdx
create mode 100644 docs/content/docs/(algorithms)/prompt-optimization-gepa.mdx
create mode 100644 docs/content/docs/(algorithms)/prompt-optimization-miprov2.mdx
create mode 100644 docs/content/docs/(algorithms)/prompt-optimization-simba.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-arc.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-bbq.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-big-bench-hard.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-bool-q.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-drop.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-gsm8k.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-hellaswag.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-human-eval.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-ifeval.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-lambada.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-logi-qa.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-math-qa.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-mmlu.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-squad.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-truthful-qa.mdx
create mode 100644 docs/content/docs/(benchmarks)/benchmarks-winogrande.mdx
create mode 100644 docs/content/docs/(benchmarks)/meta.json
create mode 100644 docs/content/docs/(community)/community-metrics-overview.mdx
create mode 100644 docs/content/docs/(community)/meta.json
create mode 100644 docs/content/docs/(community)/metrics-agent-loop-detection.mdx
create mode 100644 docs/content/docs/(community)/metrics-citation-faithfulness.mdx
create mode 100644 docs/content/docs/(community)/metrics-tool-permission.mdx
create mode 100644 docs/content/docs/(concepts)/(test-cases)/evaluation-arena-test-cases.mdx
create mode 100644 docs/content/docs/(concepts)/(test-cases)/evaluation-multiturn-test-cases.mdx
create mode 100644 docs/content/docs/(concepts)/(test-cases)/evaluation-test-cases.mdx
create mode 100644 docs/content/docs/(concepts)/(test-cases)/meta.json
create mode 100644 docs/content/docs/(concepts)/evaluation-datasets.mdx
create mode 100644 docs/content/docs/(concepts)/evaluation-llm-tracing.mdx
create mode 100644 docs/content/docs/(concepts)/evaluation-mcp.mdx
create mode 100644 docs/content/docs/(concepts)/evaluation-prompts.mdx
create mode 100644 docs/content/docs/(concepts)/meta.json
create mode 100644 docs/content/docs/(custom)/meta.json
create mode 100644 docs/content/docs/(custom)/metrics-arena-g-eval.mdx
create mode 100644 docs/content/docs/(custom)/metrics-conversational-dag.mdx
create mode 100644 docs/content/docs/(custom)/metrics-conversational-g-eval.mdx
create mode 100644 docs/content/docs/(custom)/metrics-custom.mdx
create mode 100644 docs/content/docs/(custom)/metrics-dag.mdx
create mode 100644 docs/content/docs/(custom)/metrics-llm-evals.mdx
create mode 100644 docs/content/docs/(generate-goldens)/meta.json
create mode 100644 docs/content/docs/(generate-goldens)/synthesizer-generate-from-contexts.mdx
create mode 100644 docs/content/docs/(generate-goldens)/synthesizer-generate-from-docs.mdx
create mode 100644 docs/content/docs/(generate-goldens)/synthesizer-generate-from-goldens.mdx
create mode 100644 docs/content/docs/(generate-goldens)/synthesizer-generate-from-scratch.mdx
create mode 100644 docs/content/docs/(images)/meta.json
create mode 100644 docs/content/docs/(images)/multimodal-metrics-image-coherence.mdx
create mode 100644 docs/content/docs/(images)/multimodal-metrics-image-editing.mdx
create mode 100644 docs/content/docs/(images)/multimodal-metrics-image-helpfulness.mdx
create mode 100644 docs/content/docs/(images)/multimodal-metrics-image-reference.mdx
create mode 100644 docs/content/docs/(images)/multimodal-metrics-text-to-image.mdx
create mode 100644 docs/content/docs/(mcp)/meta.json
create mode 100644 docs/content/docs/(mcp)/metrics-mcp-task-completion.mdx
create mode 100644 docs/content/docs/(mcp)/metrics-mcp-use.mdx
create mode 100644 docs/content/docs/(mcp)/metrics-multi-turn-mcp-use.mdx
create mode 100644 docs/content/docs/(metrics-others)/meta.json
create mode 100644 docs/content/docs/(metrics-others)/metrics-hallucination.mdx
create mode 100644 docs/content/docs/(metrics-others)/metrics-prompt-alignment.mdx
create mode 100644 docs/content/docs/(metrics-others)/metrics-ragas.mdx
create mode 100644 docs/content/docs/(metrics-others)/metrics-summarization.mdx
create mode 100644 docs/content/docs/(multi-turn)/meta.json
create mode 100644 docs/content/docs/(multi-turn)/metrics-conversation-completeness.mdx
create mode 100644 docs/content/docs/(multi-turn)/metrics-goal-accuracy.mdx
create mode 100644 docs/content/docs/(multi-turn)/metrics-knowledge-retention.mdx
create mode 100644 docs/content/docs/(multi-turn)/metrics-role-adherence.mdx
create mode 100644 docs/content/docs/(multi-turn)/metrics-tool-use.mdx
create mode 100644 docs/content/docs/(multi-turn)/metrics-topic-adherence.mdx
create mode 100644 docs/content/docs/(multi-turn)/metrics-turn-contextual-precision.mdx
create mode 100644 docs/content/docs/(multi-turn)/metrics-turn-contextual-recall.mdx
create mode 100644 docs/content/docs/(multi-turn)/metrics-turn-contextual-relevancy.mdx
create mode 100644 docs/content/docs/(multi-turn)/metrics-turn-faithfulness.mdx
create mode 100644 docs/content/docs/(multi-turn)/metrics-turn-relevancy.mdx
create mode 100644 docs/content/docs/(non-llm)/meta.json
create mode 100644 docs/content/docs/(non-llm)/metrics-exact-match.mdx
create mode 100644 docs/content/docs/(non-llm)/metrics-json-correctness.mdx
create mode 100644 docs/content/docs/(non-llm)/metrics-pattern-match.mdx
create mode 100644 docs/content/docs/(rag)/meta.json
create mode 100644 docs/content/docs/(rag)/metrics-answer-relevancy.mdx
create mode 100644 docs/content/docs/(rag)/metrics-contextual-precision.mdx
create mode 100644 docs/content/docs/(rag)/metrics-contextual-recall.mdx
create mode 100644 docs/content/docs/(rag)/metrics-contextual-relevancy.mdx
create mode 100644 docs/content/docs/(rag)/metrics-faithfulness.mdx
create mode 100644 docs/content/docs/(safety)/meta.json
create mode 100644 docs/content/docs/(safety)/metrics-bias.mdx
create mode 100644 docs/content/docs/(safety)/metrics-misuse.mdx
create mode 100644 docs/content/docs/(safety)/metrics-non-advice.mdx
create mode 100644 docs/content/docs/(safety)/metrics-pii-leakage.mdx
create mode 100644 docs/content/docs/(safety)/metrics-role-violation.mdx
create mode 100644 docs/content/docs/(safety)/metrics-toxicity.mdx
create mode 100644 docs/content/docs/(use-cases)/getting-started-agents.mdx
create mode 100644 docs/content/docs/(use-cases)/getting-started-chatbots.mdx
create mode 100644 docs/content/docs/(use-cases)/getting-started-llm-arena.mdx
create mode 100644 docs/content/docs/(use-cases)/getting-started-mcp.mdx
create mode 100644 docs/content/docs/(use-cases)/getting-started-rag.mdx
create mode 100644 docs/content/docs/(use-cases)/meta.json
create mode 100644 docs/content/docs/benchmarks-introduction.mdx
create mode 100644 docs/content/docs/command-line-interface.mdx
create mode 100644 docs/content/docs/conversation-simulator-custom-templates.mdx
create mode 100644 docs/content/docs/conversation-simulator-lifecycle-hooks.mdx
create mode 100644 docs/content/docs/conversation-simulator-model-callback.mdx
create mode 100644 docs/content/docs/conversation-simulator-simulation-graph.mdx
create mode 100644 docs/content/docs/conversation-simulator-stopping-logic.mdx
create mode 100644 docs/content/docs/conversation-simulator/index.mdx
create mode 100644 docs/content/docs/conversation-simulator/meta.json
create mode 100644 docs/content/docs/data-privacy.mdx
create mode 100644 docs/content/docs/environment-variables.mdx
create mode 100644 docs/content/docs/evaluation-component-level-llm-evals.mdx
create mode 100644 docs/content/docs/evaluation-end-to-end-llm-evals/index.mdx
create mode 100644 docs/content/docs/evaluation-end-to-end-llm-evals/meta.json
create mode 100644 docs/content/docs/evaluation-end-to-end-multi-turn.mdx
create mode 100644 docs/content/docs/evaluation-end-to-end-single-turn.mdx
create mode 100644 docs/content/docs/evaluation-flags-and-configs.mdx
create mode 100644 docs/content/docs/evaluation-introduction.mdx
create mode 100644 docs/content/docs/evaluation-unit-testing-in-ci-cd.mdx
create mode 100644 docs/content/docs/faq.mdx
create mode 100644 docs/content/docs/getting-started.mdx
create mode 100644 docs/content/docs/golden-synthesizer/index.mdx
create mode 100644 docs/content/docs/golden-synthesizer/meta.json
create mode 100644 docs/content/docs/introduction-comparisons.mdx
create mode 100644 docs/content/docs/introduction-design-philosophy.mdx
create mode 100644 docs/content/docs/introduction.mdx
create mode 100644 docs/content/docs/meta.json
create mode 100644 docs/content/docs/metrics-introduction.mdx
create mode 100644 docs/content/docs/miscellaneous.mdx
create mode 100644 docs/content/docs/prompt-optimization-introduction.mdx
create mode 100644 docs/content/docs/synthetic-data-generation-introduction.mdx
create mode 100644 docs/content/docs/troubleshooting.mdx
create mode 100644 docs/content/docs/vibe-coder-quickstart.mdx
create mode 100644 docs/content/docs/vibe-coding.mdx
create mode 100644 docs/content/guides/guides-ai-agent-evaluation-metrics.mdx
create mode 100644 docs/content/guides/guides-ai-agent-evaluation.mdx
create mode 100644 docs/content/guides/guides-answer-correctness-metric.mdx
create mode 100644 docs/content/guides/guides-building-custom-metrics.mdx
create mode 100644 docs/content/guides/guides-llm-observability.mdx
create mode 100644 docs/content/guides/guides-multi-turn-evaluation-metrics.mdx
create mode 100644 docs/content/guides/guides-multi-turn-evaluation.mdx
create mode 100644 docs/content/guides/guides-multi-turn-simulation.mdx
create mode 100644 docs/content/guides/guides-optimizing-hyperparameters.mdx
create mode 100644 docs/content/guides/guides-rag-evaluation.mdx
create mode 100644 docs/content/guides/guides-rag-triad.mdx
create mode 100644 docs/content/guides/guides-red-teaming.mdx
create mode 100644 docs/content/guides/guides-regression-testing-in-cicd.mdx
create mode 100644 docs/content/guides/guides-tracing-ai-agents.mdx
create mode 100644 docs/content/guides/guides-tracing-multi-turn.mdx
create mode 100644 docs/content/guides/guides-tracing-rag.mdx
create mode 100644 docs/content/guides/guides-using-custom-embedding-models.mdx
create mode 100644 docs/content/guides/guides-using-custom-llms.mdx
create mode 100644 docs/content/guides/guides-using-synthesizer.mdx
create mode 100644 docs/content/guides/meta.json
create mode 100644 docs/content/integrations/frameworks/agentcore.mdx
create mode 100644 docs/content/integrations/frameworks/anthropic.mdx
create mode 100644 docs/content/integrations/frameworks/crewai.mdx
create mode 100644 docs/content/integrations/frameworks/google-adk.mdx
create mode 100644 docs/content/integrations/frameworks/huggingface.mdx
create mode 100644 docs/content/integrations/frameworks/langchain.mdx
create mode 100644 docs/content/integrations/frameworks/langgraph.mdx
create mode 100644 docs/content/integrations/frameworks/llamaindex.mdx
create mode 100644 docs/content/integrations/frameworks/meta.json
create mode 100644 docs/content/integrations/frameworks/openai-agents.mdx
create mode 100644 docs/content/integrations/frameworks/openai.mdx
create mode 100644 docs/content/integrations/frameworks/pydanticai.mdx
create mode 100644 docs/content/integrations/frameworks/strands.mdx
create mode 100644 docs/content/integrations/index.mdx
create mode 100644 docs/content/integrations/meta.json
create mode 100644 docs/content/integrations/models/amazon-bedrock.mdx
create mode 100644 docs/content/integrations/models/anthropic.mdx
create mode 100644 docs/content/integrations/models/azure-openai.mdx
create mode 100644 docs/content/integrations/models/deepseek.mdx
create mode 100644 docs/content/integrations/models/gemini.mdx
create mode 100644 docs/content/integrations/models/grok.mdx
create mode 100644 docs/content/integrations/models/litellm.mdx
create mode 100644 docs/content/integrations/models/lmstudio.mdx
create mode 100644 docs/content/integrations/models/meta.json
create mode 100644 docs/content/integrations/models/moonshot.mdx
create mode 100644 docs/content/integrations/models/ollama.mdx
create mode 100644 docs/content/integrations/models/openai.mdx
create mode 100644 docs/content/integrations/models/openrouter.mdx
create mode 100644 docs/content/integrations/models/portkey.mdx
create mode 100644 docs/content/integrations/models/vertex-ai.mdx
create mode 100644 docs/content/integrations/models/vllm.mdx
create mode 100644 docs/content/integrations/others/meta.json
create mode 100644 docs/content/integrations/vector-databases/chroma.mdx
create mode 100644 docs/content/integrations/vector-databases/cognee.mdx
create mode 100644 docs/content/integrations/vector-databases/elasticsearch.mdx
create mode 100644 docs/content/integrations/vector-databases/meta.json
create mode 100644 docs/content/integrations/vector-databases/pgvector.mdx
create mode 100644 docs/content/integrations/vector-databases/qdrant.mdx
create mode 100644 docs/content/integrations/vector-databases/weaviate.mdx
create mode 100644 docs/content/tutorials/medical-chatbot/development.mdx
create mode 100644 docs/content/tutorials/medical-chatbot/evals-in-prod.mdx
create mode 100644 docs/content/tutorials/medical-chatbot/evaluation.mdx
create mode 100644 docs/content/tutorials/medical-chatbot/improvement.mdx
create mode 100644 docs/content/tutorials/medical-chatbot/introduction.mdx
create mode 100644 docs/content/tutorials/meta.json
create mode 100644 docs/content/tutorials/rag-qa-agent/development.mdx
create mode 100644 docs/content/tutorials/rag-qa-agent/evals-in-prod.mdx
create mode 100644 docs/content/tutorials/rag-qa-agent/evaluation.mdx
create mode 100644 docs/content/tutorials/rag-qa-agent/improvement.mdx
create mode 100644 docs/content/tutorials/rag-qa-agent/introduction.mdx
create mode 100644 docs/content/tutorials/summarization-agent/development.mdx
create mode 100644 docs/content/tutorials/summarization-agent/evals-in-prod.mdx
create mode 100644 docs/content/tutorials/summarization-agent/evaluation.mdx
create mode 100644 docs/content/tutorials/summarization-agent/improvement.mdx
create mode 100644 docs/content/tutorials/summarization-agent/introduction.mdx
create mode 100644 docs/content/tutorials/tutorial-introduction.mdx
create mode 100644 docs/content/tutorials/tutorial-setup.mdx
create mode 100644 docs/enterprise/read-me.mdx
create mode 100644 docs/home/read-me.mdx
create mode 100644 docs/lib/authors.ts
create mode 100644 docs/lib/blog-categories.ts
create mode 100644 docs/lib/cn.ts
create mode 100644 docs/lib/contributors.ts
create mode 100644 docs/lib/defaults.ts
create mode 100644 docs/lib/generated/changelog-contributors.json
create mode 100644 docs/lib/generated/contributors.json
create mode 100644 docs/lib/generated/repo-contributors.json
create mode 100644 docs/lib/lang/terms.ts
create mode 100644 docs/lib/layout.shared.tsx
create mode 100644 docs/lib/llms-route.ts
create mode 100644 docs/lib/remark-admonitions.ts
create mode 100644 docs/lib/section.tsx
create mode 100644 docs/lib/sections.tsx
create mode 100644 docs/lib/shared.ts
create mode 100644 docs/lib/source.ts
create mode 100644 docs/next.config.mjs
create mode 100644 docs/package.json
create mode 100644 docs/postcss.config.mjs
create mode 100644 docs/proxy.ts
create mode 100644 docs/public/icons/DeepEval.svg
create mode 100644 docs/public/icons/brand-icons/amdocs.svg
create mode 100644 docs/public/icons/brand-icons/bcg.svg
create mode 100644 docs/public/icons/brand-icons/epic-games.svg
create mode 100644 docs/public/icons/brand-icons/finom.svg
create mode 100644 docs/public/icons/brand-icons/humach.svg
create mode 100644 docs/public/icons/brand-icons/panasonic.svg
create mode 100644 docs/public/icons/brand-icons/phreesia.svg
create mode 100644 docs/public/icons/brand-icons/siemens.svg
create mode 100644 docs/public/icons/brand-icons/syngenta-group.svg
create mode 100644 docs/public/icons/brand-icons/toshiba.svg
create mode 100644 docs/public/icons/claudecode.svg
create mode 100644 docs/public/icons/companies/adobe.svg
create mode 100644 docs/public/icons/companies/aws.svg
create mode 100644 docs/public/icons/companies/axa.svg
create mode 100644 docs/public/icons/companies/bcg.svg
create mode 100644 docs/public/icons/companies/benz.svg
create mode 100644 docs/public/icons/companies/bosch.svg
create mode 100644 docs/public/icons/companies/cvs-health.svg
create mode 100644 docs/public/icons/companies/ey.svg
create mode 100644 docs/public/icons/companies/google.svg
create mode 100644 docs/public/icons/companies/lego.svg
create mode 100644 docs/public/icons/companies/mastercard.svg
create mode 100644 docs/public/icons/companies/microsoft.svg
create mode 100644 docs/public/icons/companies/nvidia.svg
create mode 100644 docs/public/icons/companies/openai.svg
create mode 100644 docs/public/icons/companies/pfizer.svg
create mode 100644 docs/public/icons/companies/samsung.svg
create mode 100644 docs/public/icons/companies/siemens.svg
create mode 100644 docs/public/icons/companies/toyota.svg
create mode 100644 docs/public/icons/companies/uber.svg
create mode 100644 docs/public/icons/companies/visa.svg
create mode 100644 docs/public/icons/companies/walmart.svg
create mode 100644 docs/public/icons/deepeval-icon.svg
create mode 100644 docs/public/icons/frameworks/anthropic.png
create mode 100644 docs/public/icons/frameworks/crewai.png
create mode 100644 docs/public/icons/frameworks/langchain.png
create mode 100644 docs/public/icons/frameworks/langgraph.png
create mode 100644 docs/public/icons/frameworks/llamaindex.png
create mode 100644 docs/public/icons/frameworks/pydanticai.png
create mode 100644 docs/public/icons/integrations/agentcore.svg
create mode 100644 docs/public/icons/integrations/ai-sdk.svg
create mode 100644 docs/public/icons/integrations/azure-pipelines.svg
create mode 100644 docs/public/icons/integrations/azure.svg
create mode 100644 docs/public/icons/integrations/bedrock.svg
create mode 100644 docs/public/icons/integrations/buildkite.svg
create mode 100644 docs/public/icons/integrations/circleci.svg
create mode 100644 docs/public/icons/integrations/claude.svg
create mode 100644 docs/public/icons/integrations/crewai.svg
create mode 100644 docs/public/icons/integrations/gemini.svg
create mode 100755 docs/public/icons/integrations/github.svg
create mode 100644 docs/public/icons/integrations/gitlab.svg
create mode 100644 docs/public/icons/integrations/google-adk.png
create mode 100644 docs/public/icons/integrations/jenkins.svg
create mode 100644 docs/public/icons/integrations/langchain.svg
create mode 100644 docs/public/icons/integrations/langgraph.svg
create mode 100644 docs/public/icons/integrations/litellm.svg
create mode 100755 docs/public/icons/integrations/llamaindex.svg
create mode 100644 docs/public/icons/integrations/mistral.svg
create mode 100644 docs/public/icons/integrations/openai.svg
create mode 100644 docs/public/icons/integrations/otel.svg
create mode 100644 docs/public/icons/integrations/portkey.svg
create mode 100644 docs/public/icons/integrations/pydanticai.svg
create mode 100644 docs/public/icons/integrations/strands.svg
create mode 100644 docs/public/icons/integrations/vertext_ai.svg
create mode 100644 docs/public/icons/python.svg
create mode 100644 docs/public/icons/typescript.svg
create mode 100644 docs/public/img/2025-dashboard-screenshot.png
create mode 100644 docs/public/img/confident-alerts.png
create mode 100644 docs/public/img/confident-centralized-metrics.png
create mode 100644 docs/public/img/confident-dataset-management.png
create mode 100644 docs/public/img/confident-experimentation.png
create mode 100644 docs/public/img/confident-human-annotation.png
create mode 100644 docs/public/img/confident-production-monitoring.png
create mode 100644 docs/public/img/confident-prompt-versioning.png
create mode 100644 docs/public/img/confident-regression-testing.png
create mode 100644 docs/public/img/confident-trace-to-dataset.png
create mode 100644 docs/public/img/confident-tracing-observability.png
create mode 100644 docs/public/img/dashboard-screenshot.png
create mode 100644 docs/public/img/deepeval.png
create mode 100644 docs/public/img/favicon.ico
create mode 100644 docs/public/img/social_card.png
create mode 100644 docs/public/img/white-2.png
create mode 100644 docs/public/img/white-3.png
create mode 100644 docs/public/img/white.png
create mode 100644 docs/public/llms-full.txt
create mode 100644 docs/public/llms.txt
create mode 100644 docs/scripts/build-readme-hero.mjs
create mode 100644 docs/scripts/fonts/Quicksand-SemiBold.ttf
create mode 100644 docs/scripts/generate-changelog-contributors.mjs
create mode 100644 docs/scripts/generate-contributors.mjs
create mode 100644 docs/scripts/generate-repo-contributors.mjs
create mode 100644 docs/scripts/normalize-admonition-titles.mjs
create mode 100644 docs/scripts/replace-img-with-image-displayer.mjs
create mode 100644 docs/scripts/strip-redundant-mdx-imports.mjs
create mode 100644 docs/scripts/timeline-to-steps.mjs
create mode 100644 docs/snippets/evaluation/cicd-agent-framework-tabs.mdx
create mode 100644 docs/snippets/evaluation/component-level-agent-framework-tabs.mdx
create mode 100644 docs/snippets/evaluation/end-to-end-agent-framework-tabs.mdx
create mode 100644 docs/source.config.ts
create mode 100644 docs/src/assets.ts
create mode 100644 docs/src/components/AgentTraceTerminal/AgentTraceTerminal.module.scss
create mode 100644 docs/src/components/AgentTraceTerminal/index.tsx
create mode 100644 docs/src/components/AskAIButton/index.tsx
create mode 100644 docs/src/components/BlogPostMeta/BlogPostMeta.module.scss
create mode 100644 docs/src/components/BlogPostMeta/index.tsx
create mode 100644 docs/src/components/BrandMarks/index.tsx
create mode 100644 docs/src/components/Buttons/Buttons.module.scss
create mode 100644 docs/src/components/Buttons/index.tsx
create mode 100644 docs/src/components/Callout/Callout.module.scss
create mode 100644 docs/src/components/Callout/index.tsx
create mode 100644 docs/src/components/ChangelogContributors/ChangelogContributors.module.scss
create mode 100644 docs/src/components/ChangelogContributors/index.tsx
create mode 100644 docs/src/components/CloudPlatformCallout/CloudPlatformCallout.module.scss
create mode 100644 docs/src/components/CloudPlatformCallout/index.tsx
create mode 100644 docs/src/components/CommunityCta/CommunityCta.module.scss
create mode 100644 docs/src/components/CommunityCta/index.tsx
create mode 100644 docs/src/components/ContributorDisplay/ContributorDisplay.module.scss
create mode 100644 docs/src/components/ContributorDisplay/index.tsx
create mode 100644 docs/src/components/DiscordButton/DiscordButton.module.scss
create mode 100644 docs/src/components/DiscordButton/index.tsx
create mode 100644 docs/src/components/Equation/Equation.module.scss
create mode 100644 docs/src/components/Equation/index.tsx
create mode 100644 docs/src/components/FAQ/index.tsx
create mode 100644 docs/src/components/FeatureComparisonTable/FeatureComparisonTable.module.scss
create mode 100644 docs/src/components/FeatureComparisonTable/index.tsx
create mode 100644 docs/src/components/GithubCtaButton/GithubCtaButton.module.scss
create mode 100644 docs/src/components/GithubCtaButton/index.tsx
create mode 100644 docs/src/components/GithubCtaButton/useGithubStarCount.ts
create mode 100644 docs/src/components/Hotkey/Hotkey.module.scss
create mode 100644 docs/src/components/Hotkey/index.tsx
create mode 100644 docs/src/components/ImageDisplayer/ImageDisplayer.module.scss
create mode 100644 docs/src/components/ImageDisplayer/index.tsx
create mode 100644 docs/src/components/IntegrationGrid/IntegrationGrid.module.scss
create mode 100644 docs/src/components/IntegrationGrid/index.tsx
create mode 100644 docs/src/components/IntegrationTagsDisplayer/IntegrationTagsDisplayer.module.scss
create mode 100644 docs/src/components/IntegrationTagsDisplayer/index.tsx
create mode 100644 docs/src/components/LinkCards/LinkCards.module.scss
create mode 100644 docs/src/components/LinkCards/index.tsx
create mode 100644 docs/src/components/Mermaid/index.tsx
create mode 100644 docs/src/components/MetricTagsDisplayer/MetricTagsDisplayer.module.scss
create mode 100644 docs/src/components/MetricTagsDisplayer/index.tsx
create mode 100644 docs/src/components/PageContributors/ContributorsOverflow.tsx
create mode 100644 docs/src/components/PageContributors/PageContributors.module.scss
create mode 100644 docs/src/components/PageContributors/index.tsx
create mode 100644 docs/src/components/PauseOffscreen/index.tsx
create mode 100644 docs/src/components/RedditButton/RedditButton.module.scss
create mode 100644 docs/src/components/RedditButton/index.tsx
create mode 100644 docs/src/components/SchemaInjector/SchemaInjector.tsx
create mode 100644 docs/src/components/SectionLabel/SectionLabel.module.scss
create mode 100644 docs/src/components/SectionLabel/index.tsx
create mode 100644 docs/src/components/SiteThemeSwitch/SiteThemeSwitch.module.scss
create mode 100644 docs/src/components/SiteThemeSwitch/index.tsx
create mode 100644 docs/src/components/TechStackCards/TechStackCards.module.scss
create mode 100644 docs/src/components/TechStackCards/index.tsx
create mode 100644 docs/src/components/TocFooter/TocFooter.module.scss
create mode 100644 docs/src/components/TocFooter/index.tsx
create mode 100644 docs/src/components/VideoDisplayer/VideoDisplayer.module.scss
create mode 100644 docs/src/components/VideoDisplayer/index.tsx
create mode 100644 docs/src/components/Wordmark/index.tsx
create mode 100644 docs/src/components/index.ts
create mode 100644 docs/src/global.d.ts
create mode 100644 docs/src/layouts/Footer/Footer.module.scss
create mode 100644 docs/src/layouts/Footer/index.tsx
create mode 100644 docs/src/layouts/HomeLayout/HomeLayout.module.scss
create mode 100644 docs/src/layouts/HomeLayout/HomeLayout.tsx
create mode 100644 docs/src/layouts/HomeLayout/index.ts
create mode 100644 docs/src/layouts/HomeOverflowNav/HomeOverflowNav.module.scss
create mode 100644 docs/src/layouts/HomeOverflowNav/index.tsx
create mode 100644 docs/src/layouts/HomePageShell/HomePageShell.module.scss
create mode 100644 docs/src/layouts/HomePageShell/index.tsx
create mode 100644 docs/src/layouts/NavHeader/NavHeader.module.scss
create mode 100644 docs/src/layouts/NavHeader/index.tsx
create mode 100644 docs/src/layouts/NavLinks/index.tsx
create mode 100644 docs/src/layouts/NavMenu/NavMenu.module.scss
create mode 100644 docs/src/layouts/NavMenu/index.tsx
create mode 100644 docs/src/layouts/SidebarSearch/index.tsx
create mode 100644 docs/src/layouts/SiteTopNav/SiteTopNav.module.scss
create mode 100644 docs/src/layouts/SiteTopNav/index.tsx
create mode 100644 docs/src/layouts/UtmCapture/UtmCapture.tsx
create mode 100644 docs/src/layouts/UtmCapture/index.ts
create mode 100644 docs/src/sections/enterprise/EnterpriseComparisonTable.module.scss
create mode 100644 docs/src/sections/enterprise/EnterpriseComparisonTable.tsx
create mode 100644 docs/src/sections/enterprise/EnterpriseHeroSection.module.scss
create mode 100644 docs/src/sections/enterprise/EnterpriseHeroSection.tsx
create mode 100644 docs/src/sections/enterprise/EnterprisePlatformMockup.module.scss
create mode 100644 docs/src/sections/enterprise/EnterprisePlatformMockup.tsx
create mode 100644 docs/src/sections/home/ClaudeCodeTerminal/ClaudeCodeTerminal.module.scss
create mode 100644 docs/src/sections/home/ClaudeCodeTerminal/index.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Adobe.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Aws.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Axa.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Benz.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Bosch.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/CompanyLogos.module.scss
create mode 100644 docs/src/sections/home/CompanyLogos/CvsHealth.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Ey.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Google.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Lego.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Mastercard.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Microsoft.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Nvidia.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/OpenAI.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Pfizer.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Samsung.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Siemens.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Toyota.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Uber.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Visa.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/Walmart.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/_createInlineLogo.tsx
create mode 100644 docs/src/sections/home/CompanyLogos/_loadInlineSvg.ts
create mode 100644 docs/src/sections/home/CompanyLogos/index.ts
create mode 100644 docs/src/sections/home/CompanyLogos/types.ts
create mode 100644 docs/src/sections/home/DatasetDemos/DatasetDemos.module.scss
create mode 100644 docs/src/sections/home/DatasetDemos/GoldenGenerationDemo.tsx
create mode 100644 docs/src/sections/home/DatasetDemos/MultiTurnSimulationDemo.tsx
create mode 100644 docs/src/sections/home/DatasetDemos/index.tsx
create mode 100644 docs/src/sections/home/HomeHeroSection.tsx
create mode 100644 docs/src/sections/home/HomePytestDemo/HomePytestDemo.module.scss
create mode 100644 docs/src/sections/home/HomePytestDemo/index.tsx
create mode 100644 docs/src/sections/home/HomeSection.module.scss
create mode 100644 docs/src/sections/home/JudgeCards/JudgeCards.module.scss
create mode 100644 docs/src/sections/home/JudgeCards/index.tsx
create mode 100644 docs/src/sections/home/RepoContributors/RepoContributors.module.scss
create mode 100644 docs/src/sections/home/RepoContributors/index.tsx
create mode 100644 docs/src/sections/home/SOTACards/SOTACards.module.scss
create mode 100644 docs/src/sections/home/SOTACards/index.tsx
create mode 100644 docs/src/sections/home/TraceLoopConnector/TraceLoopConnector.module.scss
create mode 100644 docs/src/sections/home/TraceLoopConnector/index.tsx
create mode 100644 docs/src/sections/home/VibeCodingLoop/VibeCodingLoop.module.scss
create mode 100644 docs/src/sections/home/VibeCodingLoop/index.tsx
create mode 100644 docs/src/utils/html-to-markdown.ts
create mode 100644 docs/src/utils/outbound-link-rel.ts
create mode 100644 docs/src/utils/schema-helpers.ts
create mode 100644 docs/src/utils/utm.ts
create mode 100644 docs/src/utils/visitor-attribution.ts
create mode 100644 docs/tsconfig.json
create mode 100644 docs/vercel.json
create mode 100644 docs/yarn.lock
create mode 100644 examples/community/chatbot_evaluation/dataset.json
create mode 100644 examples/community/chatbot_evaluation/evaluate.py
create mode 100644 examples/community/chatbot_evaluation/generate_outputs.js
create mode 100644 examples/community/chatbot_evaluation/outputs.json
create mode 100644 examples/create_tests.py
create mode 100644 examples/dag-examples/conversational_dag.ipynb
create mode 100644 examples/getting_started/test_example.py
create mode 100644 examples/mcp_evaluation/mcp_eval_multi_turn.py
create mode 100644 examples/mcp_evaluation/mcp_eval_single_turn.py
create mode 100644 examples/notebooks/crewai.ipynb
create mode 100644 examples/notebooks/langgraph.ipynb
create mode 100644 examples/notebooks/openai.ipynb
create mode 100644 examples/notebooks/pydantic_ai.ipynb
create mode 100644 examples/notebooks/static/manual.txt
create mode 100644 examples/notebooks/static/output.png
create mode 100644 examples/rag_evaluation/rag_evaluation_with_qdrant.py
create mode 100644 examples/sample.txt
create mode 100644 examples/tracing/crewai_tracing.ipynb
create mode 100644 examples/tracing/test_chatbot.py
create mode 100644 manual_after_evals_iterator.py
create mode 100644 poetry.lock
create mode 100644 pyproject.toml
create mode 100644 scripts/check_openai_model_capabilities.py
create mode 100644 scripts/compile_metric_templates.py
create mode 100644 skills/README.md
create mode 100644 skills/deepeval-otel/LICENSE
create mode 100644 skills/deepeval-otel/SKILL.md
create mode 100644 skills/deepeval-otel/references/endpoint-and-exporter.md
create mode 100644 skills/deepeval-otel/references/gen-ai-fallbacks.md
create mode 100644 skills/deepeval-otel/references/span-attributes.md
create mode 100644 skills/deepeval-otel/references/trace-attributes.md
create mode 100644 skills/deepeval-otel/templates/confident_otel_setup.py
create mode 100644 skills/deepeval-tracing/LICENSE
create mode 100644 skills/deepeval-tracing/SKILL.md
create mode 100644 skills/deepeval-tracing/references/integrations.md
create mode 100644 skills/deepeval-tracing/references/tracing.md
create mode 100644 skills/deepeval/LICENSE
create mode 100644 skills/deepeval/SKILL.md
create mode 100644 skills/deepeval/references/artifact-contracts.md
create mode 100644 skills/deepeval/references/choose-use-case.md
create mode 100644 skills/deepeval/references/confident-ai.md
create mode 100644 skills/deepeval/references/datasets.md
create mode 100644 skills/deepeval/references/intake.md
create mode 100644 skills/deepeval/references/iteration-loop.md
create mode 100644 skills/deepeval/references/metrics.md
create mode 100644 skills/deepeval/references/pytest-e2e-evals.md
create mode 100644 skills/deepeval/references/synthetic-data.md
create mode 100644 skills/deepeval/references/traced-evals.md
create mode 100644 skills/deepeval/templates/metrics.py
create mode 100644 skills/deepeval/templates/test_multi_turn_e2e.py
create mode 100644 skills/deepeval/templates/test_single_turn_no_tracing.py
create mode 100644 skills/deepeval/templates/test_single_turn_tracing.py
create mode 100644 test_agentcore_agent.py
create mode 100644 test_pydantic_agent.py
create mode 100644 tests/__init__.py
create mode 100644 tests/test_agent_loop_detection.py
create mode 100644 tests/test_confident/goldens.json
create mode 100644 tests/test_confident/goldens_multi_turn.json
create mode 100644 tests/test_confident/simulator/example_simulator.py
create mode 100644 tests/test_confident/test_annotation.py
create mode 100644 tests/test_confident/test_compare.py
create mode 100644 tests/test_confident/test_conversational_g_eval_upload.py
create mode 100644 tests/test_confident/test_dataset.py
create mode 100644 tests/test_confident/test_evaluate.py
create mode 100644 tests/test_confident/test_g_eval_upload.py
create mode 100644 tests/test_confident/test_prompt.py
create mode 100644 tests/test_confident/test_region_autodetect_request_routing.py
create mode 100644 tests/test_core/__init__.py
create mode 100644 tests/test_core/conftest.py
create mode 100644 tests/test_core/helpers.py
create mode 100644 tests/test_core/stubs.py
create mode 100644 tests/test_core/test_cli/__init__.py
create mode 100644 tests/test_core/test_cli/test_auth_command.py
create mode 100644 tests/test_core/test_cli/test_check_if_valid_file.py
create mode 100644 tests/test_core/test_cli/test_cli.py
create mode 100644 tests/test_core/test_config/__init__.py
create mode 100644 tests/test_core/test_config/test_deprecated_computed_aliases.py
create mode 100644 tests/test_core/test_config/test_settings.py
create mode 100644 tests/test_core/test_core.py
create mode 100644 tests/test_core/test_dataset_test_run_tracer.py
create mode 100644 tests/test_core/test_datasets/convo_goldens.csv
create mode 100644 tests/test_core/test_datasets/convo_goldens.json
create mode 100644 tests/test_core/test_datasets/goldens.csv
create mode 100644 tests/test_core/test_datasets/goldens.json
create mode 100644 tests/test_core/test_datasets/test_dataset.py
create mode 100644 tests/test_core/test_drop_trace_and_span.py
create mode 100644 tests/test_core/test_evaluation/test_async_trace_metric_isolation.py
create mode 100644 tests/test_core/test_evaluation/test_console_report.py
create mode 100644 tests/test_core/test_evaluation/test_end_to_end/__init__.py
create mode 100644 tests/test_core/test_evaluation/test_end_to_end/test_configs.py
create mode 100644 tests/test_core/test_evaluation/test_end_to_end/test_skip_reset.py
create mode 100644 tests/test_core/test_evaluation/test_execute/test_error_boundary.py
create mode 100644 tests/test_core/test_evaluation/test_execute/test_execute_conversational_test_case.py
create mode 100644 tests/test_core/test_evaluation/test_execute/test_execute_llm_test_case.py
create mode 100644 tests/test_core/test_evaluation/test_execute/test_execute_mllm_test_case.py
create mode 100644 tests/test_core/test_evaluation/test_execute/test_execute_timeouts.py
create mode 100644 tests/test_core/test_evaluation/test_local_store.py
create mode 100644 tests/test_core/test_evaluation/test_printing.py
create mode 100644 tests/test_core/test_evaluation/test_results_extraction.py
create mode 100644 tests/test_core/test_evaluation/test_trace_results_extraction.py
create mode 100644 tests/test_core/test_evaluation/test_trace_scope_assert_test.py
create mode 100644 tests/test_core/test_imports.py
create mode 100644 tests/test_core/test_models/test_amazon_bedrock_model.py
create mode 100644 tests/test_core/test_models/test_anthropic_model.py
create mode 100644 tests/test_core/test_models/test_azure_model.py
create mode 100644 tests/test_core/test_models/test_azure_retry_config.py
create mode 100644 tests/test_core/test_models/test_bedrock_retry_config.py
create mode 100644 tests/test_core/test_models/test_deepseek_model.py
create mode 100644 tests/test_core/test_models/test_embedding_models/__init__.py
create mode 100644 tests/test_core/test_models/test_embedding_models/test_azure_embedding_model.py
create mode 100644 tests/test_core/test_models/test_embedding_models/test_local_embedding_model.py
create mode 100644 tests/test_core/test_models/test_embedding_models/test_ollama_embedding_model.py
create mode 100644 tests/test_core/test_models/test_embedding_models/test_openai_embedding_model.py
create mode 100644 tests/test_core/test_models/test_gemini_model.py
create mode 100644 tests/test_core/test_models/test_grok_model.py
create mode 100644 tests/test_core/test_models/test_kimi_model.py
create mode 100644 tests/test_core/test_models/test_litellm_model.py
create mode 100644 tests/test_core/test_models/test_local_model.py
create mode 100644 tests/test_core/test_models/test_models_utils.py
create mode 100644 tests/test_core/test_models/test_ollama_model.py
create mode 100644 tests/test_core/test_models/test_openai_extractors.py
create mode 100644 tests/test_core/test_models/test_openai_model.py
create mode 100644 tests/test_core/test_models/test_openai_retry_policy.py
create mode 100644 tests/test_core/test_models/test_openrouter_model.py
create mode 100644 tests/test_core/test_models/test_portkey_model.py
create mode 100644 tests/test_core/test_optimization/test_copro/__init__.py
create mode 100644 tests/test_core/test_optimization/test_copro/test_configs.py
create mode 100644 tests/test_core/test_optimization/test_copro/test_loop.py
create mode 100644 tests/test_core/test_optimization/test_gepa/__init__.py
create mode 100644 tests/test_core/test_optimization/test_gepa/test_gepa_config.py
create mode 100644 tests/test_core/test_optimization/test_gepa/test_loop.py
create mode 100644 tests/test_core/test_optimization/test_miprov2/test_report_contract.py
create mode 100644 tests/test_core/test_optimization/test_mutations/__init__.py
create mode 100644 tests/test_core/test_optimization/test_mutations/test_prompt_rewriter.py
create mode 100644 tests/test_core/test_optimization/test_pareto.py
create mode 100644 tests/test_core/test_optimization/test_policies/test_tie_breaker.py
create mode 100644 tests/test_core/test_optimization/test_prompt_optimizer.py
create mode 100644 tests/test_core/test_optimization/test_simba/__init__.py
create mode 100644 tests/test_core/test_optimization/test_simba/test_configs.py
create mode 100644 tests/test_core/test_optimization/test_simba/test_loop.py
create mode 100644 tests/test_core/test_optimization/test_utils.py
create mode 100644 tests/test_core/test_prompts/test_interpolation.py
create mode 100644 tests/test_core/test_prompts/test_load.py
create mode 100644 tests/test_core/test_prompts/test_prompt.py
create mode 100644 tests/test_core/test_retry_policy.py
create mode 100644 tests/test_core/test_run/__init__.py
create mode 100644 tests/test_core/test_run/test_file_sync.py
create mode 100644 tests/test_core/test_run/test_run_manager.py
create mode 100644 tests/test_core/test_run/test_turns_table.py
create mode 100644 tests/test_core/test_sanitize_nan.py
create mode 100644 tests/test_core/test_simulator/__init__.py
create mode 100644 tests/test_core/test_simulator/helpers.py
create mode 100644 tests/test_core/test_simulator/test_conversation_simulator.py
create mode 100644 tests/test_core/test_simulator/test_conversation_simulator_controller.py
create mode 100644 tests/test_core/test_simulator/test_conversation_simulator_decision_graph.py
create mode 100644 tests/test_core/test_simulator/test_conversation_simulator_json_mode.py
create mode 100644 tests/test_core/test_stubs_contract.py
create mode 100644 tests/test_core/test_synthesizer/__init__.py
create mode 100644 tests/test_core/test_synthesizer/example_simulator.py
create mode 100644 tests/test_core/test_synthesizer/synthesizer_data/docx_example.docx
create mode 100644 tests/test_core/test_synthesizer/synthesizer_data/pdf_example.pdf
create mode 100644 tests/test_core/test_synthesizer/synthesizer_data/txt_example.txt
create mode 100644 tests/test_core/test_synthesizer/test_context_generator.py
create mode 100644 tests/test_core/test_synthesizer/test_doc_chunker.py
create mode 100644 tests/test_core/test_synthesizer/test_generate_from_goldens.py
create mode 100644 tests/test_core/test_synthesizer/test_synthesizer.py
create mode 100644 tests/test_core/test_synthesizer_bugs.py
create mode 100644 tests/test_core/test_telemetry.py
create mode 100644 tests/test_core/test_test_case/__init__.py
create mode 100644 tests/test_core/test_test_case/test_deprecated_params.py
create mode 100644 tests/test_core/test_test_case/test_multi_turn/__init__.py
create mode 100644 tests/test_core/test_test_case/test_multi_turn/test_conversational_test_case.py
create mode 100644 tests/test_core/test_test_case/test_multi_turn/test_turn.py
create mode 100644 tests/test_core/test_test_case/test_multi_turn/test_utils.py
create mode 100644 tests/test_core/test_test_case/test_single_turn.py
create mode 100644 tests/test_core/test_threadpool_tracing.py
create mode 100644 tests/test_core/test_trace_memory_leak.py
create mode 100644 tests/test_core/test_tracing/__init__.py
create mode 100644 tests/test_core/test_tracing/apps/__init__.py
create mode 100644 tests/test_core/test_tracing/apps/async_app.py
create mode 100644 tests/test_core/test_tracing/apps/sync_app.py
create mode 100644 tests/test_core/test_tracing/conftest.py
create mode 100644 tests/test_core/test_tracing/example_e2e_trace_evals.py
create mode 100644 tests/test_core/test_tracing/schemas/generators/async_streaming_concurrent_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/async_streaming_llm_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/async_streaming_nested_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/async_streaming_processor_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/async_streaming_updates_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/context_safety_async_full_consumption_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/context_safety_async_gen_yields_inner_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/context_safety_async_nested_consume_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/context_safety_async_three_level_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/context_safety_sync_full_consumption_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/context_safety_sync_gen_observe_between_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/context_safety_sync_gen_yields_inner_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/context_safety_sync_nested_consume_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/context_safety_sync_observe_between_yields_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/context_safety_sync_siblings_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/context_safety_sync_three_level_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/fastapi_basic_threadpool_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/fastapi_child_spans_threadpool_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/fastapi_deep_nesting_threadpool_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/fastapi_rag_pipeline_threadpool_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/fastapi_same_thread_sanity_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/sync_streaming_llm_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/sync_streaming_nested_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/sync_streaming_processor_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/generators/sync_streaming_updates_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/masking/comprehensive_masked_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/masking/credit_card_masked_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/masking/email_masked_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/masking/no_masking_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/metadata/agent_with_metadata_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/metadata/llm_with_metadata_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/metadata/span_basic_metadata_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/metadata/span_complex_metadata_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/metadata/trace_basic_metadata_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/metadata/trace_full_context_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/metadata/trace_nested_spans_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/metadata/trace_user_info_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/nested_spans/agent_workflow_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/nested_spans/async_nesting_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/nested_spans/deep_nesting_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/nested_spans/multiple_children_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/nested_spans/simple_nesting_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/agent_custom_name_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/agent_full_attributes_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/agent_minimal_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/agent_multiple_handoffs_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/agent_span_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/agent_with_handoffs_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/custom_processor_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/custom_validator_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/custom_with_name_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/default_span_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/llm_custom_name_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/llm_full_attributes_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/llm_minimal_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/llm_runtime_model_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/llm_span_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/llm_with_costs_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/retriever_custom_embedder_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/retriever_custom_name_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/retriever_full_attributes_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/retriever_minimal_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/retriever_override_embedder_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/retriever_span_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/span_with_only_name_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/tool_calculator_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/tool_custom_name_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/tool_description_and_name_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/tool_long_description_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/tool_minimal_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/span_types/tool_span_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/tags/basic_tags_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/tags/env_tags_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/tags/feature_tags_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/tags/name_and_tags_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/update_functions/span_context_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/update_functions/span_expected_output_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/update_functions/span_from_test_case_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/update_functions/span_input_output_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/update_functions/span_name_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/update_functions/span_override_test_case_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/update_functions/span_tools_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/update_functions/trace_context_info_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/update_functions/trace_from_test_case_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/update_functions/trace_full_context_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/update_functions/trace_identifiers_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/update_functions/trace_name_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/update_functions/trace_nested_updates_schema.json
create mode 100644 tests/test_core/test_tracing/schemas/update_functions/trace_tools_schema.json
create mode 100644 tests/test_core/test_tracing/test_configuration/test_configure_multiple.py
create mode 100644 tests/test_core/test_tracing/test_configuration/test_environment.py
create mode 100644 tests/test_core/test_tracing/test_configuration/test_masking_config.py
create mode 100644 tests/test_core/test_tracing/test_configuration/test_sampling_rate.py
create mode 100644 tests/test_core/test_tracing/test_generators/test_async_generator.py
create mode 100644 tests/test_core/test_tracing/test_generators/test_fastapi_streaming_repro.py
create mode 100644 tests/test_core/test_tracing/test_generators/test_generator_context_safety.py
create mode 100644 tests/test_core/test_tracing/test_generators/test_sync_generator.py
create mode 100644 tests/test_core/test_tracing/test_integration/test_current_golden_context.py
create mode 100644 tests/test_core/test_tracing/test_integration/test_dataset_iterator.py
create mode 100644 tests/test_core/test_tracing/test_integration/test_execute_integration.py
create mode 100644 tests/test_core/test_tracing/test_integration/test_tools_called.py
create mode 100644 tests/test_core/test_tracing/test_masking/test_masking.py
create mode 100644 tests/test_core/test_tracing/test_metadata/test_span_metadata.py
create mode 100644 tests/test_core/test_tracing/test_metadata/test_trace_metadata.py
create mode 100644 tests/test_core/test_tracing/test_nested_spans/test_nested_spans.py
create mode 100644 tests/test_core/test_tracing/test_span_types/test_agent_span.py
create mode 100644 tests/test_core/test_tracing/test_span_types/test_custom_span.py
create mode 100644 tests/test_core/test_tracing/test_span_types/test_llm_span.py
create mode 100644 tests/test_core/test_tracing/test_span_types/test_retriever_span.py
create mode 100644 tests/test_core/test_tracing/test_span_types/test_tool_span.py
create mode 100644 tests/test_core/test_tracing/test_tags/test_trace_tags.py
create mode 100644 tests/test_core/test_tracing/test_update_functions/test_update_current_span.py
create mode 100644 tests/test_core/test_tracing/test_update_functions/test_update_current_trace.py
create mode 100644 tests/test_core/test_trim_and_load_json.py
create mode 100644 tests/test_core/test_utils.py
create mode 100644 tests/test_docs/test_confident/test_integrations/test_anthropic.py
create mode 100644 tests/test_docs/test_confident/test_integrations/test_crewai.py
create mode 100644 tests/test_docs/test_confident/test_integrations/test_langchain.py
create mode 100644 tests/test_docs/test_confident/test_integrations/test_langgraph.py
create mode 100644 tests/test_docs/test_confident/test_integrations/test_litellm.py
create mode 100644 tests/test_docs/test_confident/test_integrations/test_openai.py
create mode 100644 tests/test_docs/test_confident/test_integrations/test_openai_agents.py
create mode 100644 tests/test_docs/test_confident/test_integrations/test_opentelemetry.py
create mode 100644 tests/test_docs/test_confident/test_tracing_features/test_attributes.py
create mode 100644 tests/test_docs/test_confident/test_tracing_features/test_environment.py
create mode 100644 tests/test_docs/test_confident/test_tracing_features/test_input_output.py
create mode 100644 tests/test_docs/test_confident/test_tracing_features/test_masking.py
create mode 100644 tests/test_docs/test_confident/test_tracing_features/test_metadata.py
create mode 100644 tests/test_docs/test_confident/test_tracing_features/test_name.py
create mode 100644 tests/test_docs/test_confident/test_tracing_features/test_sampling.py
create mode 100644 tests/test_docs/test_confident/test_tracing_features/test_span_types.py
create mode 100644 tests/test_docs/test_confident/test_tracing_features/test_tags.py
create mode 100644 tests/test_docs/test_confident/test_tracing_features/test_test_case.py
create mode 100644 tests/test_docs/test_confident/test_tracing_features/test_threads.py
create mode 100644 tests/test_docs/test_confident/test_tracing_features/test_users.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_crewai/test_crewai_e2e.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_e2e_crewai.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_e2e_langchain.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_e2e_langraph.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_langchain/test_langchain_e2e_async.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_langchain/test_langchain_e2e_sync.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_langchain/test_langchain_online_evals.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_langchain/test_langchain_setup.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_langgraph/test_e2e_langgraph_async.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_langgraph/test_e2e_langgraph_sync.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_langgraph/test_langgraph_component.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_langgraph/test_langgraph_online.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_langgraph/test_langgraph_setup.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_llamaindex/test_llamaindex_e2e_async.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_openai_agents/test_agent.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_overall.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_pydanticai/test_pydanticai_component_level.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_pydanticai/test_pydanticai_e2e_async.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_pydanticai/test_pydanticai_online.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_pydanticai/test_pydanticai_setup.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_pydanticai/test_pydanticai_trace_attr.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_setup_crewai.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_setup_end_to_end_python.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_setup_langchain.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_setup_langraph.py
create mode 100644 tests/test_docs/test_deepeval/test_ai_agent_evals/test_setup_llm_tracing.py
create mode 100644 tests/test_docs/test_deepeval/test_llm_evals/test_component_level_evals.py
create mode 100644 tests/test_docs/test_deepeval/test_llm_evals/test_setup_tracing.py
create mode 100644 tests/test_integrations/__init__.py
create mode 100644 tests/test_integrations/test_agentcore/apps/agentcore_eval_app.py
create mode 100644 tests/test_integrations/test_agentcore/apps/agentcore_multiple_tools_app.py
create mode 100644 tests/test_integrations/test_agentcore/apps/agentcore_simple_app.py
create mode 100644 tests/test_integrations/test_agentcore/apps/agentcore_tool_app.py
create mode 100644 tests/test_integrations/test_agentcore/conftest.py
create mode 100644 tests/test_integrations/test_agentcore/schemas/agentcore_async_parallel_tools_schema.json
create mode 100644 tests/test_integrations/test_agentcore/schemas/agentcore_async_simple_schema.json
create mode 100644 tests/test_integrations/test_agentcore/schemas/agentcore_async_tool_schema.json
create mode 100644 tests/test_integrations/test_agentcore/schemas/agentcore_features_async.json
create mode 100644 tests/test_integrations/test_agentcore/schemas/agentcore_features_sync.json
create mode 100644 tests/test_integrations/test_agentcore/schemas/agentcore_multiple_tools_time_schema.json
create mode 100644 tests/test_integrations/test_agentcore/schemas/agentcore_multiple_tools_weather_schema.json
create mode 100644 tests/test_integrations/test_agentcore/schemas/agentcore_parallel_tools_schema.json
create mode 100644 tests/test_integrations/test_agentcore/schemas/agentcore_simple_schema.json
create mode 100644 tests/test_integrations/test_agentcore/schemas/agentcore_tool_metric_collection_schema.json
create mode 100644 tests/test_integrations/test_agentcore/schemas/agentcore_tool_schema.json
create mode 100644 tests/test_integrations/test_agentcore/test_async.py
create mode 100644 tests/test_integrations/test_agentcore/test_evaluate_agent.py
create mode 100644 tests/test_integrations/test_agentcore/test_span_interceptor.py
create mode 100644 tests/test_integrations/test_agentcore/test_sync.py
create mode 100644 tests/test_integrations/test_anthropic/conftest.py
create mode 100644 tests/test_integrations/test_anthropic/simple_anthropic.py
create mode 100644 tests/test_integrations/test_anthropic/test_async_anthropic.py
create mode 100644 tests/test_integrations/test_anthropic/test_async_messages_create_with_trace.json
create mode 100644 tests/test_integrations/test_anthropic/test_sync_anthropic.py
create mode 100644 tests/test_integrations/test_anthropic/test_sync_messages_create_without_trace.json
create mode 100644 tests/test_integrations/test_crewai/apps/async_app.py
create mode 100644 tests/test_integrations/test_crewai/apps/evals_app.py
create mode 100644 tests/test_integrations/test_crewai/apps/hierarchical_app.py
create mode 100644 tests/test_integrations/test_crewai/apps/knowledge_retriever_app.py
create mode 100644 tests/test_integrations/test_crewai/apps/multi_agent_app.py
create mode 100644 tests/test_integrations/test_crewai/apps/simple_app.py
create mode 100644 tests/test_integrations/test_crewai/apps/tool_usage_app.py
create mode 100644 tests/test_integrations/test_crewai/conftest.py
create mode 100644 tests/test_integrations/test_crewai/crewai.json
create mode 100644 tests/test_integrations/test_crewai/crewai_component.json
create mode 100644 tests/test_integrations/test_crewai/knowledge_retriever.json
create mode 100644 tests/test_integrations/test_crewai/schemas/crewai_akickoff.json
create mode 100644 tests/test_integrations/test_crewai/schemas/crewai_async_kickoff.json
create mode 100644 tests/test_integrations/test_crewai/schemas/crewai_async_tool_usage.json
create mode 100644 tests/test_integrations/test_crewai/schemas/crewai_features_async.json
create mode 100644 tests/test_integrations/test_crewai/schemas/crewai_features_sync.json
create mode 100644 tests/test_integrations/test_crewai/schemas/crewai_hierarchical.json
create mode 100644 tests/test_integrations/test_crewai/schemas/crewai_kickoff_for_each.json
create mode 100644 tests/test_integrations/test_crewai/schemas/crewai_kickoff_for_each_async.json
create mode 100644 tests/test_integrations/test_crewai/schemas/crewai_knowledge_retrieval.json
create mode 100644 tests/test_integrations/test_crewai/schemas/crewai_multi_agent_sequential.json
create mode 100644 tests/test_integrations/test_crewai/schemas/crewai_simple_kickoff.json
create mode 100644 tests/test_integrations/test_crewai/schemas/crewai_tool_usage.json
create mode 100644 tests/test_integrations/test_crewai/test_async.py
create mode 100644 tests/test_integrations/test_crewai/test_crewai.py
create mode 100644 tests/test_integrations/test_crewai/test_crewai_component.py
create mode 100644 tests/test_integrations/test_crewai/test_knowledge_retriever.py
create mode 100644 tests/test_integrations/test_crewai/test_stress.py
create mode 100644 tests/test_integrations/test_crewai/test_sync.py
create mode 100644 tests/test_integrations/test_exporter/readable_spans.py
create mode 100644 tests/test_integrations/test_exporter/test_pydantic_ai.py
create mode 100644 tests/test_integrations/test_googleadk/__init__.py
create mode 100644 tests/test_integrations/test_googleadk/apps/__init__.py
create mode 100644 tests/test_integrations/test_googleadk/apps/googleadk_eval_app.py
create mode 100644 tests/test_integrations/test_googleadk/apps/googleadk_multiple_tools_app.py
create mode 100644 tests/test_integrations/test_googleadk/apps/googleadk_simple_app.py
create mode 100644 tests/test_integrations/test_googleadk/apps/googleadk_tool_app.py
create mode 100644 tests/test_integrations/test_googleadk/conftest.py
create mode 100644 tests/test_integrations/test_googleadk/schemas/README.md
create mode 100644 tests/test_integrations/test_googleadk/schemas/googleadk_async_parallel_tools_schema.json
create mode 100644 tests/test_integrations/test_googleadk/schemas/googleadk_async_simple_schema.json
create mode 100644 tests/test_integrations/test_googleadk/schemas/googleadk_async_tool_schema.json
create mode 100644 tests/test_integrations/test_googleadk/schemas/googleadk_features_async.json
create mode 100644 tests/test_integrations/test_googleadk/schemas/googleadk_features_sync.json
create mode 100644 tests/test_integrations/test_googleadk/schemas/googleadk_multiple_tools_time_schema.json
create mode 100644 tests/test_integrations/test_googleadk/schemas/googleadk_multiple_tools_weather_schema.json
create mode 100644 tests/test_integrations/test_googleadk/schemas/googleadk_parallel_tools_schema.json
create mode 100644 tests/test_integrations/test_googleadk/schemas/googleadk_simple_schema.json
create mode 100644 tests/test_integrations/test_googleadk/schemas/googleadk_tool_metric_collection_schema.json
create mode 100644 tests/test_integrations/test_googleadk/schemas/googleadk_tool_schema.json
create mode 100644 tests/test_integrations/test_googleadk/test_async.py
create mode 100644 tests/test_integrations/test_googleadk/test_evaluate_agent.py
create mode 100644 tests/test_integrations/test_googleadk/test_span_interceptor.py
create mode 100644 tests/test_integrations/test_googleadk/test_sync.py
create mode 100644 tests/test_integrations/test_langchain/__init__.py
create mode 100644 tests/test_integrations/test_langchain/apps/__init__.py
create mode 100644 tests/test_integrations/test_langchain/apps/langchain_agent_app.py
create mode 100644 tests/test_integrations/test_langchain/apps/langchain_conditional_app.py
create mode 100644 tests/test_integrations/test_langchain/apps/langchain_metric_collection_app.py
create mode 100644 tests/test_integrations/test_langchain/apps/langchain_multiple_tools_app.py
create mode 100644 tests/test_integrations/test_langchain/apps/langchain_next_span_app.py
create mode 100644 tests/test_integrations/test_langchain/apps/langchain_parallel_tools_app.py
create mode 100644 tests/test_integrations/test_langchain/apps/langchain_retriever_app.py
create mode 100644 tests/test_integrations/test_langchain/apps/langchain_simple_app.py
create mode 100644 tests/test_integrations/test_langchain/apps/langchain_single_tool_app.py
create mode 100644 tests/test_integrations/test_langchain/apps/langchain_streaming_app.py
create mode 100644 tests/test_integrations/test_langchain/conftest.py
create mode 100644 tests/test_integrations/test_langchain/langchain.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_agent_complex_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_agent_multi_step_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_agent_simple_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_agent_complex_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_agent_multi_step_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_agent_simple_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_conditional_fact_check_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_conditional_general_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_conditional_research_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_conditional_summarize_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_mixed_tools_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_multiple_tools_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_next_llm_span_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_parallel_mixed_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_parallel_stocks_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_parallel_weather_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_retriever_langchain_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_retriever_python_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_simple_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_single_tool_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_streaming_multi_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_async_streaming_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_conditional_fact_check_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_conditional_general_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_conditional_research_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_conditional_summarize_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_metric_collection_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_multiple_tools_mixed_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_multiple_tools_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_next_llm_span_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_parallel_mixed_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_parallel_stocks_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_parallel_weather_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_retriever_langchain_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_retriever_metric_collection_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_retriever_python_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_simple_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_single_tool_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_streaming_multi_schema.json
create mode 100644 tests/test_integrations/test_langchain/schemas/langchain_streaming_schema.json
create mode 100644 tests/test_integrations/test_langchain/test_async.py
create mode 100644 tests/test_integrations/test_langchain/test_langchain.py
create mode 100644 tests/test_integrations/test_langchain/test_next_span.py
create mode 100644 tests/test_integrations/test_langchain/test_sync.py
create mode 100644 tests/test_integrations/test_langgraph/__init__.py
create mode 100644 tests/test_integrations/test_langgraph/apps/__init__.py
create mode 100644 tests/test_integrations/test_langgraph/apps/langgraph_async_app.py
create mode 100644 tests/test_integrations/test_langgraph/apps/langgraph_conditional_app.py
create mode 100644 tests/test_integrations/test_langgraph/apps/langgraph_metric_collection_app.py
create mode 100644 tests/test_integrations/test_langgraph/apps/langgraph_multi_turn_app.py
create mode 100644 tests/test_integrations/test_langgraph/apps/langgraph_multiple_tools_app.py
create mode 100644 tests/test_integrations/test_langgraph/apps/langgraph_next_span_app.py
create mode 100644 tests/test_integrations/test_langgraph/apps/langgraph_parallel_tools_app.py
create mode 100644 tests/test_integrations/test_langgraph/apps/langgraph_retriever_app.py
create mode 100644 tests/test_integrations/test_langgraph/apps/langgraph_simple_app.py
create mode 100644 tests/test_integrations/test_langgraph/apps/langgraph_streaming_app.py
create mode 100644 tests/test_integrations/test_langgraph/apps/main.py
create mode 100644 tests/test_integrations/test_langgraph/conftest.py
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_async_conditional_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_async_multi_turn_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_async_multiple_tools_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_async_next_llm_span_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_async_no_tools_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_async_parallel_heavy_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_async_parallel_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_async_single_tool_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_async_streaming_multi_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_async_streaming_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_conditional_fact_check_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_conditional_general_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_conditional_research_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_conditional_summarize_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_full_flow_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_metric_collection_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_multi_turn_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_multiple_tools_mixed_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_multiple_tools_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_next_llm_span_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_parallel_mixed_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_parallel_stocks_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_parallel_weather_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_retriever_langchain_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_retriever_metric_collection_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_retriever_python_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_simple_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_stateless_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_streaming_multi_schema.json
create mode 100644 tests/test_integrations/test_langgraph/schemas/langgraph_streaming_schema.json
create mode 100644 tests/test_integrations/test_langgraph/test_async.py
create mode 100644 tests/test_integrations/test_langgraph/test_create_task.py
create mode 100644 tests/test_integrations/test_langgraph/test_next_span.py
create mode 100644 tests/test_integrations/test_langgraph/test_sync.py
create mode 100644 tests/test_integrations/test_llamaindex/apps/agent_app.py
create mode 100644 tests/test_integrations/test_llamaindex/apps/eval_app.py
create mode 100644 tests/test_integrations/test_llamaindex/apps/rag_app.py
create mode 100644 tests/test_integrations/test_llamaindex/apps/router_app.py
create mode 100644 tests/test_integrations/test_llamaindex/apps/simple_app.py
create mode 100644 tests/test_integrations/test_llamaindex/conftest.py
create mode 100644 tests/test_integrations/test_llamaindex/schemas/llama_index_async_agent_math_schema.json
create mode 100644 tests/test_integrations/test_llamaindex/schemas/llama_index_async_agent_schema.json
create mode 100644 tests/test_integrations/test_llamaindex/schemas/llama_index_async_rag_schema.json
create mode 100644 tests/test_integrations/test_llamaindex/schemas/llama_index_async_router_schema.json
create mode 100644 tests/test_integrations/test_llamaindex/schemas/llama_index_async_simple_schema.json
create mode 100644 tests/test_integrations/test_llamaindex/schemas/llama_index_features_async.json
create mode 100644 tests/test_integrations/test_llamaindex/schemas/llama_index_rag_llama_schema.json
create mode 100644 tests/test_integrations/test_llamaindex/schemas/llama_index_rag_python_schema.json
create mode 100644 tests/test_integrations/test_llamaindex/schemas/llama_index_router_math_schema.json
create mode 100644 tests/test_integrations/test_llamaindex/schemas/llama_index_simple_schema.json
create mode 100644 tests/test_integrations/test_llamaindex/test_async.py
create mode 100644 tests/test_integrations/test_llamaindex/test_sync.py
create mode 100644 tests/test_integrations/test_openai/conftest.py
create mode 100644 tests/test_integrations/test_openai/simple_openai.py
create mode 100644 tests/test_integrations/test_openai/test_async_openai.py
create mode 100644 tests/test_integrations/test_openai/test_async_openai_without_trace.json
create mode 100644 tests/test_integrations/test_openai/test_async_response_create_with_trace.json
create mode 100644 tests/test_integrations/test_openai/test_sync_openai.py
create mode 100644 tests/test_integrations/test_openai/test_sync_openai_with_trace.json
create mode 100644 tests/test_integrations/test_openai/test_sync_response_create_without_trace.json
create mode 100644 tests/test_integrations/test_openai/test_tool_call_flow_completion.json
create mode 100644 tests/test_integrations/test_openai/test_tool_call_flow_completion.py
create mode 100644 tests/test_integrations/test_openai/test_tool_call_flow_response.json
create mode 100644 tests/test_integrations/test_openai/test_tool_call_flow_response.py
create mode 100644 tests/test_integrations/test_openai_agents/apps/eval_agent.py
create mode 100644 tests/test_integrations/test_openai_agents/apps/handoff_agent.py
create mode 100644 tests/test_integrations/test_openai_agents/apps/session_agent.py
create mode 100644 tests/test_integrations/test_openai_agents/apps/simple_agent.py
create mode 100644 tests/test_integrations/test_openai_agents/apps/streaming_agent.py
create mode 100644 tests/test_integrations/test_openai_agents/apps/tool_agent.py
create mode 100644 tests/test_integrations/test_openai_agents/conftest.py
create mode 100644 tests/test_integrations/test_openai_agents/schemas/openai_agents_async_handoff_schema.json
create mode 100644 tests/test_integrations/test_openai_agents/schemas/openai_agents_async_simple_schema.json
create mode 100644 tests/test_integrations/test_openai_agents/schemas/openai_agents_async_tool_schema.json
create mode 100644 tests/test_integrations/test_openai_agents/schemas/openai_agents_eval_schema.json
create mode 100644 tests/test_integrations/test_openai_agents/schemas/openai_agents_handoff_spanish_schema.json
create mode 100644 tests/test_integrations/test_openai_agents/schemas/openai_agents_session_schema.json
create mode 100644 tests/test_integrations/test_openai_agents/schemas/openai_agents_simple_schema.json
create mode 100644 tests/test_integrations/test_openai_agents/schemas/openai_agents_tool_math_schema.json
create mode 100644 tests/test_integrations/test_openai_agents/schemas/openai_agents_tool_weather_schema.json
create mode 100644 tests/test_integrations/test_openai_agents/test_async.py
create mode 100644 tests/test_integrations/test_openai_agents/test_scenerios/multi_agents.json
create mode 100644 tests/test_integrations/test_openai_agents/test_scenerios/run.json
create mode 100644 tests/test_integrations/test_openai_agents/test_scenerios/run_streamed.json
create mode 100644 tests/test_integrations/test_openai_agents/test_scenerios/test_multi_agents.py
create mode 100644 tests/test_integrations/test_openai_agents/test_scenerios/test_run.py
create mode 100644 tests/test_integrations/test_openai_agents/test_scenerios/test_run_streamed.py
create mode 100644 tests/test_integrations/test_openai_agents/test_scenerios/test_run_sync.py
create mode 100644 tests/test_integrations/test_openai_agents/test_scenerios/test_weather_agent_patched.py
create mode 100644 tests/test_integrations/test_openai_agents/test_scenerios/test_with_trace.py
create mode 100644 tests/test_integrations/test_openai_agents/test_scenerios/test_with_trace_and_wrapped.py
create mode 100644 tests/test_integrations/test_openai_agents/test_scenerios/weather_agent_patched.json
create mode 100644 tests/test_integrations/test_openai_agents/test_scenerios/with_trace.json
create mode 100644 tests/test_integrations/test_openai_agents/test_scenerios/with_trace_and_wrapped.json
create mode 100644 tests/test_integrations/test_openai_agents/test_sync.py
create mode 100644 tests/test_integrations/test_openrouter/test_openrouter_generation.py
create mode 100644 tests/test_integrations/test_pydanticai/apps/__init__.py
create mode 100644 tests/test_integrations/test_pydanticai/apps/eval_app.py
create mode 100644 tests/test_integrations/test_pydanticai/apps/pydanticai_isolation_app.py
create mode 100644 tests/test_integrations/test_pydanticai/apps/pydanticai_metric_collection_app.py
create mode 100644 tests/test_integrations/test_pydanticai/apps/pydanticai_modes_app.py
create mode 100644 tests/test_integrations/test_pydanticai/apps/pydanticai_multiple_tools_app.py
create mode 100644 tests/test_integrations/test_pydanticai/apps/pydanticai_next_span_app.py
create mode 100644 tests/test_integrations/test_pydanticai/apps/pydanticai_simple_app.py
create mode 100644 tests/test_integrations/test_pydanticai/apps/pydanticai_streaming_app.py
create mode 100644 tests/test_integrations/test_pydanticai/apps/pydanticai_tool_app.py
create mode 100644 tests/test_integrations/test_pydanticai/conftest.py
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_async_parallel_tools_schema.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_async_simple_schema.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_async_tool_schema.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_bare_tool_enrichment_schema.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_features_async.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_features_sync.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_multiple_tools_time_schema.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_multiple_tools_weather_schema.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_next_llm_only_schema.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_next_stacked_schema.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_observe_mode_schema.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_parallel_tools_schema.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_simple_schema.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_streaming_schema.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_tool_schema.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_trace_metric_collection_schema.json
create mode 100644 tests/test_integrations/test_pydanticai/schemas/pydanticai_with_trace_mode_schema.json
create mode 100644 tests/test_integrations/test_pydanticai/test_async.py
create mode 100644 tests/test_integrations/test_pydanticai/test_evaluate_agent.py
create mode 100644 tests/test_integrations/test_pydanticai/test_span_interceptor.py
create mode 100644 tests/test_integrations/test_pydanticai/test_sync.py
create mode 100644 tests/test_integrations/test_strands/__init__.py
create mode 100644 tests/test_integrations/test_strands/apps/__init__.py
create mode 100644 tests/test_integrations/test_strands/apps/strands_eval_app.py
create mode 100644 tests/test_integrations/test_strands/apps/strands_multiple_tools_app.py
create mode 100644 tests/test_integrations/test_strands/apps/strands_simple_app.py
create mode 100644 tests/test_integrations/test_strands/apps/strands_tool_app.py
create mode 100644 tests/test_integrations/test_strands/conftest.py
create mode 100644 tests/test_integrations/test_strands/schemas/README.md
create mode 100644 tests/test_integrations/test_strands/schemas/strands_async_parallel_tools_schema.json
create mode 100644 tests/test_integrations/test_strands/schemas/strands_async_simple_schema.json
create mode 100644 tests/test_integrations/test_strands/schemas/strands_async_tool_schema.json
create mode 100644 tests/test_integrations/test_strands/schemas/strands_features_async.json
create mode 100644 tests/test_integrations/test_strands/schemas/strands_features_sync.json
create mode 100644 tests/test_integrations/test_strands/schemas/strands_multiple_tools_time_schema.json
create mode 100644 tests/test_integrations/test_strands/schemas/strands_multiple_tools_weather_schema.json
create mode 100644 tests/test_integrations/test_strands/schemas/strands_parallel_tools_schema.json
create mode 100644 tests/test_integrations/test_strands/schemas/strands_simple_schema.json
create mode 100644 tests/test_integrations/test_strands/schemas/strands_tool_metric_collection_schema.json
create mode 100644 tests/test_integrations/test_strands/schemas/strands_tool_schema.json
create mode 100644 tests/test_integrations/test_strands/test_async.py
create mode 100644 tests/test_integrations/test_strands/test_evaluate_agent.py
create mode 100644 tests/test_integrations/test_strands/test_span_interceptor.py
create mode 100644 tests/test_integrations/test_strands/test_sync.py
create mode 100644 tests/test_integrations/utils.py
create mode 100644 tests/test_metrics/images/car.png
create mode 100644 tests/test_metrics/test_answer_relevancy_metric.py
create mode 100644 tests/test_metrics/test_answer_relevancy_metric_empty_output.py
create mode 100644 tests/test_metrics/test_arena_geval_metric.py
create mode 100644 tests/test_metrics/test_bias_metric.py
create mode 100644 tests/test_metrics/test_citation_faithfulness_metric.py
create mode 100644 tests/test_metrics/test_contextual_precision_metric.py
create mode 100644 tests/test_metrics/test_contextual_recall_metric.py
create mode 100644 tests/test_metrics/test_contextual_relevancy_metric.py
create mode 100644 tests/test_metrics/test_conversation_completeness_metric.py
create mode 100644 tests/test_metrics/test_conversational_dag.py
create mode 100644 tests/test_metrics/test_conversational_g_eval.py
create mode 100644 tests/test_metrics/test_dag.py
create mode 100644 tests/test_metrics/test_dag_serialization.py
create mode 100644 tests/test_metrics/test_exact_match_metric.py
create mode 100644 tests/test_metrics/test_faithfulness_metric.py
create mode 100644 tests/test_metrics/test_g_eval_metric.py
create mode 100644 tests/test_metrics/test_g_eval_utils.py
create mode 100644 tests/test_metrics/test_goal_accuracy_metric.py
create mode 100644 tests/test_metrics/test_hallucination_metric.py
create mode 100644 tests/test_metrics/test_image_coherence_metric.py
create mode 100644 tests/test_metrics/test_image_editing_metric.py
create mode 100644 tests/test_metrics/test_image_helpfulness_metric.py
create mode 100644 tests/test_metrics/test_image_reference_metric.py
create mode 100644 tests/test_metrics/test_json_correctness_metric.py
create mode 100644 tests/test_metrics/test_knowledge_retention_metric.py
create mode 100644 tests/test_metrics/test_mcp_task_completetion_metric.py
create mode 100644 tests/test_metrics/test_mcp_use_metric.py
create mode 100644 tests/test_metrics/test_misuse_metric.py
create mode 100644 tests/test_metrics/test_multi_turn_mcp_use_metric.py
create mode 100644 tests/test_metrics/test_non_advice_metric.py
create mode 100644 tests/test_metrics/test_pattern_match_metric.py
create mode 100644 tests/test_metrics/test_pii_lekage_metric.py
create mode 100644 tests/test_metrics/test_plan_adherence_metric.py
create mode 100644 tests/test_metrics/test_plan_quality_metric.py
create mode 100644 tests/test_metrics/test_prompt_alignment_metric.py
create mode 100644 tests/test_metrics/test_role_adherence_metric.py
create mode 100644 tests/test_metrics/test_role_violation_metric.py
create mode 100644 tests/test_metrics/test_step_efficiency_metric.py
create mode 100644 tests/test_metrics/test_summarization_metric.py
create mode 100644 tests/test_metrics/test_task_completetion_metric.py
create mode 100644 tests/test_metrics/test_task_completion_tools_called.py
create mode 100644 tests/test_metrics/test_text_to_image_metric.py
create mode 100644 tests/test_metrics/test_tool_permission_metric.py
create mode 100644 tests/test_metrics/test_tool_use_metric.py
create mode 100644 tests/test_metrics/test_topic_adherence_metric.py
create mode 100644 tests/test_metrics/test_toxicity_metric.py
create mode 100644 tests/test_metrics/test_turn_contextual_precision_metric.py
create mode 100644 tests/test_metrics/test_turn_contextual_recall_async_expected_outcome.py
create mode 100644 tests/test_metrics/test_turn_contextual_recall_metric.py
create mode 100644 tests/test_metrics/test_turn_faithfulness_metric.py
create mode 100644 tests/test_metrics/test_turn_faithfulness_metric_empty_verdicts.py
create mode 100644 tests/test_metrics/test_turn_relevancy_aggregation.py
create mode 100644 tests/test_metrics/test_turn_relevancy_metric.py
create mode 100644 tests/test_metrics/turn_contextual_relevancy_metric.py
create mode 100644 tests/test_templates/test_metric_templates.py
create mode 100644 typescript/.gitignore
create mode 100644 typescript/.prettierignore
create mode 100644 typescript/.prettierrc
create mode 100644 typescript/README.md
create mode 100644 typescript/eslint.config.mts
create mode 100644 typescript/examples/annotation/example-send-annotation.ts
create mode 100644 typescript/examples/dataset/example-custom-dataset-columns.ts
create mode 100644 typescript/examples/dataset/example-dataset.ts
create mode 100644 typescript/examples/dataset/example-e2e-multiturn.ts
create mode 100644 typescript/examples/dataset/example-e2e-singleturn.ts
create mode 100644 typescript/examples/dataset/example-push-multiturn.ts
create mode 100644 typescript/examples/dataset/example-push-singleturn.ts
create mode 100644 typescript/examples/dataset/example-queue-multiturn.ts
create mode 100644 typescript/examples/dataset/example-queue-singleturn.ts
create mode 100644 typescript/examples/evaluate/example-evaluate.ts
create mode 100644 typescript/examples/integrations/langchain/prompt-template.ts
create mode 100644 typescript/examples/integrations/langchain/streaming.ts
create mode 100644 typescript/examples/integrations/langgraph/agent.ts
create mode 100644 typescript/examples/integrations/openai/completions-parse.ts
create mode 100644 typescript/examples/integrations/openai/completions-tools.ts
create mode 100644 typescript/examples/integrations/openai/completions.ts
create mode 100644 typescript/examples/integrations/openai/responses-parse.ts
create mode 100644 typescript/examples/integrations/openai/responses-tools.ts
create mode 100644 typescript/examples/integrations/openai/responses.ts
create mode 100644 typescript/examples/prompt/example-prompt-pull.ts
create mode 100644 typescript/examples/simulate/example-simulate.ts
create mode 100644 typescript/examples/tracing/example-api-key.ts
create mode 100644 typescript/examples/tracing/example-attributes.ts
create mode 100644 typescript/examples/tracing/example-environment.ts
create mode 100644 typescript/examples/tracing/example-input-output-span.ts
create mode 100644 typescript/examples/tracing/example-input-output-trace.ts
create mode 100644 typescript/examples/tracing/example-masking.ts
create mode 100644 typescript/examples/tracing/example-metadata.ts
create mode 100644 typescript/examples/tracing/example-name-span-types.ts
create mode 100644 typescript/examples/tracing/example-name-span.ts
create mode 100644 typescript/examples/tracing/example-name-trace.ts
create mode 100644 typescript/examples/tracing/example-observe.ts
create mode 100644 typescript/examples/tracing/example-online-evaluations.ts
create mode 100644 typescript/examples/tracing/example-quickstart.ts
create mode 100644 typescript/examples/tracing/example-sampling.ts
create mode 100644 typescript/examples/tracing/example-tags.ts
create mode 100644 typescript/examples/tracing/example-test-case-llm.ts
create mode 100644 typescript/examples/tracing/example-test-case-span.ts
create mode 100644 typescript/examples/tracing/example-test-case-trace.ts
create mode 100644 typescript/examples/tracing/example-thread.ts
create mode 100644 typescript/examples/tracing/example-tracing.ts
create mode 100644 typescript/examples/tracing/example-user.ts
create mode 100644 typescript/jest.config.js
create mode 100644 typescript/package-lock.json
create mode 100644 typescript/package.json
create mode 100644 typescript/src/annotation/api.ts
create mode 100644 typescript/src/annotation/index.ts
create mode 100644 typescript/src/annotation/utils.ts
create mode 100644 typescript/src/cli/index.ts
create mode 100644 typescript/src/confident/api.ts
create mode 100644 typescript/src/confident/index.ts
create mode 100644 typescript/src/confident/types.ts
create mode 100644 typescript/src/config/settings.ts
create mode 100644 typescript/src/constants.ts
create mode 100644 typescript/src/dataset/api.ts
create mode 100644 typescript/src/dataset/dataset.ts
create mode 100644 typescript/src/dataset/golden.ts
create mode 100644 typescript/src/dataset/index.ts
create mode 100644 typescript/src/dataset/utils.ts
create mode 100644 typescript/src/errors.ts
create mode 100644 typescript/src/evaluate/compare.ts
create mode 100644 typescript/src/evaluate/confident.ts
create mode 100644 typescript/src/evaluate/configs.ts
create mode 100644 typescript/src/evaluate/console-report.ts
create mode 100644 typescript/src/evaluate/evaluate.ts
create mode 100644 typescript/src/evaluate/index.ts
create mode 100644 typescript/src/evaluate/trace-eval.ts
create mode 100644 typescript/src/evaluate/types.ts
create mode 100644 typescript/src/governance/index.ts
create mode 100644 typescript/src/index.ts
create mode 100644 typescript/src/integrations/ai-sdk/index.ts
create mode 100644 typescript/src/integrations/ai-sdk/processor.ts
create mode 100644 typescript/src/integrations/index.ts
create mode 100644 typescript/src/integrations/langchain/callback-handler.ts
create mode 100644 typescript/src/integrations/langchain/index.ts
create mode 100644 typescript/src/integrations/langchain/langgraph-utils.ts
create mode 100644 typescript/src/integrations/langchain/patch-tool.ts
create mode 100644 typescript/src/integrations/langchain/utils.ts
create mode 100644 typescript/src/integrations/openai-agents/callback.ts
create mode 100644 typescript/src/integrations/openai-agents/extractors.ts
create mode 100644 typescript/src/integrations/openai-agents/index.ts
create mode 100644 typescript/src/integrations/openinference/index.ts
create mode 100644 typescript/src/integrations/openinference/processor.ts
create mode 100644 typescript/src/metrics/README.md
create mode 100644 typescript/src/metrics/answer-relevancy/answer-relevancy.ts
create mode 100644 typescript/src/metrics/answer-relevancy/index.ts
create mode 100644 typescript/src/metrics/answer-relevancy/schema.ts
create mode 100644 typescript/src/metrics/arena-g-eval/arena-g-eval.ts
create mode 100644 typescript/src/metrics/arena-g-eval/index.ts
create mode 100644 typescript/src/metrics/arena-g-eval/schema.ts
create mode 100644 typescript/src/metrics/arena-g-eval/utils.ts
create mode 100644 typescript/src/metrics/argument-correctness/argument-correctness.ts
create mode 100644 typescript/src/metrics/argument-correctness/index.ts
create mode 100644 typescript/src/metrics/argument-correctness/schema.ts
create mode 100644 typescript/src/metrics/base-arena-metric.ts
create mode 100644 typescript/src/metrics/base-conversational-metric.ts
create mode 100644 typescript/src/metrics/base-metrics.ts
create mode 100644 typescript/src/metrics/bias/bias.ts
create mode 100644 typescript/src/metrics/bias/index.ts
create mode 100644 typescript/src/metrics/bias/schema.ts
create mode 100644 typescript/src/metrics/contextual-precision/contextual-precision.ts
create mode 100644 typescript/src/metrics/contextual-precision/index.ts
create mode 100644 typescript/src/metrics/contextual-precision/schema.ts
create mode 100644 typescript/src/metrics/contextual-recall/contextual-recall.ts
create mode 100644 typescript/src/metrics/contextual-recall/index.ts
create mode 100644 typescript/src/metrics/contextual-recall/schema.ts
create mode 100644 typescript/src/metrics/contextual-relevancy/contextual-relevancy.ts
create mode 100644 typescript/src/metrics/contextual-relevancy/index.ts
create mode 100644 typescript/src/metrics/contextual-relevancy/schema.ts
create mode 100644 typescript/src/metrics/conversation-completeness/conversation-completeness.ts
create mode 100644 typescript/src/metrics/conversation-completeness/index.ts
create mode 100644 typescript/src/metrics/conversation-completeness/schema.ts
create mode 100644 typescript/src/metrics/conversational-g-eval/conversational-g-eval.ts
create mode 100644 typescript/src/metrics/conversational-g-eval/index.ts
create mode 100644 typescript/src/metrics/conversational-g-eval/utils.ts
create mode 100644 typescript/src/metrics/conversational-utils.ts
create mode 100644 typescript/src/metrics/exact-match/exact-match.ts
create mode 100644 typescript/src/metrics/exact-match/index.ts
create mode 100644 typescript/src/metrics/faithfulness/faithfulness.ts
create mode 100644 typescript/src/metrics/faithfulness/index.ts
create mode 100644 typescript/src/metrics/faithfulness/schema.ts
create mode 100644 typescript/src/metrics/g-eval/g-eval.ts
create mode 100644 typescript/src/metrics/g-eval/index.ts
create mode 100644 typescript/src/metrics/g-eval/schema.ts
create mode 100644 typescript/src/metrics/g-eval/utils.ts
create mode 100644 typescript/src/metrics/goal-accuracy/goal-accuracy.ts
create mode 100644 typescript/src/metrics/goal-accuracy/index.ts
create mode 100644 typescript/src/metrics/goal-accuracy/schema.ts
create mode 100644 typescript/src/metrics/hallucination/hallucination.ts
create mode 100644 typescript/src/metrics/hallucination/index.ts
create mode 100644 typescript/src/metrics/hallucination/schema.ts
create mode 100644 typescript/src/metrics/index.ts
create mode 100644 typescript/src/metrics/json-correctness/index.ts
create mode 100644 typescript/src/metrics/json-correctness/json-correctness.ts
create mode 100644 typescript/src/metrics/json-correctness/schema.ts
create mode 100644 typescript/src/metrics/knowledge-retention/index.ts
create mode 100644 typescript/src/metrics/knowledge-retention/knowledge-retention.ts
create mode 100644 typescript/src/metrics/knowledge-retention/schema.ts
create mode 100644 typescript/src/metrics/mcp-use-metric/index.ts
create mode 100644 typescript/src/metrics/mcp-use-metric/mcp-use-metric.ts
create mode 100644 typescript/src/metrics/mcp-use-metric/schema.ts
create mode 100644 typescript/src/metrics/mcp/index.ts
create mode 100644 typescript/src/metrics/mcp/mcp-task-completion.ts
create mode 100644 typescript/src/metrics/mcp/multi-turn-mcp-use.ts
create mode 100644 typescript/src/metrics/mcp/schema.ts
create mode 100644 typescript/src/metrics/mcp/utils.ts
create mode 100644 typescript/src/metrics/misuse/index.ts
create mode 100644 typescript/src/metrics/misuse/misuse.ts
create mode 100644 typescript/src/metrics/misuse/schema.ts
create mode 100644 typescript/src/metrics/multimodal-metrics/image-coherence/image-coherence.ts
create mode 100644 typescript/src/metrics/multimodal-metrics/image-coherence/index.ts
create mode 100644 typescript/src/metrics/multimodal-metrics/image-editing/image-editing.ts
create mode 100644 typescript/src/metrics/multimodal-metrics/image-editing/index.ts
create mode 100644 typescript/src/metrics/multimodal-metrics/image-helpfulness/image-helpfulness.ts
create mode 100644 typescript/src/metrics/multimodal-metrics/image-helpfulness/index.ts
create mode 100644 typescript/src/metrics/multimodal-metrics/image-reference/image-reference.ts
create mode 100644 typescript/src/metrics/multimodal-metrics/image-reference/index.ts
create mode 100644 typescript/src/metrics/multimodal-metrics/index.ts
create mode 100644 typescript/src/metrics/multimodal-metrics/schema.ts
create mode 100644 typescript/src/metrics/multimodal-metrics/text-to-image/index.ts
create mode 100644 typescript/src/metrics/multimodal-metrics/text-to-image/text-to-image.ts
create mode 100644 typescript/src/metrics/multimodal-metrics/utils.ts
create mode 100644 typescript/src/metrics/non-advice/index.ts
create mode 100644 typescript/src/metrics/non-advice/non-advice.ts
create mode 100644 typescript/src/metrics/non-advice/schema.ts
create mode 100644 typescript/src/metrics/pattern-match/index.ts
create mode 100644 typescript/src/metrics/pattern-match/pattern-match.ts
create mode 100644 typescript/src/metrics/pii-leakage/index.ts
create mode 100644 typescript/src/metrics/pii-leakage/pii-leakage.ts
create mode 100644 typescript/src/metrics/pii-leakage/schema.ts
create mode 100644 typescript/src/metrics/plan-adherence/index.ts
create mode 100644 typescript/src/metrics/plan-adherence/plan-adherence.ts
create mode 100644 typescript/src/metrics/plan-adherence/schema.ts
create mode 100644 typescript/src/metrics/plan-quality/index.ts
create mode 100644 typescript/src/metrics/plan-quality/plan-quality.ts
create mode 100644 typescript/src/metrics/plan-quality/schema.ts
create mode 100644 typescript/src/metrics/prompt-alignment/index.ts
create mode 100644 typescript/src/metrics/prompt-alignment/prompt-alignment.ts
create mode 100644 typescript/src/metrics/prompt-alignment/schema.ts
create mode 100644 typescript/src/metrics/role-adherence/index.ts
create mode 100644 typescript/src/metrics/role-adherence/role-adherence.ts
create mode 100644 typescript/src/metrics/role-adherence/schema.ts
create mode 100644 typescript/src/metrics/role-violation/index.ts
create mode 100644 typescript/src/metrics/role-violation/role-violation.ts
create mode 100644 typescript/src/metrics/role-violation/schema.ts
create mode 100644 typescript/src/metrics/step-efficiency/index.ts
create mode 100644 typescript/src/metrics/step-efficiency/schema.ts
create mode 100644 typescript/src/metrics/step-efficiency/step-efficiency.ts
create mode 100644 typescript/src/metrics/summarization/index.ts
create mode 100644 typescript/src/metrics/summarization/schema.ts
create mode 100644 typescript/src/metrics/summarization/summarization.ts
create mode 100644 typescript/src/metrics/task-completion/index.ts
create mode 100644 typescript/src/metrics/task-completion/schema.ts
create mode 100644 typescript/src/metrics/task-completion/task-completion.ts
create mode 100644 typescript/src/metrics/tool-correctness/index.ts
create mode 100644 typescript/src/metrics/tool-correctness/schema.ts
create mode 100644 typescript/src/metrics/tool-correctness/tool-correctness.ts
create mode 100644 typescript/src/metrics/tool-use/index.ts
create mode 100644 typescript/src/metrics/tool-use/schema.ts
create mode 100644 typescript/src/metrics/tool-use/tool-use.ts
create mode 100644 typescript/src/metrics/topic-adherence/index.ts
create mode 100644 typescript/src/metrics/topic-adherence/schema.ts
create mode 100644 typescript/src/metrics/topic-adherence/topic-adherence.ts
create mode 100644 typescript/src/metrics/toxicity/index.ts
create mode 100644 typescript/src/metrics/toxicity/schema.ts
create mode 100644 typescript/src/metrics/toxicity/toxicity.ts
create mode 100644 typescript/src/metrics/turn-contextual-precision/index.ts
create mode 100644 typescript/src/metrics/turn-contextual-precision/schema.ts
create mode 100644 typescript/src/metrics/turn-contextual-precision/turn-contextual-precision.ts
create mode 100644 typescript/src/metrics/turn-contextual-recall/index.ts
create mode 100644 typescript/src/metrics/turn-contextual-recall/schema.ts
create mode 100644 typescript/src/metrics/turn-contextual-recall/turn-contextual-recall.ts
create mode 100644 typescript/src/metrics/turn-contextual-relevancy/index.ts
create mode 100644 typescript/src/metrics/turn-contextual-relevancy/schema.ts
create mode 100644 typescript/src/metrics/turn-contextual-relevancy/turn-contextual-relevancy.ts
create mode 100644 typescript/src/metrics/turn-faithfulness/index.ts
create mode 100644 typescript/src/metrics/turn-faithfulness/schema.ts
create mode 100644 typescript/src/metrics/turn-faithfulness/turn-faithfulness.ts
create mode 100644 typescript/src/metrics/turn-relevancy/index.ts
create mode 100644 typescript/src/metrics/turn-relevancy/schema.ts
create mode 100644 typescript/src/metrics/turn-relevancy/turn-relevancy.ts
create mode 100644 typescript/src/metrics/utils.ts
create mode 100644 typescript/src/models/README.md
create mode 100644 typescript/src/models/base-model.ts
create mode 100644 typescript/src/models/gateways/openrouter-model.ts
create mode 100644 typescript/src/models/gateways/portkey-model.ts
create mode 100644 typescript/src/models/index.ts
create mode 100644 typescript/src/models/multimodal.ts
create mode 100644 typescript/src/models/openai-compatible-model.ts
create mode 100644 typescript/src/models/providers/ai-sdk-model.ts
create mode 100644 typescript/src/models/providers/anthropic-model.ts
create mode 100644 typescript/src/models/providers/azure-model.ts
create mode 100644 typescript/src/models/providers/bedrock-model.ts
create mode 100644 typescript/src/models/providers/deepseek-model.ts
create mode 100644 typescript/src/models/providers/gemini-model.ts
create mode 100644 typescript/src/models/providers/grok-model.ts
create mode 100644 typescript/src/models/providers/kimi-model.ts
create mode 100644 typescript/src/models/providers/local-model.ts
create mode 100644 typescript/src/models/providers/ollama-model.ts
create mode 100644 typescript/src/models/providers/openai-model.ts
create mode 100644 typescript/src/models/utils.ts
create mode 100644 typescript/src/openai/extractor.ts
create mode 100644 typescript/src/openai/index.ts
create mode 100644 typescript/src/openai/patch.ts
create mode 100644 typescript/src/openai/types.ts
create mode 100644 typescript/src/openai/utils.ts
create mode 100644 typescript/src/prompt/index.ts
create mode 100644 typescript/src/prompt/types.ts
create mode 100644 typescript/src/prompt/utils.ts
create mode 100644 typescript/src/simulate/index.ts
create mode 100644 typescript/src/telemetry.ts
create mode 100644 typescript/src/telemetry.txt
create mode 100644 typescript/src/templates/index.ts
create mode 100644 typescript/src/templates/metrics/templates.json
create mode 100644 typescript/src/templates/resolver.ts
create mode 100644 typescript/src/test-case/arena-test-case.ts
create mode 100644 typescript/src/test-case/conversational-test-case.ts
create mode 100644 typescript/src/test-case/index.ts
create mode 100644 typescript/src/test-case/llm-test-case.ts
create mode 100644 typescript/src/test-case/mcp.ts
create mode 100644 typescript/src/test-case/mllm-image.ts
create mode 100644 typescript/src/test-case/utils.ts
create mode 100644 typescript/src/tracing/api.ts
create mode 100644 typescript/src/tracing/index.ts
create mode 100644 typescript/src/tracing/logging.ts
create mode 100644 typescript/src/tracing/offline-evals/api.ts
create mode 100644 typescript/src/tracing/offline-evals/index.ts
create mode 100644 typescript/src/tracing/offline-evals/span.ts
create mode 100644 typescript/src/tracing/offline-evals/thread.ts
create mode 100644 typescript/src/tracing/offline-evals/trace.ts
create mode 100644 typescript/src/tracing/trace-context.ts
create mode 100644 typescript/src/tracing/tracing.ts
create mode 100644 typescript/src/tracing/utils.ts
create mode 100644 typescript/src/utils.ts
create mode 100644 typescript/test/test-core/annotation.test.ts
create mode 100644 typescript/test/test-core/api.test.ts
create mode 100644 typescript/test/test-core/dataset.test.ts
create mode 100644 typescript/test/test-core/evaluate.test.ts
create mode 100644 typescript/test/test-core/prompt.test.ts
create mode 100644 typescript/test/test-core/tracing.test.ts
create mode 100644 typescript/test/test-integrations/test-ai-sdk/ai-sdk-tools.test.ts
create mode 100644 typescript/test/test-integrations/test-ai-sdk/ai-sdk.test.ts
create mode 100644 typescript/test/test-integrations/test-ai-sdk/fixtures/expected_ai_sdk_confident_attributes.json
create mode 100644 typescript/test/test-integrations/test-ai-sdk/fixtures/expected_ai_sdk_generate_text.json
create mode 100644 typescript/test/test-integrations/test-ai-sdk/fixtures/expected_ai_sdk_stream_text.json
create mode 100644 typescript/test/test-integrations/test-ai-sdk/fixtures/expected_ai_sdk_structured_output.json
create mode 100644 typescript/test/test-integrations/test-ai-sdk/fixtures/expected_ai_sdk_tools.json
create mode 100644 typescript/test/test-integrations/test-langchain/apps/langchain-agent-app.ts
create mode 100644 typescript/test/test-integrations/test-langchain/apps/langchain-conditional-app.ts
create mode 100644 typescript/test/test-integrations/test-langchain/apps/langchain-metric-collection-app.ts
create mode 100644 typescript/test/test-integrations/test-langchain/apps/langchain-multiple-tools-app.ts
create mode 100644 typescript/test/test-integrations/test-langchain/apps/langchain-parallel-tools-app.ts
create mode 100644 typescript/test/test-integrations/test-langchain/apps/langchain-retriever-app.ts
create mode 100644 typescript/test/test-integrations/test-langchain/apps/langchain-simple-app.ts
create mode 100644 typescript/test/test-integrations/test-langchain/apps/langchain-single-tool-app.ts
create mode 100644 typescript/test/test-integrations/test-langchain/apps/langchain-streaming-app.ts
create mode 100644 typescript/test/test-integrations/test-langchain/fixtures/langchain_agent_multi_step_schema.json
create mode 100644 typescript/test/test-integrations/test-langchain/fixtures/langchain_conditional_research_schema.json
create mode 100644 typescript/test/test-integrations/test-langchain/fixtures/langchain_metric_collection_schema.json
create mode 100644 typescript/test/test-integrations/test-langchain/fixtures/langchain_multiple_tools_schema.json
create mode 100644 typescript/test/test-integrations/test-langchain/fixtures/langchain_parallel_weather_schema.json
create mode 100644 typescript/test/test-integrations/test-langchain/fixtures/langchain_retriever_python_schema.json
create mode 100644 typescript/test/test-integrations/test-langchain/fixtures/langchain_simple_schema.json
create mode 100644 typescript/test/test-integrations/test-langchain/fixtures/langchain_single_tool_schema.json
create mode 100644 typescript/test/test-integrations/test-langchain/fixtures/langchain_streaming_schema.json
create mode 100644 typescript/test/test-integrations/test-langchain/test-langchain-apps.test.ts
create mode 100644 typescript/test/test-integrations/test-openai-agents/apps/openai-agents-eval-app.ts
create mode 100644 typescript/test/test-integrations/test-openai-agents/apps/openai-agents-handoff-app.ts
create mode 100644 typescript/test/test-integrations/test-openai-agents/apps/openai-agents-simple-app.ts
create mode 100644 typescript/test/test-integrations/test-openai-agents/apps/openai-agents-streaming-app.ts
create mode 100644 typescript/test/test-integrations/test-openai-agents/apps/openai-agents-tool-app.ts
create mode 100644 typescript/test/test-integrations/test-openai-agents/fixtures/openai_agents_evals_app_schema.json
create mode 100644 typescript/test/test-integrations/test-openai-agents/fixtures/openai_agents_handoff_schema.json
create mode 100644 typescript/test/test-integrations/test-openai-agents/fixtures/openai_agents_simple_schema.json
create mode 100644 typescript/test/test-integrations/test-openai-agents/fixtures/openai_agents_streaming_schema.json
create mode 100644 typescript/test/test-integrations/test-openai-agents/fixtures/openai_agents_tool_schema.json
create mode 100644 typescript/test/test-integrations/test-openai-agents/openai-agents.test.ts
create mode 100644 typescript/test/test-integrations/test-openai/fixtures/expected_openai_confident_attributes.json
create mode 100644 typescript/test/test-integrations/test-openai/fixtures/expected_openai_with_trace.json
create mode 100644 typescript/test/test-integrations/test-openai/fixtures/expected_openai_without_trace.json
create mode 100644 typescript/test/test-integrations/test-openai/fixtures/expected_response_create_with_trace.json
create mode 100644 typescript/test/test-integrations/test-openai/fixtures/expected_tool_call_flow_completion.json
create mode 100644 typescript/test/test-integrations/test-openai/fixtures/expected_tool_call_flow_response.json
create mode 100644 typescript/test/test-integrations/test-openai/openai-tool-calls.test.ts
create mode 100644 typescript/test/test-integrations/test-openai/openai.test.ts
create mode 100644 typescript/test/test-integrations/utils.ts
create mode 100644 typescript/tsconfig.json
diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json
new file mode 100644
index 0000000..702daff
--- /dev/null
+++ b/.claude-plugin/marketplace.json
@@ -0,0 +1,24 @@
+{
+ "name": "deepeval-plugins",
+ "owner": {
+ "name": "Confident AI",
+ "email": "open-source@confident-ai.com"
+ },
+ "description": "DeepEval plugins for LLM evaluation, tracing, and testing in Claude Code.",
+ "plugins": [
+ {
+ "name": "deepeval",
+ "source": "./",
+ "description": "Skills for adding DeepEval evaluations, tracing, datasets, Confident AI reports, and iterative improvement loops to AI applications.",
+ "category": "developer-tools",
+ "keywords": [
+ "deepeval",
+ "llm",
+ "evaluation",
+ "tracing",
+ "datasets",
+ "confident-ai"
+ ]
+ }
+ ]
+}
diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json
new file mode 100644
index 0000000..7a7eac9
--- /dev/null
+++ b/.claude-plugin/plugin.json
@@ -0,0 +1,22 @@
+{
+ "name": "deepeval",
+ "displayName": "DeepEval",
+ "version": "1.0.0",
+ "description": "Skills for adding DeepEval evaluations, tracing, datasets, Confident AI reports, and iterative improvement loops to AI applications.",
+ "author": {
+ "name": "Confident AI",
+ "email": "open-source@confident-ai.com"
+ },
+ "homepage": "https://deepeval.com",
+ "repository": "https://github.com/confident-ai/deepeval",
+ "license": "Apache-2.0",
+ "keywords": [
+ "deepeval",
+ "llm",
+ "evaluation",
+ "tracing",
+ "datasets",
+ "confident-ai"
+ ],
+ "skills": "./skills/"
+}
diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json
new file mode 100644
index 0000000..4aa3492
--- /dev/null
+++ b/.cursor-plugin/plugin.json
@@ -0,0 +1,23 @@
+{
+ "name": "deepeval",
+ "displayName": "DeepEval",
+ "version": "1.0.0",
+ "description": "Skills for adding DeepEval evaluations, tracing, datasets, Confident AI reports, and iterative improvement loops to AI applications.",
+ "author": {
+ "name": "Confident AI",
+ "email": "open-source@confident-ai.com"
+ },
+ "homepage": "https://deepeval.com",
+ "repository": "https://github.com/confident-ai/deepeval",
+ "license": "Apache-2.0",
+ "keywords": [
+ "deepeval",
+ "llm",
+ "evaluation",
+ "tracing",
+ "datasets",
+ "confident-ai"
+ ],
+ "category": "developer-tools",
+ "skills": "./skills/"
+}
diff --git a/.env.example b/.env.example
new file mode 100644
index 0000000..2c23616
--- /dev/null
+++ b/.env.example
@@ -0,0 +1,113 @@
+# =========================
+# OpenAI (non-Azure)
+# =========================
+OPENAI_API_KEY=
+
+# Optional, if you use Azure OpenAI / custom gateways
+# OPENAI_BASE_URL=https://api.openai.com/v1
+
+# Optional: local cost overrides (used for cost reporting; not secrets)
+# OPENAI_COST_PER_INPUT_TOKEN=0.0
+# OPENAI_COST_PER_OUTPUT_TOKEN=0.0
+
+# =========================
+# Azure OpenAI
+# =========================
+AZURE_OPENAI_API_KEY=
+AZURE_OPENAI_ENDPOINT=
+AZURE_DEPLOYMENT_NAME=
+OPENAI_API_VERSION=
+AZURE_MODEL_NAME=
+AZURE_MODEL_VERSION=
+
+# =========================
+# Google / Gemini (google-genai)
+# =========================
+GOOGLE_API_KEY=
+
+# =========================
+# xAI Grok
+# =========================
+GROK_API_KEY=
+
+# =========================
+# Moonshot
+# =========================
+MOONSHOT_API_KEY=
+
+# =========================
+# DeepSeek
+# =========================
+DEEPSEEK_API_KEY=
+
+# =========================
+# LiteLLM (proxy to many providers)
+# =========================
+LITELLM_API_KEY=
+# LITELLM_API_BASE=
+
+# =========================
+# Anthropic
+# =========================
+ANTHROPIC_API_KEY=
+
+# =========================
+# Openrouter
+# =========================
+OPENROUTER_API_KEY=
+
+# =========================
+# Confident
+# =========================
+CONFIDENT_API_KEY= # Confident AI cloud
+
+# Ollama (local LLMs)
+# Defaults to http://127.0.0.1:11434; override if you run on another host/port.
+# OLLAMA_HOST=http://127.0.0.1:11434
+
+# =================
+# DeepEval settings
+# =================
+
+# Disable automatic dotenv loading (set to 1 to disable)
+# DEEPEVAL_DISABLE_DOTENV=1
+
+# gRPC logs (1 to enable)
+# DEEPEVAL_GRPC_LOGGING=1
+
+# Show PyPI update warning (1 to enable)
+# DEEPEVAL_UPDATE_WARNING_OPT_IN=1
+
+# Change results output folder (absolute or relative path)
+# DEEPEVAL_RESULTS_FOLDER=.deepeval_results
+
+# File-system mode (rare; READ_ONLY for constrained envs)
+# DEEPEVAL_FILE_SYSTEM=READ_ONLY
+
+# Telemetry (opt out globally if desired)
+# DEEPEVAL_TELEMETRY_OPT_OUT=1
+# ERROR_REPORTING=1
+
+# ===================
+# Test run overrides
+# ===================
+# Optional: name a pytest test dynamically
+# PYTEST_RUN_TEST_NAME=my_test_name
+
+# ==========================
+# Integrations & test suites
+# ==========================
+
+# Google Cloud (Gemini tests)
+# GOOGLE_CLOUD_PROJECT=
+# GOOGLE_CLOUD_LOCATION=
+
+# Vector DB docs examples
+# ELASTIC_PASSWORD=
+# WEAVIATE_API_KEY=
+# PG_PASSWORD=
+
+# Tracing test IDs (used by certain test files)
+# SPAN_ID=
+# TRACE_ID=
+# THREAD_ID=
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 0000000..10879ef
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,41 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**❗BEFORE YOU BEGIN❗**
+Are you on discord? 🤗 We'd love to have you asking questions on discord instead: https://discord.com/invite/a3K9c8GRGt
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 0000000..4e13e31
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,23 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**❗BEFORE YOU BEGIN❗**
+Are you on discord? 🤗 We'd love to have you asking questions on discord instead: https://discord.com/invite/a3K9c8GRGt
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.
diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml
new file mode 100644
index 0000000..b07d3c5
--- /dev/null
+++ b/.github/workflows/black.yml
@@ -0,0 +1,20 @@
+name: Lint Lint Lint
+
+on:
+ pull_request:
+ paths-ignore:
+ - 'typescript/**'
+
+jobs:
+ lint:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ os: [ubuntu-latest]
+ steps:
+ - uses: actions/checkout@v3
+ - uses: psf/black@stable
+ with:
+ options: "--check --verbose"
+ src: "."
+ jupyter: true
diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml
new file mode 100644
index 0000000..daf505d
--- /dev/null
+++ b/.github/workflows/changelog.yml
@@ -0,0 +1,76 @@
+name: Generate Changelog
+
+on:
+ workflow_dispatch:
+ inputs:
+ tag:
+ description: "Release tag to generate for (e.g. python-v4.0.8). Overrides mode/year/range."
+ required: false
+ mode:
+ description: "Mode: year or range"
+ required: true
+ default: "year"
+ year:
+ description: "Year (e.g. 2025)"
+ required: false
+ from_tag:
+ description: "From tag (e.g. v3.7.0)"
+ required: false
+ to_tag:
+ description: "To tag (e.g. v3.9.0)"
+ required: false
+
+jobs:
+ changelog:
+ runs-on: ubuntu-latest
+
+ permissions:
+ contents: write
+ pull-requests: write
+
+ steps:
+ - name: Checkout repo
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Fetch tags
+ run: git fetch --tags --force
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Install dependencies
+ run: |
+ pip install rich pydantic deepeval
+
+ - name: Run changelog generator
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ run: |
+ if [ -n "${{ github.event.inputs.tag }}" ]; then
+ python .scripts/changelog/generate.py \
+ --tag "${{ github.event.inputs.tag }}" \
+ --github --ai
+ elif [ "${{ github.event.inputs.mode }}" = "year" ]; then
+ python .scripts/changelog/generate.py \
+ --year ${{ github.event.inputs.year }} \
+ --github --ai
+ else
+ python .scripts/changelog/generate.py \
+ --range ${{ github.event.inputs.from_tag }} ${{ github.event.inputs.to_tag }} \
+ --github --ai
+ fi
+
+ - name: Create PR
+ uses: peter-evans/create-pull-request@v6
+ with:
+ add-paths: docs/content/changelog
+ branch: chore/changelog-${{ github.event.inputs.tag || 'update' }}
+ delete-branch: true
+ title: "chore: update changelog${{ github.event.inputs.tag && format(' for {0}', github.event.inputs.tag) || '' }}"
+ commit-message: "chore: update changelog${{ github.event.inputs.tag && format(' for {0}', github.event.inputs.tag) || '' }}"
+ body: "Auto-generated changelog updates"
\ No newline at end of file
diff --git a/.github/workflows/full_test_core_for_pr.yml b/.github/workflows/full_test_core_for_pr.yml
new file mode 100644
index 0000000..03b186f
--- /dev/null
+++ b/.github/workflows/full_test_core_for_pr.yml
@@ -0,0 +1,106 @@
+name: Full Core Tests (maintainer only)
+
+on:
+ workflow_dispatch:
+ inputs:
+ pr:
+ description: "PR number"
+ required: true
+ ref_kind:
+ description: "Which ref to test (merge|head)"
+ required: false
+ default: "merge"
+
+permissions:
+ contents: read
+
+concurrency:
+ group: full-tests-pr-${{ github.event.inputs.pr }}-${{ github.event.inputs.ref_kind }}
+ cancel-in-progress: true
+
+jobs:
+ full-tests:
+ if: ${{ github.repository_owner == 'confident-ai' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ environment: ci-secrets
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ DEEPEVAL_TELEMETRY_OPT_OUT: 1
+ DEEPEVAL_DEBUG_ASYNC: 1
+ LOG_LEVEL: "info"
+ PYTHONFAULTHANDLER: "1"
+ PYTHONASYNCIODEBUG: "1"
+ PYTHONUNBUFFERED: "1"
+ PYTEST_ADDOPTS: >-
+ -vv -rA --maxfail=1 --capture=tee-sys
+ --durations=25
+ -o log_cli=true -o log_cli_level=INFO
+ --log-cli-format="%(asctime)s %(levelname)s [%(name)s] %(message)s"
+
+ steps:
+ - name: Resolve ref
+ id: refsel
+ run: |
+ if [ "${{ github.event.inputs.ref_kind }}" = "head" ]; then
+ echo "ref=refs/pull/${{ github.event.inputs.pr }}/head" >> $GITHUB_OUTPUT
+ else
+ # test what would merge
+ echo "ref=refs/pull/${{ github.event.inputs.pr }}/merge" >> $GITHUB_OUTPUT
+ fi
+
+ - name: Checkout PR ref
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ steps.refsel.outputs.ref }}
+ fetch-depth: 0
+
+ - name: Set up Python
+ id: setup-python
+ uses: actions/setup-python@v4
+ with:
+ python-version: "3.11"
+
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ virtualenvs-create: true
+ virtualenvs-in-project: true
+ installer-parallel: true
+
+ - name: Cache virtualenv
+ id: cached-poetry-dependencies
+ uses: actions/cache@v3
+ with:
+ path: .venv
+ key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+ # Core deps only (main)
+ - name: Install dependencies (main)
+ if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+ run: poetry install --no-interaction --no-root --only main
+
+ - name: Install project (main)
+ run: poetry install --no-interaction --only main
+ # Install dev dependencies
+ - name: Install dev dependencies
+ run: poetry install --no-interaction --with dev
+ #----------------------------------------------
+ # run test suite
+ #----------------------------------------------
+
+ # Run Core tests
+ - name: Run core tests
+ if: ${{ env.OPENAI_API_KEY != '' }}
+ run: |
+ poetry run pytest \
+ tests/test_core/ \
+ --ignore=tests/test_core/test_synthesizer/ \
+ --ignore=tests/test_core/test_datasets/
+
+ - name: Run dev tests (with secrets)
+ if: ${{ env.OPENAI_API_KEY != '' }}
+ run: |
+ poetry run pytest \
+ tests/test_core/test_synthesizer/ tests/test_core/test_datasets/ tests/test_core/test_simulator/
+
diff --git a/.github/workflows/pr-title-check.yml b/.github/workflows/pr-title-check.yml
new file mode 100644
index 0000000..b2a88b1
--- /dev/null
+++ b/.github/workflows/pr-title-check.yml
@@ -0,0 +1,27 @@
+name: PR title check
+
+on:
+ pull_request:
+ types: [opened, edited, synchronize, reopened]
+
+permissions:
+ pull-requests: read
+
+jobs:
+ pr-title-check:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: amannn/action-semantic-pull-request@v5
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ types: |
+ feat
+ fix
+ docs
+ refactor
+ chore
+ test
+ perf
+ ci
+ build
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 0000000..5e4638e
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,115 @@
+name: Release
+
+on:
+ push:
+ branches: [main]
+ paths:
+ - "pyproject.toml"
+ - "typescript/package.json"
+ workflow_dispatch:
+ inputs:
+ tag_name:
+ description: "Existing tag to release (e.g. python-v4.0.8 or typescript-v0.1.29)."
+ required: true
+
+permissions:
+ contents: write
+ actions: write
+ pull-requests: read
+
+concurrency:
+ group: release-${{ github.event.inputs.tag_name || github.ref_name }}
+
+jobs:
+ release:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout repo
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Determine tags to release
+ id: tags
+ run: |
+ set -euo pipefail
+ git config user.name "github-actions[bot]"
+ git config user.email "github-actions[bot]@users.noreply.github.com"
+
+ declare -a TAGS=()
+
+ add_tag() {
+ local tag="$1"
+ if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then
+ echo "Tag $tag already exists; skipping."
+ else
+ echo "Creating tag $tag."
+ git tag "$tag"
+ git push origin "$tag"
+ TAGS+=("$tag")
+ fi
+ }
+
+ if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+ TAG="${{ github.event.inputs.tag_name }}"
+ case "$TAG" in
+ python-v*|typescript-v*) TAGS+=("$TAG") ;;
+ *) echo "::error::Tag '$TAG' must match python-v* or typescript-v*."; exit 1 ;;
+ esac
+ else
+ PY_VERSION=$(grep -m1 -E '^version = ' pyproject.toml | sed -E 's/^version = "([^"]+)".*/\1/')
+ TS_VERSION=$(jq -r .version typescript/package.json)
+ add_tag "python-v$PY_VERSION"
+ add_tag "typescript-v$TS_VERSION"
+ fi
+
+ echo "list=${TAGS[*]:-}" >> "$GITHUB_OUTPUT"
+
+ - name: Set up Python
+ if: steps.tags.outputs.list != ''
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Install dependencies
+ if: steps.tags.outputs.list != ''
+ run: pip install rich pydantic deepeval
+
+ - name: Create draft releases
+ if: steps.tags.outputs.list != ''
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ REPO: ${{ github.repository }}
+ TAGS: ${{ steps.tags.outputs.list }}
+ run: |
+ set -euo pipefail
+
+ for TAG in $TAGS; do
+ case "$TAG" in
+ python-v*) SDK="Python"; PREFIX="python-v"; SDK_SLUG="python" ;;
+ typescript-v*) SDK="TypeScript"; PREFIX="typescript-v"; SDK_SLUG="typescript" ;;
+ esac
+
+ PREV=$(
+ { echo "$TAG"; git tag --list "${PREFIX}*"; } \
+ | grep -v '^$' | sort -u -V -r \
+ | awk -v cur="$TAG" 'found{print; exit} $0==cur{found=1}'
+ )
+
+ API_ARGS=(-f tag_name="$TAG")
+ if [ -n "$PREV" ]; then
+ API_ARGS+=(-f previous_tag_name="$PREV")
+ fi
+ BODY=$(gh api "repos/$REPO/releases/generate-notes" "${API_ARGS[@]}" --jq .body)
+
+ printf '%s' "$BODY" \
+ | python .scripts/changelog/release_notes.py --sdk "$SDK_SLUG" \
+ > notes.md
+
+ gh release create "$TAG" --title "$SDK ${TAG#$PREFIX}" --draft --notes-file notes.md
+
+ if [ "$SDK" = "Python" ]; then
+ gh workflow run changelog.yml -f tag="$TAG"
+ fi
+ done
diff --git a/.github/workflows/test_confident.yml b/.github/workflows/test_confident.yml
new file mode 100644
index 0000000..25dca57
--- /dev/null
+++ b/.github/workflows/test_confident.yml
@@ -0,0 +1,72 @@
+name: Confident Tests
+
+on:
+ push:
+ paths-ignore:
+ - 'typescript/**'
+ pull_request:
+ paths-ignore:
+ - 'typescript/**'
+ workflow_dispatch:
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
+ steps:
+ #----------------------------------------------
+ # check-out repo and set-up python
+ #----------------------------------------------
+ - name: Check out repository
+ uses: actions/checkout@v3
+ - name: Set up python
+ id: setup-python
+ uses: actions/setup-python@v4
+ with:
+ python-version: "3.11"
+ #----------------------------------------------
+ # ----- install & configure poetry -----
+ #----------------------------------------------
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ virtualenvs-create: true
+ virtualenvs-in-project: true
+ installer-parallel: true
+
+ #----------------------------------------------
+ # load cached venv if cache exists
+ #----------------------------------------------
+ - name: Load cached venv
+ id: cached-poetry-dependencies
+ uses: actions/cache@v3
+ with:
+ path: .venv
+ key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+ #----------------------------------------------
+ # install dependencies if cache does not exist
+ #----------------------------------------------
+ - name: Install dependencies
+ if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+ run: poetry install --no-interaction --no-root --only main
+ #----------------------------------------------
+ # install your root project, if required
+ #----------------------------------------------
+
+ - name: Install project
+ run: poetry install --no-interaction --only main
+ # Install dev dependencies
+ - name: Install dev dependencies
+ run: poetry install --no-interaction --with dev
+ #----------------------------------------------
+ # run test suite
+ #----------------------------------------------
+ # Only run when a Confident API key is available (e.g. not on fork PRs).
+ # Without the key these tests can't run, so skip the step outright to
+ # keep contributor PRs green.
+ - name: Run tests
+ if: ${{ env.CONFIDENT_API_KEY != '' }}
+ run: |
+ poetry run pytest tests/test_confident/
diff --git a/.github/workflows/test_core.yml b/.github/workflows/test_core.yml
new file mode 100644
index 0000000..283873b
--- /dev/null
+++ b/.github/workflows/test_core.yml
@@ -0,0 +1,107 @@
+name: Core Tests
+
+on:
+ push:
+ paths-ignore:
+ - 'typescript/**'
+ pull_request:
+ paths-ignore:
+ - 'typescript/**'
+ workflow_dispatch:
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ env:
+ # Expose once at job level because forked PRs can't use secrets.* in `if:` conditions.
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ DEEPEVAL_TELEMETRY_OPT_OUT: 1
+ steps:
+ #----------------------------------------------
+ # check-out repo and set-up python
+ #----------------------------------------------
+ - name: Check out repository
+ uses: actions/checkout@v3
+ - name: Set up python
+ id: setup-python
+ uses: actions/setup-python@v4
+ with:
+ python-version: "3.11"
+ #----------------------------------------------
+ # ----- install & configure poetry -----
+ #----------------------------------------------
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ virtualenvs-create: true
+ virtualenvs-in-project: true
+ installer-parallel: true
+
+ #----------------------------------------------
+ # load cached venv if cache exists
+ #----------------------------------------------
+ - name: Load cached venv
+ id: cached-poetry-dependencies
+ uses: actions/cache@v3
+ with:
+ path: .venv
+ key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+ #----------------------------------------------
+ # install dependencies if cache does not exist
+ #----------------------------------------------
+ - name: Install dependencies
+ if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+ run: poetry install --no-interaction --no-root --only main
+ #----------------------------------------------
+ # install your root project, if required
+ #----------------------------------------------
+
+ - name: Install project
+ run: poetry install --no-interaction --only main
+
+ #----------------------------------------------
+ # install dev dependencies (including chromadb and model deps)
+ #----------------------------------------------
+ - name: Install dev dependencies
+ run: poetry install --no-interaction --with dev
+
+ #----------------------------------------------
+ # run test suite
+ #----------------------------------------------
+
+ # Run tests (with secrets): full suite
+ - name: Run tests
+ if: ${{ env.OPENAI_API_KEY != '' }}
+ run: |
+ poetry run pytest -vv -rA --maxfail=1 --capture=tee-sys \
+ tests/test_core/ \
+ --ignore=tests/test_core/test_synthesizer/ \
+ --ignore=tests/test_core/test_datasets/
+
+ # Run tests (no secrets): skip e2e that require API keys
+ - name: Run tests (no secrets)
+ if: ${{ env.OPENAI_API_KEY == '' }}
+ run: |
+ poetry run pytest -vv -rA --maxfail=1 --capture=tee-sys tests/test_core/ \
+ --ignore=tests/test_core/test_synthesizer/ \
+ --ignore=tests/test_core/test_datasets/ \
+ --ignore=tests/test_core/test_tracing/test_dataset_iterator.py \
+ --ignore=tests/test_core/test_evaluation/test_end_to_end/test_configs.py
+
+ # Dev tests (with secrets)
+ - name: Run dev tests
+ if: ${{ env.OPENAI_API_KEY != '' }}
+ run: |
+ poetry run pytest -vv -rA --maxfail=1 --capture=tee-sys -o faulthandler_timeout=300 \
+ tests/test_core/test_synthesizer/ tests/test_core/test_datasets/ tests/test_core/test_simulator/
+
+ # Dev tests (no secrets)
+ - name: Run dev tests (no secrets)
+ if: ${{ env.OPENAI_API_KEY == '' }}
+ run: |
+ poetry run pytest -vv -rA --maxfail=1 --capture=tee-sys tests/test_core/test_synthesizer/ tests/test_core/test_datasets/ tests/test_core/test_simulator/ \
+ --ignore=tests/test_core/test_tracing/test_dataset_iterator.py \
+ --ignore=tests/test_core/test_synthesizer/test_context_generator.py \
+ --ignore=tests/test_core/test_simulator/test_conversation_simulator.py \
+ --ignore=tests/test_core/test_synthesizer/test_generate_from_goldens.py \
+ --ignore=tests/test_core/test_synthesizer/test_synthesizer.py
diff --git a/.github/workflows/test_integrations.yml b/.github/workflows/test_integrations.yml
new file mode 100644
index 0000000..773fad9
--- /dev/null
+++ b/.github/workflows/test_integrations.yml
@@ -0,0 +1,344 @@
+name: All Integration Tests
+
+on:
+ push:
+ paths-ignore:
+ - 'typescript/**'
+ pull_request:
+ paths-ignore:
+ - 'typescript/**'
+ workflow_dispatch:
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ # ===========================================================================
+ # 1. LangChain / LangGraph Tests
+ # ===========================================================================
+ langchain:
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ DEEPEVAL_TELEMETRY_OPT_OUT: "1"
+ PYTHONUNBUFFERED: "1"
+ PYTEST_ADDOPTS: "-vv -rA --maxfail=1 --capture=tee-sys"
+
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout@v4
+
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ virtualenvs-create: true
+ virtualenvs-in-project: true
+ installer-parallel: true
+
+ - name: Install Dependencies
+ run: |
+ poetry install --no-interaction --no-root --only main
+ poetry run pip install -U langgraph langchain langchain-openai
+
+ - name: Install Project
+ run: poetry install --no-interaction --only main
+
+ - name: Run LangChain/LangGraph Tests
+ if: ${{ env.OPENAI_API_KEY != '' }}
+ run: |
+ poetry run pytest tests/test_integrations/test_langchain/
+ poetry run pytest tests/test_integrations/test_langgraph/
+
+ # ===========================================================================
+ # 2. CrewAI Tests
+ # ===========================================================================
+ crewai:
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ DEEPEVAL_TELEMETRY_OPT_OUT: "1"
+ PYTHONUNBUFFERED: "1"
+
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout@v4
+
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ virtualenvs-create: true
+ virtualenvs-in-project: true
+
+ - name: Install Dependencies
+ run: |
+ poetry install --no-interaction --no-root --only main
+ poetry install --with integrations
+ poetry run pip install -U crewai
+ # Pin to pydantic-ai-slim[openai] to avoid the meta-package's
+ # mistral extra (mistralai is currently quarantined on PyPI;
+ # see comment in pyproject.toml integrations group).
+ poetry run pip install -U "pydantic-ai-slim[openai]"
+
+ - name: Install Project
+ run: poetry install --no-interaction --only main
+
+ - name: Run CrewAI Tests
+ if: ${{ env.OPENAI_API_KEY != '' }}
+ run: poetry run pytest tests/test_integrations/test_crewai/
+
+ # ===========================================================================
+ # 3. Pydantic AI Tests
+ # ===========================================================================
+ pydantic-ai:
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ DEEPEVAL_TELEMETRY_OPT_OUT: "1"
+ PYTHONUNBUFFERED: "1"
+
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout@v4
+
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ virtualenvs-create: true
+ virtualenvs-in-project: true
+
+ - name: Install Dependencies (Pydantic AI)
+ run: |
+ poetry install --no-interaction --no-root --only main
+ poetry install --with integrations
+ # Pin to pydantic-ai-slim[openai] to avoid the meta-package's
+ # mistral extra (mistralai is currently quarantined on PyPI;
+ # see comment in pyproject.toml integrations group).
+ poetry run pip install -U "pydantic-ai-slim[openai]"
+
+ - name: Run Pydantic AI Tests
+ if: ${{ env.OPENAI_API_KEY != '' }}
+ run: poetry run pytest tests/test_integrations/test_pydanticai/
+
+ # ===========================================================================
+ # 4. LlamaIndex Tests
+ # ===========================================================================
+ llama-index:
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ DEEPEVAL_TELEMETRY_OPT_OUT: "1"
+ PYTHONUNBUFFERED: "1"
+
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout@v4
+
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ virtualenvs-create: true
+ virtualenvs-in-project: true
+
+ - name: Install Dependencies (LlamaIndex)
+ run: |
+ poetry install --no-interaction --no-root --only main
+ poetry install --with integrations
+ poetry run pip install -U llama-index
+
+ - name: Run LlamaIndex Tests
+ if: ${{ env.OPENAI_API_KEY != '' }}
+ run: poetry run pytest tests/test_integrations/test_llamaindex/
+
+ # ===========================================================================
+ # 5. OpenAI Agents Tests
+ # ===========================================================================
+ openai-agents:
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ DEEPEVAL_TELEMETRY_OPT_OUT: "1"
+ PYTHONUNBUFFERED: "1"
+
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout@v4
+
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ virtualenvs-create: true
+ virtualenvs-in-project: true
+
+ - name: Install Dependencies (OpenAI Agents)
+ run: |
+ poetry install --no-interaction --no-root --only main
+ poetry install --with integrations
+ poetry run pip install -U openai-agents
+
+ - name: Run OpenAI Agents Tests
+ if: ${{ env.OPENAI_API_KEY != '' }}
+ run: poetry run pytest tests/test_integrations/test_openai_agents/
+
+ # ===========================================================================
+ # 5. OpenAI Tests
+ # ===========================================================================
+ openai:
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ DEEPEVAL_TELEMETRY_OPT_OUT: "1"
+ PYTHONUNBUFFERED: "1"
+
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout@v4
+
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ virtualenvs-create: true
+ virtualenvs-in-project: true
+
+ - name: Install Dependencies (OpenAI)
+ run: |
+ poetry install --no-interaction --no-root --only main
+ poetry install --with integrations
+ poetry run pip install -U openai
+
+ - name: Run OpenAI Tests
+ if: ${{ env.OPENAI_API_KEY != '' }}
+ run: poetry run pytest tests/test_integrations/test_openai/
+
+ # ===========================================================================
+ # 6. AgentCore Tests
+ # ===========================================================================
+ agentcore:
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ DEEPEVAL_TELEMETRY_OPT_OUT: "1"
+ PYTHONUNBUFFERED: "1"
+
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout@v4
+
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ virtualenvs-create: true
+ virtualenvs-in-project: true
+
+ - name: Install Dependencies
+ run: |
+ poetry install --no-interaction --no-root --only main
+ poetry install --with integrations
+ poetry run pip install -U bedrock-agentcore strands-agents strands-agents-tools
+
+ - name: Install Project
+ run: poetry install --no-interaction --only main
+
+ - name: Run AgentCore Tests
+ if: ${{ env.OPENAI_API_KEY != '' }}
+ run: |
+ poetry run pytest tests/test_integrations/test_agentcore/test_async.py
+ poetry run pytest tests/test_integrations/test_agentcore/test_sync.py
+
+ # ===========================================================================
+ # 7. Strands Tests
+ # ===========================================================================
+ strands:
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ DEEPEVAL_TELEMETRY_OPT_OUT: "1"
+ PYTHONUNBUFFERED: "1"
+
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout@v4
+
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ virtualenvs-create: true
+ virtualenvs-in-project: true
+
+ - name: Install Dependencies
+ run: |
+ poetry install --no-interaction --no-root --only main
+ poetry install --with integrations
+ poetry run pip install -U strands-agents
+
+ - name: Install Project
+ run: poetry install --no-interaction --only main
+
+ - name: Run Strands Tests
+ if: ${{ env.OPENAI_API_KEY != '' }}
+ run: |
+ poetry run pytest tests/test_integrations/test_strands/
diff --git a/.github/workflows/test_metric_templates.yml b/.github/workflows/test_metric_templates.yml
new file mode 100644
index 0000000..3815c39
--- /dev/null
+++ b/.github/workflows/test_metric_templates.yml
@@ -0,0 +1,42 @@
+name: Metric Templates
+
+# Guards that the compiled `templates.json` bundles (Python + TypeScript) stay in
+# sync with the metric template `.txt` sources. The check spans both packages, so
+# it lives in its own workflow rather than the Python-only Core Tests workflow.
+
+on:
+ push:
+ paths:
+ - 'deepeval/**/templates/**'
+ - 'typescript/src/templates/metrics/**'
+ - 'scripts/compile_metric_templates.py'
+ - 'tests/test_templates/test_metric_templates.py'
+ - '.github/workflows/test_metric_templates.yml'
+ pull_request:
+ paths:
+ - 'deepeval/**/templates/**'
+ - 'typescript/src/templates/metrics/**'
+ - 'scripts/compile_metric_templates.py'
+ - 'tests/test_templates/test_metric_templates.py'
+ - '.github/workflows/test_metric_templates.yml'
+ workflow_dispatch:
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Check out repository
+ uses: actions/checkout@v3
+
+ - name: Set up python
+ uses: actions/setup-python@v4
+ with:
+ python-version: "3.11"
+
+ # The test only uses the standard library + the compile script (also
+ # stdlib-only), so no project install is needed — just pytest.
+ - name: Install pytest
+ run: pip install pytest
+
+ - name: Check templates.json is in sync with the .txt sources
+ run: pytest -vv -rA tests/test_templates/test_metric_templates.py
diff --git a/.github/workflows/test_metrics.yml b/.github/workflows/test_metrics.yml
new file mode 100644
index 0000000..9e87a29
--- /dev/null
+++ b/.github/workflows/test_metrics.yml
@@ -0,0 +1,75 @@
+name: Metrics Tests
+
+on:
+ push:
+ paths-ignore:
+ - 'typescript/**'
+ pull_request:
+ paths-ignore:
+ - 'typescript/**'
+ workflow_dispatch:
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ env:
+ # Expose once at job level because forked PRs can't use secrets.* in `if:` conditions.
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ DEEPEVAL_TELEMETRY_OPT_OUT: 1
+ steps:
+ #----------------------------------------------
+ # check-out repo and set-up python
+ #----------------------------------------------
+ - name: Check out repository
+ uses: actions/checkout@v3
+ - name: Set up python
+ id: setup-python
+ uses: actions/setup-python@v4
+ with:
+ python-version: "3.11"
+ #----------------------------------------------
+ # ----- install & configure poetry -----
+ #----------------------------------------------
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ virtualenvs-create: true
+ virtualenvs-in-project: true
+ installer-parallel: true
+
+ #----------------------------------------------
+ # load cached venv if cache exists
+ #----------------------------------------------
+ - name: Load cached venv
+ id: cached-poetry-dependencies
+ uses: actions/cache@v3
+ with:
+ path: .venv
+ key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+ #----------------------------------------------
+ # install dependencies if cache does not exist
+ #----------------------------------------------
+ - name: Install dependencies
+ if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+ run: poetry install --no-interaction --no-root --only main
+ #----------------------------------------------
+ # install your root project, if required
+ #----------------------------------------------
+
+ - name: Install project
+ run: poetry install --no-interaction --only main
+
+ #----------------------------------------------
+ # Install Pillow for PIL and mcp[cli]
+ #----------------------------------------------
+ - name: Install metric dependencies
+ run: poetry run pip install Pillow mcp["cli"]
+
+ #----------------------------------------------
+ # run test suite
+ #----------------------------------------------
+
+ - name: Run metric tests
+ run: |
+ poetry run pytest -vv -rA --maxfail=1 --capture=tee-sys -o faulthandler_timeout=300 \
+ tests/test_metrics/
diff --git a/.github/workflows/typescript_lint.yml b/.github/workflows/typescript_lint.yml
new file mode 100644
index 0000000..7754f45
--- /dev/null
+++ b/.github/workflows/typescript_lint.yml
@@ -0,0 +1,35 @@
+name: TypeScript Lint
+
+on:
+ pull_request:
+ paths:
+ - 'typescript/**'
+ - '.github/workflows/typescript_lint.yml'
+ push:
+ branches:
+ - main
+ paths:
+ - 'typescript/**'
+ - '.github/workflows/typescript_lint.yml'
+ workflow_dispatch:
+
+jobs:
+ lint:
+ runs-on: ubuntu-latest
+ container:
+ image: node:22
+ defaults:
+ run:
+ working-directory: typescript
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Install dependencies
+ run: npm ci
+
+ - name: Run ESLint
+ run: npm run lint -- .
+
+ - name: Run Prettier (check)
+ run: npx prettier --check "src/**/*.ts" "test/**/*.ts"
diff --git a/.github/workflows/typescript_test.yml b/.github/workflows/typescript_test.yml
new file mode 100644
index 0000000..5c91e8d
--- /dev/null
+++ b/.github/workflows/typescript_test.yml
@@ -0,0 +1,36 @@
+name: TypeScript Tests
+
+on:
+ pull_request:
+ paths:
+ - 'typescript/**'
+ - '.github/workflows/typescript_test.yml'
+ push:
+ branches:
+ - main
+ paths:
+ - 'typescript/**'
+ - '.github/workflows/typescript_test.yml'
+ workflow_dispatch:
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ container:
+ image: node:22
+ defaults:
+ run:
+ working-directory: typescript
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Install dependencies
+ run: npm ci
+
+ - name: Run Jest tests
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }}
+ CONFIDENT_TRACE_VERBOSE: 0
+ run: npm test
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..0da76bf
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,178 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+/lib/
+/lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+# For a library or package, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+# This is especially recommended for binary packages to ensure reproducibility, and is more
+# commonly ignored for libraries.
+# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+# in version control.
+# https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.env.local
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+# and can be added to the global gitignore or merged into this file. For a more nuclear
+# option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+docs/.docusaurus
+node_modules
+.deepeval
+.deepeval-cache.json
+.deepeval_telemetry.txt
+.vector_db
+*/cache
+
+# deepeval tests
+secrets
+
+# Mac OS system files
+**/.DS_Store
+
+# Cursor IDE local config (rules, etc.)
+.cursor/
+
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 0000000..6da1897
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,10 @@
+repos:
+ - repo: https://github.com/psf/black
+ rev: 24.8.0
+ hooks:
+ - id: black
+ - repo: https://github.com/astral-sh/ruff-pre-commit
+ rev: v0.6.9
+ hooks:
+ - id: ruff
+ args: [--fix] # auto-fix lint issues
diff --git a/.scripts/changelog/generate.py b/.scripts/changelog/generate.py
new file mode 100644
index 0000000..e327df7
--- /dev/null
+++ b/.scripts/changelog/generate.py
@@ -0,0 +1,1548 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import time
+import urllib.request
+import urllib.error
+from rich import print
+from rich.console import Console, Group
+from rich.markup import escape
+from rich.progress import (
+ Progress,
+ SpinnerColumn,
+ BarColumn,
+ TextColumn,
+ TimeElapsedColumn,
+)
+from rich.live import Live
+
+from dataclasses import dataclass
+from typing import Callable, Dict, Iterable, List, Optional, Tuple
+from pydantic import BaseModel, Field, field_validator
+
+#################
+# Configuration #
+#################
+
+OWNER = "confident-ai"
+REPO = "deepeval"
+
+START_MARKER = "{/* DeepEval release notes start */}"
+LEGACY_START_MARKER = ""
+
+CATEGORY_ORDER = [
+ "Backward Incompatible Change",
+ "New Feature",
+ "Experimental Feature",
+ "Improvement",
+ "Bug Fix",
+ "Security",
+]
+
+MONTH_NAMES = [
+ "January",
+ "February",
+ "March",
+ "April",
+ "May",
+ "June",
+ "July",
+ "August",
+ "September",
+ "October",
+ "November",
+ "December",
+]
+MONTH_INDEX = {name: i for i, name in enumerate(MONTH_NAMES, start=1)}
+AI_MAX_DIFF_LENGTH = 12000 # max chars for diff
+CLEAR_PROGRESS_BAR_ON_COMPLETION = False
+
+
+##############
+# Data types #
+##############
+
+
+@dataclass
+class Commit:
+ sha: str
+ subject: str
+
+
+@dataclass
+class Pull:
+ number: int
+ title: str
+ body: str
+ merged_at: str
+ html_url: str
+ user_login: str
+ user_html_url: str
+ diff_url: str
+
+
+class AiReleaseNote(BaseModel):
+ entry: str = Field(
+ ...,
+ description="User-facing changelog entry. Plain text. No markdown. No PR numbers/links.",
+ min_length=10,
+ max_length=500,
+ )
+ category: str
+ confidence: Optional[float] = Field(
+ default=None,
+ ge=0.0,
+ le=1.0,
+ description="Optional confidence score.",
+ )
+ notes: Optional[str] = Field(
+ default=None,
+ description="Optional internal notes; not written to changelog.",
+ max_length=400,
+ )
+
+ @field_validator("category")
+ @classmethod
+ def validate_category(cls, category: str) -> str:
+ if category not in CATEGORY_ORDER:
+ raise ValueError(f"category must be one of: {CATEGORY_ORDER}")
+ return category
+
+
+class AiMonthSummary(BaseModel):
+ summary: str = Field(
+ ...,
+ description="Short prose summary for the month. Plain text. No lists. No headings.",
+ min_length=40,
+ max_length=700,
+ )
+
+
+#######################
+# Git and PR parsing #
+#######################
+
+PR_NUM_RE = re.compile(r"\(#(\d+)\)|pull request #(\d+)", re.IGNORECASE)
+MERGE_SUBJECT_RE = re.compile(r"^Merge pull request #(\d+)\b", re.IGNORECASE)
+user_cache: Dict[str, Tuple[str, str]] = (
+ {}
+) # maps login to (display_name, html_url)
+tag_to_date: Dict[str, str] = {}
+
+###################################
+# Changelog index and MDX parsing #
+###################################
+
+ChangelogIndex = Dict[str, Dict[str, Dict[str, Dict[int, str]]]]
+# month -> category -> version -> pr_number -> bullet_line
+
+MONTH_RE = re.compile(r"^##\s+(.+?)\s*$")
+CATEGORY_RE = re.compile(r"^###\s+(.+?)\s*$")
+VERSION_RE = re.compile(r"^####\s+(v[0-9].+?)\s*$")
+
+# Bullet PR extraction:
+# - Prefer the stable marker (lets humans edit the visible link/text)
+# - Fall back to parsing the link if the marker is missing
+BULLET_PR_RE = re.compile(r"\[#(\d+)\]\(")
+BULLET_PR_MARKER_RE = re.compile(
+ r"(?:|\{/\*\s*pr:(\d+)\s*\*/\})"
+)
+BULLET_TAIL_RE = re.compile(
+ r"\s*\(\[#\d+\]\([^)]+\)\)\s*(?:|\{/\*\s*pr:\d+\s*\*/\}).*$"
+)
+
+# Optional ignore list to be placed right after START_MARKER to avoid confusing the parser:
+# add a list of PR numbers you would like to be excluded from the generated changelog.
+# {/* changelog-ignore:
+# - 1234
+# - 5678
+# */}
+IGNORE_BLOCK_TOP_RE = re.compile(
+ r"(?is)^\s*(?:|\{/\*\s*changelog-ignore:.*?\*/\})\s*\n*"
+)
+IGNORE_BLOCK_ANY_RE = re.compile(
+ r"(?is)(?:|\{/\*\s*changelog-ignore:(.*?)\*/\})"
+)
+
+###############
+# Git helpers #
+###############
+
+
+def sh(cmd: List[str]) -> str:
+ out = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+ return out.decode("utf-8", errors="replace").strip()
+
+
+def git_tag_date_ymd(tag: str) -> str:
+ if tag not in tag_to_date:
+ try:
+ date_value = sh(["git", "log", "-1", "--format=%cs", tag])
+ tag_to_date[tag] = date_value
+ except subprocess.CalledProcessError:
+ import datetime
+
+ tag_to_date[tag] = datetime.datetime.now().strftime("%Y-%m-%d")
+ return tag_to_date[tag]
+
+
+def tag_match_pattern(tag: str) -> str:
+ """Return the git ``--match`` glob for a tag's release scheme.
+
+ Monorepo releases are tagged per SDK (``python-v*`` / ``typescript-v*``);
+ legacy releases use ``v*``. This keeps the previous-tag lookup within the
+ same scheme so a Python release diffs against the prior Python release.
+ """
+ for prefix in ("python-v", "typescript-v"):
+ if tag.startswith(prefix):
+ return f"{prefix}*"
+ return "v*"
+
+
+def get_prev_tag(tag: str) -> str:
+ return sh(
+ [
+ "git",
+ "describe",
+ "--tags",
+ "--abbrev=0",
+ "--match",
+ tag_match_pattern(tag),
+ f"{tag}^",
+ ]
+ )
+
+
+def list_tags_between(from_tag: str, to_tag: str) -> List[str]:
+ # Ordered by tag date ascending
+ # Uses creatordate which works for lightweight tags too.
+ raw = sh(
+ [
+ "git",
+ "for-each-ref",
+ "--format=%(refname:short)%09%(creatordate:short)",
+ "--sort=creatordate",
+ "refs/tags/v*",
+ ]
+ )
+ tags: List[Tuple[str, str]] = []
+ for line in raw.splitlines():
+ if not line.strip():
+ continue
+ tag, date = line.split("\t", 1)
+ tags.append((tag.strip(), date.strip()))
+ # filter in [from..to] by order in this sorted list
+ tag_names = [tag for tag, _ in tags]
+ if from_tag not in tag_names or to_tag not in tag_names:
+ raise SystemExit(
+ f"from/to tag not found in local tags: {from_tag} -> {to_tag}"
+ )
+ from_index = tag_names.index(from_tag)
+ to_index = tag_names.index(to_tag)
+ if from_index > to_index:
+ from_index, to_index = to_index, from_index
+ return tag_names[from_index : to_index + 1]
+
+
+def list_all_tags() -> List[str]:
+ """Return all version tags sorted by tag creation date (ascending)."""
+ raw = sh(
+ [
+ "git",
+ "for-each-ref",
+ "--sort=creatordate",
+ "--format",
+ "%(refname:short)\t%(creatordate:short)",
+ "refs/tags/v*",
+ ]
+ )
+ tags: List[str] = []
+ for line in raw.splitlines():
+ if not line.strip():
+ continue
+ tag = line.split("\t", 1)[0].strip()
+ if tag:
+ tags.append(tag)
+ return tags
+
+
+def list_tags_for_year(year: int) -> List[str]:
+ """
+ Return all tags whose effective tag date falls within `year`.
+ """
+ all_tags = list_all_tags()
+ out: List[str] = []
+ for tag in all_tags:
+ ymd = git_tag_date_ymd(tag)
+ if ymd.startswith(f"{year}-"):
+ out.append(tag)
+ # keep chronological order (oldest -> newest)
+ out.sort(key=lambda t: git_tag_date_ymd(t))
+ return out
+
+
+def latest_tag() -> str:
+ return sh(["git", "describe", "--tags", "--abbrev=0", "--match", "v*"])
+
+
+def commits_in_range(base: str, head: str) -> List[Commit]:
+ # get sha and subject for commit subjects in range
+ raw = sh(
+ [
+ "git",
+ "log",
+ "--first-parent",
+ "--merges",
+ "--format=%H%x00%s",
+ f"{base}..{head}",
+ ]
+ )
+ commits: List[Commit] = []
+ for line in raw.splitlines():
+ if "\x00" not in line:
+ continue
+ sha, subj = line.split("\x00", 1)
+ commits.append(Commit(sha=sha.strip(), subject=subj.strip()))
+ return commits
+
+
+def extract_pr_numbers(commits: Iterable[Commit]) -> Dict[int, Commit]:
+ # Map PR to representative commit
+ prs: Dict[int, Commit] = {}
+ for commit in commits:
+ subj_match = PR_NUM_RE.search(commit.subject)
+ if not subj_match:
+ continue
+ n = subj_match.group(1) or subj_match.group(2)
+ if not n:
+ continue
+ pr_num = int(n)
+ # prefer merge commit subjects if multiple commits mention same PR
+ if pr_num not in prs:
+ prs[pr_num] = commit
+ else:
+ if MERGE_SUBJECT_RE.match(
+ commit.subject
+ ) and not MERGE_SUBJECT_RE.match(prs[pr_num].subject):
+ prs[pr_num] = commit
+ return prs
+
+
+def offline_pr_title_from_merge_commit(
+ commit_sha: str, fallback_subject: str
+) -> str:
+ """
+ GitHub merge commits look like:
+ Merge pull request #1234 from ...
+
+ PR Title here
+
+ So, we don't need to use the api to get the PR title from the commit message body
+ """
+ if not MERGE_SUBJECT_RE.match(fallback_subject):
+ return fallback_subject
+
+ full = sh(["git", "show", "-s", "--format=%B", commit_sha])
+ lines = [ln.rstrip() for ln in full.splitlines()]
+ # the first line is merge subject, so find first non empty line after it
+ for ln in lines[1:]:
+ if ln.strip():
+ return ln.strip()
+ return fallback_subject
+
+
+def stitch_truncated_title(title: str, body: str) -> str:
+ t = (title or "").strip()
+ if not body:
+ return t
+
+ # If title ends with ellipsis, try to append the first non-empty line of the body.
+ if t.endswith("…") or t.endswith("..."):
+ first_line = next(
+ (ln.strip() for ln in body.splitlines() if ln.strip()), ""
+ )
+ if first_line:
+ t2 = t[:-1].rstrip() if t.endswith("…") else t[:-3].rstrip()
+ # Avoid doubling if body starts with same prefix
+ if not first_line.lower().startswith(t2.lower()):
+ return f"{t2} {first_line}"
+ return first_line
+ return t
+
+
+def sanitize_for_multimodal_sentinel(prompt: str) -> str:
+ # DeepEval's GPTModel treats [DEEPEVAL:IMAGE:...] and [DEEPEVAL:PDF:...] tokens in a
+ # plain-text prompt as real image/PDF inputs (see check_if_multimodal). PR titles,
+ # bodies, and diffs can contain these tokens literally, which makes the model try to
+ # load a bogus URL. Break the sentinel so it can never be parsed as multimodal.
+ return re.sub(r"\[DEEPEVAL:(IMAGE|PDF):", r"[DEEPEVAL:\1_:", prompt)
+
+
+######################
+# GitHub API helpers #
+######################
+
+
+def gh_get(
+ url: str, *, accept: Optional[str] = None, timeout_s: int = 20
+) -> bytes:
+ token = os.getenv("GITHUB_TOKEN") or os.getenv("GH_TOKEN")
+ req = urllib.request.Request(url)
+ req.add_header("User-Agent", "deepeval-changelog-generator")
+ if accept:
+ req.add_header("Accept", accept)
+ if token:
+ req.add_header("Authorization", f"Bearer {token}")
+ with urllib.request.urlopen(req, timeout=timeout_s) as resp:
+ return resp.read()
+
+
+def gh_request(path: str, timeout_s: int = 20) -> dict:
+ data = gh_get(
+ f"https://api.github.com{path}",
+ accept="application/vnd.github+json",
+ timeout_s=timeout_s,
+ )
+ return json.loads(data.decode("utf-8"))
+
+
+def fetch_pr(pr_number: int) -> Pull:
+ data = gh_request(f"/repos/{OWNER}/{REPO}/pulls/{pr_number}")
+ user_data = data.get("user") or {}
+ return Pull(
+ number=pr_number,
+ title=data.get("title") or "",
+ body=data.get("body") or "",
+ merged_at=data.get("merged_at") or "",
+ html_url=data.get("html_url")
+ or f"https://github.com/{OWNER}/{REPO}/pull/{pr_number}",
+ user_login=user_data.get("login") or "",
+ user_html_url=user_data.get("html_url") or "",
+ diff_url=data.get("diff_url") or "",
+ )
+
+
+def fetch_pr_diff(diff_url: str, timeout_s: int = 20) -> str:
+ data = gh_get(
+ diff_url, accept="application/vnd.github.v3.diff", timeout_s=timeout_s
+ )
+ return data.decode("utf-8", errors="replace")
+
+
+def fetch_user_display(login: str) -> Tuple[str, str]:
+ """
+ Returns (display_name, html_url). display_name falls back to login.
+ Cached per-login to avoid repeated requests.
+ """
+ login = (login or "").strip()
+ if not login:
+ return "", ""
+ if login in user_cache:
+ return user_cache[login]
+
+ data = gh_request(f"/users/{login}")
+ name = (data.get("name") or "").strip()
+ html_url = (data.get("html_url") or "").strip()
+ display = name or login
+ user_cache[login] = (display, html_url)
+ return user_cache[login]
+
+
+###############
+# LLM Helpers #
+###############
+
+
+def get_ai_model(model_name: str):
+ from deepeval.models import GPTModel
+
+ return GPTModel(model=model_name)
+
+
+def build_ai_prompt(*, title: str, body: str) -> str:
+ # Keep the instructions short + strict; rely on the schema for structure.
+ return f"""
+You are writing release notes for an open-source Python developer tool.
+
+Task:
+Given a PR title and PR body, produce:
+- entry: one short, ClickHouse-style release note line (no markdown, no PR refs, no URLs)
+- category: choose the best match from the allowed categories
+
+Style rules (very important):
+- Focus on the user-visible change and outcome.
+- Use plain language; avoid internal jargon, code names, branch names, and "merge pull request".
+- Prefer an action verb: "Add", "Fix", "Improve", "Reduce", "Prevent", "Support".
+- Keep it to 1-4 sentences, plain text, target 120-500 chars not exceeding 500.
+- If PR body provides enough detail, write 2-4 sentences. Otherwise keep to 1 sentence.
+- Don’t mention "DeepEval" unless it is essential for clarity. Use your existing confidence to decide if you should fall back to title-only.
+- If PR body is empty, write a single sentence based on title only.
+- No version numbers, no PR numbers.
+- You may use backticks for inline code (like_this) when appropriate.
+- Do not use any other markdown (no lists, headers, links).
+
+If the PR is unclear, write the safest high-level improvement without guessing details.
+IMPORTANT: Output only valid JSON with no code fences or comments.
+
+Allowed categories:
+- Backward Incompatible Change
+- New Feature
+- Experimental Feature
+- Improvement
+- Bug Fix
+- Security
+
+PR title:
+{title.strip()}
+
+PR body (may include templates/checklists):
+{(body or "").strip()}
+""".strip()
+
+
+def build_month_summary_prompt(*, month: str, entries: list[str]) -> str:
+ # entries are your bullet texts (ideally without the PR link tail)
+ joined = "\n".join(f"- {e}" for e in entries)
+ return f"""
+You are writing a short monthly release summary for an open-source Python developer tool.
+
+Write 2–5 sentences of prose summarizing the themes and highlights for the month.
+- No lists, no headings, no links, no PR numbers.
+- Plain text.
+- You may use backticks for inline code identifiers when appropriate.
+
+Month:
+{month}
+
+Release note entries:
+{joined}
+
+IMPORTANT: Output only valid JSON.
+""".strip()
+
+
+def ai_month_summary(model, *, month: str, entries: list[str]) -> str:
+ compact = entries[:80]
+ prompt = sanitize_for_multimodal_sentinel(
+ build_month_summary_prompt(month=month, entries=compact)
+ )
+ parsed, _cost = model.generate(prompt, schema=AiMonthSummary)
+ assert isinstance(parsed, AiMonthSummary)
+ return parsed.summary.strip()
+
+
+def ai_release_note_for_pr(
+ model,
+ *,
+ pr_number: int,
+ title: str,
+ body: str,
+) -> tuple[AiReleaseNote, float]:
+ prompt = sanitize_for_multimodal_sentinel(
+ build_ai_prompt(title=title, body=body)
+ )
+ try:
+ parsed, cost = model.generate(prompt, schema=AiReleaseNote)
+ # GPTModel returns (BaseModel, cost) when schema is provided
+ assert isinstance(parsed, AiReleaseNote)
+ except Exception as e:
+ raise RuntimeError(
+ f"--ai failed for PR #{pr_number}. "
+ f"Title={title!r}. Error={type(e).__name__}: {e}"
+ ) from e
+ return parsed, cost
+
+
+def clean_pr_body_for_ai(body: str, *, max_chars: int = 2000) -> str:
+ if not body:
+ return ""
+
+ s = body
+
+ # Remove HTML comments (often template hints)
+ s = re.sub(r"(?s)", "", s)
+
+ # Remove blocks (often long checklists / screenshots)
+ s = re.sub(r"(?is).*? ", "", s)
+
+ lines: list[str] = []
+ for raw in s.splitlines():
+ line = raw.strip()
+
+ if not line:
+ continue
+
+ # Drop common checklist/template noise
+ if re.match(r"^-\s*\[[ xX]\]\s+", line):
+ continue
+ if re.match(
+ r"^(##|###)\s*(Checklist|Changelog|Testing|Test Plan|Screenshots|Notes)\b",
+ line,
+ re.I,
+ ):
+ continue
+ if re.match(r"^(Closes|Fixes|Resolves)\s+#\d+", line, re.I):
+ continue
+
+ # Drop link dumps
+ if re.match(r"^https?://\S+$", line):
+ continue
+
+ lines.append(line)
+
+ out = "\n".join(lines).strip()
+
+ if len(out) > max_chars:
+ out = out[:max_chars].rstrip() + "\n\n[TRUNCATED]"
+ return out
+
+
+def clean_diff_for_ai(diff_text: str) -> str:
+ """
+ Light cleanup to make diffs more model-friendly, then truncate.
+
+ - Drops very large/binary-ish sections (e.g., 'GIT binary patch').
+ - Removes extremely long lines (often minified / generated).
+ """
+ if not diff_text:
+ return ""
+
+ lines: list[str] = []
+ for ln in diff_text.splitlines():
+ # Skip binary patches / obvious noise
+ if "GIT binary patch" in ln:
+ continue
+ if ln.startswith("Binary files "):
+ continue
+
+ # Drop absurdly long lines (minified/compiled)
+ if len(ln) > 2000:
+ lines.append(ln[:2000] + " [LINE TRUNCATED]")
+ continue
+
+ lines.append(ln)
+
+ cleaned = "\n".join(lines).strip()
+ max_chars = AI_MAX_DIFF_LENGTH
+ return truncate_text(
+ cleaned,
+ max_chars=max_chars,
+ head_chars=int(max_chars * 0.6),
+ tail_chars=int(max_chars * 0.25),
+ marker="\n\n[... DIFF TRUNCATED ...]\n\n",
+ )
+
+
+#############
+# Utilities #
+#############
+
+
+def truncate_text(
+ text: str,
+ *,
+ max_chars: int = 12000,
+ head_chars: int = 6000,
+ tail_chars: int = 3000,
+ marker: str = "\n\n[... TRUNCATED ...]\n\n",
+) -> str:
+ """
+ Truncate large text safely.
+
+ - If <= max_chars: return as is.
+ - Otherwise: keep head_chars plus tail_chars with a marker between.
+ """
+ if not text:
+ return ""
+ if max_chars <= 0:
+ return ""
+ if len(text) <= max_chars:
+ return text
+
+ # Ensure sane values
+ head_chars = max(0, min(head_chars, max_chars))
+ tail_chars = max(0, min(tail_chars, max_chars - head_chars))
+ if head_chars == 0 and tail_chars == 0:
+ return marker.strip()
+
+ head = text[:head_chars].rstrip()
+ tail = text[-tail_chars:].lstrip() if tail_chars else ""
+ return f"{head}{marker}{tail}".strip()
+
+
+def strip_entry_tail(line: str) -> str:
+ s = line.strip()
+ if s.startswith("- "):
+ s = s[2:]
+ s = BULLET_TAIL_RE.sub("", s).strip()
+ return s
+
+
+#################################
+# Classification / sanitization #
+#################################
+
+
+def clean_title(title: str) -> str:
+ title = title.strip()
+ title = re.sub(
+ r"^(feat|fix|docs|perf|refactor|ci|chore)(\([^)]+\))?:\s*",
+ "",
+ title,
+ flags=re.I,
+ )
+ return title.strip()
+
+
+def classify(title: str, body: str) -> str:
+ title_lower = title.lower()
+ body_lower = (body or "").lower()
+
+ if any(
+ key_word in title_lower or key_word in body_lower
+ for key_word in [
+ "breaking",
+ "backward incompatible",
+ "incompatible",
+ "breaking change",
+ ]
+ ):
+ return "Backward Incompatible Change"
+ if any(
+ key_word in title_lower or key_word in body_lower
+ for key_word in ["security", "vuln", "cve"]
+ ):
+ return "Security"
+ if any(
+ key_word in title_lower or key_word in body_lower
+ for key_word in [
+ "poc",
+ "prototype",
+ "spike",
+ "experimental",
+ "preview",
+ "beta",
+ ]
+ ):
+ return "Experimental Feature"
+ if any(
+ key_word in title_lower or key_word in body_lower
+ for key_word in [
+ "fix",
+ "bug",
+ "crash",
+ "regression",
+ "error",
+ "fails",
+ "failure",
+ ]
+ ):
+ return "Bug Fix"
+ if any(
+ key_word in title_lower or key_word in body_lower
+ for key_word in [
+ "feat",
+ "add",
+ "introduce",
+ "support",
+ "enable",
+ "flag",
+ "option",
+ "new",
+ ]
+ ):
+ return "New Feature"
+ return "Improvement"
+
+
+def mdx_escape(s: str) -> str:
+ # Prevent MDX JSX parsing issues
+ s = s.replace("&", "&")
+ s = s.replace("<", "<").replace(">", ">")
+ s = s.replace("{", "\\{").replace("}", "\\}")
+ return s
+
+
+############################
+# File parsing / rendering #
+############################
+
+
+def split_prefix_and_body(text: str) -> Tuple[str, str]:
+ """
+ Return (prefix_with_marker, body_after_marker).
+
+ - The prefix includes any YAML frontmatter (the leading `--- ... ---` block),
+ plus the `START_MARKER` line.
+ - If an ignore block is present immediately after the marker, we keep it in the
+ prefix as well so it won't be interpreted as changelog bullets.
+
+ If the marker is missing, we preserve frontmatter (if present) and inject the
+ marker into the prefix.
+ """
+
+ def _pull_top_ignore_block(s: str) -> Tuple[str, str]:
+ s2 = s.lstrip("\n")
+ matched = IGNORE_BLOCK_TOP_RE.match(s2)
+ if not matched:
+ return "", s
+ ignore_block = s2[: matched.end()]
+ rest = s2[matched.end() :]
+ return ignore_block.rstrip("\n") + "\n", rest
+
+ marker_in_text = next(
+ (
+ marker
+ for marker in (START_MARKER, LEGACY_START_MARKER)
+ if marker in text
+ ),
+ None,
+ )
+ if marker_in_text:
+ before, _, after = text.partition(marker_in_text)
+ ignore_block, rest = _pull_top_ignore_block(after)
+ prefix = before.rstrip() + "\n\n" + START_MARKER + "\n"
+ if ignore_block:
+ prefix += ignore_block
+ body = rest.lstrip("\n")
+ return prefix, body
+
+ # Try to keep frontmatter if present
+ if text.startswith("---"):
+ matched = re.match(r"^---\n.*?\n---\n", text, flags=re.S)
+ if matched:
+ front = matched.group(0).rstrip()
+ rest = text[matched.end() :]
+ ignore_block, rest2 = _pull_top_ignore_block(rest)
+ prefix = front + "\n\n" + START_MARKER + "\n"
+ if ignore_block:
+ prefix += ignore_block
+ return prefix, rest2.lstrip("\n")
+
+ # No frontmatter, just inject marker at top
+ ignore_block, rest = _pull_top_ignore_block(text)
+ prefix = START_MARKER + "\n"
+ if ignore_block:
+ prefix += ignore_block
+ return prefix, rest.lstrip("\n")
+
+
+def parse_ignore_prs(text: str) -> set[int]:
+ """
+ Parse PR numbers from one or more changelog-ignore comment blocks.
+
+ Should be placed immediately after the `START_MARKER`, for example:
+
+ {/* changelog-ignore:
+ - 1234
+ - 5678
+ */}
+
+ Lines may contain comments which can be used to document why a PR is being ignored
+ Any integers found in the block are treated as PR numbers.
+ """
+ ignored: set[int] = set()
+ for matched in IGNORE_BLOCK_ANY_RE.finditer(text):
+ block = next(group for group in matched.groups() if group is not None)
+ for line in block.splitlines():
+ line = line.strip()
+ if not line or line.startswith("#"):
+ continue
+ for pr_num in re.findall(r"\b\d+\b", line):
+ try:
+ ignored.add(int(pr_num))
+ except ValueError:
+ pass
+ return ignored
+
+
+def prune_ignored(idx: ChangelogIndex, ignore_prs: set[int]) -> int:
+ """
+ Remove any PR entries whose number is in `ignore_prs`.
+
+ This is what makes deletions persist across updates: add the PR number to the ignore block, re-run
+ the generator, and the entry will be removed and it won't be re-added by future generator updates.
+ """
+ removed = 0
+ for month, categories in list(idx.items()):
+ for category, versions in list(categories.items()):
+ for version, prs in list(versions.items()):
+ for pr in list(prs.keys()):
+ if pr in ignore_prs:
+ del prs[pr]
+ removed += 1
+ return removed
+
+
+def parse_body(body: str) -> ChangelogIndex:
+ idx: ChangelogIndex = {}
+ month = None
+ category = None
+ version = None
+
+ for line in body.splitlines():
+ matched = MONTH_RE.match(line)
+ if matched:
+ month = matched.group(1).strip()
+ idx.setdefault(month, {})
+ category = None
+ version = None
+ continue
+ matched = CATEGORY_RE.match(line)
+ if matched:
+ category = matched.group(1).strip()
+ if month is None:
+ continue
+ idx[month].setdefault(category, {})
+ version = None
+ continue
+ matched = VERSION_RE.match(line)
+ if matched:
+ version = matched.group(1).strip()
+ if month is None or category is None:
+ continue
+ idx[month][category].setdefault(version, {})
+ continue
+
+ if line.startswith("- "):
+ if month is None or category is None or version is None:
+ continue
+ matched = BULLET_PR_RE.search(line) or BULLET_PR_MARKER_RE.search(
+ line
+ )
+ if not matched:
+ continue
+ pr = int(next(group for group in matched.groups() if group))
+ idx[month][category][version][pr] = line.rstrip()
+
+ return idx
+
+
+def month_sort_key(name: str) -> int:
+ return MONTH_INDEX.get(name, 0)
+
+
+def render_changelog_body(
+ idx: ChangelogIndex,
+ version_date: Dict[str, str],
+ *,
+ use_ai: bool = False,
+ ai_model: str = "gpt-5.2",
+) -> str:
+ """
+ Render an ChangelogIndex into an MDX/Markdown changelog body.
+
+ Output structure:
+ - "## {Month}" sections (newest month first)
+ - "### {Category}" subsections in CATEGORY_ORDER. Empty categories are omitted
+ - "#### {Version}" blocks ordered by version_date desc
+ - bullet entries under each version, sorted by PR number
+
+ Returns the rendered body text with a trailing newline.
+ """
+ months = sorted(idx.keys(), key=month_sort_key, reverse=True)
+
+ out: List[str] = []
+ ai = get_ai_model(ai_model) if use_ai else None
+ for month in months:
+ out.append(f"## {month}")
+ out.append("")
+
+ # Monthly summary
+ if use_ai and ai is not None:
+ month_entries: list[str] = []
+ for _category, versions in idx[month].items():
+ for _version, prs in versions.items():
+ for _pr, line in prs.items():
+ month_entries.append(strip_entry_tail(line))
+
+ if month_entries:
+ try:
+ summary = ai_month_summary(
+ ai, month=month, entries=month_entries
+ )
+ summary = mdx_escape(summary)
+ out.append(summary)
+ except Exception as e:
+ # Don't kill changelog rendering if summary fails
+ print(f"[month summary] {month}: {type(e).__name__}: {e}")
+ out.append("")
+
+ for category in CATEGORY_ORDER:
+ if category not in idx[month]:
+ continue
+ # only render those that actually have entries
+ has_any = any(idx[month][category].values())
+ if not has_any:
+ continue
+
+ out.append(f"### {category}")
+ out.append("")
+
+ # version DESC by tag date
+ versions = list(idx[month][category].keys())
+ versions.sort(key=lambda v: version_date.get(v, ""), reverse=True)
+
+ for version in versions:
+ entries = idx[month][category][version]
+ if not entries:
+ continue
+ out.append(f"#### {version}")
+ # ascending by PR number
+ for pr in sorted(entries.keys()):
+ out.append(entries[pr])
+ out.append("") # blank line after each version block
+
+ out.append("") # blank line after category
+
+ out.append("") # blank line after month
+
+ # Trim trailing blank lines
+ while out and out[-1] == "":
+ out.pop()
+ return "\n".join(out) + ("\n" if out else "")
+
+
+###################
+# Build and merge #
+###################
+
+JUNK_TITLE_RE = re.compile(
+ r"^(merge pull request|merge branch|bump |release |main$|master$|patch-\d+|hotfix|wip)\b",
+ re.IGNORECASE,
+)
+
+
+def title_needs_github(title: str) -> bool:
+ title = (title or "").strip()
+ if not title:
+ return True
+ if MERGE_SUBJECT_RE.match(title):
+ return True
+ if JUNK_TITLE_RE.match(title):
+ return True
+ if re.fullmatch(r"[\w.-]+/[\w.-]+", title):
+ return True
+ return False
+
+
+def month_name_from_ymd(ymd: str) -> Tuple[int, str]:
+ year, month, _ = map(int, ymd.split("-"))
+ return year, MONTH_NAMES[month - 1]
+
+
+def build_release_entries(
+ tag: str,
+ use_github: bool,
+ use_ai: bool = False,
+ ai_model: str = "gpt-5.2",
+ sleep_s: float = 0.0,
+ ignore_prs: Optional[set[int]] = None,
+ existing_keys: Optional[set[tuple[str, int]]] = None,
+ overwrite_existing: bool = False,
+ status_cb: Optional[Callable[[str], None]] = None,
+ tick_cb: Optional[Callable[[], None]] = None,
+) -> Tuple[int, str, ChangelogIndex, Dict[str, str], float]:
+ prev = get_prev_tag(tag)
+ tag_date = git_tag_date_ymd(tag)
+ year, month = month_name_from_ymd(tag_date)
+
+ commits = commits_in_range(prev, tag)
+ pr_map = extract_pr_numbers(commits)
+ if ignore_prs:
+ pr_map = {
+ pr: commit for pr, commit in pr_map.items() if pr not in ignore_prs
+ }
+
+ # collect entries for this tag into an index shape
+ idx: ChangelogIndex = {month: {}}
+ version_date = {tag: tag_date}
+ ai = None
+ ai_cache: dict[int, AiReleaseNote] = {}
+ ai_total_cost = 0.0
+ if use_ai:
+ ai = get_ai_model(ai_model)
+
+ def _status(msg: str) -> None:
+ if status_cb is not None:
+ status_cb(msg)
+
+ def _tick() -> None:
+ if tick_cb:
+ tick_cb()
+
+ for pr_num, commit in sorted(pr_map.items(), key=lambda kv: kv[0]):
+ _status(f"[{tag}] PR #{pr_num}: preparing…")
+ key = (tag, pr_num)
+ if existing_keys and (key in existing_keys) and not overwrite_existing:
+ _status(f"[{tag}] PR #{pr_num}: skipping (already present)")
+ _tick()
+ # Preserve manual edits/moves and avoid useless LLM calls
+ continue
+
+ # offline title from merge commit body if possible
+ title = offline_pr_title_from_merge_commit(commit.sha, commit.subject)
+ body = ""
+ user_login = ""
+ user_html_url = ""
+ user_display = ""
+ user_profile_url = ""
+ diff_url = ""
+
+ if use_github and (use_ai or title_needs_github(title)):
+ _status(f"[{tag}] PR #{pr_num}: fetching from GitHub…")
+ try:
+ pr = fetch_pr(pr_num)
+ diff_url = pr.diff_url
+ except urllib.error.HTTPError as e:
+ msg = (
+ f"Unable to fetch PR #{pr_num} for tag {tag} (commit {commit.sha[:8]}): "
+ f"HTTP {e.code} {e.reason}"
+ )
+ _status(f"[{tag}] PR #{pr_num}: error: HTTP {e.code}")
+ print(msg)
+ if e.code == 404:
+ _status(f"[{tag}] PR #{pr_num}: 404 (skipped)")
+ _tick()
+ continue
+ raise
+ except Exception as e:
+ msg = (
+ f"Unable to fetch PR #{pr_num} for tag {tag} (commit {commit.sha[:8]}): "
+ f"{type(e).__name__}: {e}"
+ )
+ _status(f"[{tag}] PR #{pr_num}: error: {type(e).__name__}")
+ print(msg)
+ raise
+
+ title = pr.title or title
+ body = pr.body or ""
+ if sleep_s:
+ time.sleep(sleep_s)
+ user_login = pr.user_login
+ user_html_url = pr.user_html_url
+ user_display, user_profile_url = fetch_user_display(user_login)
+ # prefer profile url from user endpoint if present
+ user_profile_url = user_profile_url or user_html_url
+
+ body_clean = clean_pr_body_for_ai(body)
+ has_detail = len(body_clean) >= 200
+ title = stitch_truncated_title(title, body_clean)
+
+ diff = ""
+ if use_ai and use_github and (not has_detail) and diff_url:
+ try:
+ _status(f"[{tag}] PR #{pr_num}: fetching diff…")
+ diff = fetch_pr_diff(diff_url)
+ diff = clean_diff_for_ai(diff)
+ except Exception:
+ diff = ""
+
+ # Use AI to generate a higher-quality bullet.
+ if use_ai:
+ if pr_num in ai_cache:
+ note = ai_cache[pr_num]
+ else:
+ body_for_ai = body_clean if has_detail else ""
+ if diff:
+ body_for_ai = (
+ (body_for_ai + "\n\n" if body_for_ai else "")
+ + "PR diff (for context; may be truncated):\n"
+ + diff
+ )
+
+ note, cost = ai_release_note_for_pr(
+ ai,
+ pr_number=pr_num,
+ title=title,
+ body=body_for_ai,
+ )
+ ai_cache[pr_num] = note
+ ai_total_cost += cost if cost is not None else 0
+
+ bullet = mdx_escape(clean_title(note.entry.strip()))
+ if not bullet.endswith("."):
+ bullet += "."
+ category = note.category
+ title_out = bullet
+ else:
+ title_out = mdx_escape(clean_title(title))
+ if not title_out.endswith("."):
+ title_out += "."
+ category = classify(title, body)
+
+ idx[month].setdefault(category, {}).setdefault(tag, {})
+ author = ""
+ if user_display:
+ if user_profile_url:
+ author = f" ([{user_display}]({user_profile_url}))"
+ else:
+ author = f" ({user_display})"
+ line = (
+ f"- {title_out} ([#{pr_num}](https://github.com/{OWNER}/{REPO}/pull/{pr_num})) "
+ f"{{/* pr:{pr_num} */}}{author}"
+ )
+ idx[month][category][tag][pr_num] = line
+ _status(f"[{tag}] PR #{pr_num}: done")
+ _tick()
+ return year, month, idx, version_date, ai_total_cost
+
+
+def collect_existing_keys(idx: ChangelogIndex) -> set[tuple[str, int]]:
+ out: set[tuple[str, int]] = set()
+ for _month, categories in idx.items():
+ for _category, versions in categories.items():
+ for version, prs in versions.items():
+ for pr in prs.keys():
+ out.add((version, pr))
+ return out
+
+
+def merge_idx(
+ existing: ChangelogIndex,
+ updates: ChangelogIndex,
+ overwrite_existing: bool = False,
+) -> int:
+ """Merge `updates` entries into `existing` (in-place).
+
+ Entries are keyed by PR number and version tag. If an entry for the same
+ (version,PR) already exists anywhere in `existing`, that location is treated
+ as the correct location so manual moves between categories and months persist across
+ updates.
+
+ If `overwrite_existing` is False, existing entries are left untouched.
+ If True, the existing bullet line is updated in-place.
+
+ Returns the number of newly-added entries."""
+ added = 0
+
+ # Build a quick lookup of where each (version, PR) currently lives.
+ loc_by_key: Dict[Tuple[str, int], Tuple[str, str]] = {}
+ for month, categories in existing.items():
+ for category, versions in categories.items():
+ for version, prs in versions.items():
+ for pr in prs.keys():
+ loc_by_key[(version, pr)] = (month, category)
+
+ for month, categories in updates.items():
+ existing.setdefault(month, {})
+ for category, versions in categories.items():
+ existing[month].setdefault(category, {})
+ for version, prs in versions.items():
+ existing[month][category].setdefault(version, {})
+ for pr, line in prs.items():
+ key = (version, pr)
+ if key in loc_by_key:
+ month0, category0 = loc_by_key[key]
+ if not overwrite_existing:
+ continue
+ existing[month0].setdefault(category0, {})
+ existing[month0][category0].setdefault(version, {})
+ existing[month0][category0][version][pr] = line
+ continue
+
+ # new entry
+ added += 1
+ existing[month][category][version][pr] = line
+ loc_by_key[key] = (month, category)
+
+ return added
+
+
+def run_with_overall_progress(
+ tags: list[str],
+ args,
+ per_year,
+ per_year_prefix,
+ per_year_ignore,
+ version_date_entries,
+) -> float:
+ ai_total_cost = 0.0
+
+ # If --silent, skip all rich UI and just run normally.
+ if args.silent:
+ for tag in tags:
+ y, _m = month_name_from_ymd(git_tag_date_ymd(tag))
+ out_path = os.path.join(args.output_dir, f"changelog-{y}.mdx")
+
+ if y not in per_year:
+ if os.path.exists(out_path):
+ existing_text = open(out_path, "r", encoding="utf-8").read()
+ prefix, body = split_prefix_and_body(existing_text)
+ per_year_prefix[y] = prefix
+ per_year_ignore[y] = parse_ignore_prs(prefix)
+ per_year[y] = parse_body(body)
+ for _month, categories in per_year[y].items():
+ for _cat, versions in categories.items():
+ for version in versions.keys():
+ if version not in version_date_entries:
+ version_date_entries[version] = (
+ git_tag_date_ymd(version)
+ )
+ else:
+ os.makedirs(args.output_dir, exist_ok=True)
+ per_year_prefix[y] = (
+ f"---\n"
+ f"id: changelog-{y}\n"
+ f"title: {y}\n"
+ f"sidebar_label: {y}\n"
+ f"---\n\n"
+ f"{START_MARKER}\n"
+ )
+ per_year_ignore[y] = set()
+ per_year[y] = {}
+
+ existing_keys = collect_existing_keys(per_year[y])
+ year, month, idx_update, vd, ai_cost = build_release_entries(
+ tag,
+ use_github=args.github,
+ use_ai=args.ai,
+ ai_model=args.ai_model,
+ sleep_s=args.sleep,
+ ignore_prs=per_year_ignore[y],
+ existing_keys=existing_keys,
+ overwrite_existing=args.overwrite_existing,
+ )
+ ai_total_cost += ai_cost
+ version_date_entries.update(vd)
+ merge_idx(
+ per_year[year],
+ idx_update,
+ overwrite_existing=args.overwrite_existing,
+ )
+
+ return ai_total_cost
+
+ console = Console(stderr=True)
+
+ overall = Progress(
+ SpinnerColumn(),
+ TextColumn("[progress.description]{task.description}"),
+ BarColumn(bar_width=26),
+ TextColumn("{task.completed}/{task.total}"),
+ TimeElapsedColumn(),
+ transient=CLEAR_PROGRESS_BAR_ON_COMPLETION,
+ console=console,
+ )
+
+ # We would like the per tag progress indicator to remain only if it fails to cmplete due to an error, other wise we would like it to be removed at the end of the run.
+ # The Key trick to getting the behavior we want is to make this transient=False like the overall indicator, but remove tasks on success.
+ # That way, if there’s an error, the last per tag line stays visible.
+ per_tag = Progress(
+ SpinnerColumn(),
+ TextColumn("[progress.description]{task.description}"),
+ BarColumn(bar_width=40),
+ TextColumn("{task.completed}/{task.total}"),
+ TimeElapsedColumn(),
+ transient=False,
+ console=console,
+ )
+
+ with Live(Group(overall, per_tag), console=console, refresh_per_second=10):
+ overall_task = overall.add_task("Processing releases…", total=len(tags))
+
+ for tag in tags:
+ tag_task = per_tag.add_task(f"{tag}: preparing…", total=0)
+ # Determine the output year early so we can load existing content and the ignore list
+ y, _m = month_name_from_ymd(git_tag_date_ymd(tag))
+ out_path = os.path.join(args.output_dir, f"changelog-{y}.mdx")
+
+ if y not in per_year:
+ if os.path.exists(out_path):
+ existing_text = open(out_path, "r", encoding="utf-8").read()
+ prefix, body = split_prefix_and_body(existing_text)
+ per_year_prefix[y] = prefix
+ per_year_ignore[y] = parse_ignore_prs(
+ prefix
+ ) # ignore block is kept in prefix
+ per_year[y] = parse_body(body)
+ for _month, categories in per_year[y].items():
+ for _cat, versions in categories.items():
+ for version in versions.keys():
+ if version not in version_date_entries:
+ version_date_entries[version] = (
+ git_tag_date_ymd(version)
+ )
+ else:
+ os.makedirs(args.output_dir, exist_ok=True)
+ per_year_prefix[y] = (
+ f"---\n"
+ f"id: changelog-{y}\n"
+ f"title: {y}\n"
+ f"sidebar_label: {y}\n"
+ f"---\n\n"
+ f"{START_MARKER}\n"
+ )
+ per_year_ignore[y] = set()
+ per_year[y] = {}
+
+ existing_keys = collect_existing_keys(per_year[y])
+
+ # Compute total PRs we expect to handle for this tag so the bar is the right length.
+ prev = get_prev_tag(tag)
+ commits = commits_in_range(prev, tag)
+ pr_map = extract_pr_numbers(commits)
+ if per_year_ignore.get(y):
+ pr_map = {
+ pr: c
+ for pr, c in pr_map.items()
+ if pr not in per_year_ignore[y]
+ }
+ per_tag.update(tag_task, total=len(pr_map), completed=0)
+
+ def status_cb(msg: str) -> None:
+ per_tag.update(tag_task, description=escape(msg))
+
+ def tick_cb() -> None:
+ per_tag.advance(tag_task, 1)
+
+ per_tag.update(tag_task, description=f"{tag}: generating entries…")
+ try:
+ year, month, idx_update, vd, ai_cost = build_release_entries(
+ tag,
+ use_github=args.github,
+ use_ai=args.ai,
+ ai_model=args.ai_model,
+ sleep_s=args.sleep,
+ ignore_prs=per_year_ignore[y],
+ existing_keys=existing_keys,
+ overwrite_existing=args.overwrite_existing,
+ status_cb=status_cb,
+ tick_cb=tick_cb,
+ )
+ except Exception as e:
+ # Leave the tag line visible (do NOT remove task), show concise error.
+ per_tag.update(
+ tag_task,
+ description=f"{tag}: error: {type(e).__name__}: {e}",
+ )
+ raise
+ else:
+ ai_total_cost += ai_cost
+ version_date_entries.update(vd)
+ merge_idx(
+ per_year[year],
+ idx_update,
+ overwrite_existing=args.overwrite_existing,
+ )
+
+ per_tag.update(tag_task, description=f"{tag}: done")
+ per_tag.remove_task(tag_task) # remove on success
+ overall.advance(overall_task, 1)
+
+ return ai_total_cost
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ g = ap.add_mutually_exclusive_group(required=True)
+ g.add_argument("--tag", help="Release tag like v3.7.6")
+ g.add_argument(
+ "--latest", action="store_true", help="Generate for latest tag only"
+ )
+ g.add_argument(
+ "--range",
+ nargs=2,
+ metavar=("FROM", "TO"),
+ help="Generate for an inclusive tag range",
+ )
+ g.add_argument(
+ "--year",
+ type=int,
+ help="Generate for all tags whose tag date falls within YEAR",
+ )
+
+ ap.add_argument(
+ "--output-dir",
+ default="docs/content/changelog",
+ help="Docs changelog dir",
+ )
+ ap.add_argument(
+ "--github",
+ action="store_true",
+ help="Enrich titles/bodies from GitHub API (needs token for speed)",
+ )
+ ap.add_argument(
+ "--ai",
+ action="store_true",
+ help="Use an LLM to generate release-note bullets",
+ )
+ ap.add_argument("--ai-model", default="gpt-5.2", help="Model name for --ai")
+ ap.add_argument(
+ "--overwrite-existing",
+ action="store_true",
+ help="Overwrite existing entries for the same PR (default preserves manual edits)",
+ )
+ ap.add_argument(
+ "--sleep",
+ type=float,
+ default=0.0,
+ help="Sleep between GitHub API calls (seconds)",
+ )
+ ap.add_argument(
+ "--silent",
+ action="store_true",
+ help="Disable progress indicator output.",
+ )
+ ap.add_argument("--dry-run", action="store_true")
+ args = ap.parse_args()
+
+ ai_total_cost = 0.0
+ if args.latest:
+ tags = [latest_tag()]
+ elif args.tag:
+ tags = [args.tag]
+ elif args.year is not None:
+ tags = list_tags_for_year(args.year)
+ else:
+ tags = list_tags_between(args.range[0], args.range[1])
+
+ # Load existing year files into memory, merge updates, then write once per year.
+ per_year: Dict[int, ChangelogIndex] = {}
+ per_year_prefix: Dict[int, str] = {}
+ version_date_entries: Dict[str, str] = {}
+ per_year_ignore: Dict[int, set[int]] = {}
+
+ ai_total_cost = run_with_overall_progress(
+ tags,
+ args,
+ per_year,
+ per_year_prefix,
+ per_year_ignore,
+ version_date_entries,
+ )
+
+ # Write outputs
+ for year, idx in per_year.items():
+ prune_ignored(idx, per_year_ignore.get(year, set()))
+ out_path = os.path.join(args.output_dir, f"changelog-{year}.mdx")
+ body = render_changelog_body(
+ idx,
+ version_date=version_date_entries,
+ use_ai=args.ai,
+ ai_model=args.ai_model,
+ )
+ text = per_year_prefix[year].rstrip() + "\n\n" + body
+
+ if args.dry_run:
+ print(f"Would write {out_path}")
+ continue
+
+ with open(out_path, "w", encoding="utf-8") as f:
+ f.write(text)
+
+ print(f"Wrote {out_path}")
+
+ if args.ai:
+ print(f"AI total cost: ${ai_total_cost:.4f}")
+
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.scripts/changelog/release_notes.py b/.scripts/changelog/release_notes.py
new file mode 100644
index 0000000..86144eb
--- /dev/null
+++ b/.scripts/changelog/release_notes.py
@@ -0,0 +1,333 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import sys
+import urllib.error
+from dataclasses import dataclass, field
+from typing import Dict, List, Literal, Tuple
+
+from pydantic import BaseModel, Field, field_validator
+
+try:
+ from generate import (
+ gh_request,
+ get_ai_model,
+ sanitize_for_multimodal_sentinel,
+ clean_title,
+ classify,
+ OWNER,
+ REPO,
+ CATEGORY_ORDER,
+ )
+except ImportError:
+ sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+ from generate import (
+ gh_request,
+ get_ai_model,
+ sanitize_for_multimodal_sentinel,
+ clean_title,
+ classify,
+ OWNER,
+ REPO,
+ CATEGORY_ORDER,
+ )
+
+HEADER = "New to `deepeval`? Get started [here](https://deepeval.com/docs/introduction)."
+FOOTER = "A huge thank you to everyone who contributed to this release ❤️"
+
+CATEGORY_EMOJI = {
+ "Backward Incompatible Change": "💥",
+ "New Feature": "✨",
+ "Experimental Feature": "🧪",
+ "Improvement": "⚡",
+ "Bug Fix": "🐛",
+ "Security": "🔒",
+}
+
+TS_PREFIXES = ("typescript/",)
+PY_PREFIXES = ("deepeval/",)
+PY_FILES = ("pyproject.toml", "poetry.lock")
+
+BATCH_MAX = 60
+
+
+@dataclass
+class PrLine:
+ number: int
+ title: str
+ login: str
+ url: str
+
+
+@dataclass
+class ParsedNotes:
+ prs: List[PrLine] = field(default_factory=list)
+ new_contributors: Dict[int, str] = field(default_factory=dict)
+
+
+class CleanedItem(BaseModel):
+ pr_number: int
+ entry: str = Field(..., min_length=1, max_length=500)
+ category: Literal[
+ "Backward Incompatible Change",
+ "New Feature",
+ "Experimental Feature",
+ "Improvement",
+ "Bug Fix",
+ "Security",
+ ]
+
+ @field_validator("category", mode="before")
+ @classmethod
+ def _coerce_category(cls, value):
+ return value if value in CATEGORY_ORDER else "Improvement"
+
+
+class CleanedNotes(BaseModel):
+ items: List[CleanedItem]
+
+
+WHATS_CHANGED_LINE_RE = re.compile(
+ r"^\*\s+(?P.+?)\s+by\s+@(?P[A-Za-z0-9-]+)\s+in\s+"
+ r"(?Phttps?://\S+/pull/(?P\d+))\s*$"
+)
+NEW_CONTRIB_RE = re.compile(
+ r"^\*\s+@(?P[A-Za-z0-9-]+)\s+made their first contribution in\s+"
+ r"\S+/pull/(?P\d+)"
+)
+
+SECTION_WHATS_CHANGED = "## What's Changed"
+SECTION_NEW_CONTRIB = "## New Contributors"
+
+
+def parse_release_body(body: str) -> ParsedNotes:
+ parsed = ParsedNotes()
+ section = None
+ for line in body.splitlines():
+ stripped = line.strip()
+
+ if stripped.startswith("## "):
+ section = stripped
+ continue
+
+ if section == SECTION_WHATS_CHANGED:
+ match = WHATS_CHANGED_LINE_RE.match(stripped)
+ if match:
+ parsed.prs.append(
+ PrLine(
+ number=int(match.group("num")),
+ title=match.group("title").strip(),
+ login=match.group("login"),
+ url=match.group("url"),
+ )
+ )
+ elif section == SECTION_NEW_CONTRIB:
+ match = NEW_CONTRIB_RE.match(stripped)
+ if match:
+ parsed.new_contributors[int(match.group("num"))] = match.group(
+ "login"
+ )
+
+ return parsed
+
+
+def _is_ts(path: str) -> bool:
+ return any(path.startswith(p) for p in TS_PREFIXES)
+
+
+def _is_py(path: str) -> bool:
+ return any(path.startswith(p) for p in PY_PREFIXES) or path in PY_FILES
+
+
+def classify_sdk(files: List[str]) -> str:
+ ts = any(_is_ts(f) for f in files)
+ py = any(_is_py(f) for f in files)
+ if ts and not py:
+ return "typescript"
+ if py and not ts:
+ return "python"
+ return "shared"
+
+
+def pr_scope(number: int) -> str:
+ ts = py = False
+ page = 1
+ try:
+ while True:
+ batch = gh_request(
+ f"/repos/{OWNER}/{REPO}/pulls/{number}/files?per_page=100&page={page}"
+ )
+ if not isinstance(batch, list) or not batch:
+ break
+ for item in batch:
+ name = item.get("filename") or ""
+ ts = ts or _is_ts(name)
+ py = py or _is_py(name)
+ if (ts and py) or len(batch) < 100:
+ break
+ page += 1
+ except urllib.error.HTTPError as exc:
+ log(
+ f"warning: could not fetch files for #{number} ({exc}); treating as shared"
+ )
+ return "shared"
+
+ if ts and not py:
+ return "typescript"
+ if py and not ts:
+ return "python"
+ return "shared"
+
+
+def select_in_scope(prs: List[PrLine], sdk: str) -> List[PrLine]:
+ kept = []
+ for pr in prs:
+ scope = pr_scope(pr.number)
+ if scope in (sdk, "shared"):
+ kept.append(pr)
+ log(
+ f"#{pr.number}: {scope} -> {'keep' if scope in (sdk, 'shared') else 'drop'}"
+ )
+ return kept
+
+
+def build_cleanup_prompt(prs: List[PrLine]) -> str:
+ listing = "\n".join(f"{pr.number}: {pr.title}" for pr in prs)
+ return f"""
+You are writing release notes for an open-source developer tool.
+
+You are given a list of merged pull requests as ": ".
+For EACH pr_number, produce one item with:
+- pr_number: the same number, unchanged.
+- entry: one clean, user-facing release-note line. Plain text, an action verb
+ ("Add", "Fix", "Improve", "Support"...), 1 sentence, no markdown headers/lists,
+ no PR numbers, no URLs, no author names. Inline `backticks` for code are fine.
+- category: the single best match from the allowed categories.
+
+Return exactly one item per input pr_number. Do not invent pr_numbers.
+
+Allowed categories:
+- Backward Incompatible Change
+- New Feature
+- Experimental Feature
+- Improvement
+- Bug Fix
+- Security
+
+Pull requests:
+{listing}
+""".strip()
+
+
+def ai_cleanup(model, prs: List[PrLine]) -> Dict[int, Tuple[str, str]]:
+ cleaned: Dict[int, Tuple[str, str]] = {}
+ total_cost = 0.0
+ for start in range(0, len(prs), BATCH_MAX):
+ chunk = prs[start : start + BATCH_MAX]
+ prompt = sanitize_for_multimodal_sentinel(build_cleanup_prompt(chunk))
+ parsed, cost = model.generate(prompt, schema=CleanedNotes)
+ assert isinstance(parsed, CleanedNotes)
+ total_cost += cost or 0.0
+ for item in parsed.items:
+ cleaned[item.pr_number] = (item.entry.strip(), item.category)
+ log(f"AI cleanup cost: ${total_cost:.4f}")
+ return cleaned
+
+
+def fallback_entry(pr: PrLine) -> Tuple[str, str]:
+ return clean_title(pr.title), classify(pr.title, "")
+
+
+def render_markdown(
+ prs: List[PrLine],
+ cleaned: Dict[int, Tuple[str, str]],
+ new_contributors: Dict[int, str],
+) -> str:
+ buckets: Dict[str, List[str]] = {c: [] for c in CATEGORY_ORDER}
+ for pr in prs:
+ entry, category = cleaned.get(pr.number) or fallback_entry(pr)
+ if category not in buckets:
+ category = "Improvement"
+ buckets[category].append(f"* {entry} (#{pr.number} by @{pr.login})")
+
+ parts: List[str] = [HEADER, ""]
+
+ if any(buckets[c] for c in CATEGORY_ORDER):
+ for category in CATEGORY_ORDER:
+ lines = buckets[category]
+ if not lines:
+ continue
+ emoji = CATEGORY_EMOJI.get(category, "")
+ parts.append(f"### {emoji} {category}".strip())
+ parts.extend(lines)
+ parts.append("")
+ else:
+ parts.append("_No user-facing changes in this release._")
+ parts.append("")
+
+ contributors = [f"@{login}" for login in new_contributors.values()]
+ if contributors:
+ parts.append("**New contributors:** " + ", ".join(contributors))
+ parts.append("")
+
+ parts.append(FOOTER)
+ return "\n".join(parts).rstrip() + "\n"
+
+
+_SILENT = False
+
+
+def log(message: str) -> None:
+ if not _SILENT:
+ print(message, file=sys.stderr)
+
+
+def parse_args() -> argparse.Namespace:
+ ap = argparse.ArgumentParser(
+ description="Clean up and SDK-scope GitHub release notes from stdin."
+ )
+ ap.add_argument("--sdk", required=True, choices=["python", "typescript"])
+ ap.add_argument("--ai-model", default="gpt-5.2")
+ ap.add_argument("--dry-run", action="store_true")
+ ap.add_argument("--no-ai", action="store_true")
+ ap.add_argument("--silent", action="store_true")
+ return ap.parse_args()
+
+
+def main() -> int:
+ global _SILENT
+ args = parse_args()
+ _SILENT = args.silent
+
+ parsed = parse_release_body(sys.stdin.read())
+ log(f"Parsed {len(parsed.prs)} PRs from release body.")
+
+ in_scope = select_in_scope(parsed.prs, args.sdk)
+ log(f"{len(in_scope)} PRs in scope for '{args.sdk}'.")
+
+ in_scope_numbers = {pr.number for pr in in_scope}
+ new_contributors = {
+ num: login
+ for num, login in parsed.new_contributors.items()
+ if num in in_scope_numbers
+ }
+
+ cleaned: Dict[int, Tuple[str, str]] = {}
+ if in_scope and not (args.dry_run or args.no_ai):
+ try:
+ cleaned = ai_cleanup(get_ai_model(args.ai_model), in_scope)
+ except Exception as exc:
+ log(
+ f"warning: AI cleanup failed ({exc}); using deterministic titles."
+ )
+ cleaned = {}
+
+ sys.stdout.write(render_markdown(in_scope, cleaned, new_contributors))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000..b0236c3
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,13 @@
+{
+ "editor.formatOnSave": true,
+ "editor.defaultFormatter": "esbenp.prettier-vscode",
+ "[javascript]": {
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
+ },
+ "[javascriptreact]": {
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
+ },
+ "[css]": {
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
+ }
+}
diff --git a/CITATION.cff b/CITATION.cff
new file mode 100644
index 0000000..bf36767
--- /dev/null
+++ b/CITATION.cff
@@ -0,0 +1,15 @@
+cff-version: 1.2.0
+message: If you use this software, please cite it as below.
+authors:
+ - family-names: Ip
+ given-names: Jeffrey
+ - family-names: Vongthongsri
+ given-names: Kritin
+title: deepeval
+version: 4.1.0
+date-released: "2026-07-11"
+url: https://confident-ai.com
+repository-code: https://github.com/confident-ai/deepeval
+license: Apache-2.0
+type: software
+description: The Open-Source LLM Evaluation Framework
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..4a0311c
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,57 @@
+# Contributing to DeepEval 🥳
+
+Thanks for thinking about contributing to DeepEval! We accept fixes, improvements, or even entire new features. Some reasons why you might want to contribute:
+
+- there's a bug that you want fixed
+- there's a cool new feature you're thinking about that might be useful for DeepEval
+- there's a metric or benchmark that you want implemented
+- there's room for improvement in the docs
+
+## How to contribute
+
+We follow fork and pull request workflow. To know more about it, check out this [guide](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork).
+
+### Set up your development environment
+
+1. Create a python virtual environment.
+2. We recommend using Poetry to install dependencies. If you haven't already, see the [Poetry docs](https://python-poetry.org/docs/).
+3. Install the dependencies using:
+
+```bash
+poetry install
+```
+
+## Our expectations (not a lot :)
+
+To contribute, all we ask for is to follow existing patterns within the codebase. For example, if you're looking to add a new benchmark, look at how the different modules in the existing benchmarks are structured and implemented, and we encourage you to reuse helper functions and methods shared by similar modules.
+
+Other than that, there are no strict rules to follow, except for optionally running `black` to ensure good formatting. Also, there's no need to worry about failing test cases in GitHub Actions, as these are mostly for internal use and will only pass if triggered by a user with the correct permissions within Confident AI.
+
+## Contributing new metrics
+
+Metrics contributed by the community go through a staged process:
+
+1. **Start in the community section.** New metrics land under `deepeval/metrics/community/` in the codebase and under the "Community" section of the metrics docs. Community metrics are **not** exported from `deepeval.metrics` — users import them explicitly:
+
+```python
+from deepeval.metrics.community import YourNewMetric
+```
+
+2. **Promotion over time.** Metrics that prove useful — through adoption, feedback, and stability — will be promoted out of the community section and into `deepeval.metrics` proper, alongside the built-in metrics.
+
+When adding a community metric, mirror the structure of existing metrics (a folder containing the metric class, `schema.py`, and `template.py`), export it from `deepeval/metrics/community/__init__.py`, add a docs page under the community section, and include tests.
+
+Thank you and come ask any questions or discuss any new PRs you have in mind on our [Discord](https://discord.com/invite/a3K9c8GRGt)!
+
+
+## Issue lifecycle & staleness policy
+
+- **Stale closure:** We close issues with no activity for **≥ 12 months**.
+- **Reopening:** If your issue is still relevant:
+
+ 1. Leave a comment mentioning one or more maintainers from [MAINTAINERS.md](./MAINTAINERS.md) and include any new details (version, repro steps, logs).
+ 2. If you don’t get a response in a few days, open a **new issue** and reference the old one.
+
+**Exclusions:** Labeled issues.
+
+**Why:** Keeps the tracker actionable and reflects the current roadmap. If your issue still matters, please comment and we’ll re-open.
diff --git a/LICENSE.md b/LICENSE.md
new file mode 100644
index 0000000..a27de83
--- /dev/null
+++ b/LICENSE.md
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [2024] [Confident AI Inc.]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
new file mode 100644
index 0000000..0dc6358
--- /dev/null
+++ b/MAINTAINERS.md
@@ -0,0 +1,8 @@
+# Maintainers
+
+For issues in this repo you can mention one or more of:
+
+- @trevor-cai
+- @A-Vamshi
+- @jeffreyip
+- @kritinv
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..fa01868
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1 @@
+recursive-include deepeval/benchmarks *.txt
\ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..3eb303a
--- /dev/null
+++ b/README.md
@@ -0,0 +1,611 @@
+
+
+
+
+
+
+
+
+
The LLM Evaluation Framework
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Documentation |
+ Metrics and Features |
+ Getting Started |
+ Integrations |
+ Confident AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Deutsch |
+ Español |
+ français |
+ 日本語 |
+ 한국어 |
+ Português |
+ Русский |
+ 中文
+
+
+**DeepEval** is a simple-to-use, open-source LLM evaluation framework, for evaluating large-language model systems. It is similar to Pytest but specialized for unit testing LLM apps. DeepEval incorporates the latest research to run evals via metrics such as G-Eval, task completion, answer relevancy, hallucination, etc., which uses LLM-as-a-judge and other NLP models that run **locally on your machine**.
+
+Whether you're building AI agents, RAG pipelines, or chatbots, implemented via LangChain or OpenAI, DeepEval has you covered. With it, you can easily determine the optimal models, prompts, and architecture to improve your AI quality, prevent prompt drifting, or even transition from OpenAI to Claude with confidence.
+
+> [!IMPORTANT]
+> Need a place for your DeepEval testing data to live 🏡❤️? [Sign up to Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=signup_callout) to compare iterations of your LLM app, generate & share testing reports, and more.
+>
+> 
+
+> Want to talk LLM evaluation, need help picking metrics, or just to say hi? [Come join our discord.](https://discord.com/invite/3SEyvpgu2f)
+
+
+
+# 🔥 Metrics and Features
+
+- 📐 Large variety of ready-to-use LLM eval metrics (all with explanations) powered by **ANY** LLM of your choice, statistical methods, or NLP models that run **locally on your machine** covering all use cases:
+
+ - **Custom, All-Purpose Metrics:**
+
+ - [G-Eval](https://deepeval.com/docs/metrics-llm-evals) — a research-backed LLM-as-a-judge metric for evaluating on any custom criteria with human-like accuracy
+ - [DAG](https://deepeval.com/docs/metrics-dag) — DeepEval's graph-based deterministic LLM-as-a-judge metric builder
+
+ -
+ Agentic Metrics
+
+ - [Task Completion](https://deepeval.com/docs/metrics-task-completion) — evaluate whether an agent accomplished its goal
+ - [Tool Correctness](https://deepeval.com/docs/metrics-tool-correctness) — check if the right tools were called with the right arguments
+ - [Goal Accuracy](https://deepeval.com/docs/metrics-goal-accuracy) — measure how accurately the agent achieved the intended goal
+ - [Step Efficiency](https://deepeval.com/docs/metrics-step-efficiency) — evaluate whether the agent took unnecessary steps
+ - [Plan Adherence](https://deepeval.com/docs/metrics-plan-adherence) — check if the agent followed the expected plan
+ - [Plan Quality](https://deepeval.com/docs/metrics-plan-quality) — evaluate the quality of the agent's plan
+ - [Tool Use](https://deepeval.com/docs/metrics-tool-use) — measure quality of tool usage
+ - [Argument Correctness](https://deepeval.com/docs/metrics-argument-correctness) — validate tool call arguments
+
+
+
+ -
+ RAG Metrics
+
+ - [Answer Relevancy](https://deepeval.com/docs/metrics-answer-relevancy) — measure how relevant the RAG pipeline's output is to the input
+ - [Faithfulness](https://deepeval.com/docs/metrics-faithfulness) — evaluate whether the RAG pipeline's output factually aligns with the retrieval context
+ - [Contextual Recall](https://deepeval.com/docs/metrics-contextual-recall) — measure how well the RAG pipeline's retrieval context aligns with the expected output
+ - [Contextual Precision](https://deepeval.com/docs/metrics-contextual-precision) — evaluate whether relevant nodes in the RAG pipeline's retrieval context are ranked higher
+ - [Contextual Relevancy](https://deepeval.com/docs/metrics-contextual-relevancy) — measure the overall relevance of the RAG pipeline's retrieval context to the input
+ - [RAGAS](https://deepeval.com/docs/metrics-ragas) — average of answer relevancy, faithfulness, contextual precision, and contextual recall
+
+
+
+ -
+ Multi-Turn Metrics
+
+ - [Knowledge Retention](https://deepeval.com/docs/metrics-knowledge-retention) — evaluate whether the chatbot retains factual information throughout a conversation
+ - [Conversation Completeness](https://deepeval.com/docs/metrics-conversation-completeness) — measure whether the chatbot satisfies user needs throughout a conversation
+ - [Turn Relevancy](https://deepeval.com/docs/metrics-turn-relevancy) — evaluate whether the chatbot generates consistently relevant responses throughout a conversation
+ - [Turn Faithfulness](https://deepeval.com/docs/metrics-turn-faithfulness) — check if the chatbot's responses are factually grounded in retrieval context across turns
+ - [Role Adherence](https://deepeval.com/docs/metrics-role-adherence) — evaluate whether the chatbot adheres to its assigned role throughout a conversation
+
+
+
+ -
+ MCP Metrics
+
+ - [MCP Task Completion](https://deepeval.com/docs/metrics-mcp-task-completion) — evaluate how effectively an MCP-based agent accomplishes a task
+ - [MCP Use](https://deepeval.com/docs/metrics-mcp-use) — measure how effectively an agent uses its available MCP servers
+ - [Multi-Turn MCP Use](https://deepeval.com/docs/metrics-multi-turn-mcp-use) — evaluate MCP server usage across conversation turns
+
+
+
+ -
+ Multimodal Metrics
+
+ - [Text to Image](https://deepeval.com/docs/multimodal-metrics-text-to-image) — evaluate image generation quality based on semantic consistency and perceptual quality
+ - [Image Editing](https://deepeval.com/docs/multimodal-metrics-image-editing) — evaluate image editing quality based on semantic consistency and perceptual quality
+ - [Image Coherence](https://deepeval.com/docs/multimodal-metrics-image-coherence) — measure how well images align with their accompanying text
+ - [Image Helpfulness](https://deepeval.com/docs/multimodal-metrics-image-helpfulness) — evaluate how effectively images contribute to user comprehension of the text
+ - [Image Reference](https://deepeval.com/docs/multimodal-metrics-image-reference) — evaluate how accurately images are referred to or explained by accompanying text
+
+
+
+ -
+ Other Metrics
+
+ - [Hallucination](https://deepeval.com/docs/metrics-hallucination) — check whether the LLM generates factually correct information against provided context
+ - [Summarization](https://deepeval.com/docs/metrics-summarization) — evaluate whether summaries are factually correct and include necessary details
+ - [Bias](https://deepeval.com/docs/metrics-bias) — detect gender, racial, or political bias in LLM outputs
+ - [Toxicity](https://deepeval.com/docs/metrics-toxicity) — evaluate toxicity in LLM outputs
+ - [JSON Correctness](https://deepeval.com/docs/metrics-json-correctness) — check whether the output matches an expected JSON schema
+ - [Prompt Alignment](https://deepeval.com/docs/metrics-prompt-alignment) — measure whether the output aligns with instructions in the prompt template
+
+
+
+- 🎯 Supports both end-to-end and component-level LLM evaluation.
+- 🧩 Build your own custom metrics that are automatically integrated with DeepEval's ecosystem.
+- 🔮 Generate both single and multi-turn synthetic datasets for evaluation.
+- 🔗 Integrates seamlessly with **ANY** CI/CD environment.
+- 🧬 Optimize prompts automatically based on evaluation results.
+- 🏆 Easily benchmark **ANY** LLM on popular LLM benchmarks in [under 10 lines of code.](https://deepeval.com/docs/benchmarks-introduction?utm_source=GitHub), including MMLU, HellaSwag, DROP, BIG-Bench Hard, TruthfulQA, HumanEval, GSM8K.
+
+
+
+# 🔌 Integrations
+
+DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and more. To scale evals across your team — or let anyone run them without writing code — **Confident AI** gives you a native platform integration.
+
+## Frameworks
+
+- [OpenAI](https://www.deepeval.com/integrations/frameworks/openai?utm_source=GitHub) — evaluate and trace OpenAI applications via a client wrapper
+- [OpenAI Agents](https://www.deepeval.com/integrations/frameworks/openai-agents?utm_source=GitHub) — evaluate OpenAI Agents end-to-end in under a minute
+- [LangChain](https://www.deepeval.com/integrations/frameworks/langchain?utm_source=GitHub) — evaluate LangChain applications with a callback handler
+- [LangGraph](https://www.deepeval.com/integrations/frameworks/langgraph?utm_source=GitHub) — evaluate LangGraph agents with a callback handler
+- [Pydantic AI](https://www.deepeval.com/integrations/frameworks/pydanticai?utm_source=GitHub) — evaluate Pydantic AI agents with type-safe validation
+- [CrewAI](https://www.deepeval.com/integrations/frameworks/crewai?utm_source=GitHub) — evaluate CrewAI multi-agent systems
+- [Anthropic](https://www.deepeval.com/integrations/frameworks/anthropic?utm_source=GitHub) — evaluate and trace Claude applications via a client wrapper
+- [AWS AgentCore](https://www.deepeval.com/integrations/frameworks/agentcore?utm_source=GitHub) — evaluate agents deployed on Amazon AgentCore
+- [LlamaIndex](https://www.deepeval.com/integrations/frameworks/llamaindex?utm_source=GitHub) — evaluate RAG applications built with LlamaIndex
+
+## ☁️ Platform + Ecosystem
+
+[Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=platform_section) is an all-in-one platform that integrates natively with DeepEval.
+
+- Manage datasets, trace LLM applications, run evaluations, and monitor responses in production — all from one platform.
+- Don't need a UI? Confident AI can also be your data persistent layer - run evals, pull datasets, and inspect traces straight from claude code, cursor, via Confident AI's [MCP server](https://github.com/confident-ai/confident-mcp-server).
+
+
+
+
+
+
+
+# 🤖 Vibe-Coder QuickStart
+
+Want your coding agent to add evals and fix failures for you? Install the DeepEval skill, point it at your agent, RAG pipeline, or chatbot, and ask it to generate a dataset, write the eval suite, run `deepeval test run`, and iterate on the failing metrics.
+
+[Start with the 5-minute vibe-coder guide](https://deepeval.com/docs/vibe-coder-quickstart?utm_source=GitHub).
+
+
+
+# 🚀 Human QuickStart
+
+Let's pretend your LLM application is a RAG based customer support chatbot; here's how DeepEval can help test what you've built.
+
+## Installation
+
+Deepeval works with **Python>=3.9+**.
+
+```
+pip install -U deepeval
+```
+
+## Create an account (highly recommended)
+
+Using the `deepeval` platform will allow you to generate sharable testing reports on the cloud. It is free, takes no additional code to setup, and we highly recommend giving it a try.
+
+To login, run:
+
+```
+deepeval login
+```
+
+Follow the instructions in the CLI to create an account, copy your API key, and paste it into the CLI. All test cases will automatically be logged (find more information on data privacy [here](https://deepeval.com/docs/data-privacy?utm_source=GitHub)).
+
+## Write your first test case
+
+Create a test file:
+
+```bash
+touch test_chatbot.py
+```
+
+Open `test_chatbot.py` and write your first test case to run an **end-to-end** evaluation using DeepEval, which treats your LLM app as a black-box:
+
+```python
+import pytest
+from deepeval import assert_test
+from deepeval.metrics import GEval
+from deepeval.test_case import LLMTestCase, SingleTurnParams
+
+def test_case():
+ correctness_metric = GEval(
+ name="Correctness",
+ criteria="Determine if the 'actual output' is correct based on the 'expected output'.",
+ evaluation_params=[SingleTurnParams.ACTUAL_OUTPUT, SingleTurnParams.EXPECTED_OUTPUT],
+ threshold=0.5
+ )
+ test_case = LLMTestCase(
+ input="What if these shoes don't fit?",
+ # Replace this with the actual output from your LLM application
+ actual_output="You have 30 days to get a full refund at no extra cost.",
+ expected_output="We offer a 30-day full refund at no extra costs.",
+ retrieval_context=["All customers are eligible for a 30 day full refund at no extra costs."]
+ )
+ assert_test(test_case, [correctness_metric])
+```
+
+Set your `OPENAI_API_KEY` as an environment variable (you can also evaluate using your own custom model, for more details visit [this part of our docs](https://deepeval.com/docs/metrics-introduction#using-a-custom-llm?utm_source=GitHub)):
+
+```
+export OPENAI_API_KEY="..."
+```
+
+And finally, run `test_chatbot.py` in the CLI:
+
+```
+deepeval test run test_chatbot.py
+```
+
+**Congratulations! Your test case should have passed ✅** Let's break down what happened.
+
+- The variable `input` mimics a user input, and `actual_output` is a placeholder for what your application's supposed to output based on this input.
+- The variable `expected_output` represents the ideal answer for a given `input`, and [`GEval`](https://deepeval.com/docs/metrics-llm-evals) is a research-backed metric provided by `deepeval` for you to evaluate your LLM outputs on any custom with human-like accuracy.
+- In this example, the metric `criteria` is correctness of the `actual_output` based on the provided `expected_output`.
+- All metric scores range from 0 - 1, which the `threshold=0.5` threshold ultimately determines if your test has passed or not.
+
+[Read our documentation](https://deepeval.com/docs/getting-started?utm_source=GitHub) for more information!
+
+
+
+## Evals With Full Traceability
+
+Use `evals_iterator()` to run the same dataset through your app, whether you instrument it manually or through one of DeepEval's framework integrations.
+
+Here's an example of manual instrumentation:
+
+```python
+from deepeval.tracing import observe, update_current_span
+from deepeval.test_case import LLMTestCase
+from deepeval.metrics import TaskCompletionMetric
+
+@observe()
+def inner_component(input: str):
+ output = "result"
+ update_current_span(test_case=LLMTestCase(input=input, actual_output=output))
+ return output
+
+@observe()
+def app(input: str):
+ return inner_component(input)
+
+# This metric will be run on your trace end to end.
+for golden in dataset.evals_iterator(metrics=[TaskCompletionMetric()]):
+ app(golden.input)
+```
+
+
+OpenAI
+
+```python
+from deepeval.openai import OpenAI
+from deepeval.tracing import trace
+from deepeval.metrics import TaskCompletionMetric
+
+client = OpenAI()
+
+# This metric will be run on your trace end to end.
+for golden in dataset.evals_iterator():
+ with trace(metrics=[TaskCompletionMetric()]):
+ client.chat.completions.create(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": golden.input}],
+ )
+```
+
+
+
+
+OpenAI Agents
+
+```python
+from agents import Runner
+from deepeval.metrics import TaskCompletionMetric
+
+# This metric will be run on your trace end to end.
+for golden in dataset.evals_iterator(metrics=[TaskCompletionMetric()]):
+ Runner.run_sync(agent, golden.input)
+```
+
+
+
+
+Anthropic
+
+```python
+from deepeval.anthropic import Anthropic
+from deepeval.tracing import trace
+from deepeval.metrics import TaskCompletionMetric
+
+client = Anthropic()
+
+# This metric will be run on your trace end to end.
+for golden in dataset.evals_iterator():
+ with trace(metrics=[TaskCompletionMetric()]):
+ client.messages.create(
+ model="claude-sonnet-4-5",
+ max_tokens=1024,
+ messages=[{"role": "user", "content": golden.input}],
+ )
+```
+
+
+
+
+LangChain
+
+```python
+from deepeval.integrations.langchain import CallbackHandler
+from deepeval.metrics import TaskCompletionMetric
+
+# This metric will be run on your trace end to end.
+for golden in dataset.evals_iterator():
+ llm.invoke(
+ golden.input,
+ config={"callbacks": [CallbackHandler(metrics=[TaskCompletionMetric()])]},
+ )
+```
+
+
+
+
+LangGraph
+
+```python
+from deepeval.integrations.langchain import CallbackHandler
+from deepeval.metrics import TaskCompletionMetric
+
+# This metric will be run on your trace end to end.
+for golden in dataset.evals_iterator():
+ agent.invoke(
+ {"messages": [{"role": "user", "content": golden.input}]},
+ config={"callbacks": [CallbackHandler(metrics=[TaskCompletionMetric()])]},
+ )
+```
+
+
+
+
+Pydantic AI
+
+```python
+from deepeval.metrics import TaskCompletionMetric
+
+# This metric will be run on your trace end to end.
+for golden in dataset.evals_iterator(metrics=[TaskCompletionMetric()]):
+ agent.run_sync(golden.input)
+```
+
+
+
+
+CrewAI
+
+```python
+from deepeval.integrations.crewai import instrument_crewai
+from deepeval.metrics import TaskCompletionMetric
+
+instrument_crewai()
+
+# This metric will be run on your trace end to end.
+for golden in dataset.evals_iterator(metrics=[TaskCompletionMetric()]):
+ crew.kickoff({"input": golden.input})
+```
+
+
+
+
+AWS AgentCore
+
+```python
+from deepeval.integrations.agentcore import instrument_agentcore
+from deepeval.metrics import TaskCompletionMetric
+
+instrument_agentcore()
+
+# This metric will be run on your trace end to end.
+for golden in dataset.evals_iterator(metrics=[TaskCompletionMetric()]):
+ invoke({"prompt": golden.input})
+```
+
+
+
+
+LlamaIndex
+
+```python
+import asyncio
+from deepeval.evaluate.configs import AsyncConfig
+from deepeval.metrics import TaskCompletionMetric
+
+# This metric will be run on your trace end to end.
+for golden in dataset.evals_iterator(
+ async_config=AsyncConfig(run_async=True),
+ metrics=[TaskCompletionMetric()],
+):
+ task = asyncio.create_task(agent.run(golden.input))
+ dataset.evaluate(task)
+```
+
+
+
+
+Google ADK
+
+```python
+import asyncio
+from deepeval.evaluate.configs import AsyncConfig
+from deepeval.integrations.google_adk import instrument_google_adk
+from deepeval.metrics import TaskCompletionMetric
+
+instrument_google_adk()
+
+# This metric will be run on your trace end to end.
+for golden in dataset.evals_iterator(
+ async_config=AsyncConfig(run_async=True),
+ metrics=[TaskCompletionMetric()],
+):
+ task = asyncio.create_task(run_agent(golden.input))
+ dataset.evaluate(task)
+```
+
+
+
+
+Strands
+
+```python
+from deepeval.integrations.strands import instrument_strands
+from deepeval.metrics import TaskCompletionMetric
+
+instrument_strands()
+
+# This metric will be run on your trace end to end.
+for golden in dataset.evals_iterator(metrics=[TaskCompletionMetric()]):
+ agent(golden.input)
+```
+
+
+
+Learn more about component-level evaluations [here.](https://www.deepeval.com/docs/evaluation-component-level-llm-evals)
+
+
+
+## Evaluate Without Pytest Integration
+
+Alternatively, you can evaluate without Pytest, which is more suited for a notebook environment.
+
+```python
+from deepeval import evaluate
+from deepeval.metrics import AnswerRelevancyMetric
+from deepeval.test_case import LLMTestCase
+
+answer_relevancy_metric = AnswerRelevancyMetric(threshold=0.7)
+test_case = LLMTestCase(
+ input="What if these shoes don't fit?",
+ # Replace this with the actual output from your LLM application
+ actual_output="We offer a 30-day full refund at no extra costs.",
+ retrieval_context=["All customers are eligible for a 30 day full refund at no extra costs."]
+)
+evaluate([test_case], [answer_relevancy_metric])
+```
+
+## Using Standalone Metrics
+
+DeepEval is extremely modular, making it easy for anyone to use any of our metrics. Continuing from the previous example:
+
+```python
+from deepeval.metrics import AnswerRelevancyMetric
+from deepeval.test_case import LLMTestCase
+
+answer_relevancy_metric = AnswerRelevancyMetric(threshold=0.7)
+test_case = LLMTestCase(
+ input="What if these shoes don't fit?",
+ # Replace this with the actual output from your LLM application
+ actual_output="We offer a 30-day full refund at no extra costs.",
+ retrieval_context=["All customers are eligible for a 30 day full refund at no extra costs."]
+)
+
+answer_relevancy_metric.measure(test_case)
+print(answer_relevancy_metric.score)
+# All metrics also offer an explanation
+print(answer_relevancy_metric.reason)
+```
+
+Note that some metrics are for RAG pipelines, while others are for fine-tuning. Make sure to use our docs to pick the right one for your use case.
+
+## A Note on Env Variables (.env / .env.local)
+
+DeepEval auto-loads `.env.local` then `.env` from the current working directory **at import time**.
+**Precedence:** process env -> `.env.local` -> `.env`.
+Opt out with `DEEPEVAL_DISABLE_DOTENV=1`.
+
+```bash
+cp .env.example .env.local
+# then edit .env.local (ignored by git)
+```
+
+# DeepEval With Confident AI
+
+[Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=cli_login_section) is an all-in-one platform to manage datasets, trace LLM applications, and run evaluations in production. Log in from the CLI to get started:
+
+```bash
+deepeval login
+```
+
+Then run your tests as usual — results are automatically synced to the platform:
+
+```bash
+deepeval test run test_chatbot.py
+```
+
+
+
+Prefer to stay in your IDE? Use DeepEval via [Confident AI's MCP server](https://github.com/confident-ai/confident-mcp-server) as the persistent layer to run evals, pull datasets, and inspect traces without leaving your editor.
+
+
+
+
+
+Everything on Confident AI is available [here](https://www.confident-ai.com/docs?utm_source=deepeval&utm_medium=github&utm_content=cloud_docs).
+
+
+
+# Contributing
+
+Please read [CONTRIBUTING.md](https://github.com/confident-ai/deepeval/blob/main/CONTRIBUTING.md) for details on our code of conduct, and the process for submitting pull requests to us.
+
+
+
+# Roadmap
+
+Features:
+
+- [x] Integration with Confident AI
+- [x] Implement G-Eval
+- [x] Implement RAG metrics
+- [x] Implement Conversational metrics
+- [x] Evaluation Dataset Creation
+- [x] Red-Teaming
+- [ ] DAG custom metrics
+- [ ] Guardrails
+
+
+
+# Authors
+
+Built by the founders of Confident AI. Contact jeffreyip@confident-ai.com for all enquiries.
+
+
+
+# License
+
+DeepEval is licensed under Apache 2.0 - see the [LICENSE.md](https://github.com/confident-ai/deepeval/blob/main/LICENSE.md) file for details.
diff --git a/README.wehub.md b/README.wehub.md
new file mode 100644
index 0000000..6fc1234
--- /dev/null
+++ b/README.wehub.md
@@ -0,0 +1,7 @@
+# WeHub 来源说明
+
+- 原始项目:`confident-ai/deepeval`
+- 原始仓库:https://github.com/confident-ai/deepeval
+- 导入方式:上游默认分支的最新快照
+- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
+- 本文件仅用于记录来源,不代表 WeHub 是原项目作者
diff --git a/assets/confident-mcp-architecture.png b/assets/confident-mcp-architecture.png
new file mode 100644
index 0000000000000000000000000000000000000000..5052960141aa1d653045c1df7d9dc65ed128e1e5
GIT binary patch
literal 544437
zcmbSUcOaGR8;&$3N+LT#MTjVweL~|HrQ_I?ke$7baZ*A?GO{UU92}96E!kT(q3pf)
z{5`J``c)s_U!Qp2=Y5|0xu5&K?(4qZ_j8mG6Tv%7dU(&CJ$N^7T)(?#&yk!xdk*y;
z+z)<3b8pai&mMw3H?Lp2XNEP{awy$ca_-7#5678f=T3MKiq^_Gz#-kHIS
zx+bldp0t@pk>CpJVZUxTKh8k{Y606lSlDYn2=rmM({A}U^oL?%g3}v+U%yZy)AryY
z0-v+2qO+&BTrE(A{ba{?gU4o{!E$8vAYY_kAH(w;_4)OQY|JPGiFNGw))|67bm2O@
z-1xh)lkIh%b0;{VWrSS98r!y-U#?wW=2<0yr(2gDLHzQdWbA!^0WNHst*e0^@yE4q
zn-Yp1g-v+hk0#;vrX^P=c;YPenH?@D`}@8#IY(Mck8PbAi5yla;kHL@U;QsDVukD}
z_`r~0W4p4Y50SjyFx4*?Z{4>!Y??Fc|2_g2pedE;A{SY`Tv#dFVe5pzTm-gUykYHM
z%=CrekC;xv$61v)3enj|Ig0ZXAj+YdW!DR
zm-V~g*u@+7t=skw^Z#e_nV}5S$J8dHU9Q{iK?!47?tV{j4mzX3QMuE_+3*l++^h^OgTp!-uY;&rl01eDE
zi1_2Wb6LQ%(+_Q%(w={_DGEdM@?dphcz!O}j7m#vHQ4_<(3TBYwb-xGD8wHeTBC`^
zi^+<|SrMDN_1lY~;eX>Ny#}bGU=Z`CVQWWj(>06)1c-OIQK(kn>D3E85;$9Ka89sY
z-i1-l{sU}!Ec_0^`Vc60!b%uzuwE-K3(QuDI$qSB63KaZZzDR^B>+>#RHWacWjd
z;2F*ZADPwGN64N%Nw6yyKjGPrKO6wV8GxKro->ouOf1UBxB4O!j9K`0a@i5TW%vSbaw<@YptwM2Q{heg1Q_
zwIBXylaOlcZrzZ|SFl~_Huq?Cdw;tC
z2tKY)U04|SXyK=RFKE8@^xx0!W&-Pj(9(nKnES{Wru+fBY3I3ptMRgL95xc6DZL>a
zq4(V1wPE+6db>JbbWTxiZ#r2;6NMW$wI}Z}k7q$Jsk|k?wc8y5
zJETtwBFDD?P1H1z#6LSuN82v!!iBj=(3IZ8mD|3fAxv0R8$oWvGQYupYkPF*;Eksm
z-#-X$j(q+e(ewrvAaD-5eIpud4EcY)vBd=iJ_0vUJGQVsA%G4Euy$!2iPOO0v4xI7
zw@t>|ru>Jf8Udh#8wT81(E}Wuoo*M2D=@~7
zUM}y=kc@>L%AmZ{%6v61oda&-V|$cgBWi%MP-SB$Hsl
zJlq?&bwXfazdx-%D|0YP2rZPeb0P?_afELPWNr7itypJcO1~j6dOd+T#BQ2oQt|$c
zSXp<(Kg@|-2mql_2c7N7I)eCv=%xd&uI|rKRoMta$e84+pL0D3J3hLKbvJ+WFE$ZY
z0I4#DXPaZ7#18kq*!<_JvHx&Npz0Tu
z1M=JnlYgFVg!W!%Tg5F7{{J}P|NKnA9PDf{cQ_41cA$P7zOn^e{@df%eIeuqa8jWl
zscXkbQf+B>y83OFmw@P_NcFmt>qBlFwSUDuKi89^&jg|Zm2O@>btvfaRQPVV9op`x;?A&%D1y6*}XYv&g6VL3<9=-0CJm+cIurG
zz^x)Ldd}aL|4+BCV$EiVzQTB#Y#X7{3R?|Ogxj|0j}b#anE;q#KkEItpVy;wkFyf*
z&YTdq{%o$X9{7!(+sf=NFM?<9zX1HA^5Ic%7P6&rNrJb}|6g*zdZ>N_LdGd4Kb3s3
z?80p(+3B7Dqh=oj?55gNW#eBQ(w^o!$xe*|STg`h4U#N2ni{|w%8#*Sx41kw#eYZZ
z%GaUPcs90t!!w5tKiWjvKhM)&$`!!&CV+i$KUv2Q?d=h!kbOV*c=~@*FB>M~m2v
zH>QEAB4Q=ozWx7F6;CDsCvD}kQCl;F@a^m?yFI(8Ac#+!2$$Fcd_LXw*zUIk#PMrj
zQx2)?pHZp2ysP>DbVpAXRxuh+h$v0iPLzho3T#H#wXy$r^0H?E<-%@Vo_J&PTwKC!
z_*rv6Pyr0_x56*`({V5aMl4x*Z3t8(cG4~_f*OO!wDdszF#zDG_oeUhc4&>xg5a!m
zjJ|W5d71a0TpS9I_n=PGEumjH+Fk`L_8lAK5mdwObAGVlpkQtmcd`REt-HE^e~dbi
zD^|(^*U%#4*oaWA$64jJ3lcO_trrSXVWG^lutReYECgG~=<@iy4t0^7Wo73y+Kgnb#Ps@jDCenybB#}r%#1+C8qvTZdB=;Czl
ze8~J;U#(oy>3bE@u|3s@)cWIQddwh=zHxz&)rj
z8fvj$-Rv*)_fLP!mxpW-{A8TrZ^Z0R!~B+g2_Xhixf)l!hk(COXwOZgJ2`Fc0_x4-E4v^IP!9n9IAN)|4>aReCqj~LOS`)z2^4|)=N4bu
z1E^Ds`h4)wFSvgPkwfAWKrlJtA4U%*FoY2OVvwB$j$!~{B$v5P{wlKCH2L}h|HLw|
z-T?>YI?l@0#Y@E7Hf;Sjw7}nsWLB#PpEDrmzp`ul>C>u*9ky)E>bm~)`1N_hAn@`m
zXaCx@q5?cW$J)X9Sc5iGU_1Ibv(8~l=a(;)-MjM@*+cfcp?!2y`@|}RoxiI#7&s0f&!Yo6)(nBF
zB4(x8t$!(#0E^nAdL%&Pat^bN`!6;n2?FN$F~?93<$Cwr%}6m$e`MmP>v{mM#o9uQE-<24PH3TdL%kk3BeFg*XR*W?SwYx=bl5NC{
z{~*Sy3XtdqBjs4%nSgdDJL4e@A_mp%il?M3)?>XhKKV5F^
zhd+54N(@Y-FCk)i>oH?RV)1TGme800#aO&~;=REI{Mz^2IMuA!z+r{=z+eA-{+DBq
znRe%MBv5%z+ksw_wX$Bq64_l{4n57;2kg*d;;NtCj3wkJa(852%3!gnmwNOfNagd%BUmH5)&JGh!y^QCt8G+`5_4AYOcDO(R_h$%$N52vtCRGnnq
z88}Tnn?ok<)L?Ay5Os9e=I-nKdHrkAls)-}^);EkjtopnzpdkJUZKS}Nbx5JQR-J(
z(oEZ^N1ZI*K6wGt!wFsos{>+tMDYRcZM?txq~T#r;gJKZXL9Ja`aakJ8N
zWPDv9Y(;kDV<%?zjy!3wt6}>WeMA6N8NW?B&4c=SA)b}sZqlCutDxhoGSC*29DtC*
z=bI-tO0|EJwe{{t#b$D~dsi=%ceYD2Sf7Z>ZEG*EV&s+|NAeuIoNKKw#Y85a$NzS@
zr;C_&+Ds)^My~dC69hlDsyntx;RHgwYz;Po)Q0E%!ni3r?=$f73Z%czGhxkVBeEf{mi{O762b6P-mwdtSs!%2=mE$#f)3U(Arr@CsWZzpTI5kH~Uj2r_${jLrBcse*4on`g38P`_TmH17*O
zzx=7jsMt)>3f>v{ht0+N%bwAGTImgS%il*VPw-m7L}syvf92F?ccT_YZ1a^*ov&`U
zr%#l>I6#GbToV>p|G?Eej|h>=Uctkz*-?k@8C>)xQ_-UD{XFJseBjYOcV;)9@f#xZ
zm^nT%L+8DCZs_4_+EnR~gGAc7sdF
zyszuFC({Lo-Tb|!-Qbw7YJ5p#co3<-PLd2$O6RM%%71Y4;!BvxsFr#~RD4yv9Vtv2
z?mOA=owXVLe4qe1QAS+Cl7)=X394Tm_3h*7M)msc^b+Q@-05aLsuxW0H=Jwb^Kr66
ztZ>*)pC6%8n*aLx!Ay&(TbH*S#`|%7`2FX&Vpsd#H6{9}=%x)Njb+Yc=-rFWgPoDL
z97@y*E?PW--!+9X&0y>Xw)JbAABd%TPHu2R-n`38Pz;tnRVSwfZN+Vt3?p)z#W4zR
zmgKYoDw}lg&g5Dtxx5@)7V;O&sgI%)zTR^0F>f)hcX{tEd;Scum*uh$q^wsYCW!@6
z*}Ap#*H#3Hw7_p`vtCV9IHmIyGxz*+d?#&O*;OLV&IRd2lXPT|bqwgMYS*#8&~kIW
zVpwAT7Sm)74Bg?(ULHZ0sh!#vmiX}X!f=|NgyG^s+lrzbhgzr$r;`^#F{Q4+W@Vx6
z+SD6;Efh?m$*jXgtB4u8bg`W{%R#>C?VJNx;-%%;0{%KD=xh|sWUVYUO?M`lD<*$4
zC~3O2FVX4FqH(vc`t+#_9xC%E&XW*h&wMi!zi=AO)d2gHHr=G<|AISp=_I4jknJyaGrA8mZy`i87mRl&<}Px
z-L>m~6u^-QCe|?#yn)P9if@W@TJ
z;!IF1uGNVf@p=nY*upg0ZU`mTvEmX*^||ZcC_r`?y9cZY+m=lec&xN^@MtyBjuCy4t#
z89V*B_s#R9BbPif#+&pSl}U(ZS}pZDdC@5^
zUSg~iv~=WN;#Eod#&-z*=Pm>2i3)!H@r8*dtyrCs`|mJ)N&D*~#Iot3P{?bXVAE`1qYVtQrPd26*=G_Qr~d>$H@3Z9Iy~;f?HOi=S~4}0x#eT8N17!rsd3Le
zol$LXU)Bk2v48}zDw)Me_3lCy5rVmFQKB71|*Awg}zpr~`d1R%D&n(PW&FLAj
z;-|XIETYqtVBcl^tL8aK5A6>nx}EIoQ6!X-dP9++g)Q8q$GwOMA+Ik~(rqqYOynZV
zNg^zSv90YNt@I!r-Ri
z5TP%N%ioOa736B9h-2f6&({*&T21TOB#_XmgHYLQH9Y=%TMjjR^2(x}%Y-G(@<~Ji
zF&S=}*$DsGN_)C=yNm$*AWkzv2>Bz01p52!*)I
zZO92*_ND9td;|kA$%+>u``TZyxWep<>{^$GO?%|J1_DQr>Z<0E&*<{JqHbwtEltPu
zzI~COI)EWFUpA}_QFOY9U^?|E69k(@0nwV|(=VgE22*;1I+#ygM0}Fb&m-zkp<0mc
zT(~+AO$eRXW&*NLMS3<1%}JU36CZtxLVQ_0CiL4V8l+ak3<()HEh-(l_ki4XM}19&
z3!@%-*T2~e)STm^ax2IiD~jC58$YGZ)1=Wf`jw-HP^qJ#cvAO%nvtna`cnE>^Gwx9>gOC{IC7;HhP|*fV#BYTc9#@6_Cm>-OfOwOyB_cI
zHwo?)bMaQatR))43-k9=D&AQgO2DXT3z
zdgZ&V_fqw1a%
zyRtTeX|zp*7SoV*@d>WVQ2rCjCdkul++Kv2&(=Cn>0UB6y_C#`>H^tl#I$hVNoi^*
z=oCJ$G<@$^0romBpHwUN_-&?R6Q%6pOi2w7;z0B*c%3=<{>tL$7h1&Wxh@tPnP$By
zfoVcGw$=5ycv<};!!go1Hxq1R<8C95fE
z-IOlPL${#}{jePtHF2-Xru1cqKtRuTVk#4cu+dXj6(FXAZ=W5cYM5Lg_%$cib74LN^(!w%>H=Z0Z_tVbpjtM
zT3pqCpYK77g`s3bQfm@Lnc@Z)08+K4<6DsCwI@yWFg(($?W7};*+0HOjTv&@f^9wA
z?A}8poA2MQMgDd6pjRI?!cBmyG+gUmTjF2xh9lltvXsL7IW>_Y3a=olp401wKkeX0g
zjCZQ0*n2Ky%3>e}A%lQ_`g)78R;sPAmYVH;U-n*n=j4nb9k{61ltVUf9|TvmdXRZo
ze7gT(Lm{+$(YqF|u3Pl@yps}XWd%BEFxy7!g&{3Q6IB{fqU#tE&mg$*doSLV#R;#|
zU7FXui0sa4dj)tghN`lCovaq;cWAvtS9uz%1q5MNVE
z-P(x9dfUSDN3Vb57_;dSztGzlgXnOUj=w+6E#CZr`z`mJ@+b&!4I?BXR8~fV3Dsg3
z@8>+VZk|GLf#fcXx#|(cr~T=NeWDkx!M7HZ=9sK6HMNJn*{m!XTe|o+Ykw3~LGfbq
zo_Rb~izu+e=W9^UH&+;mE1`DFA!?RZttjMB<{
zeevlTkk9U0nM#?U*UpdgD?q;7dBS;Qtps3wh;?9_3xG}ER@w?w;F8b%$R}WSH4gO*
ze%&!=#!RVouqX14>>xE_Z=}T@tMP~Bz+a`+1C8lNjxH?OQ7tNy1MsQesQNl@IBkS7
z<-z9<+!PEHivWm3I~xrp*1IArGve?5FeB_OG84b@<7u(E&Gg4$7g#_Pny2rbM|-1s
za18pc)aehv@$>5B^c3z8Ce~)Mor?!I*(jZEpzH-7`KaNYQ$5`6^cAq
zp6hV~E8yj9P__$eilC!}g0NLFQNl@S6&DU!sp;1QjpNir94M(M&{(
zIW+Im;l|I$
z_9iy7)I>ub4VQ9^_eaI%L%HYj
zx5t9%lV5KyiLHRM@`8ILogI@tUVH`Qw~lXy6)swslrYY4u=w~kmF7q!0FdYXRlnX<
z%{xn+Iotli9sg;YQMKq;2i3|awU;MtX1`FqB^<-23m~5nx0#n5;t9~tKb5FO*JD(2
zAvqT$hl`%f3*;V*Q=R4jpDwf4FN4A%A3B4DI0+xiuQD3&alLNV%i@&{Z055gl6o23
z0m_{S@p~!7RZfgz3UpdfrHACWTzWlIX?gs0S>CkV!Os4xF~b8Yfd?lBKUN6SFSLMi
zBXOYs0OGp~&hl1MweB-7#jF(NDvw0Xf=3~`#`+XwE;{EdmKrsiDz&OlWP0}uMufVe
ztq$;k65~SXVu{l7@wt>T!&dYi>)_6*(I};*^W}UCgH0LS1)o5*5SrD2o-eU9w8meY
z4CjxrVgXxio;bDw3UK_{48_{u_&6fgtKdy3Mo=poWw>l%D#3SKvbK{yTf_zV&$6rC
zF4>#kCjTleQQ7LDFU~`d<#)WwS5B;Pa8?-e_9ecTlyLz5t=Oa{O*mXd)&yb(PCq9<
z9u*yb3)w43<%GxCK%F_8$C8i8ny+oJ9LZ1tc+gcG*-jV-wz&Y9dwI4lIg@Cva!`UH
z5Wl?p%0To-G3|sZ;W*&ic@@^vAMJ;OY?kMm1e>*DGfa@*_fk0^S8|9m384B7*~O^j
zC|^s{<5>XJw}}q}VHl0xcNz=}JWzEGa`hvo^wAtiMc+$RH3>;+AX1BJNVa9R3_(Bw
zigtMBsxGkM2K8X8_LdQA)94R5hsS&?kn|?4<~@GbWTO-7WI57!^oZpjU+dgkp6KXS
z8H)^Afy#cU#^xLb*i42SUumJKUbFsIc^@clj4s5{#J+*ak`CI{a!nyCEtjV1-Czxc
z6X_7b1wQW!msB*z#AYqeH>mmRX!i6%NcP=~qU8_~S8zZHgn^;h)oDpz_%Nr`^XPfb
z@iz;5k6sDk)Lr5#&y8~Gc2}uQj%rghD5C2CzG1L9otzbO)hM@w4HcvLvg~pjPgtNy
zlRY{nsR@1GX1v^yA?S*E^#Uu-{#%n|!u4LyyUnsdS*kTtlul19`{<1{DEA?1Bf5nj
z@CsQ?>Srn}TLsyUo#&7B74UY%^9rmZ>gLq}d=bWKIDQ4$l@Wp|@+?~{RkeyjwAQ=v
z&G*VaQME1~H&kll?78~>W2y`6NTl9nb8dc9?!ga&-%OfyjAP{VGN)VW;cZst6{Qu5
ze%&*2t@U3|XQhp2ntanQm^#$^<2e<^DmEr_zTZ8Hr;DrN%Xs-uA*{-V+O@0h40%HF+#9pRd8P@zSjNFLFglH+sm$1m%)
ztM!e(Sb4E*3L*x0X207>I(@8w`T5AdjH-G#UIGxIyxbu*1M{dEM$D(T-q9Y8jE#sW
zles;n%kU-SELXFrb)sf)mq+uFE?%mWt*c0>?V-|8=J=a#BrBH(H-O~{SokFqNx*Sx
zUh%QzW8srLv>|grM>P3L}XxDO$B~O!P~9VJI-V;NT{6k*;C
znshH2P2X9vCJ~YP#E-xRq;v;B^L)kap;q&pED9Xy=
zuffE-As=6*%AtT!!G{3B&{RkyH~ynp-i4k@n-Uh>$VvEsZ>~r@33lBt{;CKkWnO(P
zLaAl?(cGvZ-=hFZ9jZy$FUZj=RCoCS>QQ1mwJb+((uKwo9AfVQSe9FeuVHDtge95d
zjZ|FETv}h4KTU=Hgkal{QNSU=Q@2SGi6bK)`UKGK!v->5zCvN6Us*Qp)i)b7@#!`nyET4Wg5!-+60y8e8z
z6wlZJCQ<6eCu_xWgl;S@Fh$cwJvQ|zSOCJCd!v6SDJ9r
zJY}ZnG1F#a^YCreo9C2%LUtBbfbNKmdBiyv0`o40fBR};Aj3UkZl(5BJ8Hp#+X`w7
zfL4>>xlkQpXfetW={lv&ZRPo3GgFT&-2Mug_v`WvzA=93HMU
zQ!S3`vC=436S+M8c}9u?1Cm4xS~!tsr5_1)3_rr;!FQN^5fOLO&;a0PBDDRiI=>`)
zdFh-~dy)$(>pP7Wfa;EBqUy^rLDYT%iQ@q%w8+2ORZd+*f^yzPzVThA_SxpS$5Wb7
zF;a4;RnHdGQyI29GUNwY4$COj7%aSht%!~;dHS{j+0A}|Fz`IS=J`@BYl!MaU*sEi
zce|8GxvY!Pp0vN5et(uCu~CqojKT(K?5#w9JE5O`d+=UvHvV>rCcMo`2h7
zcl~r&rtyHO5TO%+nsB9{>S3xglagc6s&9OY(m1sS1fTEbfOK^Fvz3OnBb
z9H4ww0mem-=pr3@mwnEz83bDa;=|5TG%#X&-CMt6&gc7#e&3Jn#}Shb*H!oN=TGY!
zG+V6YhnX)eCQX3k?W^aLAzaE=;IHE4c7gwW8+peYbv-vm8>{@63)bhtFr}mYH32@S
zXAs8U0b_p`POfKOB6l945~0oWYYk~BYFrQB{-~uX|9Xwq
z%La7s(+eK{(*!Wg$-Q*@yVd7F2-4}j>XFJ;#EiHXnKevC$}v=vtkqh+Fu%MoqLlU;
z3bS$J#f-^QjU}i37K)b^zp36uV4l3v=~tRdshAxo@k_4OWXfCsjgH{%$jQcv+40Qa
zp7-LaFP^{_Bhkf{*DyTZZO`CSQ>rQF@6ij_zOL2l$xsDFwi|37&7#*@UTZH)*T<2Z
zC>|n+oqGB=Z&G7nXV5B#{pxQ&zl7(44SH7mqglDHa0@Qjq%bXy&AN=mB}!?Me{#u?
zQ48d-f?8e
zTz%HE@ggW@99C)WYkeTQGYta_G~0bSt5MH}Si$H{@H`0K%t-*)%C-wHKU*k@MLHyl
zcR22wW1Z^c97<1?byRzR*t=Y!GI8yP6dwOnq__~hf2Um;R7}#Yq*_~ardv<=x7EC2
zmZjis81D(T$@C`p+W%=UJ`bxwse(x(-B)I+u<`>@hC*IUC8?lA^=5oj-sQby{SChB
zHc+y~lqKABgC)#?iIPA7X9zW4d=ti0I`625w*tH)sc^2Vn7!}noR^~r9-dNjQAI%~
zXW2-~KO*#>-<4>UiF94yPloT+iOV|)&%;yoovf1>Z4#r8=;9b_!X#fs$;(7!tPG6+!AxA9Rq7>RvibbsL#rc;eKLHP;-p1^B%yfTGZhdJ4-Np0Lo6VxCc@q8T06dzwb*!(0>u)t6X$7O{b-Y`BYYl^Icj>sjysGtbQcj03Pp4FdEvvTKx-LZYsb7$ZkkC5
zXY79{;(Np^>Sj8qKq^{_5p_@&ad)JF?w3+~}hp$0WDo?e!7A>$#ZPjsi0E!=l~U2kd4$>w_zKu?YL{Sz0>ePC~k
zI-knlR5g%UvSeuy8R%(AZ((e+;u?{^-$k4?E41oqbXPzZb;M<{yd?EarU5(|HY#UW
z#z8w$F}2h=6*quM6Q;MWd!5|W3Ydrv&>?jeIw^BzwIEY7kL3cbby~v9Wakx|iyl_M
zg3NX5$)LI<5!5C6R`dH-S+O0wxFkXMxGhHVwhe`ne=0k++zXbJ$K<9>rxSw^vxRJT
zrizOX5z^<{`(r%(L#OA<4le4COEKnj?tL42QaVonYUJw(c{!n*&a?R58%;{ih(V{m
zVd^M^@0aEM9$0j4u6}^*C~2oyx$1kb{feT!3L_Q>?T=-gUKu@KlJI#(1ymK1i4;L%
zxe%VvX1ySUv965&tVJ}J1ezXoQsqTMZ!$q+$gvQ<7wuY6E6jEZjmDt0qhgYrU^g@{
zCADY9Ewq{SO!=uA%He1HsB(1qmR6~_xZe5FcrF@(oVNU9TQwY;8B#Jkf@|M5I
z$WQjMg3nUyO-wzYjHy!m5bzG?M0EP=;&DZfC?R+mS60RNLYQ3Uf&pNpV>C18x}*Gt
zUly}n>-Urta|CfVt*wLITopKae{xwdV=OsTMFl_fo3|U!{H7`NLHqTghjYV10CJ8o
z17;;%39Frfl?O{?3lsgP7Zx4~ejoc3)Smz;{!fYn3jaMKlvQ|zn
zW2z=FEM{?FLR3g0c>zp0*Ue_Jqu9Kw|L73?XV|bZh}W+2ryqcqj-G1-phe}cV$~yxH$64|
zOuB}F1cbaN5Roxo2UHHy<}mmV&>SF|`;IhhQ7~;Gs#mhHGSn9zQ8%bK9~FIkDE*|O
z*$-*$xI7v}P2s~gKsu7z)^NGitV6chn#}l47Z99`6*nfxUW<;`e|@rh=1!|2I;>u_
z%?c7hAp34Bkxuusl<9Yy_D6p#Z<7C}&hOF1rhyR`Qn-ehgeNx}7=GSI91VL?x|hN*JlO=nhK*(}3XBIzWP4U9b^En8Bi0J+5{8$&Z$
z8*9xm{-g7AITFxg6oaQFn?T&3V!JuSw}AgnTIdzp
zWUkt5(os5K1wfrnvNNh#6-XSs<##4)M(8YwTfW=gJ2?W$xB)g@hvKS(iA8#Ik)N(9UVkF4H;Ec
zc0)Ah7wa@`s9DbF&j;!OudbGRv!h?|5n>D;BKGm5rJdz`qbPh>tM&pf;e9k|ETF>S
zc{;D*bXDH~g{wowpYns|1rwXOp~Q%@PJwYE-M0_I~M7E7^R<*u`Bu}S|1mOy}}
zB#`b>p>o=Lw9&X&D+4cz3axwe{j$$DUChX{B`HT%kkqn$L|3XrsC2ldA1@!aqSs8Y
zsGJ6Dtp=A`l#^-3fI~Glu7#l+%{}h%Df4z1Z8aG8JnB
zr!)kZUxcrdzoz~WLqKJC+cKu8o^;0C5>Tqu;n9IlWYm1$$IGzDQ+W}_G9tUY5bX^r
zZZV9$sVRUL5NkQrS=a5ar)ZuF`US_`_?OL8Tg({cfi6x3^oiXj%DGvr$322`9~0&a
z=dkdrrxaI%Ceg_`E}s5~&~`{80*GR2&<~Go0KeC~_>eqC3Xjau+(jz*lucAzYG0b(
z6+;aoCGQGQR?gK;YXLXb1#D8Mtd!Kf$$Es4p%93OB&Lw|vE>{wD;D+=Zp6R8?ADu}
zoiLV45)*7WGG}vViZo`-zrbA3c=@{F$7@3spk?YcWGLh<<+$JFfRXhG;G-i36KZdr
z@^X`z=*1Bubv06ZQTB&2Rf5&fhahXMu1Qeim`m&9(H#q`Js^K=&{u}S~Kmh@Ie~|-a@-l*JgA3qlWyfjhExH$WUUjZXlJWD_8fU#y%G7u@~OPv@&Bg
zlHzcaw(}a{@%k@vq>2s!gKJcY#+U(gACuu0t`zesR5r$-8Y3Fz
z>+ou4Og=>gRPt`}cu@d{k7O=90Abk~2=ch5+iX@Kg-nikuGz})K&Ns4@&KWbZMMTf
z=MFKt9On)X7Nb?Q(z-#MFJnR9uvioOt{?UFj(6xeR!Id+{^UhT=bDQ8nX5FdY(onj
zMGQf@S`HbeyqBcpOhS{IOx{)v2vbiYrBiFaVN(0iQgD^K%Yc49V)+de*;8It4S^;+l*0
z6PmSSp>BMnkb)~pvn!P8C1XuAsEyni>zv0cdAS2dGJ|-Hg+%39L7k@u)p-kxFMhs2
zp}cs|VcRDH!13GkhU~MrF!m?7dOVAjSse~~6jUEYrO&wXBv>RqS%a;h+e=08Z^6JgM)BHjoFQNT3U4ipozqI
z`TL;)QHB#ht#OHyytCCqoR6!JmZo*SE;)%u4IGOXVi~R=HD3X$Upd$JZ+TIGetMr_
zPg~g!#C&8ZMPn{$)FNNw{pS=kh&}xy4U1y3fYwz3Jia5u96&VA6+pdrK!FLsMIK0o
zvwU>Ks3l^mzJIwtO|M1_g<5@|=@}@1PO>_!#`8U>S2($UNrr%14`ulvcz~FGuI-Br
z!JMq#l#b>hCfu?_1M*KZcPI{N#?A4a)C*FplFi~0&MfDfdV6|?*EE{7t3gBL@kJ-l
z3$B*T*Jy<_$$l?{kv3qSdR)R$*S)%N0T+i5atl?O9&oC%DM&zH{4fcWl
zj-K==;i`%lM0{6JI5Jz0+EL|xK0rkdaeh$26$fUK
zmrtfgPNx0g8nRz9*a_wtIT-OA7jSm#-|IO6N{_HnLM9$?UU0Ct9-38d`zsdd^1$t_A$nj&Wb
zBh(=F(Vi2?4oL81yd?it$o!e*
zfXeZUUr7kVSJeYBAJ#Vp!XPtcF_ws=jzS_)aL!T;d{n((EJD(V;N{8i;v}g0O#gwW7ZI%mzY@@yc8CzoPdm5tvR*6d
zqDbmD)3zJ91FMO}Z(mbOa9E3d({^nKcyEV6`Je5ET7O9!h+mg`lf!^gZQ
zFd2PI#v>c#%^zW}Zl;0G@FOTUdYJ5=Tj&c9ru;fJ1*)CP=;MY4?;XQavhwcv#V>OzG3P{L
z(X&Gee@RQnKXDzBiYOy#FaPk4%I0-o-S-Ff)
zHK_%NOGk+0AO93k0$fT)24V37XigMQwXe#^#@(4p;Q~O#_Nn+ZClKVCwG1dCWG)Bf
zq%#8q?qAF-24q}j{gL57V>He3AsZX_9_OP>Qv|1<^eh-gY*Arp1Zrnt9^J73wPSLG
zM(n4o>pqslDd8Dn!V$H-$Qe<;^pVV)N39{oBRd;_#{HaMH$JxTZyzv31rhj|yE@co
z>G$6wexVaSd?Ja%2Kn~%+h-w5-!{fa1${x!fu(cdlK8fBw$HwyA>dV%nH)17Jve3l3YmqgNv`7wm@uZVRFU
zy|D@DVGckeogB}HSSlpqJI^S0V!2P;#!3EjsL+&cUQG9^C?V)wn7x#3N_NP{{Dcwn
zTGcMbPyhoN?-MA#MU0Ctw-`xwa$!g>5&l3IbKjYc6t_aB`a8Zi&+?t^Zlm4@!g*#f
zEhMljd>UR{oSFlr3rH6T36;rk=Nr@7ofsc{VqAbigX%wRAgeS~^wKJIrRjCLxU8Za
zJBYY2V;6z24iIo}{RsW4k8VPQHNj^u9SYA?Ct%3G!^&mfv%wgW=VFeX7jUmLEsmW_
zqfNaCKtKV|t^++|O6r%xjL{Oe6=#8jxJ9pdz8Fbjy|Ogp2I>V)X3*uCM>m;F?41W9
zODk<~))P~^m3$w&+i^qfvpI;H!Tv&7S5L`Db$XC@
z3@EKwkONJ?7$ClTbhBb16<3{-&60KVk$YSfJQKP7Q3!RXK!7LXK(nysRm}eW@XTdQ
z+OU(=bR)w_NRR?TFVrG+E}*oG&b*1xV*#&1T<9?cdM{!6*#zy%A5J_6;0l=JR*Ny9
z9!f-`X;mR1u3Rg>EC2HBeYy$bhfjDLZd%u&!vcdlf)_g(;BY)u_EJ{Cl1>lkN}WKD
zf!HiQsBdC#{XBKz4Zc;xcqNpBAzHgQwd(9-%VpBydC2-h6Cd?P1qQ-k092A9-f8d+gM{x4$>t!!
zNA~UKm8G_oJg9UA0fS64iobIKpePFHwHyc$xEIm;!bYLjOj4q(XV@L@L!_6sN73OZ
zOsb~5QI*gI4`!&KX#lSebpQksJ4lP01?1R_qkI;+G=
zAk*TYI)nB7dqzGpv0&2Zx59U9utzNF(`2Q3mZ3jkkT!XbJsjsBu}`?IkMOH(ll7&jK-^*`YG-Af}M_@oE;vZ4#TV>N?L4Q
z@PfR%nenfo+$hnu=-12$6%zu!*i_mU))s-ESc~U;2}T^e$Z}-ftps@5Cj~Zff%^IZ
zD8coj(qNaa41W0X0BECKT{xQ}k}QNXM*zCfAB-!gqkNhL>YV+9THpn^zG`{|L1C@X
z{Jk0?>dz)kPTv#mj1>=G31P0f+q7h1%)fL4Q4cZndVrvB;53d2D9gN~
zxF6zY0qtBnq|Uu?h%VMeUZdB6*42SWDFs^`v&Vht>R9m^pY%G}DKGpO8^Dk{Az$(n
z#vAaND|B64?J|OR<0el*Llv)=csv&fKbp}C7$!Zh$1gpW0M?k~wb1_Xc|Sy;
z3DF039lipz7$Tfx(*uA4j~t}sH>8@)K5%~Es*KN=RVOV~_D*{GeYwQKILxPo+*e#KElIiJh$d%A&7*%bjLlpxvS&
zlo`MFy^7?L2W?WT#X#{J1JxJu_us|GV@3h=c1C{U^&m6o@e+BPMA?jjtgcFg4RL
z5?{Tu$y!J43m?8hk*XiRhdJrlLr)lOurC;`lGLo5hHg(P@dj1UmEI7${o27%X`RS%
zzv6d5wYm)G85eoFcUg8HS3nUI!;JW+pGD*TII@}+;TY-0@6l}VETBloeta2X@PTW|
znfE@j9%h^W5)!k)1)L7mJns7ldM#24eopH&YuTmh-1^Mv#FL4VFD6?3ix^G6>5G6i
zG*{w7@1jKTU-hiuYQVo;V&l3%n8L=T;{}&;*G*%O=AL48k7SYd7SlFrI0y9Y6Vx#xAGPb2^_%~b1{CMM)sHu9keN_U
z+WKmh5h`QlE6EQ{eR-8@_`_w#UK9+*HF)}pUlz=rWEX3s@NikgodvJIh(8E2?>^-|
z_OT6WjRVzSvJW7sOD?i$bV09dodD)3F1sK=KFf0!*K$Px)ob;VnjG6M*b6xI?@%Qc}Dy#v
zO%|l^Ku)iQ-iVo3hzzTZiz%M?rfvYoU{|5bjI*o)4za?DiN`$(bNBh`4
z@E6o+N;>C`bS(jOgWF*ET*KrdV2xVkc^)jRg1`j?J?A~VZ^iu0YHT+$-8u(A#I;hL
zi793zbCdW1f#gSt#bQyX{bTJG1BL?7kv<-mElNUfZwbt|HG0GXzed_J~c5c=Te
zt&=`so!@iUbWj%8voCE0y3l5Be@{L83W3iA@3E+%iyW1{D#No9NEOX%ZaSib)ddCq
zQg(CYhE+uTbBI1@A_X@Y0#86`N2EIeSS}wbT=|#!#O3%(zwuN;%1i!*#)=1k;?%`x
zTJo!K0hz-CfZa8@57Bdjmu+e%Vt#nt~!su-!$)hwef&2!f(azlBqe_zTM6GMS^#SZa=QqNU8BR~clDC1o-}
z8(4^5F9tvZuc@Jv-E&KjyckkmKvLc%%DQ5$h^WHOL`4qeD1CI3nv8Y%*9SnwUD1-H
zG{@C9f;C}s6@BY?VF&c~)1xT@WS!}}mp(ZVvU{~a>TnEGMa2|0?xcb$6QP+fEg&bY
zOr8-*MFfmIBDMxV$W4mT3|`g+k?g_kfI6Xt(0cN8%7|p6_%%g;U*zkVPV_>TykS`_
z+gKz#d5)_M;8hXsq}$dw2;~Xrndbb-cT7@xw)l
z<+L@~9>c8+W$2&7TZkv-3oEP*7nY|d(n8UhwenF-(ElrK^J{pOgxK`|{$c{FHBKGZ
zrFJ~MAkvX1an_~8TEZ;L+T@oxV3uDxNKO1rgl$YgGyR2|^@om!XyJ?Z6f=Vtso4EfnOiII+71ga{aVmjaGD=a3`8_ZqJki
zC4I#^CsFi=7eK};G+
zk=}@+(j}b&(jd|)8(&dLgYNF`77ReTr0bP#DQWpG*(}IhfRlo#^c_eDPTQ0(u!invUs2XTDiPsNQE>Ys~x9
zU0UIE=!xHwO#A36-rf7DQT>WT8%HBaQ0?T@ZF6pZ0_5b@mey2L-}2lcT08)wSaW52
zk$;;25{RnPz!;M9wUK=+8J5oT?6i1ERUMRD<|dtR92acfEF`OOw1pV=Rh3SQg467y
zlx~cz&6=159*KG}Brd5gT3_&Bm|I)cdD2HwJ6Bdh)z)YYxiOE5D}in2!h6=bY6BkG
zFVOLu2X?a9zU^C4(M(tiIX03FoW`@0RrOQc+Z&N(bvtD0WWek)c@``G>EPXyli5iJ
zvK^UzB&by(JRcfxm5-uyDO*5=thMqe^O|nai{RSFnhR@jR4ozydAz;B_wkW3vPkx!Q2-Jdc6=m
zf>}6h^u)VKL{WhXJ`{oY77$BTAyNR{Zrn59n8EdJr=Wa`2pS+f0Wy<<^JQ??`Svz~
z#r4;E=vneaAJ&)$eiH9`EEs*w8cC&nfD~`3(qmF}HG;se3-zW>*;c$Us+B=Q!|&VH
z2F)Bh)-cu(kf(dCroaiPjtREdI*!&b1|Ig^8qE#txYNKu5c4Qa
z|7FJnf~W-Qu>~9|Wn~@&oG49osOxcPxVOhKM^E-
z`#qq9L;HRka)f->DkN2d^)sT3+p3=p%iSg!^O1Ol<_UrG?5`
z82N+O)S4|s-B}Wu-X2`{&$&dRdrvh^?NU52Uaqi7?1yg6lToyjwM+F#}sNxUqoIyHj2;r
zY%{HTie`I5^ZH8PbfY^4`uO4TC_~T^w5U&HI=L5#^r51>u=oFa5LuKunsb0)K9~QR
zUpnD4AgxALzGLj`q5qE0TG{q2Q=MPrgLT+c%O^QeIsz^PkZ+gkt$rI6c9#hDz{jgj
z%nJew^vVW_v1jDYzBvAU_(>CN6?J+bkj?`fLA!8%JVdbU(JimD#h>5TJP8%545Gu=
zzF$c;uv%F`q1N@Ccg|JbS^+)qc`>Z`*h?
zNM-&?x+Xg#wRcnK8zyAPCwo5(H%pgovnJAoJgzDyYxsGw|DB*AHUD}aP
zE5C<@+&zs{0nMtO8pgG0wnW7exT{zhsWAdq)5
zX7@oh=&SJ5&5~me>Pm!I63?f9axjKSIG2yrHj97ztUV%6!SnVBY5KjMvtZwB&(+ut
z(~Z4encO_CH>NfZ1h;6b&|~cppF@IbZ*LTvBB{d?(>j1B>t#q6ocho4tyY|9Unid}
zti2hna*rWByq97j<_bl?0Ff>4$YXvW#|3W8elU5q)F=Qb>%X)qKt)`_%-mxWcYvNf
zb(>AngQ;`wol7~DlM)}+m`xp0GBe!2#d!zbA1dISeEcAo>Pr*Vln98bB7U1073I&C
zrV+Ks+Uq*Pj4%oRlWqp(C@hi3jQ9KAlPRa3y)AU*!%hFK5*lI;&o7k$d=K2#!?)%V
z2eJVh-6wB{o+Su~+lbQ^LY~bGv-STkgQzE?;oEn?sxUHh9@cj!tnGC7z-Th3cJJ_y
zTxOc0&l9~*)iaqH+%7%CtZ5+SQG5<_A9V^PkGuLdN!3=#&RY6`_`c*Hv_ky%P5!Lp
zG|EOU;vKK8!xn#2c92@b-peX`AHEF3%b
zOCVNaH(Iy9byek!@_>Jfnj`s2`aHD{PSz_kcHb4auXzr9e3Yp^P;-*jR)qe{RHaU)
zv;$Sp<2)YzFV6PCA~V0Aj#?arf^KXXaDr0s*~a~TIOwJ2@y|sI8kcbTvrr$T(uGp8
z3i>wgF#fBeFutla8W?JyTeYlHg+b({vqPj6pGcF6Xtj39}b7Tqp)Kj&}t{I56|B{!b}!^HnUeQ
zGUFY!sT@ar@*qhGg$rY*-}I)u4}N{nB9S1){5*YGx|&1vAR*yK>evkTX_
z`0f{x+Tp+V*$ppuMY@KBHG<5DQhzqFfNzOA-1&daGinY`bldfLA#vc6(zg9c_0sI^
z4n_{4y6wO3&gedtpR2mMljGbYA5?CGkb(Le3T%+?{-4@hR7S!gFtcfS%F~bY;g?{{
z&(-)3hGst<0XnZOeUhU8Wt$57oK4JTXKohO8q_dm3`K%?PEI(jK44C8-qZIOd0EFOz7F}ZK@f10J;
zeTo04>N*m0a3k%8SiS7_oBCHLo9K2I{Lj6%%U9fEfqp_GXfH%#ktEjZx7n~zLLtU%
z?>hdyZ+`#cHw4yn7dBsLTiZ{TWD_SQ@4rK!_wIYR%>E<~7bu;@y^knD;(IFWr9Il&
zJ0JT`mMLlsO5-g0Wxd}MJY@Ti2VmbDp*_wOM8}uF>dC$xP`*lp{~Hke@sWQV({cr*
z@d=&n(CP}0;Lqc0NCcJhd*pZK=Kr1&xI`y%3FXJpa0!i)
z!_WU~VP&{P0HUq5D5J4of8)nI?w`JYyrm)m4Q&F=qVqS*_pT@85<+sQx&QE)&=g3I
z=ZjP|d-1(-am?(0uhzKls=Lz^VGWyhsiBZ%x6vdj|46`zeY5}n?VvJF0}GwkvPg8N
zMQhRRa?$JLE3lTQn8OQ&+u|Yv)ya`yy*XT*GeK@
ztJ-@60joKQ|39>D!Nd}S=LkYg`%zLJ-2d$VKwSDQNbu_>HNGNNxFqn8J}F?+NOo(;
zb{E}0K9R;9xDVG4qm+AYqEfiLVBd87V4@-&ulCNhV%^tLfJJ_-X9{)%A1T5{QU
z1>}OCPHVXSd1cC%4ww&~|9UIFqBAO>`Y*Pf?~*jZ?7qEoy*+=Shw+#McfaS?>vzg9
zSa1INA9534@G0@H9KY2K%LOaPRhV);)HbE29)%Cx)
zHfjV0Dlvqe_x$d~cgDjBv*G8!{eA?>HLN;(WGsKb1V6pLT7L8-@)9jAic<7e;l$Bj
zZz(4lB>458-4FeG|K98DyL3Vh2Aav2IG+EUQ1Xj*<)2FIf4W`sT-
zll`AJj{mPS^iP*gx(JVUxo-mRe|VZ-H;+8Hp&Trxpi=guzXu5!mjr%Hq5pD0v}M5{
zkDBdGXa{v642~p(Ql_e8rt(s)lVJdIk5Xj$MN85so!@0ASv?a
zPB&}BHh=76g$TQCZ?7=8D0&qKHtT>7@kvxf;t?kGe}0CagNaT$lA)1ouw>zZY9-L9
z{9^}Oo`*YE?TzauzjRS>ciHXU_P<>^26rQFds7Ix4PeNK+Ws*#HV0oXAI&lNoaFaC
z{j8P%2Kvm&J2gy)|GHZfs!j37H;>>)i2d7}Pq~Q6Fg)2`T5L(BS!r
z-ydV<53{<@hVFT~{pR+PINZ{QKItoe!G&+0j==fzp(k$*Hu7&HRi>$GuF5h)+Dv<{
z_2Vn|TUo^TT#|yTUh2O$N_fk;=-<9S03U5ORH^7r%wPlw$~-8AzIZ+xq+Iqt1oM%m
zqrW!At|`1ETTXEJO%o{#!=Lwx|Ml?0=O|kozU@u(;A_T8dQ`}5iv_Y=UQ$vR1tAmD
zH&hag)R1jxjeo{2vs?mM)LJNjJXQ?EGovpbUg(f{I~+Pnq5&j14vQok2xs=B
zC}szJh=~^uHCRtGKJ`ZD_0LDc7jRDc<0%jSIHyppM3ne(6aDrNymlWI=J409&>^x}
zYQAmk&XAWOm#Wk?-W=`d=y{I6Un$?j*tk0
zl2*t@DZD%GfF&)bUtEiSm^^Q>T|N{CR?EZuw?o
z;3wqNx{N1z@}CWJ5OSJ`t$$;+{W>E{rHs4!nK*SY6V~YN2rJThF~x#ui9;&yndqJ5EmTn?Fi?a
ze=7!-?@38Y(PX*QmeA#?j>azXAz-pfjHN=|P=h$bga_7%?tD|zZnKN@GqCF0i!2S5
zK|w7$4`J#FQ-u?
z^)C@Ky5EL$RjFu$ae>59N
zIWr@hIhT?Nme>Ytm(WPPs?(;FH6LCOggR_Bd0`@|Q`}Eraibmwjd}|swlR8-B%cqrPT$~{q`6IPbX;UUqW)OlkaX0Eg#Q$%`HV;J_RJZ
zd8=?fxraB_|2lwupI8C-Fk%8|yYCw-bEfO!p&)yEe>V2(=)1D_L>3A*cs@J6JhhN>
z)*mzV(+(rgDp>%g=BcvOsa@-WipawzIrei%WED{rS8ty+`zsZ=${@X%tO)LsQK)5m
zCc;Iq(6%0M%r{pZk#$0hZ$l_Iav0pGz5Pi=^C?kCpO%X;R39Cxpl_HGMY+#~J1pEC
zNXYLo;U2K%lt7r^9-~AU(W(Y^OXIK1JdYwP4_%G*BWgix!LQ}DH_SjuVPU@`{CG?A
z$3iu@yg24y9m?NlgFiX)8mRDmJ^Yt?}fa1z!U3@5a|
zO?Q0*Mtm0>!1qMn)7sEO)v9?(@rL%q8Wg3>t9C){i{qH@b>OdxOv75wgQBqduFraD
zg$66$inFAIPyJ^a2u88d-CJbcr>?XTCNbUKYbACZ+&g6`-uj2-&Y0I8
z4d1Z`QtdR<-sghp?L@eHTu%j$bAABID+3v?T}~-f^TdVn%9=(#CB|hqcY72Ru4W8J
z;i;EH4I`JKTuTSC@?N>71N|-n#AWZ$Z$MwLraRYgtj$~k;G>mVg%9RH8O-}5O0RG82JDxa5)l?LdFrpI
zid%if*4F))`KpeSd@Ivkt!!~v!3#vin+vt%hdfu{tzzoI%v!EMn{{m9VgCl?!&;vu
zQR~UJPe3`ovH>bjEhFxnP4)|ePfl>neS&ItE!?^{Z$_g-J%HZC1h-Ztszsfn>Upm*
zzQ6eI_s?`wzXA37x@%Asct3%R@%B-nC0^IR93FFQ=4~xbo;$~T5!9Ew7J0+c205D?
z)|aPr?TCGH$X<_SnIn5DE>V`A1HI!^o4C$~GmNv&oB@le$Z|u1s?7rHNn_LCO
zGeaKs;1f68Fj7a;Q{jpi0>@Nz#xUO#Bue}GfgDC3%bK10bqfY12UMvl*1_CP_9zR_
zAc{u~Up7Iz<1^pdFu~kNrx!S-GuW4D3Q$!~m!dH!8NTq(!W%oXM>d+6)>o=&E;HpVx+=>*q1O*j^matBs$&9H?gm8vx=|$WU8T{(`
zs&h!mQ*e(?zeH~AAT=*HS9>wHVLLq;pF@E$jIvcFsjjw7VhYv%ve+!jLF^j{O4joygTAmJtGFXDO&u}a8B1f~Dr;yefnHm-l0F@$x-cF@$3oUU4`
zqvUP{%I*PSOe~9uH02kl;54=KM%2G$Hj>hri*+mpbP~tp#Bu%Bz^Ce~b=Wa*ZTT3{
z{|JUf7{Srvz+yxb$j+d8l5+TjF8+TI?rrB^?Gcs}Q8*16#C>@T=r_Ct*_t1V2h&Ilr|(
z1`dBoym*rj);&V}q2iY#H_=zX9lN2I;hOJeH6T?kpEd?8xeA4DO6=@NvD$2+Z=YhPILb3ug{!@!Ea+xD{UE*8*lIzu009ow{WbA(VKZtnJsownll
z(?|1KOJyEvW)!&ix-3eUhxVT?eN(ZZIR*z_D56zeCc>)QuoQaQK59-o-^)cKXT!tl
zrP(6xnrucL(z~Gd{f!!#gXnG|4ej5)z|WudhT!n=ay)-pho^Xwd)%2>9H_L%
zX?Q$kO4@5a7w;0v_AJl1ryII(
zOq33v1!8$^gRajxTsbpqBeR|^*nJ`IL7cp?>)t!gwT)*k4|4D-2jV&T!7nfLc)AQ~
zN|p^=xsRk6Z&TRD5X781+X`O05iXnLOK+>g%?)JzEVyaXWgJ&KGOfNHKIo5x2XLu5^#v;piDQ>ZKr-Wh299uQdi;(`W+31x`S;nVwDOzR(*(F6Ywt#6I<<1WuSu
zt^OwNQE5~I)O$>q`tY@<$#o0RhMs&Bp;4Y;HWK9CQwR1=xBJCM-V;v58x3|-hKDE>
z?H7@gvXwj7xU5@o0yeLhJSWteg!KJo(dgT9n&;3uV-vfnG=I05a50+m7TM$R;!AbX
zjV+)oa=$w_UOC?+ZsUA$CGCW|ot#UB*o6DpJucf1PY-2$mYL6i`xXG|6gGHxL1C`MjE2u}b@a>%iEmFS+%4uD&koW5I#@wkOq%c(DyHt7V}E1!
zZCby3egq!ClDOQ^-|f>Is$k;QLx&PS4ipl9I#gR_n?{ZzR5)d14r&D%z;A9vyv3j=
zSU47l>gfj7X+mxM6y@o{?q$Z|jxX!Ja!NFPb#SPL@mZZ1o}d_8F98`@2Y(_GNiP(M
zIu&y$b@d1sqm|q^XIh*>>HrO}q~
zRgFP8w`2u_A`-~`Cd>IEC7b#F(@%-EPi-O2ic7>$EC-bjy_GKZC$#db&?$86%F
zEtE4OK~c!i4JV}V962}xuuheX$Ri#LC2c@DWg#i+0~>d)D%{y31clI=JAuPQ4wZ=C
zKy#`#*cgvK!#P{#c^L1&(^C?J<54U0^eAzGb4Bfug3mnK-&dxL(bGGp`V@{kX+mh3@XJgDjSPXCCo#Mx{li!ZO?Rz74g9~Gs-XzKBC3C+T(W5}w$5Jz4~*c%&h
z24|KYx*GIPp54J?xoxEri;G#R}ywh(G2Vb~L<`o@!4;io=*0hTCUPS413SPbcRC
z4*gm3_M6=>_o*`R;+$2WjTPL9XxU2%3E+?h>eP|2=}A_4kr%x!8YnChi;?;>a6Rn3UYhGIruDLP_g8WP~
zdP9CkHrldJ=PimVbg9^OYu*(g?y8~29|JyrZ2|p?#$XdzNn-*k@g-jdOU9up@B9;d
ztK}VXXQ{daAAkc@+ua6Ib^)3Y$IR(6s82oxk$$e|B$oshmII`ARYWB*&SB-3#eX;=
z2`4dK(fd7LFq%JtVo7M6`L3-dRO9TP{RF5q=T2^njS~}xPkTdtE11L7q?DeSy*ZVPO9{#r`at}
z#(KNOLV(e6#K4SO1sat&pwF5(jWvY!yVHl0>m!b+rh7X>9?hbIPf?Ff;b9Dpj769gzuqKmugf9o#U>35yqwLg5uAGYaLOy
z@YG4`Kl`72{|UI-ubPEBvtLR*2UH=!7*pukKMWso__V!*pq<@bu^$k5>_(93v<*v6Kjvn7PSjIVn4KeN~X#9((aP$bo6diG^*0_MZ5|
z0PBZOv8@V55G^<`wjtntEe1|KvyxK+sK=bhERDwI1LxPxRlp^AbQ@(MBAy)20y|9gE%T=SL#Q?137`kB
z1hE^A(j}9hfr$R9x*DfMrT4ixLpAQ{RTkQZr;aKh>OWcE?5fW{(^jzuV2O3X=Vb4(
zILD1|?IpJ6s!3~0aDM1|%tLie-bJtLQ4to7`Mx~(w@_?r22w;jCbD?
z0C|g_O*9TNvOPU$Sa2)_y4cNXHizSm)8@2*Xu(SX8!P$*qJ&MNF`B}E(K
zq!1y~SxOg^s-lAkT6olgW1W^Q;$2pS=(4n?UB_)o8eUPYjHv#VK>cwdzQj1P8{_W$
zCV>-OXkc>l=wWoxX^lNN*Uv4CpI=^%J}?9AHR^&8XXL=aqZzOX>OgV8W~wRj$y0`L
zFH@D+{2T~j3GX4zP3x%B`%ti5UotzcTR=i6C|zd;_e(AH;tQ)|^9Mx}wp9dp6Jncqcg|;T7x3E`VEgH@wP%lh1%dsUJgVnRqb+T
zk0B~&afI!DWsO92Hqd$_(CS0Wh6CsbbOtbMy>olBXIG*Ear6*hZ~w>{NKy8q0+d3|
zYu)&lqsRtF?-(Zbg(G!A?8yo1P{#XEXbrG3){E(9Nj15d>)7RWw(po#zdT=-2)?pO
z*n+E{@fdPm<`>n6aI!j59L;KZdcRwKjoMsl%T6RwwgzI;sY^5CzJ{CK
zI8%PgCYj=)T#8`grk|a2s=gOuxI44)7{>ca3&=*=VSrfDN_M@?O_RCgx*Nv)_)G!T
zkI0ph5=|{|INb4DU=a0nM*vSrb?9=71s~48Jt$q>aGv(Gyd3~J4q`J<-HSY{uOfi#
zj{YEhmbO%AC37UtzD?@3BCW5mf8z3!3{_#|pe+WoM@0G;
z*!cwxKL42(l&r$qq)@Kn@Q%FbjQWUE!p*BawFxp&4OTR!-{Hjf$@j=FAO7^FG(K+7
z2P{jKzkCD1jL#Qi{_7NTxdq-j>}Fp+SY+dx=r>2*TIcCOJAgpgoq-XlXrV5cqVeHl
zBuwRN7LThfz~{G?nEGN4H97SpW8iL@AmLf_-DQ?Dbml^|zFC9flpo9F-&_%+we|w4H6#sjU+FoHFGMK
zGQgs`Cc?4O4R0fYg=1T6XLDU$4;og#>CIXfjUCO2)mn~@aY|-2w7Z3YRw8b)e)77E
zs^sp#xn;lJc|*QjqlV5&K~&?beza`O_QoSx_OTTOUeje}0wE*|=Odv3yR}Yl6Dpq?
zs_`D^MOrbXH!T7<)E*}usw8PPb8{Y~$X1ip1}t{5fEwa6IbQ)DS)KH?J9YFgQf{jh
zB7&Qe25*x%EXE}rW8PSKH$|;k2zfn$gn_P#;~@HwXdDz#C(m_5)H{t$vc_V#c~d~3
z;yhxHDpG>@OxsfeO&!2PDuTqMX<3O8mAcbfETn|_dPYjtqHlXQNMB*yfa1-%Y89x)
zb+Coz2ySyclPb85jgFymtCyx#J+ef+xi`Ta^(>WUgVswYdxwgapzG03HvA(~eq(=T
zxzJy5683pq%Zw0?mJSG1dZR95j~_q_Vb0y?mZkMQ{zyv0J*_Y?Q*ZihTi&|V0^{iC
zFRHPf+%W>MqRB8F`N!=m#MZ;UUQtcYANEo=|H7SSLh&)^Y(m)i%cTdp4knB-_zH7!
z7wFIl@*(qWfrjVc(Q<>0t|%17q1B$Eh#bpy*=~I8UuJL6(i|QIG{zszkh{?cp&_IqtL-R(*U(f&5PW;9!d6-I#LhW$hn;6*Kjs`z$Y;BF)u0k8
z_|Hvs=O(8IB0jKn!x4DSUn`vOJWA?Ms}BrYO3|?3Y907RL?WKvdhb4;wK<
z|2|0So5F5N^c#phFfXz`B(P
zfqhu6q^kHolEIJ37TngI4L!T-j(>+Jpx(k57SrLA5*NC}Pa4J@Dp_;})T^W#|L*
ztd(t+H5ohhb@c$3vo75JdS;t|WxSHQY$UJ_v!ID=YiebqO3;}65$fk6Fni88@6Zjc
z%B6``F-&Y+6-6E2dqx2Gcou%#Is=r3beEDGJh!W?jh|o}7$R!Z4bm{VHVsp`4pPOh
zk#28g=MAcYVIn)ONQ6z-u7m+rcQtNI3lVLGb_$=Aa`ZI8wl5GTR+}NYPEZC}UXInc
zjN=OqLZ$y)m
zuol}e;)2JRf#81~atvYA*Nq_vT0ACJVTiO~Y|x;eS;~RZRIa&E(8YQHG4FEjNuD(b
z2N(+IyvvkbMy~Gw*41>Ve>q5h(E)=z;A9XRm19NrQiKgALkVM<+_IsbnFB356{YK~
z($8I~_g8*W0C;8l)9(ihTSG7C1BjOnG~x
z6?1jD~s06J05>;qo(4MaTHU-T=N_2n}>9L3)nrIdC%I-uN
zs;b)p%-syPxnCPW=-)6$XF}NsQoMmu9O|xEuS;}t%-vT?nud@jIfha=M8+TmrI`Lq
zbrk~XRKt$6Jg5-YjzLZqSt&{33?s<1gpAQfKYT?BWP^N~>mry(`je2OB=H1qRnbb8
zY?JTo6Dv|_jCw!|bX()ZMC9$EabQc8FHci3(X>RPXIL_VcEPzq+-e^=CNc+}`1JiE
ziWHg2H4y!uQV0dp5Z4#c=8vTTZYK8>0RKYS>t#(Ffo74+f&!1a>tpAcwnVfT9x&o!
z82~Ax$SL)l#dIgL*7I)t+jCQpRG4v?Lj+{v2G92i5?LJp9fvV%=w@P8W;(JBxlv95
z6d60PQT+0*U@B=KL~He#y=Hz3_Ou&*lZzq4
zbA~A4dLkr`>RK67*w#B}po#LpL~{`}*{jA2db1fhQqkQq_^al{>F4YA6padB&Yiph
zo}3~!ggn@I2Tvu8L-y1Z^tWFiNLs`F%@+hzv^{r0HWa<6SmO
zBI~&@OMfDqiobB83$!FN-0Y{fWbywZjx!V}bU9bjfy;~ZT%3Vcz%`w`x0iDjR4Z_C
z2PZx!`^$(6@D+{50VdYKOFzB{FhwSKYq7PH-+hX{I)Xsf3G)0i@*1~jeOoju?z*q$
zzIi@MBHDH4h+nG-gIyZJFEF}?+n^50=wfIx7F@X&1
z_06GgF{PrISeh(i)xZgzJWLPp1u6N}+%{=aTQeO50Yud7S6yTt<(4)Jkp&W49XkE^
zDi94KKl>>@AXP83vM0IzP@`fiaRG>SdXQ#%dptW1L5zu?>oPRi{0*jFZp*~ooU-xA^tWhT*2nsfUwoVC>46-*
zXruSGo8s3lh9%xMZXzR*{tBKImEx3MVAIU%_e=2RBT)%bonMyq5nF|ZdKxgU#m|kF
zQDbVwABTgUP12!;z%R_c|70Ym_N7ZZl1gSL2zuAwc-c?kLi9A)?toYRroARl2YLOL
zLE&XgtzlJG9TElE2)04m;X-P{-90e|0v(^uL=dNp@pkYc$a9I}9xxR-7Y}0Y^?P;*
zj|dC0&-?*Gqh?dY#QBi%yyR2A(-kBK&UOTehHi)Ho;AKnfqe|#eiqBJuM#45<9?j2
zZc>d{Upd;T5T!h$32fV8+-uD@=AdD0HHFO}rZR{ZA${&eyzjc#WzQ%;XM&BGT79WP
zt0+myPXND&aZ^%XRi%6wfiuv3R};L;<0?n^@7dxU)*ExnIMrW%0>=W;;GVe8(g}9I
zbxiY6(AZ8^XhLA|Fx}874Tey0AA)k-x@seja_;fs!wY>|;~I&8XosI16fGQioI+6TO{I9sBYf-<@#3R+Z*SnxH)Gh48F(
zTxGll$;`S1a5`4Y=AKygE+PaMHM@z+V_}81KO=U;$-R!XxgXL1gahGozsHA-ctf%p
z!8L-4P_LidI&F~xaeUiF4tk^(zS!f$P`N-E@$~Qa
zuE$s}&y-2O;7l#};${T~r3d74LYOB_qnQaGXG+2(RJ7G{~})_dXp{`YR1*4Pjq+
zQt)u6dDUg5sOe=~flMo>bEv4b@
ze*3vV7aFz)x#5>*^ubU
z49K$N0EK{8cY8KFnOv()EzdbHLx}AlFB$LwVrXKe6Ep6Y@yxhB0twn~SZO-%pPjIF
zkyxUbByUq$N*gb2Esl1x^Vd$a7HQ0|@{@96zAIo|6mFo5sbH6ud!Q8n@dwZvGl&=X
z9&s2fUBsxQBXfKj+-8%-MQthYgW?=GL+BPTS#z?m7z0WzbQ_mcCt0#ECF?d
z9^b&t3@#=U$y+cfD;tni(~{i+0if1WpGBFKo80`Lq#6Uu@Yn`Gy!#__+akeHwzdF5
zrr5Hj(8xI%@+n1xX)L;e8b}51N*+RABAoU$$5+`Q`02S|iIV=48-RWY>+GFsHR1r^
z-^p+$ZlxxPv!3fe5_VZekTMX(rMWY!cmDGCu-@OLtK2y}q=(mvM6RDjtKgVvL}(
z(B8KOu-OMnofK!?iDykgp)0bZM8_X|T*J>v_qd){+KbI3IpWwSG^XZ9^J-4*Vgdc#
zNBo*yEKCdUd*|9eb>U?z1B-v%{BA7Elao2lszY>nj=mkUCvfAM0(4?wk#vMul(yx-
z@{LbMgv%pMCa`XuWi##X&9T2F+jF5PMM;XcDky!{-ReLyOn-4l-ph)KuBMHMZ@4sb
z4AHt;urBjuX%`C@{qQL0Q1_AS60d|E)zWbTM$Jaw+1P#tPu5Ox<}teyIp&@&@;M^4
zkcQENhDV~zV%iMJc^yiI46Q0le6(Ljw!hD9XZbZ+3sflBy@O*#Apx&_t;Q}r
zxuh~-y?^tDm!WA#4bppg>v~;^2%Ev0SU6e0d#n
zr}W@==!^WchuQe@9)MvfY$`kE;!Y7tQ1Hpgqi<4*>x}U46gd+&Tz2%T`$d&{>1_3>
zso@*~sgHCo-_W^xn6haJJAirr5d-#Te-Q-bOA+`I6)qKo$U1r3`IR4aQ&W@1G?UZ?
ziqH5785na(iZJPPDdN_@0!B;hoHx(0G!XFFJ~RZYQ$TkY6r9
zMET~>^V(&~auTFuMh^>6Mpoj^+pICyunx-#4Z_i%EPDHAjT=e~mx=n%tNw~`_I(=|
z(3|1s(M-+9^f8fT9fFcMiaP-`xi7O932tDZwU6i>O4hD_4p|(aixMKGL4SV!AFJQc
z3P=^+(^o<}pO{odRq%I{#`TX?te(7+RsQ_s#n(KgA?AiXd8zCpeHxFxJlw2Fm
zhTh|GKLJCo2irOKK3(+m)(8-GYXw&HUq=cCp5i)7;nE3G?0JGu_z89Npg?aBeBCVS
z4KOE_0>vkA{up9R?C->b2ll=A96@_+c=eB=ZLsxlpix!`95`_wd-`Ka96mAd+kZ1M
zf8!$KlfzVUifuCa)K=3WSR`uYTWX2R5-Y_S(gXIsZwZ!ms~o*B_#wcF!h$m_oaSDR
z4A*{c_P+VvxeatDjsYY7m2adils|{Mpn_G2B<+y58rw16*eJBbOZ^`2kKh}^dch&vi-HvXN0Be$0u%{
zj9}a|qN!6lRyrrBSZ@yrl!0yt3hJ*H1R`d^ohMI_Kq9I;;-xM
ze%+7yx7UnJ`Yo3bF0WBc6x8yWZHWiBU>Yx>+b
zWQucTyAlgC9LNzXO(POJddaj5%g#rZD(-xnHfvD$YwOurGunIMj6vS7XTwLMBx0pa
zox7!_l`CSbN)y}o66DFSG*=W?@~;VT=n|fZ=BWRa8XA5kxXxuxn092FPm1)iZ+d7m
z>nQ+zT`*Mla;bNw^XDYCWtIbqG!3aVPqrt3VSjYmoUgiLobVk0i8G0wD!vGP8wf4;
z(k|!}$Jy#TNpY>hVLyW;A$$HBO(+~?P|!vLwQz^wv0fYw9QBV6>VxiS{7Z|)4(7i`yK
zWDm>^z8^nP;6^53D{P3)CDw|UPPF*yzEJY1C!0f^dxW}hStikE@sKs`P^ye~RnQae
ztB$lZR|9$rZ5MT-Vz?69LY~eP4bTqxrH{r}MrG3Ihn(@}^Oa8ITwxkmmA$ynYJ7|XruXJ_sFws;XvIIc%Y??d;T%r{HA>#=p(
zgXL(Xccc>3<4uEmdo}h*F=U%V*8y$`gm75oa_+0-7T{+aKRI@dDHn#bYK->GMlR-o
zgPkBfI8-+WU@dLp7Q$?W*h?Q!34}pQWB64N>N-L=4bmX3PLr5!*dGtC)!KN8kJY2Ix3pObtK%CG3c4Y;r;i{W
z1gJ-l8tejmD=&YB
z<6Qrm*9DGRqU=0gW2v$17X@M#sTQ-sjTAqT%taa7jvMZ?mT83fuXx(h)*o&p59{_W
zCJuMfHT%!H&q_r)PN>Os=L9etFpf}!cl$qi?S+?17Z)=6O1OT_>GNivY}$7>QZk)A
zc>!u!=STZa3sfl{;&hbJ5R&oCL(Nv)0nxUX#OtSvqupI7WIM#bL>1VR^o)Uhb1=qf
zbUdxaH$STvh4E54XeBRly<#z=1jx^KyE0<}S#^UroV6fnM3dA9rD(+W`jr(LeXJ&$
zysq~%+tm7OTo>NRT5RudC>e`AdnjBTr^lz|xFrit$x4-dh~n$ByZb%D3T%F2pZ)u4
za0K#jNL0xW=RkzmeJzfQ#K7+P`)7?^6EPv~S*@W@&jPutO&cXD^Qig4+Dqj(9CVO$
zTM=K8RI{bKNgcrlCJJxkBAei65}c*p@iM}mm`NgKD&e*JRSIeI6XeY+DX+bv>k^xa
zf{Hm3=VaYh5+wO+emDn`w=j2V&vy^2AHD|mVFYI?Q-m&57yB1Ws*p}tLV;9aHWw~l
zm9GSLhoe8FpzH@YsXm{!fhrM4ObG}XRAU5YARA`|QOrgQ^qn&SJRle8Lx;49Z~ErV
zJ$oIo8G`Ar_LWGQgJTdlRNfecO^rZFZQkX8WuLCqsLdtm;l92c`Lx+r$!BJU+a~2>
zcP@)x+;CK34WA9a!Lo|hXLIS~Aw%JzmQre+vh^}4+E|lJ%O!`?%D1&-X!F$47S22i
z>nVWpA>A~?BI`3UX)HUTLlPCLao_UzCzXc^IW
z>K&FQh2xSGUQgzwl+>$btM7o@;#-!35Sf8E9mqEpaSa>Ujuh2^1{>I!w2)ptix9?u
z>3B%J8^~v=^66?;U>LibA_a{R3?o*=-=iI92*PyNI%%U%vh10IJ9wAoz?~WHlXE@yy4zA-v=%VmuCC8g_k{wX5USF;IRitebNY^EhdaTm+wBR)`nq_@-X$#nKlx6qxUZ^R$rQ(
zLU?yt-0Cc_-(vtwd+r|!wVU@$16Jj$KX0FDZCyU}xxH#0rS
zxvx1=SbYT+T{lcpco`xH&;@juV7|w|ZkmY?cM=YNMxGE3+n3w%ZgGXp6McjM-;HzA
z5^*)wH{p{!@*>=_WI$AWIJyY=n=T-x7?0Kk#s9
zPOShAkpWEL)rEKBInnh{4Pt^jZVT<=W?;ZG;m(*-^aO0-2rLEf!LVU5YOIp%lC0!`
zup5Z82dVZpA#aMAoOrFqk=MuB2WVvC0>X4kqY>umOJ?HUylx~|L#SQQ*ZQ7JPi)ya
z1G<{hCZrs?L<wJhfU3wVW@c
zw}6~>fg{HH>Z6M~29z}``J=JAS)zh;BH3rm70<-|q5@Ddl>Y&jLtjfSgVa4&bs(B8Qqggf!5DfKLEY=P#Cw7ed*iWk}w^qa>0KF?_@B
z`NF!JF$Q*87RVk~=}S~i4j~vVP$(!joNuIwhV%77qql8Es8A7mERm$E*tjIlJ)?a_
zD_yPBs3qp^d3M%pi-EbKDaKZNhPOzv1mv*)#x71YEKgmTx!33NOg9ksrEn0RX|lRoq5bz8E0ii^UP?Z
zhu*qec0^aUhnH6JCe0*ut=T|ZTp34{M`~N)=P7Hqx+oi|)5wuZ>5bvP$3OY8n|H!b
z^jWNmx7LzhpAb*!CPz)pvNK#V1+2~(a{YOb+%N8m>mzNUI)4b)#4J0PKomUjzEzX=9=^+vlWG{+#T5?Dfw@^PHg|hHZMXWykCuXXn#{?d)
zjJF?ist#bx2UBi_3g)fHHh5aI1XHhxDLNPaI#_;v$*%Sl$^;E9i^{uv
z-!a675+ilOD{U)lz;f;Z^^aCtsQmXg=K)f57o=h)&Q4W?
z3`Adiy+%K?v>QpWI<9rs3!s_1Jq0n@W^ZlvwLqP$aUqsXZ&tU|J2IvS3<{iuf{s23
zhK!C)LcPIJ_E?Fz27c%p;fin(?u9ymO?23j2xR$aokoE3%4RNgVJT$%J0jRGUhW4)
zRb#T;83kkg5~|!2F>gu~k$yg1;Oy(+=78`*YNBzzbAPZ(1qW?BCCkjgXsyeiJJN2h
zw-pAn9m28Jk3Y?EwI|+1u4FEzyJhz7`qB30Y<}SFXylov@mY6j9~wzEApjP0Q@hj+Hj0LDQ}EHtQ$r9ht#w>V3hGe&tqNXGEm+MbWI{u4B(eqd5N`
zV{aW7Ww*T#D}n(k27(}E&@BxT2BLI#DIyKR5YnNDihu$`H_|Xj49yTqh%^I3qjV!N
zjLZ=4zA>IS=X`(fKYmmgxbMB!UVFuLUCXUVq3TmhU0m})L0EeC1n)!QGrVzR#J^(a
ztax9_uiA*Rbu0z$256T^39yfIPT1>iysYiR+o;~WfOJt7q=&YXNC-66`WiN_rj;j(
zw9EFZ4sjq+3P*PP+b!d9$#%@`(*Hh{FJ*nT{&)m`VlkJcIgU)jbDq>bNfr)-@xYE|
z;HRcM7i593$mTnxK^Gl}(NX#4G??8X?DZpTPf4cAr91|7LegtMMCAg&SUM1r76?d`
zLLI7VKT^G~`#=byQghuKD%$B@ld?jy_I5A;+)fd|0e5LDlTAX1Y-1yJQ8xgJ?NL7^
z$uLiL;l}Z!4bMsRkQUunBvOHvNi>miQ}CtPk#SHT=-x(;i?e-b<`MxQMPrl5xB^4v
z5mPU#W7HD+?sviHXwNEuy#n`#5kE;PC+LZ@hn=;UeDq>R<$I8eV$T2wTx|eACUF#m
zIjA}sk|0a%#J3A6xUyS}Oi?RZ
zcd5<9H?*QXgAOc)*=t|H`<*`J9<*U3;Bp=>U(Z=tKuR2zvJ$MXf|I;>FH<@hRE1-}
zPR}P`5ZCSKB+q8S4!;OJC^q>5<*zp%>kjH_e(%ZVj9FiqhTxy2sttEDmDzbL?BS@
z&V%q7X7a~L@R(U?~r
z^hs*#Suz=qld7$o2_!lJ#`6ist{(u|aIUu)yfRI}&W1H;6%~Ra050iGQyP$VNm-kf
zsx66o>CZ7jOVU>YqzEJweHIeBm9E%Vue~_0kY;G!Lsd)*4K{$t^3js&**;CorV`;}
zRt%os{cr+{g+$g6XHFkPQi`30`X2yRFT+_t`1PnP?)B0N`o&G+=z2D^-5{k>;jG(8!m;l$p%;-Xrf}
z>5iMZjWZ4Gifm6h<<*r)f!tLH}G-S_3Pg)^Inwy7}^Sqap9pa%N$c1~_168LG62WENjhxAz@
zy)!5fRAvBh8Ua+#Vk9wslwH=Y$LTnh3!CZj-=XRlhZi)DYyhgUfG@k$eLt#`wPOI=
zjXqpx;eEa4Rpj@@~csu06jjyK=-Hoxk9;Bxb
zC-2vvw<{*S6BgrPl;5ebkd*w+z*OZ)A84Aq&3y@t3Pdfs+f`H7Gk*9mYF<4JN}O&-
z?p9;`A!Kb~JBW|#ZDwzji--9)#FM#sTa?k*e+ViocX1H~%>HWK3_}a>Syv@-9=5C|
z{U(Wi*G53cUr7{88apJ)GP%m&FcMT7x9*h(AkO{8=E}xhp=G`0hbZ+2Eu;
zFF>PLn%e%-Uja6j{p-GuHdc(R32*_Zi5`J+yzL1K2bYnnJr7r*gTEI*$wbFt$yXic
zgqSk2x3G9=dl7#`C7tzD+&w_EJubLz|l%%){4ghw9$oRCUm%~punV;U-fAGnE_9LKX!NpqXCujEntaEzt
z(Lopv=^?lx!a*TghGBjaV8P#qI-jw*YO_}1`VK|8HcQr^0{~d)-plqr1H#qeBMGIPPb>2q!@1RqK2(6#QpgHAxQ*mLYG)*$l_}6>C<+TAT2O)|5m%#
zv6d1@@5mG5Z_?mmsrucQa;A!e!ENd+t_v(Q&&CREVH1tD4sVOhu7zri{(bRYi(|nC
z^Pv24Ao!O55Wr6r83O4n&id$}gbUm@X2#SBpxWm-8<5xrgfdX6^$B-9sfy!wKfKNbmzL;Xko8mF8aB|<@wSdjJ
z?vItui!&aN8;A`2>#yeYA`HcQH!`GCz#!)qw+z_nD^DB%K#p|kQKI{JJxGPPilrGr
zg4BNk{13Q|cQP1hadNymZI_Ox9ZCW7t(&-FvFfV@xCw1zuW1P^ooo#is)1?pMexT|o>W!ra3kSL|))D4zmh2&@CFitPq>;wnfs!x^?r0YF90
zJQRSGAt{ns16zeSp?1-Ja(iPjZ9F(T*l+S6F-Zf&8q#flEq}5fHO`}x>5+P^5>ArO
z$QBdYh`b@V0!l5tWD;L-3StMZJkXU4JV_m9r!5{)0)i$lqxNSej0(IupXr9bfBTpZ
zhC-d@(1WPhYNQWUM&?)zs-g!6>7aL%jVyY%&-h75D1+k34B&nbIQ1%C@|oF3Hjr}s
zAoT+6UKA|hlQR!GQkZ01c~y|kV>WGLe`i(--x5O!a`|`+L_F#{ldKG_r=J?Ajk3uE
zi~UYx0Ng?4&iwaOJ*Rxe;P1KV(>XO+ro9X^tnlmpSzfuF*D5WHrthu095LQJ2A3Q2
z-n_y3S;MRbGa;gte2aEj#P<8uOy3pL4k8i*v`4;%boZMps!0reJT9km!uj4_yuO(@
zm!Q%U?X+U6HQS1*(nRn*B~1R*BmKR>7HUKa0Lt}yxBae@In8y|W1hL7RGPqWZQ#0k
zIxfzK*VRKB7+>5#)hS`aF7QI+S<0KUB-d59*r%!D28sDxAxhkeI|-WnsnkN`;cl0Em0=EIwwiLA<)!epRU
zUt4>=#!J-!gtXmmj%N?s?7}4J-^PP;yIikH>f#344qmq%
zW*Iya{l!;*u)8Vwcc$ZXd75j!J0;+r0AZ&scsFeM%W=_zGKk~nKb*`32swbSGJkW?
zFN3|V`fQfGUsh16rg2d27g4v}Z`u)Cf#eKEPS_VOUZgHQ<;e0rXIKq;v#5hB)8Odi
z5;&gxb}e!}kD`a$>^jLrxqbQwPfc7*8XI@!z^q`1WX;ux1fu%abb&)EbTz_a%)dYx
zVDuy)rTle{ep<{F8M^HxL~PyEDGB;?LEGQ~-GliIDB
z&4{R;dAH)Hf9LRBSi2@F-4?1m4#Us07J-UhW@oPRcd!WqK%i%CaqocmF`G0Vl_`C}
zz6mIyJ%Y-FK=BO&&BS)&83cTvw#9|2=;=o>W|4)U71Du5_#2IRZYYWam+G0Gd-nKw
zPc5$_cmhZWCASF{9f2;5F|bD+AyQ+mN>%fq3-WP&SNX8j*mVebgfAJvT^Uu4JTl6$
z%GCbZgWkp9Ud#vgsXsUtB(n-?zf8rNKYwwJ_vyhI(>rScg6R6&uT0H}xpKJ6Kb|AA
z>Mm1z)t6w^$QjL(k=OgQCG_FS?R!}NqDQG3D&IH*CZeq)Gss$jFf>~9Q$?0kh-+%{
zE0*^v5gt?+S6RN!C$u>2P8+^5o)F1ryN%mz$A)Jhz*q~9^oc)i@At<2Y}Whfof`o3
z7tfDC8$rKkc1Y$SsC>#LnTroVJhv!_tUA68)eXdPiBA`;j=_P}A>ebb7rLB+_{YVi
zX?2&DFHUelG#d|PGD#ya9o*AQkYSm7;!B%js&p_=^!wzBe9L&vIN*wMqUZW&r(n=&
z8-}pGn{I-qOXOuiebUR&9^`W_b%&4rSa&)%LTv^b8=5W_>nrZ(^^$$tn4%1NAjXe1
zAYxRO__Mt};UuU1=E-+3NK{;WY|82(L^l4CuV1#6z){y6mr^IV>h@hnd}&VCWdj+6
zzOm6ZnW&e(DpNUGOAa8i#kHX`o)#<&%Qk|uoIBuYn%OI2odw?s+3nRZPo&iFJHvgN_pp#Mw2IUzF0bMa;hOzU@dN9E0C%ANX
zJKMi~+cBx&_%3C51`OAnqu1I|7Y{Q4LVe6Qs8)^H2NUCIXp)QPQiJfVB+vLpf}qBT
zAt>FhSoSNExI>&S-GVBrSYi7WdajU76aK8Nl-CZi6HwgJ6c8Ix1@^+pGvTo4Md_m4Y@5YG#n0f2ctF+o%gx7kAXg8T<
zjvRTWX1%4e|e8y)58LzJ?fNh=Uk$mTYUfN)@gJ(6fU+x4mCE=sFAeHzpHN^%r#_JPP$b)dgOJF=CtegZGBk`6jv~*EnrzJL3AT>oV%?q@}AjKbo!heV)p1lgh
zEe9bkAeEuhJ2oBGi-!&{6l(HZ)7C72r_DK0nzP$vR9w)