""" High-value edge case tests for benchmarks/metrics/calculation.py. Complements test_metrics_calculation.py with additional edge cases: - JSONL with blank lines and whitespace - Mixed graded/ungraded results - Confidence with falsy values (0, empty string) - calculate_combined_score with partial metrics - calculate_resource_metrics with extreme values """ import json from local_deep_research.benchmarks.metrics.calculation import ( calculate_combined_score, calculate_metrics, calculate_resource_metrics, ) # --------------------------------------------------------------------------- # calculate_metrics edge cases # --------------------------------------------------------------------------- class TestCalculateMetricsEdgeCases: """Additional edge case tests for calculate_metrics.""" def test_blank_lines_in_jsonl_skipped(self, tmp_path): """Blank lines between JSON entries are ignored.""" results_file = tmp_path / "results.jsonl" results_file.write_text( json.dumps({"is_correct": True}) + "\n" "\n" " \n" + json.dumps({"is_correct": False}) + "\n" ) metrics = calculate_metrics(str(results_file)) assert metrics["total_examples"] == 2 assert metrics["graded_examples"] == 2 def test_ungraded_results_not_counted_for_accuracy(self, tmp_path): """Results without 'is_correct' are excluded from accuracy calculation.""" results_file = tmp_path / "results.jsonl" results = [ {"is_correct": True}, {"question": "What is X?"}, # ungraded {"is_correct": False}, ] with open(results_file, "w") as f: for r in results: f.write(json.dumps(r) + "\n") metrics = calculate_metrics(str(results_file)) assert metrics["total_examples"] == 3 assert metrics["graded_examples"] == 2 assert metrics["accuracy"] == 0.5 def test_confidence_zero_string_is_included(self, tmp_path): """String '0' is truthy so it passes the 'if r.get(confidence)' check. int('0') = 0, so both values are included: (0 + 80) / 2 = 40.""" results_file = tmp_path / "results.jsonl" results = [ {"confidence": "0"}, {"confidence": "80"}, ] with open(results_file, "w") as f: for r in results: f.write(json.dumps(r) + "\n") metrics = calculate_metrics(str(results_file)) # "0" is truthy string, int("0")=0, so (0+80)/2 = 40 assert metrics["average_confidence"] == 40 def test_confidence_empty_string_skipped(self, tmp_path): """Empty string confidence is falsy and skipped.""" results_file = tmp_path / "results.jsonl" results = [ {"confidence": ""}, {"confidence": "90"}, ] with open(results_file, "w") as f: for r in results: f.write(json.dumps(r) + "\n") metrics = calculate_metrics(str(results_file)) assert metrics["average_confidence"] == 90 def test_processing_time_only_from_entries_that_have_it(self, tmp_path): results_file = tmp_path / "results.jsonl" results = [ {"processing_time": 2.0}, {"question": "no time"}, {"processing_time": 4.0}, ] with open(results_file, "w") as f: for r in results: f.write(json.dumps(r) + "\n") metrics = calculate_metrics(str(results_file)) assert metrics["average_processing_time"] == 3.0 def test_all_results_are_errors(self, tmp_path): results_file = tmp_path / "results.jsonl" results = [ {"error": "timeout"}, {"error": "connection failed"}, ] with open(results_file, "w") as f: for r in results: f.write(json.dumps(r) + "\n") metrics = calculate_metrics(str(results_file)) assert metrics["error_count"] == 2 assert metrics["error_rate"] == 1.0 assert metrics["graded_examples"] == 0 assert metrics["accuracy"] == 0 def test_category_with_zero_correct(self, tmp_path): results_file = tmp_path / "results.jsonl" results = [ {"is_correct": False, "category": "hard"}, {"is_correct": False, "category": "hard"}, ] with open(results_file, "w") as f: for r in results: f.write(json.dumps(r) + "\n") metrics = calculate_metrics(str(results_file)) assert metrics["categories"]["hard"]["accuracy"] == 0.0 def test_single_result(self, tmp_path): results_file = tmp_path / "results.jsonl" results_file.write_text( json.dumps({"is_correct": True, "confidence": "95"}) + "\n" ) metrics = calculate_metrics(str(results_file)) assert metrics["total_examples"] == 1 assert metrics["accuracy"] == 1.0 assert metrics["average_confidence"] == 95 def test_is_correct_false_counted_in_graded(self, tmp_path): results_file = tmp_path / "results.jsonl" results = [{"is_correct": False}] with open(results_file, "w") as f: for r in results: f.write(json.dumps(r) + "\n") metrics = calculate_metrics(str(results_file)) assert metrics["graded_examples"] == 1 assert metrics["correct"] == 0 assert metrics["accuracy"] == 0.0 def test_timestamp_present_in_metrics(self, tmp_path): results_file = tmp_path / "results.jsonl" results_file.write_text(json.dumps({"is_correct": True}) + "\n") metrics = calculate_metrics(str(results_file)) assert "timestamp" in metrics assert "T" in metrics["timestamp"] # ISO format # --------------------------------------------------------------------------- # calculate_combined_score edge cases # --------------------------------------------------------------------------- class TestCalculateCombinedScoreEdgeCases: """Additional edge cases for combined score calculation.""" def test_empty_metrics_dict(self): score = calculate_combined_score({}) assert score == 0.0 def test_only_quality_metric(self): metrics = {"quality": {"quality_score": 0.9}} score = calculate_combined_score(metrics) # Only quality contributes: 0.9 * (0.6/1.0) = 0.54 expected = 0.9 * 0.6 assert abs(score - expected) < 0.001 def test_only_speed_metric(self): metrics = {"speed": {"speed_score": 0.8}} score = calculate_combined_score(metrics) expected = 0.8 * 0.3 assert abs(score - expected) < 0.001 def test_only_resource_metric(self): metrics = {"resource": {"resource_score": 0.5}} score = calculate_combined_score(metrics) expected = 0.5 * 0.1 assert abs(score - expected) < 0.001 def test_unrecognized_metric_keys_ignored(self): metrics = { "quality": {"quality_score": 1.0}, "custom_metric": {"score": 0.5}, } score = calculate_combined_score(metrics) # custom_metric has no weight, so ignored expected = 1.0 * 0.6 assert abs(score - expected) < 0.001 def test_all_scores_zero(self): metrics = { "quality": {"quality_score": 0.0}, "speed": {"speed_score": 0.0}, "resource": {"resource_score": 0.0}, } assert calculate_combined_score(metrics) == 0.0 def test_negative_weights_still_normalize(self): """Negative weights are technically allowed by the function.""" metrics = { "quality": {"quality_score": 1.0}, "speed": {"speed_score": 1.0}, } weights = {"quality": 1.0, "speed": -0.5, "resource": 0.0} score = calculate_combined_score(metrics, weights) # total_weight = 0.5, norm: quality=2.0, speed=-1.0 # score = 1.0 * 2.0 + 1.0 * (-1.0) = 1.0 assert abs(score - 1.0) < 0.001 def test_missing_score_key_defaults_to_zero(self): metrics = { "quality": {}, # no quality_score key } score = calculate_combined_score(metrics) assert score == 0.0 # --------------------------------------------------------------------------- # calculate_resource_metrics edge cases # --------------------------------------------------------------------------- class TestCalculateResourceMetricsEdgeCases: """Additional edge cases for resource metrics.""" def test_very_high_complexity(self): config = { "iterations": 10, "questions_per_iteration": 10, "max_results": 200, } metrics = calculate_resource_metrics(config) assert ( metrics["resource_score"] < 0.1 ) # Very high complexity -> low score def test_minimal_complexity(self): config = { "iterations": 1, "questions_per_iteration": 1, "max_results": 10, } metrics = calculate_resource_metrics(config) assert ( metrics["resource_score"] > 0.8 ) # Very low complexity -> high score def test_zero_iterations(self): config = {"iterations": 0} metrics = calculate_resource_metrics(config) assert metrics["estimated_complexity"] == 0 assert metrics["resource_score"] == 1.0 # 1/(1+0) = 1.0 def test_complexity_formula(self): """Verify the exact formula: iterations * questions * (max_results/50).""" config = { "iterations": 3, "questions_per_iteration": 4, "max_results": 100, } metrics = calculate_resource_metrics(config) expected_complexity = 3 * 4 * (100 / 50) assert metrics["estimated_complexity"] == expected_complexity def test_resource_score_formula(self): """Verify: resource_score = 1/(1 + complexity/4).""" config = { "iterations": 2, "questions_per_iteration": 2, "max_results": 50, } metrics = calculate_resource_metrics(config) # complexity = 2 * 2 * (50/50) = 4.0 expected_score = 1.0 / (1.0 + 4.0 / 4.0) assert abs(metrics["resource_score"] - expected_score) < 0.001