""" Coverage tests for benchmarks/graders.py. Targets the 37 missing lines: - get_evaluation_llm() with/without custom_config, with/without settings_snapshot - get_evaluation_llm() openai_endpoint branch with api_key from snapshot - extract_answer_from_response() for browsecomp and simpleqa - grade_single_result() simpleqa success path (invoke without chat_messages) - grade_single_result() browsecomp success path - grade_single_result() grading error path - grade_results() basic flow with progress_callback - human_evaluation() non-interactive mode """ import json from unittest.mock import Mock, patch MODULE = "local_deep_research.benchmarks.graders" # --------------------------------------------------------------------------- # get_evaluation_llm # --------------------------------------------------------------------------- class TestGetEvaluationLlm: @patch(f"{MODULE}.get_llm") def test_default_config_uses_claude_sonnet(self, mock_get_llm): mock_get_llm.return_value = Mock() from local_deep_research.benchmarks.graders import get_evaluation_llm get_evaluation_llm() call_kwargs = mock_get_llm.call_args[1] assert call_kwargs["model_name"] == "anthropic/claude-3.7-sonnet" assert call_kwargs["provider"] == "openai_endpoint" assert call_kwargs["temperature"] == 0 @patch(f"{MODULE}.get_llm") def test_custom_config_overrides_default(self, mock_get_llm): mock_get_llm.return_value = Mock() from local_deep_research.benchmarks.graders import get_evaluation_llm get_evaluation_llm( custom_config={"model_name": "gpt-4", "provider": "openai"} ) call_kwargs = mock_get_llm.call_args[1] assert call_kwargs["model_name"] == "gpt-4" assert call_kwargs["provider"] == "openai" @patch(f"{MODULE}.get_llm") def test_settings_snapshot_dict_api_key_suppresses_warning( self, mock_get_llm ): mock_get_llm.return_value = Mock() from local_deep_research.benchmarks.graders import get_evaluation_llm snapshot = {"llm.openai_endpoint.api_key": {"value": "sk-test-key"}} with patch(f"{MODULE}.logger") as mock_logger: get_evaluation_llm(settings_snapshot=snapshot) # With a valid api_key from snapshot, no "no API key found" warning no_key_warnings = [ c for c in mock_logger.warning.call_args_list if "no api key" in str(c).lower() ] assert len(no_key_warnings) == 0 @patch(f"{MODULE}.get_llm") def test_settings_snapshot_string_api_key_suppresses_warning( self, mock_get_llm ): mock_get_llm.return_value = Mock() from local_deep_research.benchmarks.graders import get_evaluation_llm snapshot = {"llm.openai_endpoint.api_key": "sk-direct-key"} with patch(f"{MODULE}.logger") as mock_logger: get_evaluation_llm(settings_snapshot=snapshot) no_key_warnings = [ c for c in mock_logger.warning.call_args_list if "no api key" in str(c).lower() ] assert len(no_key_warnings) == 0 @patch(f"{MODULE}.get_llm") def test_no_settings_snapshot_logs_warning(self, mock_get_llm): mock_get_llm.return_value = Mock() from local_deep_research.benchmarks.graders import get_evaluation_llm with patch(f"{MODULE}.logger") as mock_logger: get_evaluation_llm() # Warning about missing settings snapshot (provider is openai_endpoint by default) warning_calls = [str(c) for c in mock_logger.warning.call_args_list] assert any( "settings snapshot" in w.lower() or "api key" in w.lower() for w in warning_calls ) # --------------------------------------------------------------------------- # extract_answer_from_response # --------------------------------------------------------------------------- class TestExtractAnswerFromResponse: def test_browsecomp_extracts_exact_answer(self): from local_deep_research.benchmarks.graders import ( extract_answer_from_response, ) response = "Exact Answer: Paris\nConfidence: 95%" result = extract_answer_from_response( response, dataset_type="browsecomp" ) assert result["extracted_answer"] == "Paris" assert result["confidence"] == "95" def test_browsecomp_no_match_returns_none(self): from local_deep_research.benchmarks.graders import ( extract_answer_from_response, ) response = "I don't know" result = extract_answer_from_response( response, dataset_type="browsecomp" ) assert result["extracted_answer"] == "None" def test_simpleqa_returns_full_response(self): from local_deep_research.benchmarks.graders import ( extract_answer_from_response, ) response = "The answer is 42." result = extract_answer_from_response(response, dataset_type="simpleqa") assert result["extracted_answer"] == "The answer is 42." assert result["confidence"] == "100" def test_citations_removed_from_response(self): from local_deep_research.benchmarks.graders import ( extract_answer_from_response, ) response = "Paris[1] is the capital[2]." result = extract_answer_from_response(response, dataset_type="simpleqa") assert "[1]" not in result["extracted_answer"] assert "[2]" not in result["extracted_answer"] # --------------------------------------------------------------------------- # grade_single_result – simpleqa success # --------------------------------------------------------------------------- class TestGradeSingleResultSimpleqa: @patch(f"{MODULE}.get_evaluation_llm") @patch("local_deep_research.utilities.resource_utils.safe_close") def test_simpleqa_correct_answer(self, mock_safe_close, mock_get_llm): mock_llm = Mock(spec=[]) # no chat_messages attr, has invoke mock_llm.invoke = Mock( return_value="Extracted Answer: Paris\nReasoning: Correct.\nCorrect: yes" ) mock_get_llm.return_value = mock_llm from local_deep_research.benchmarks.graders import grade_single_result result = grade_single_result( result_data={ "problem": "Capital of France?", "correct_answer": "Paris", "response": "Paris is the capital of France.", }, dataset_type="simpleqa", ) assert result["is_correct"] is True @patch(f"{MODULE}.get_evaluation_llm") @patch("local_deep_research.utilities.resource_utils.safe_close") def test_simpleqa_incorrect_answer(self, mock_safe_close, mock_get_llm): mock_llm = Mock(spec=[]) mock_llm.invoke = Mock( return_value="Extracted Answer: London\nReasoning: Wrong city.\nCorrect: no" ) mock_get_llm.return_value = mock_llm from local_deep_research.benchmarks.graders import grade_single_result result = grade_single_result( result_data={ "problem": "Capital of France?", "correct_answer": "Paris", "response": "London is the capital.", }, dataset_type="simpleqa", ) assert result["is_correct"] is False @patch(f"{MODULE}.get_evaluation_llm") @patch("local_deep_research.utilities.resource_utils.safe_close") def test_simpleqa_llm_with_content_attribute( self, mock_safe_close, mock_get_llm ): response_obj = Mock() response_obj.content = ( "Extracted Answer: Tokyo\nReasoning: Correct.\nCorrect: yes" ) mock_llm = Mock(spec=[]) mock_llm.invoke = Mock(return_value=response_obj) mock_get_llm.return_value = mock_llm from local_deep_research.benchmarks.graders import grade_single_result result = grade_single_result( result_data={ "problem": "Capital of Japan?", "correct_answer": "Tokyo", "response": "Tokyo.", }, dataset_type="simpleqa", ) assert result["is_correct"] is True # --------------------------------------------------------------------------- # grade_single_result – browsecomp success # --------------------------------------------------------------------------- class TestGradeSingleResultBrowsecomp: @patch(f"{MODULE}.get_evaluation_llm") @patch("local_deep_research.utilities.resource_utils.safe_close") def test_browsecomp_correct_answer(self, mock_safe_close, mock_get_llm): mock_llm = Mock(spec=[]) mock_llm.invoke = Mock( return_value=( "extracted_final_answer: Eiffel Tower\n" "reasoning: The answer matches.\n" "correct: yes\n" "confidence: 90" ) ) mock_get_llm.return_value = mock_llm from local_deep_research.benchmarks.graders import grade_single_result result = grade_single_result( result_data={ "problem": "Famous Paris landmark?", "correct_answer": "Eiffel Tower", "response": "The Eiffel Tower is in Paris.", }, dataset_type="browsecomp", ) assert result["is_correct"] is True assert result["extracted_by_grader"] == "Eiffel Tower" @patch(f"{MODULE}.get_evaluation_llm") @patch("local_deep_research.utilities.resource_utils.safe_close") def test_browsecomp_incorrect_returns_false( self, mock_safe_close, mock_get_llm ): mock_llm = Mock(spec=[]) mock_llm.invoke = Mock( return_value=( "extracted_final_answer: None\n" "reasoning: No match.\n" "correct: no\n" "confidence: 50" ) ) mock_get_llm.return_value = mock_llm from local_deep_research.benchmarks.graders import grade_single_result result = grade_single_result( result_data={ "problem": "Q?", "correct_answer": "A", "response": "wrong", }, dataset_type="browsecomp", ) assert result["is_correct"] is False # --------------------------------------------------------------------------- # grade_single_result – error path # --------------------------------------------------------------------------- class TestGradeSingleResultError: @patch(f"{MODULE}.get_evaluation_llm") @patch("local_deep_research.utilities.resource_utils.safe_close") def test_llm_error_returns_error_dict(self, mock_safe_close, mock_get_llm): mock_llm = Mock(spec=[]) mock_llm.invoke = Mock(side_effect=RuntimeError("LLM crashed")) mock_get_llm.return_value = mock_llm from local_deep_research.benchmarks.graders import grade_single_result result = grade_single_result( result_data={ "problem": "Q?", "correct_answer": "A", "response": "R", } ) assert result["is_correct"] is False assert "grading_error" in result assert "LLM crashed" in result["grading_error"] # --------------------------------------------------------------------------- # grade_results – basic flow # --------------------------------------------------------------------------- class TestGradeResults: @patch(f"{MODULE}.get_evaluation_llm") @patch("local_deep_research.utilities.resource_utils.safe_close") def test_grade_results_processes_all_lines( self, mock_safe_close, mock_get_llm, tmp_path ): mock_llm = Mock(spec=[]) mock_llm.invoke = Mock( return_value="Extracted Answer: A\nReasoning: ok.\nCorrect: yes" ) mock_get_llm.return_value = mock_llm results_file = tmp_path / "results.jsonl" output_file = tmp_path / "graded.jsonl" data = [ {"problem": "Q1", "correct_answer": "A1", "response": "A1"}, {"problem": "Q2", "correct_answer": "A2", "response": "A2"}, ] results_file.write_text("\n".join(json.dumps(d) for d in data)) from local_deep_research.benchmarks.graders import grade_results graded = grade_results( results_file=str(results_file), output_file=str(output_file), dataset_type="simpleqa", ) assert len(graded) == 2 @patch(f"{MODULE}.get_evaluation_llm") @patch("local_deep_research.utilities.resource_utils.safe_close") def test_grade_results_with_progress_callback( self, mock_safe_close, mock_get_llm, tmp_path ): mock_llm = Mock(spec=[]) mock_llm.invoke = Mock( return_value="Extracted Answer: A\nReasoning: ok.\nCorrect: no" ) mock_get_llm.return_value = mock_llm results_file = tmp_path / "results.jsonl" output_file = tmp_path / "graded.jsonl" data = [{"problem": "Q1", "correct_answer": "A1", "response": "wrong"}] results_file.write_text(json.dumps(data[0])) callback = Mock() from local_deep_research.benchmarks.graders import grade_results grade_results( results_file=str(results_file), output_file=str(output_file), dataset_type="simpleqa", progress_callback=callback, ) assert callback.call_count >= 1 # --------------------------------------------------------------------------- # human_evaluation – non-interactive mode # --------------------------------------------------------------------------- class TestHumanEvaluation: def test_non_interactive_returns_false_for_each(self, tmp_path): results_file = tmp_path / "results.jsonl" output_file = tmp_path / "human_graded.jsonl" data = [ {"problem": "Q1", "correct_answer": "A1", "response": "R1"}, {"problem": "Q2", "correct_answer": "A2", "response": "R2"}, ] results_file.write_text("\n".join(json.dumps(d) for d in data)) from local_deep_research.benchmarks.graders import human_evaluation graded = human_evaluation( results_file=str(results_file), output_file=str(output_file), interactive=False, ) assert len(graded) == 2 # Non-interactive mode always marks as incorrect for item in graded: assert item["is_correct"] is False assert item["human_evaluation"] is True