"""Extra coverage tests for benchmarks/graders.py targeting uncovered branches. Targets: - LLM invoke __call__ fallback (line 203) - BrowseComp extraction edge cases (missing fields, multiline, malformed) - SimpleQA "no" judgment - Batch _grade_results_inner per-item exception - grade_single_result chat_messages path - extract_answer_from_response: SimpleQA path, BrowseComp with/without fields - grade_single_result: exception path (grading error) - _grade_results_inner: browsecomp dataset, progress callback, with existing output file - get_evaluation_llm: custom config override, openai_endpoint API key from snapshot """ import json import tempfile from pathlib import Path from unittest.mock import MagicMock, patch MODULE = "local_deep_research.benchmarks.graders" def _make_llm_with_invoke(response_text): """Create a mock LLM with invoke() returning an object with .content.""" llm = MagicMock() result = MagicMock() result.content = response_text llm.invoke.return_value = result # Ensure callable interface also exists llm.return_value = response_text return llm def _make_llm_callable_only(response_text): """Create a mock LLM without invoke attribute — uses __call__ fallback.""" llm = MagicMock(spec=[]) # No attributes llm.return_value = response_text return llm def _make_llm_with_chat_messages(response_text): """Create a mock LLM with both invoke and chat_messages.""" llm = MagicMock() llm.chat_messages = True result = MagicMock() result.content = response_text llm.invoke.return_value = result return llm # --------------------------------------------------------------------------- # LLM invoke __call__ fallback # --------------------------------------------------------------------------- class TestLLMCallFallback: def test_callable_only_llm(self): """LLM without invoke uses __call__ fallback in grade_single_result.""" from local_deep_research.benchmarks.graders import grade_single_result llm = _make_llm_callable_only( "Extracted Answer: Paris\nReasoning: Capital of France\nCorrect: yes" ) with patch(f"{MODULE}.get_evaluation_llm", return_value=llm): result = grade_single_result( { "problem": "Capital of France?", "correct_answer": "Paris", "response": "Paris is the capital.", }, dataset_type="simpleqa", ) assert result["is_correct"] is True assert result["extracted_by_grader"] == "Paris" # --------------------------------------------------------------------------- # BrowseComp extraction edge cases # --------------------------------------------------------------------------- class TestBrowseCompExtraction: def test_missing_reasoning_field(self): """Response without reasoning → still extracts answer and correct.""" from local_deep_research.benchmarks.graders import grade_single_result response = "extracted_final_answer: The Moon\ncorrect: yes" llm = _make_llm_with_invoke(response) with patch(f"{MODULE}.get_evaluation_llm", return_value=llm): result = grade_single_result( { "problem": "What orbits Earth?", "correct_answer": "The Moon", "response": "The Moon orbits Earth", }, dataset_type="browsecomp", ) assert result["is_correct"] is True assert result["extracted_by_grader"] == "The Moon" def test_multiline_reasoning(self): """Response with multiline reasoning (re.DOTALL) → extracts correctly.""" from local_deep_research.benchmarks.graders import grade_single_result response = ( "extracted_final_answer: 42\n" "reasoning: The answer is 42\nbecause of the ultimate question\n\n" "correct: yes\nconfidence: 95" ) llm = _make_llm_with_invoke(response) with patch(f"{MODULE}.get_evaluation_llm", return_value=llm): result = grade_single_result( { "problem": "Answer to life?", "correct_answer": "42", "response": "42", }, dataset_type="browsecomp", ) assert result["is_correct"] is True assert result["graded_confidence"] == "95" def test_completely_malformed_response(self): """Malformed response → returns defaults (None/False).""" from local_deep_research.benchmarks.graders import grade_single_result llm = _make_llm_with_invoke("This is totally garbled output xyz") with patch(f"{MODULE}.get_evaluation_llm", return_value=llm): result = grade_single_result( { "problem": "Test?", "correct_answer": "A", "response": "B", }, dataset_type="browsecomp", ) assert result["is_correct"] is False assert result["extracted_by_grader"] == "None" # --------------------------------------------------------------------------- # SimpleQA "no" judgment # --------------------------------------------------------------------------- class TestSimpleQANoJudgment: def test_correct_no(self): """SimpleQA with 'Correct: no' → is_correct=False.""" from local_deep_research.benchmarks.graders import grade_single_result response = ( "Extracted Answer: Wrong\nReasoning: Completely wrong\nCorrect: no" ) llm = _make_llm_with_invoke(response) with patch(f"{MODULE}.get_evaluation_llm", return_value=llm): result = grade_single_result( { "problem": "What is 2+2?", "correct_answer": "4", "response": "5", }, dataset_type="simpleqa", ) assert result["is_correct"] is False assert result["extracted_by_grader"] == "Wrong" # --------------------------------------------------------------------------- # Batch _grade_results_inner per-item exception # --------------------------------------------------------------------------- class TestBatchGradePerItemException: def test_one_item_throws_others_still_graded(self): """One result throwing during grading → error recorded, others graded.""" from local_deep_research.benchmarks.graders import _grade_results_inner # Create results file with 2 entries with tempfile.NamedTemporaryFile( mode="w", suffix=".jsonl", delete=False ) as f: f.write( json.dumps( { "problem": "Q1", "correct_answer": "A1", "response": "R1", } ) + "\n" ) f.write( json.dumps( { "problem": "Q2", "correct_answer": "A2", "response": "R2", } ) + "\n" ) results_file = f.name output_file = results_file + ".graded" # LLM that fails on first call, succeeds on second llm = MagicMock() call_count = 0 def invoke_side_effect(prompt): nonlocal call_count call_count += 1 if call_count == 1: raise RuntimeError("LLM failed") result = MagicMock() result.content = ( "Extracted Answer: A2\nReasoning: Correct\nCorrect: yes" ) return result llm.invoke = invoke_side_effect try: results = _grade_results_inner( llm, results_file, output_file, "simpleqa", None ) assert len(results) == 2 assert "grading_error" in results[0] assert results[1].get("is_correct") is True finally: Path(results_file).unlink(missing_ok=True) Path(output_file).unlink(missing_ok=True) # --------------------------------------------------------------------------- # grade_single_result chat_messages path # --------------------------------------------------------------------------- class TestChatMessagesPath: def test_chat_messages_format(self): """LLM with chat_messages attribute → uses HumanMessage format.""" from local_deep_research.benchmarks.graders import grade_single_result response_text = ( "Extracted Answer: Paris\nReasoning: Capital\nCorrect: yes" ) llm = _make_llm_with_chat_messages(response_text) with patch(f"{MODULE}.get_evaluation_llm", return_value=llm): result = grade_single_result( { "problem": "Capital of France?", "correct_answer": "Paris", "response": "Paris", }, dataset_type="simpleqa", ) assert result["is_correct"] is True # Verify HumanMessage was used (invoke called with list) call_args = llm.invoke.call_args[0][0] assert isinstance(call_args, list) # --------------------------------------------------------------------------- # extract_answer_from_response # --------------------------------------------------------------------------- class TestExtractAnswerFromResponse: def test_simpleqa_strips_citations(self): """SimpleQA: citations [1] [2] removed, whole response returned.""" from local_deep_research.benchmarks.graders import ( extract_answer_from_response, ) result = extract_answer_from_response( "Paris is the capital [1] of France [2].", "simpleqa" ) assert result["extracted_answer"] == "Paris is the capital of France ." assert result["confidence"] == "100" def test_browsecomp_extracts_answer_and_confidence(self): """BrowseComp: extracts Exact Answer and Confidence.""" from local_deep_research.benchmarks.graders import ( extract_answer_from_response, ) result = extract_answer_from_response( "Exact Answer: The Moon\nConfidence: 95%", "browsecomp" ) assert result["extracted_answer"] == "The Moon" assert result["confidence"] == "95" def test_browsecomp_no_match(self): """BrowseComp: no matching fields → defaults.""" from local_deep_research.benchmarks.graders import ( extract_answer_from_response, ) result = extract_answer_from_response("Random text here", "browsecomp") assert result["extracted_answer"] == "None" assert result["confidence"] == "100" def test_browsecomp_no_confidence(self): """BrowseComp: answer found but no confidence → default 100.""" from local_deep_research.benchmarks.graders import ( extract_answer_from_response, ) result = extract_answer_from_response( "Exact Answer: Jupiter", "browsecomp" ) assert result["extracted_answer"] == "Jupiter" assert result["confidence"] == "100" # --------------------------------------------------------------------------- # grade_single_result — exception path # --------------------------------------------------------------------------- class TestGradeSingleResultException: def test_llm_exception_returns_error(self): """LLM raises exception → returns grading_error dict.""" from local_deep_research.benchmarks.graders import grade_single_result llm = MagicMock() llm.invoke.side_effect = RuntimeError("API down") with patch(f"{MODULE}.get_evaluation_llm", return_value=llm): result = grade_single_result( { "problem": "Q?", "correct_answer": "A", "response": "R", }, dataset_type="simpleqa", ) assert "grading_error" in result assert result["is_correct"] is False assert result["graded_confidence"] == "0" # --------------------------------------------------------------------------- # _grade_results_inner — browsecomp + progress callback + existing output # --------------------------------------------------------------------------- class TestGradeResultsInnerBrowsecomp: def test_browsecomp_batch(self): """Batch grading with browsecomp dataset type.""" from local_deep_research.benchmarks.graders import _grade_results_inner with tempfile.NamedTemporaryFile( mode="w", suffix=".jsonl", delete=False ) as f: f.write( json.dumps( { "problem": "Q1", "correct_answer": "A1", "response": "R1", } ) + "\n" ) results_file = f.name output_file = results_file + ".graded" llm = MagicMock() result_obj = MagicMock() result_obj.content = ( "extracted_final_answer: A1\n" "reasoning: Looks correct\n\n" "correct: yes\nconfidence: 90" ) llm.invoke.return_value = result_obj try: results = _grade_results_inner( llm, results_file, output_file, "browsecomp", None ) assert len(results) == 1 assert results[0]["is_correct"] is True assert results[0]["graded_confidence"] == "90" finally: Path(results_file).unlink(missing_ok=True) Path(output_file).unlink(missing_ok=True) def test_with_progress_callback(self): """Progress callback called for each result.""" from local_deep_research.benchmarks.graders import _grade_results_inner with tempfile.NamedTemporaryFile( mode="w", suffix=".jsonl", delete=False ) as f: f.write( json.dumps( { "problem": "Q1", "correct_answer": "A1", "response": "R1", } ) + "\n" ) results_file = f.name output_file = results_file + ".graded" llm = MagicMock() result_obj = MagicMock() result_obj.content = ( "Extracted Answer: A1\nReasoning: Good\nCorrect: yes" ) llm.invoke.return_value = result_obj callback_calls = [] def progress_cb(idx, total, meta): callback_calls.append((idx, total, meta["status"])) try: _grade_results_inner( llm, results_file, output_file, "simpleqa", progress_cb ) # Should be called twice per item: "grading" and "graded" assert len(callback_calls) == 2 assert callback_calls[0][2] == "grading" assert callback_calls[1][2] == "graded" finally: Path(results_file).unlink(missing_ok=True) Path(output_file).unlink(missing_ok=True) def test_existing_output_file_removed(self): """Existing output file gets removed before grading starts.""" from local_deep_research.benchmarks.graders import _grade_results_inner with tempfile.NamedTemporaryFile( mode="w", suffix=".jsonl", delete=False ) as f: f.write( json.dumps( { "problem": "Q", "correct_answer": "A", "response": "R", } ) + "\n" ) results_file = f.name output_file = results_file + ".graded" # Create pre-existing output file Path(output_file).write_text("old data\n") llm = MagicMock() result_obj = MagicMock() result_obj.content = ( "Extracted Answer: A\nReasoning: Correct\nCorrect: yes" ) llm.invoke.return_value = result_obj try: _grade_results_inner( llm, results_file, output_file, "simpleqa", None ) # Output file should only contain the new result, not "old data" content = Path(output_file).read_text() assert "old data" not in content finally: Path(results_file).unlink(missing_ok=True) Path(output_file).unlink(missing_ok=True) # --------------------------------------------------------------------------- # get_evaluation_llm # --------------------------------------------------------------------------- class TestGetEvaluationLLM: def test_custom_config_override(self): """Custom config merges with defaults.""" from local_deep_research.benchmarks.graders import get_evaluation_llm with patch(f"{MODULE}.get_llm") as mock_get_llm: mock_get_llm.return_value = MagicMock() get_evaluation_llm(custom_config={"model_name": "custom-model"}) call_kwargs = mock_get_llm.call_args.kwargs assert call_kwargs["model_name"] == "custom-model" def test_api_key_from_settings_snapshot(self): """API key extracted from settings snapshot dict format.""" from local_deep_research.benchmarks.graders import get_evaluation_llm snapshot = {"llm.openai_endpoint.api_key": {"value": "sk-test-key-123"}} with patch(f"{MODULE}.get_llm") as mock_get_llm: mock_get_llm.return_value = MagicMock() get_evaluation_llm(settings_snapshot=snapshot) # Should not raise or warn about missing API key mock_get_llm.assert_called_once() def test_api_key_from_settings_snapshot_plain_string(self): """API key as plain string (not dict) from snapshot.""" from local_deep_research.benchmarks.graders import get_evaluation_llm snapshot = {"llm.openai_endpoint.api_key": "sk-plain-key"} with patch(f"{MODULE}.get_llm") as mock_get_llm: mock_get_llm.return_value = MagicMock() get_evaluation_llm(settings_snapshot=snapshot) mock_get_llm.assert_called_once() def test_no_snapshot_no_api_key(self): """No snapshot, no API key → warns but proceeds.""" from local_deep_research.benchmarks.graders import get_evaluation_llm with patch(f"{MODULE}.get_llm") as mock_get_llm: mock_get_llm.return_value = MagicMock() get_evaluation_llm() mock_get_llm.assert_called_once()