chore: import upstream snapshot with attribution

tests/web/queue/test_processor_v2_coverage.py

@@ -0,0 +1,525 @@
+"""
+Coverage tests for processor_v2.py – error recovery paths.
+
+Targets ~24 missing statements in:
+- notify_research_queued: exception path, no-password path, queue fallback exception
+- _start_research_directly: active-record creation failure, thread-ID update failure,
+  start_research_process exception + active-record cleanup failure
+- notify_research_completed / notify_research_failed: outer exception paths
+- _process_queue_loop: cleanup_dead_threads path, finally-block import error
+- _process_user_queue: engine=None path, outer exception path
+- _start_queued_researches: processing-flag reset on error, task-status update on error
+- _start_research: research-not-found ValueError
+- process_pending_operations_for_user: rollback failure path, error_update with report_path
+"""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+MODULE = "local_deep_research.web.queue.processor_v2"
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_processor():
+    """Return a fresh QueueProcessorV2 without starting the background thread."""
+    with patch(f"{MODULE}.logger"):
+        from local_deep_research.web.queue.processor_v2 import QueueProcessorV2
+
+        return QueueProcessorV2(check_interval=1)
+
+
+# ---------------------------------------------------------------------------
+# notify_research_queued – no password → falls through to queue fallback
+# ---------------------------------------------------------------------------
+
+
+class TestNotifyResearchQueuedFallbackPaths:
+    def test_no_password_goes_to_queue_fallback(self):
+        """When session_password_store returns None the code skips direct exec
+        and calls the queue fallback path."""
+        proc = _make_processor()
+        mock_session = MagicMock()
+        mock_qs = MagicMock()
+
+        with (
+            patch(f"{MODULE}.session_password_store") as mock_store,
+            patch(f"{MODULE}.get_user_db_session") as mock_ctx,
+            patch(f"{MODULE}.UserQueueService", return_value=mock_qs),
+        ):
+            mock_store.get_session_password.return_value = None
+            mock_ctx.return_value.__enter__ = MagicMock(
+                return_value=mock_session
+            )
+            mock_ctx.return_value.__exit__ = MagicMock(return_value=False)
+
+            proc.notify_research_queued(
+                "alice", "r1", session_id="sess1", query="q"
+            )
+
+        mock_qs.add_task_metadata.assert_called_once_with(
+            task_id="r1", task_type="research", priority=0
+        )
+
+    def test_queue_fallback_exception_is_swallowed(self):
+        """Exception in the queue fallback must not propagate."""
+        proc = _make_processor()
+
+        with (
+            patch(f"{MODULE}.session_password_store") as mock_store,
+            patch(
+                f"{MODULE}.get_user_db_session", side_effect=RuntimeError("db")
+            ),
+            patch(f"{MODULE}.logger"),
+        ):
+            mock_store.get_session_password.return_value = None
+            # Should not raise
+            proc.notify_research_queued("alice", "r1", session_id="sess1")
+
+    def test_direct_exec_exception_falls_back_to_queue(self):
+        """Exception inside direct-exec block triggers queue fallback."""
+        proc = _make_processor()
+        mock_session = MagicMock()
+        mock_qs = MagicMock()
+
+        with (
+            patch(f"{MODULE}.session_password_store") as mock_store,
+            patch(f"{MODULE}.db_manager") as mock_db,
+            patch(f"{MODULE}.get_user_db_session") as mock_ctx,
+            patch(f"{MODULE}.UserQueueService", return_value=mock_qs),
+            patch(f"{MODULE}.logger"),
+        ):
+            mock_store.get_session_password.return_value = "secret"
+            # First call (inside direct-exec) raises; second call (fallback) succeeds
+            mock_ctx.return_value.__enter__ = MagicMock(
+                side_effect=[RuntimeError("boom"), mock_session]
+            )
+            mock_ctx.return_value.__exit__ = MagicMock(return_value=False)
+            mock_db.open_user_database.return_value = MagicMock()
+
+            proc.notify_research_queued("alice", "r1", session_id="sess1")
+
+        # Fallback queue path must have been called
+        mock_qs.add_task_metadata.assert_called_once()
+
+
+# ---------------------------------------------------------------------------
+# _start_research_directly – active-record creation failure
+# ---------------------------------------------------------------------------
+
+
+class TestStartResearchDirectlyErrors:
+    def test_active_record_creation_failure_returns_early(self):
+        """If creating the active-research DB record fails the method returns
+        without calling start_research_process."""
+        proc = _make_processor()
+
+        with (
+            patch(
+                f"{MODULE}.get_user_db_session", side_effect=RuntimeError("db")
+            ),
+            patch(f"{MODULE}.start_research_process") as mock_start,
+            patch(f"{MODULE}.logger"),
+        ):
+            proc._start_research_directly("alice", "r1", "secret", query="q")
+
+        mock_start.assert_not_called()
+
+    def test_thread_id_update_failure_is_swallowed(self):
+        """Failure to update thread_id must not crash the method."""
+        proc = _make_processor()
+
+        mock_session = MagicMock()
+        mock_session.query.return_value.filter_by.return_value.count.return_value = 0
+        mock_thread = MagicMock()
+        mock_thread.ident = 9999
+
+        def ctx_side_effect(*args, **kwargs):
+            cm = MagicMock()
+            cm.__enter__ = MagicMock(return_value=mock_session)
+            cm.__exit__ = MagicMock(return_value=False)
+            return cm
+
+        call_count = {"n": 0}
+
+        def ctx_raiser(*args, **kwargs):
+            call_count["n"] += 1
+            cm = MagicMock()
+            if call_count["n"] == 1:
+                # First call: active record creation succeeds
+                cm.__enter__ = MagicMock(return_value=mock_session)
+            else:
+                # Second call: thread-id update fails
+                cm.__enter__ = MagicMock(side_effect=RuntimeError("tid"))
+            cm.__exit__ = MagicMock(return_value=False)
+            return cm
+
+        with (
+            patch(f"{MODULE}.get_user_db_session", side_effect=ctx_raiser),
+            patch(f"{MODULE}.start_research_process", return_value=mock_thread),
+            patch(f"{MODULE}.UserQueueService"),
+            patch(f"{MODULE}.UserActiveResearch"),
+            patch(f"{MODULE}.logger"),
+        ):
+            # Should complete without raising even though thread-id update failed
+            proc._start_research_directly("alice", "r1", "secret", query="q")
+
+    def test_duplicate_research_error_leaves_state_intact(self):
+        """When start_research_process raises DuplicateResearchError, the
+        active record must NOT be deleted and the ResearchHistory row must
+        NOT be marked FAILED — that state belongs to the live thread that
+        already owns this research_id. Mutating it would terminate a
+        running thread from the user's perspective."""
+        from local_deep_research.database.models import (
+            ResearchHistory,
+            UserActiveResearch,
+        )
+        from local_deep_research.exceptions import DuplicateResearchError
+
+        proc = _make_processor()
+
+        active_record = MagicMock()
+        research_row = MagicMock()
+        initial_status = MagicMock()
+        research_row.status = initial_status
+
+        mock_session = MagicMock()
+
+        def _query(model):
+            q = MagicMock()
+            q.filter_by.return_value = q
+            q.filter.return_value = q
+            if model is UserActiveResearch:
+                q.first.return_value = active_record
+            elif model is ResearchHistory:
+                q.first.return_value = research_row
+            else:
+                q.first.return_value = None
+            return q
+
+        mock_session.query.side_effect = _query
+
+        def ctx_side_effect(*args, **kwargs):
+            cm = MagicMock()
+            cm.__enter__ = MagicMock(return_value=mock_session)
+            cm.__exit__ = MagicMock(return_value=False)
+            return cm
+
+        with (
+            patch(f"{MODULE}.get_user_db_session", side_effect=ctx_side_effect),
+            patch(
+                f"{MODULE}.start_research_process",
+                side_effect=DuplicateResearchError(
+                    "research r1 already has a live thread"
+                ),
+            ),
+            patch(f"{MODULE}.UserQueueService"),
+            patch(f"{MODULE}.logger"),
+        ):
+            proc._start_research_directly("alice", "r1", "secret", query="q")
+
+        # Critical invariants: no delete of active record, no status
+        # mutation on ResearchHistory. The live thread owns that state.
+        mock_session.delete.assert_not_called()
+        assert research_row.status is initial_status
+
+    def test_start_research_process_exception_cleans_up_active_record(self):
+        """When start_research_process raises the active record should be
+        deleted AND the ResearchHistory row marked FAILED."""
+        from local_deep_research.constants import ResearchStatus
+        from local_deep_research.database.models import (
+            ResearchHistory,
+            UserActiveResearch,
+        )
+
+        proc = _make_processor()
+
+        active_record = MagicMock()
+        research_row = MagicMock()
+
+        mock_session = MagicMock()
+
+        def _query(model):
+            q = MagicMock()
+            q.filter_by.return_value = q
+            q.filter.return_value = q
+            if model is UserActiveResearch:
+                q.first.return_value = active_record
+            elif model is ResearchHistory:
+                q.first.return_value = research_row
+            else:
+                q.first.return_value = None
+            return q
+
+        mock_session.query.side_effect = _query
+
+        def ctx_side_effect(*args, **kwargs):
+            cm = MagicMock()
+            cm.__enter__ = MagicMock(return_value=mock_session)
+            cm.__exit__ = MagicMock(return_value=False)
+            return cm
+
+        with (
+            patch(f"{MODULE}.get_user_db_session", side_effect=ctx_side_effect),
+            patch(
+                f"{MODULE}.start_research_process",
+                side_effect=RuntimeError("thread error"),
+            ),
+            patch(f"{MODULE}.UserQueueService"),
+            patch(f"{MODULE}.logger"),
+        ):
+            proc._start_research_directly("alice", "r1", "secret", query="q")
+
+        # Cleanup path: delete active record AND mark research FAILED.
+        mock_session.delete.assert_called_once_with(active_record)
+        assert research_row.status == ResearchStatus.FAILED
+        mock_session.commit.assert_called()
+
+
+# ---------------------------------------------------------------------------
+# notify_research_completed / notify_research_failed – outer exception
+# ---------------------------------------------------------------------------
+
+
+class TestNotifyCompletedFailedExceptions:
+    def test_notify_completed_outer_exception_swallowed(self):
+        """Exception in get_user_db_session must not propagate from notify_research_completed."""
+        proc = _make_processor()
+
+        with (
+            patch(
+                f"{MODULE}.get_user_db_session",
+                side_effect=RuntimeError("boom"),
+            ),
+            patch(f"{MODULE}.logger"),
+        ):
+            proc.notify_research_completed("alice", "r1", user_password="pw")
+
+    def test_notify_failed_outer_exception_swallowed(self):
+        """Exception in get_user_db_session must not propagate from notify_research_failed."""
+        proc = _make_processor()
+
+        with (
+            patch(
+                f"{MODULE}.get_user_db_session",
+                side_effect=RuntimeError("boom"),
+            ),
+            patch(f"{MODULE}.logger"),
+        ):
+            proc.notify_research_failed(
+                "alice", "r1", error_message="oops", user_password="pw"
+            )
+
+
+# ---------------------------------------------------------------------------
+# _process_queue_loop – cleanup_dead_threads periodic sweep + import error
+# ---------------------------------------------------------------------------
+
+
+class TestProcessQueueLoopCleanup:
+    def test_cleanup_dead_threads_called_every_6_iterations(self):
+        """After 6 loop iterations cleanup_dead_threads should have been called once."""
+        proc = _make_processor()
+        proc.running = True
+
+        call_counts = {"iterations": 0}
+
+        def fake_wait(t):
+            call_counts["iterations"] += 1
+            if call_counts["iterations"] >= 7:
+                proc.running = False
+
+        mock_cleanup_current = MagicMock()
+        mock_cleanup_dead = MagicMock()
+
+        with (
+            patch.object(proc._stop_event, "wait", side_effect=fake_wait),
+            patch(f"{MODULE}.logger"),
+            patch.object(proc, "_users_to_check", set()),
+            patch(
+                f"{MODULE}.cleanup_current_thread",
+                mock_cleanup_current,
+                create=True,
+            ),
+        ):
+            # Patch the import inside the finally block
+            with patch.dict(
+                "sys.modules",
+                {
+                    "local_deep_research.database.thread_local_session": MagicMock(
+                        cleanup_current_thread=mock_cleanup_current,
+                        cleanup_dead_threads=mock_cleanup_dead,
+                    )
+                },
+            ):
+                proc._process_queue_loop()
+
+        # cleanup_dead_threads should have been invoked at least once (at iteration 6)
+        assert mock_cleanup_dead.call_count >= 1
+
+    def test_cleanup_import_error_swallowed(self):
+        """ImportError in the finally cleanup block must not crash the loop."""
+        proc = _make_processor()
+        proc.running = True
+
+        call_counts = {"n": 0}
+
+        def fake_wait(t):
+            call_counts["n"] += 1
+            if call_counts["n"] >= 2:
+                proc.running = False
+
+        with (
+            patch.object(proc._stop_event, "wait", side_effect=fake_wait),
+            patch(f"{MODULE}.logger"),
+            patch.object(proc, "_users_to_check", set()),
+            patch.dict(
+                "sys.modules",
+                {"local_deep_research.database.thread_local_session": None},
+            ),
+        ):
+            # Should not raise
+            proc._process_queue_loop()
+
+
+# ---------------------------------------------------------------------------
+# _process_user_queue – engine=None and outer exception
+# ---------------------------------------------------------------------------
+
+
+class TestProcessUserQueueEdgeCases:
+    def test_engine_none_returns_false(self):
+        """When db_manager.open_user_database returns None the method returns False."""
+        proc = _make_processor()
+
+        with (
+            patch(f"{MODULE}.session_password_store") as mock_store,
+            patch(f"{MODULE}.db_manager") as mock_db,
+            patch(f"{MODULE}.logger"),
+        ):
+            mock_store.get_session_password.return_value = "pw"
+            mock_db.open_user_database.return_value = None
+
+            result = proc._process_user_queue("alice", "sess1")
+
+        assert result is False
+
+    def test_outer_exception_returns_false(self):
+        """Unexpected exception in _process_user_queue must return False."""
+        proc = _make_processor()
+
+        with (
+            patch(f"{MODULE}.session_password_store") as mock_store,
+            patch(f"{MODULE}.db_manager") as mock_db,
+            patch(
+                f"{MODULE}.get_user_db_session", side_effect=RuntimeError("db")
+            ),
+            patch(f"{MODULE}.logger"),
+        ):
+            mock_store.get_session_password.return_value = "pw"
+            mock_db.open_user_database.return_value = MagicMock()
+
+            result = proc._process_user_queue("alice", "sess1")
+
+        assert result is False
+
+
+# ---------------------------------------------------------------------------
+# _start_research – research not found raises ValueError
+# ---------------------------------------------------------------------------
+
+
+class TestStartResearchNotFound:
+    def test_research_not_found_raises_value_error(self):
+        """_start_research must raise ValueError when research record is missing."""
+        proc = _make_processor()
+
+        mock_session = MagicMock()
+        mock_session.query.return_value.filter_by.return_value.first.return_value = None
+
+        queued = MagicMock()
+        queued.research_id = "r99"
+        queued.settings_snapshot = None
+
+        with pytest.raises(ValueError, match="r99"):
+            proc._start_research(mock_session, "alice", "pw", queued)
+
+
+# ---------------------------------------------------------------------------
+# process_pending_operations_for_user – rollback failure + error_update with report_path
+# ---------------------------------------------------------------------------
+
+
+class TestProcessPendingOperationsEdgeCases:
+    def test_rollback_failure_after_operation_error_is_swallowed(self):
+        """If both the operation AND its rollback raise the method must not crash."""
+        proc = _make_processor()
+
+        proc.pending_operations["op1"] = {
+            "username": "alice",
+            "operation_type": "progress_update",
+            "research_id": "r1",
+            "progress": 50.0,
+        }
+
+        mock_session = MagicMock()
+        mock_session.query.return_value.filter_by.return_value.first.return_value = MagicMock()
+        mock_session.commit.side_effect = RuntimeError("commit failed")
+        mock_session.rollback.side_effect = RuntimeError("rollback failed too")
+
+        with patch(f"{MODULE}.logger"):
+            count = proc.process_pending_operations_for_user(
+                "alice", mock_session
+            )
+
+        # processed_count stays 0 because commit raised before increment
+        assert count == 0
+
+    def test_error_update_with_report_path_sets_report_path(self):
+        """error_update operation sets report_path on the research record."""
+        proc = _make_processor()
+
+        proc.pending_operations["op2"] = {
+            "username": "alice",
+            "operation_type": "error_update",
+            "research_id": "r2",
+            "status": "failed",
+            "error_message": "Something went wrong",
+            "metadata": {"key": "val"},
+            "completed_at": "2026-03-18T00:00:00",
+            "report_path": "/reports/r2.html",
+        }
+
+        mock_research = MagicMock()
+        mock_session = MagicMock()
+        mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research
+
+        with patch(f"{MODULE}.logger"):
+            count = proc.process_pending_operations_for_user(
+                "alice", mock_session
+            )
+
+        assert count == 1
+        assert mock_research.report_path == "/reports/r2.html"
+
+    def test_no_pending_operations_returns_zero(self):
+        """When there are no pending operations for the user return 0 immediately."""
+        proc = _make_processor()
+        # Operations exist for a different user
+        proc.pending_operations["op3"] = {
+            "username": "bob",
+            "operation_type": "progress_update",
+            "research_id": "r3",
+            "progress": 10.0,
+        }
+
+        mock_session = MagicMock()
+        count = proc.process_pending_operations_for_user("alice", mock_session)
+
+        assert count == 0
+        # bob's operation must still be there
+        assert "op3" in proc.pending_operations