learningcircuit--local-deep-research/tests/web/queue/test_processor_v2_real.py

"""
Tests for QueueProcessorV2 — real integration tests for core queue orchestration.

These tests exercise the actual class methods with proper mocking of external
dependencies (database, password store, research service), unlike the fake tests
in test_processor_v2_core/operations/research.py that only test Python primitives.

Tests cover:
- notify_research_queued: direct vs queue mode branching
- notify_research_completed: status updates + notifications
- notify_research_failed: error message defaulting, sanitization
- _process_queue_loop: user:session parsing, removal logic
- _process_user_queue: slot calculation, empty queue, no password
- _start_research: new vs legacy settings format
- process_pending_operations_for_user: progress/error updates, rollback
"""

import time
import threading
from unittest.mock import Mock, MagicMock, patch

import pytest


MODULE = "local_deep_research.web.queue.processor_v2"
SETTINGS_MGR = "local_deep_research.settings.manager.SettingsManager"


def _make_processor():
    """Create a QueueProcessorV2 instance without triggering module-level side effects."""
    from local_deep_research.web.queue.processor_v2 import QueueProcessorV2

    return QueueProcessorV2(check_interval=1)


# ---------------------------------------------------------------------------
# notify_research_queued — direct vs queue mode branching
# ---------------------------------------------------------------------------


class TestNotifyResearchQueuedDirectMode:
    """Tests for notify_research_queued when queue_mode='direct'."""

    @patch(f"{MODULE}.get_user_db_session")
    @patch(f"{MODULE}.db_manager")
    @patch(f"{MODULE}.session_password_store")
    def test_direct_mode_slots_available_starts_directly(
        self, mock_pw_store, mock_db_mgr, mock_get_session
    ):
        """direct mode + slots available -> calls _start_research_directly."""
        processor = _make_processor()

        mock_pw_store.get_session_password.return_value = "pw"
        mock_db_mgr.open_user_database.return_value = Mock()

        mock_session = MagicMock()
        mock_session.__enter__ = Mock(return_value=mock_session)
        mock_session.__exit__ = Mock(return_value=False)
        mock_get_session.return_value = mock_session

        # SettingsManager says direct mode, max 3 concurrent
        mock_settings = Mock()
        mock_settings.get_setting.side_effect = lambda key, default: {
            "app.queue_mode": "direct",
            "app.max_concurrent_researches": 3,
        }.get(key, default)

        # 0 active researches -> has slots
        mock_session.query.return_value.filter_by.return_value.count.return_value = 0

        with patch(SETTINGS_MGR, return_value=mock_settings):
            with patch.object(
                processor, "_start_research_directly"
            ) as mock_start:
                processor.notify_research_queued(
                    "alice",
                    "r-001",
                    session_id="sess1",
                    query="test query",
                )
                mock_start.assert_called_once()

    @patch(f"{MODULE}.get_user_db_session")
    @patch(f"{MODULE}.db_manager")
    @patch(f"{MODULE}.session_password_store")
    def test_direct_mode_slots_full_falls_back_to_queue(
        self, mock_pw_store, mock_db_mgr, mock_get_session
    ):
        """direct mode + slots full -> queues instead."""
        processor = _make_processor()

        mock_pw_store.get_session_password.return_value = "pw"
        mock_db_mgr.open_user_database.return_value = Mock()

        mock_session = MagicMock()
        mock_session.__enter__ = Mock(return_value=mock_session)
        mock_session.__exit__ = Mock(return_value=False)
        mock_get_session.return_value = mock_session

        mock_settings = Mock()
        mock_settings.get_setting.side_effect = lambda key, default: {
            "app.queue_mode": "direct",
            "app.max_concurrent_researches": 2,
        }.get(key, default)

        # 2 active = full at max 2
        mock_session.query.return_value.filter_by.return_value.count.return_value = 2

        with patch(SETTINGS_MGR, return_value=mock_settings):
            with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
                mock_qs = Mock()
                mock_qs_class.return_value = mock_qs

                processor.notify_research_queued(
                    "alice",
                    "r-002",
                    session_id="sess1",
                    query="test",
                )
                mock_qs.add_task_metadata.assert_called_once()

    @patch(f"{MODULE}.get_user_db_session")
    @patch(f"{MODULE}.db_manager")
    @patch(f"{MODULE}.session_password_store")
    def test_queue_mode_always_queues(
        self, mock_pw_store, mock_db_mgr, mock_get_session
    ):
        """queue_mode='queue' -> always queues regardless of slots."""
        processor = _make_processor()

        mock_pw_store.get_session_password.return_value = "pw"
        mock_db_mgr.open_user_database.return_value = Mock()

        mock_session = MagicMock()
        mock_session.__enter__ = Mock(return_value=mock_session)
        mock_session.__exit__ = Mock(return_value=False)
        mock_get_session.return_value = mock_session

        mock_settings = Mock()
        mock_settings.get_setting.side_effect = lambda key, default: {
            "app.queue_mode": "queue",
            "app.max_concurrent_researches": 3,
        }.get(key, default)

        with patch(SETTINGS_MGR, return_value=mock_settings):
            with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
                mock_qs = Mock()
                mock_qs_class.return_value = mock_qs

                with patch.object(
                    processor, "_start_research_directly"
                ) as mock_start:
                    processor.notify_research_queued(
                        "alice",
                        "r-003",
                        session_id="sess1",
                        query="test",
                    )
                    mock_start.assert_not_called()
                    mock_qs.add_task_metadata.assert_called_once()

    @patch(f"{MODULE}.get_user_db_session")
    def test_missing_session_id_falls_back_to_queue(self, mock_get_session):
        """No session_id kwarg -> falls back to queue mode."""
        processor = _make_processor()

        mock_session = MagicMock()
        mock_session.__enter__ = Mock(return_value=mock_session)
        mock_session.__exit__ = Mock(return_value=False)
        mock_get_session.return_value = mock_session

        with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
            mock_qs = Mock()
            mock_qs_class.return_value = mock_qs

            # No session_id in kwargs
            processor.notify_research_queued("alice", "r-004", query="test")
            mock_qs.add_task_metadata.assert_called_once()

    @patch(f"{MODULE}.get_user_db_session")
    @patch(f"{MODULE}.session_password_store")
    def test_missing_password_falls_back_to_queue(
        self, mock_pw_store, mock_get_session
    ):
        """No password for session -> falls back to queue mode."""
        processor = _make_processor()

        mock_pw_store.get_session_password.return_value = None

        mock_session = MagicMock()
        mock_session.__enter__ = Mock(return_value=mock_session)
        mock_session.__exit__ = Mock(return_value=False)
        mock_get_session.return_value = mock_session

        with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
            mock_qs = Mock()
            mock_qs_class.return_value = mock_qs

            processor.notify_research_queued(
                "alice", "r-005", session_id="sess1", query="test"
            )
            mock_qs.add_task_metadata.assert_called_once()

    @patch(f"{MODULE}.get_user_db_session")
    @patch(f"{MODULE}.db_manager")
    @patch(f"{MODULE}.session_password_store")
    def test_exception_in_direct_falls_back_to_queue(
        self, mock_pw_store, mock_db_mgr, mock_get_session
    ):
        """Exception during direct execution -> falls back to queue."""
        processor = _make_processor()

        mock_pw_store.get_session_password.return_value = "pw"
        mock_db_mgr.open_user_database.side_effect = RuntimeError("DB error")

        mock_session = MagicMock()
        mock_session.__enter__ = Mock(return_value=mock_session)
        mock_session.__exit__ = Mock(return_value=False)
        mock_get_session.return_value = mock_session

        with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
            mock_qs = Mock()
            mock_qs_class.return_value = mock_qs

            processor.notify_research_queued(
                "alice", "r-006", session_id="sess1", query="test"
            )
            mock_qs.add_task_metadata.assert_called_once()

    @patch(f"{MODULE}.get_user_db_session")
    def test_no_kwargs_queues_directly(self, mock_get_session):
        """No kwargs at all -> goes straight to queue fallback."""
        processor = _make_processor()

        mock_session = MagicMock()
        mock_session.__enter__ = Mock(return_value=mock_session)
        mock_session.__exit__ = Mock(return_value=False)
        mock_get_session.return_value = mock_session

        with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
            mock_qs = Mock()
            mock_qs_class.return_value = mock_qs

            processor.notify_research_queued("alice", "r-007")
            mock_qs.add_task_metadata.assert_called_once()


# ---------------------------------------------------------------------------
# notify_research_completed — updates status + sends notification
# ---------------------------------------------------------------------------


class TestNotifyResearchCompleted:
    """Tests for notify_research_completed."""

    @patch(f"{MODULE}.send_research_completed_notification_from_session")
    @patch(f"{MODULE}.get_user_db_session")
    def test_updates_status_and_sends_notification(
        self, mock_get_session, mock_send_notif
    ):
        """Updates queue status to COMPLETED and sends notification."""
        processor = _make_processor()

        mock_session = MagicMock()
        mock_session.__enter__ = Mock(return_value=mock_session)
        mock_session.__exit__ = Mock(return_value=False)
        mock_get_session.return_value = mock_session

        with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
            mock_qs = Mock()
            mock_qs_class.return_value = mock_qs

            processor.notify_research_completed(
                "alice", "r-010", user_password="pw"
            )

            mock_qs.update_task_status.assert_called_once()
            mock_send_notif.assert_called_once_with(
                username="alice",
                research_id="r-010",
                db_session=mock_session,
            )

    @patch(f"{MODULE}.get_user_db_session")
    def test_exception_does_not_propagate(self, mock_get_session):
        """Exception in status update is caught silently."""
        processor = _make_processor()

        mock_get_session.side_effect = RuntimeError("DB unavailable")

        # Should not raise
        processor.notify_research_completed("alice", "r-011")

    @patch(f"{MODULE}.send_research_completed_notification_from_session")
    @patch(f"{MODULE}.get_user_db_session")
    def test_passes_password_to_session(
        self, mock_get_session, mock_send_notif
    ):
        """User password is passed to get_user_db_session."""
        processor = _make_processor()

        mock_session = MagicMock()
        mock_session.__enter__ = Mock(return_value=mock_session)
        mock_session.__exit__ = Mock(return_value=False)
        mock_get_session.return_value = mock_session

        with patch(f"{MODULE}.UserQueueService"):
            processor.notify_research_completed(
                "bob", "r-012", user_password="secret"
            )

        mock_get_session.assert_called_once_with("bob", "secret")


# ---------------------------------------------------------------------------
# notify_research_failed — error message defaulting, sanitization
# ---------------------------------------------------------------------------


class TestNotifyResearchFailed:
    """Tests for notify_research_failed."""

    @patch(f"{MODULE}.send_research_failed_notification_from_session")
    @patch(f"{MODULE}.get_user_db_session")
    def test_sends_error_message_to_queue_service(
        self, mock_get_session, mock_send_notif
    ):
        """Passes error_message to queue service update."""
        processor = _make_processor()

        mock_session = MagicMock()
        mock_session.__enter__ = Mock(return_value=mock_session)
        mock_session.__exit__ = Mock(return_value=False)
        mock_get_session.return_value = mock_session

        with patch(f"{MODULE}.UserQueueService") as mock_qs_class:
            mock_qs = Mock()
            mock_qs_class.return_value = mock_qs

            processor.notify_research_failed(
                "alice", "r-020", error_message="OOM", user_password="pw"
            )

            mock_qs.update_task_status.assert_called_once()
            args, kwargs = mock_qs.update_task_status.call_args
            assert (
                kwargs.get("error_message") == "OOM" or args[2]
                if len(args) > 2
                else True
            )

    @patch(f"{MODULE}.send_research_failed_notification_from_session")
    @patch(f"{MODULE}.get_user_db_session")
    def test_defaults_error_message_to_unknown(
        self, mock_get_session, mock_send_notif
    ):
        """None error_message defaults to 'Unknown error' for notification."""
        processor = _make_processor()

        mock_session = MagicMock()
        mock_session.__enter__ = Mock(return_value=mock_session)
        mock_session.__exit__ = Mock(return_value=False)
        mock_get_session.return_value = mock_session

        with patch(f"{MODULE}.UserQueueService"):
            processor.notify_research_failed(
                "alice", "r-021", user_password="pw"
            )

        mock_send_notif.assert_called_once_with(
            username="alice",
            research_id="r-021",
            error_message="Unknown error",
            db_session=mock_session,
        )

    @patch(f"{MODULE}.get_user_db_session")
    def test_exception_does_not_propagate(self, mock_get_session):
        """Exception in failure handling is caught."""
        processor = _make_processor()

        mock_get_session.side_effect = RuntimeError("DB error")

        # Should not raise
        processor.notify_research_failed("alice", "r-022", error_message="bad")


# ---------------------------------------------------------------------------
# _process_queue_loop — user:session parsing, removal logic
# ---------------------------------------------------------------------------


class TestProcessQueueLoop:
    """Tests for _process_queue_loop."""

    def test_parses_user_session_format(self):
        """Correctly parses (username, session_id) tuple entries."""
        processor = _make_processor()
        processor._users_to_check.add(("alice", "sess-abc"))

        with patch.object(
            processor, "_process_user_queue", return_value=True
        ) as mock_pq:
            # Run one iteration then stop
            processor.running = True

            def stop_after_one():
                time.sleep(0.05)
                processor.running = False

            threading.Thread(target=stop_after_one).start()
            processor._process_queue_loop()

            mock_pq.assert_called_with("alice", "sess-abc")

    def test_removes_users_with_empty_queues(self):
        """Users whose queues are empty get removed from check set."""
        processor = _make_processor()
        processor._users_to_check.add(("alice", "sess1"))
        processor._users_to_check.add(("bob", "sess2"))

        def side_effect(username, session_id):
            return username == "alice"  # alice's queue is empty

        with patch.object(
            processor, "_process_user_queue", side_effect=side_effect
        ):
            processor.running = True

            def stop_after_one():
                time.sleep(0.05)
                processor.running = False

            threading.Thread(target=stop_after_one).start()
            processor._process_queue_loop()

        assert ("alice", "sess1") not in processor._users_to_check
        assert ("bob", "sess2") in processor._users_to_check

    def test_keeps_users_on_error(self):
        """Users are kept if _process_user_queue raises."""
        processor = _make_processor()
        processor._users_to_check.add(("alice", "sess1"))

        with patch.object(
            processor,
            "_process_user_queue",
            side_effect=RuntimeError("transient"),
        ):
            processor.running = True

            def stop_after_one():
                time.sleep(0.05)
                processor.running = False

            threading.Thread(target=stop_after_one).start()
            processor._process_queue_loop()

        assert ("alice", "sess1") in processor._users_to_check


# ---------------------------------------------------------------------------
# _process_user_queue — slot calculation, empty queue, no password
# ---------------------------------------------------------------------------


class TestProcessUserQueue:
    """Tests for _process_user_queue."""

    @patch(f"{MODULE}.session_password_store")
    def test_no_password_returns_true(self, mock_pw_store):
        """No password -> returns True (remove from checking)."""
        processor = _make_processor()
        mock_pw_store.get_session_password.return_value = None

        result = processor._process_user_queue("alice", "sess1")
        assert result is True

    @patch(f"{MODULE}.get_user_db_session")
    @patch(f"{MODULE}.db_manager")
    @patch(f"{MODULE}.session_password_store")
    def test_empty_queue_returns_true(
        self, mock_pw_store, mock_db_mgr, mock_get_session
    ):
        """Empty queue -> returns True."""
        processor = _make_processor()

        mock_pw_store.get_session_password.return_value = "pw"
        mock_db_mgr.open_user_database.return_value = Mock()

        mock_session = MagicMock()
        mock_session.__enter__ = Mock(return_value=mock_session)
        mock_session.__exit__ = Mock(return_value=False)
        mock_get_session.return_value = mock_session

        mock_settings = Mock()
        mock_settings.get_setting.return_value = 3

        mock_qs = Mock()
        mock_qs.get_queue_status.return_value = {
            "active_tasks": 0,
            "queued_tasks": 0,
        }

        with patch(SETTINGS_MGR, return_value=mock_settings):
            with patch(f"{MODULE}.UserQueueService", return_value=mock_qs):
                result = processor._process_user_queue("alice", "sess1")

        assert result is True

    @patch(f"{MODULE}.get_user_db_session")
    @patch(f"{MODULE}.db_manager")
    @patch(f"{MODULE}.session_password_store")
    def test_no_slots_available_returns_false(
        self, mock_pw_store, mock_db_mgr, mock_get_session
    ):
        """No available slots -> returns False (keep checking)."""
        processor = _make_processor()

        mock_pw_store.get_session_password.return_value = "pw"
        mock_db_mgr.open_user_database.return_value = Mock()

        mock_session = MagicMock()
        mock_session.__enter__ = Mock(return_value=mock_session)
        mock_session.__exit__ = Mock(return_value=False)
        mock_get_session.return_value = mock_session

        mock_settings = Mock()
        mock_settings.get_setting.return_value = 2

        mock_qs = Mock()
        mock_qs.get_queue_status.return_value = {
            "active_tasks": 2,
            "queued_tasks": 1,
        }

        with patch(SETTINGS_MGR, return_value=mock_settings):
            with patch(f"{MODULE}.UserQueueService", return_value=mock_qs):
                result = processor._process_user_queue("alice", "sess1")

        assert result is False

    @patch(f"{MODULE}.db_manager")
    @patch(f"{MODULE}.session_password_store")
    def test_failed_db_open_returns_false(self, mock_pw_store, mock_db_mgr):
        """Failed database open -> returns False (keep checking)."""
        processor = _make_processor()

        mock_pw_store.get_session_password.return_value = "pw"
        mock_db_mgr.open_user_database.return_value = None

        result = processor._process_user_queue("alice", "sess1")
        assert result is False


# ---------------------------------------------------------------------------
# _start_research — new vs legacy settings format
# ---------------------------------------------------------------------------


class TestStartResearch:
    """Tests for _start_research."""

    @patch(f"{MODULE}.start_research_process")
    def test_new_format_extracts_submission_and_settings(
        self, mock_start_research
    ):
        """New format: {submission: {...}, settings_snapshot: {...}}."""
        processor = _make_processor()

        mock_thread = Mock()
        mock_thread.ident = 12345
        mock_start_research.return_value = mock_thread

        mock_session = MagicMock()
        mock_research = Mock()
        mock_research.status = "queued"
        mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research

        queued = Mock()
        queued.research_id = "r-100"
        queued.query = "test query"
        queued.mode = "standard"
        queued.settings_snapshot = {
            "submission": {
                "model_provider": "openai",
                "model": "gpt-4",
                "strategy": "graph-based",
            },
            "settings_snapshot": {"setting_a": "value_a"},
        }

        with patch(f"{MODULE}.UserActiveResearch"):
            processor._start_research(mock_session, "alice", "pw", queued)

        # Verify start_research_process was called with new-format extracted params
        call_kwargs = mock_start_research.call_args.kwargs
        assert call_kwargs["model_provider"] == "openai"
        assert call_kwargs["model"] == "gpt-4"
        assert call_kwargs["strategy"] == "graph-based"
        assert call_kwargs["settings_snapshot"] == {"setting_a": "value_a"}

    @patch(f"{MODULE}.start_research_process")
    def test_legacy_format_uses_flat_dict(self, mock_start_research):
        """Legacy format: flat dict used as submission_params."""
        from local_deep_research.constants import ResearchStatus

        processor = _make_processor()

        mock_thread = Mock()
        mock_thread.ident = 12345
        mock_start_research.return_value = mock_thread

        mock_session = MagicMock()
        mock_research = Mock()
        mock_research.status = ResearchStatus.QUEUED
        mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research

        queued = Mock()
        queued.research_id = "r-101"
        queued.query = "legacy query"
        queued.mode = "standard"
        queued.settings_snapshot = {
            "model_provider": "anthropic",
            "model": "claude-3",
        }

        with patch(f"{MODULE}.UserActiveResearch"):
            processor._start_research(mock_session, "alice", "pw", queued)

        call_kwargs = mock_start_research.call_args.kwargs
        assert call_kwargs["model_provider"] == "anthropic"
        assert call_kwargs["model"] == "claude-3"
        assert call_kwargs["settings_snapshot"] == {}

    @patch(f"{MODULE}.start_research_process")
    def test_legacy_format_seeds_search_tool_from_search_engine(
        self, mock_start_research
    ):
        """A legacy-flat row that carries a search_engine seeds the (otherwise
        empty) settings_snapshot with search.tool, so the worker's egress build
        (resolve_run_primary_engine) doesn't fail closed on an empty snapshot
        and refuse a replayed pre-upgrade queued run."""
        from local_deep_research.constants import ResearchStatus

        processor = _make_processor()
        mock_thread = Mock()
        mock_thread.ident = 1
        mock_start_research.return_value = mock_thread

        mock_session = MagicMock()
        mock_research = Mock()
        mock_research.status = ResearchStatus.QUEUED
        mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research

        queued = Mock()
        queued.research_id = "r-102"
        queued.query = "legacy query"
        queued.mode = "standard"
        queued.settings_snapshot = {
            "model_provider": "anthropic",
            "search_engine": "pubmed",
        }

        with patch(f"{MODULE}.UserActiveResearch"):
            processor._start_research(mock_session, "alice", "pw", queued)

        call_kwargs = mock_start_research.call_args.kwargs
        assert call_kwargs["settings_snapshot"] == {"search.tool": "pubmed"}
        assert call_kwargs["search_engine"] == "pubmed"

    @patch(f"{MODULE}.start_research_process")
    def test_none_settings_snapshot_handled(self, mock_start_research):
        """None settings_snapshot treated as empty dict."""
        from local_deep_research.constants import ResearchStatus

        processor = _make_processor()

        mock_thread = Mock()
        mock_thread.ident = 12345
        mock_start_research.return_value = mock_thread

        mock_session = MagicMock()
        mock_research = Mock()
        mock_research.status = ResearchStatus.QUEUED
        mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research

        queued = Mock()
        queued.research_id = "r-102"
        queued.query = "test"
        queued.mode = "standard"
        queued.settings_snapshot = None

        with patch(f"{MODULE}.UserActiveResearch"):
            processor._start_research(mock_session, "alice", "pw", queued)

        call_kwargs = mock_start_research.call_args.kwargs
        assert call_kwargs["settings_snapshot"] == {}

    def test_research_not_found_raises(self):
        """Raises ValueError when research record not found."""
        processor = _make_processor()

        mock_session = MagicMock()
        mock_session.query.return_value.filter_by.return_value.first.return_value = None

        queued = Mock()
        queued.research_id = "r-999"
        queued.settings_snapshot = {}

        with pytest.raises(ValueError, match="not found"):
            processor._start_research(mock_session, "alice", "pw", queued)

    @patch(f"{MODULE}.start_research_process")
    def test_status_set_in_progress_before_spawn(self, mock_start_research):
        """IN_PROGRESS must be committed BEFORE start_research_process is
        called, so a fast thread cannot race ahead and have its terminal
        status overwritten by our post-spawn commit."""
        from local_deep_research.constants import ResearchStatus

        processor = _make_processor()

        mock_thread = Mock()
        mock_thread.ident = 999
        mock_start_research.return_value = mock_thread

        mock_session = MagicMock()
        mock_research = Mock()
        mock_research.status = ResearchStatus.QUEUED
        mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research

        # Record the status at the moment start_research_process is called.
        status_at_spawn = {}

        def _capture_status(*args, **kwargs):
            status_at_spawn["value"] = mock_research.status
            # Capture the commit count at the moment of spawn so we can
            # assert the IN_PROGRESS commit already happened, not merely
            # that some commit happened at some point in the method.
            status_at_spawn["commit_count"] = mock_session.commit.call_count
            return mock_thread

        mock_start_research.side_effect = _capture_status

        queued = Mock()
        queued.research_id = "r-race"
        queued.query = "q"
        queued.mode = "standard"
        queued.settings_snapshot = {}

        with patch(f"{MODULE}.UserActiveResearch"):
            processor._start_research(mock_session, "alice", "pw", queued)

        assert status_at_spawn["value"] == ResearchStatus.IN_PROGRESS
        # Exactly one commit (the IN_PROGRESS write) must have already
        # run by the time start_research_process is called.
        assert status_at_spawn["commit_count"] == 1

    @patch(f"{MODULE}.start_research_process")
    def test_spawn_failure_resets_status_to_queued(self, mock_start_research):
        """On genuine spawn failure (non-duplicate), status must be rolled
        back to QUEUED so the next retry sees a clean row and the
        research is not stuck IN_PROGRESS if the retry budget is
        eventually exhausted."""
        from local_deep_research.constants import ResearchStatus

        processor = _make_processor()

        mock_start_research.side_effect = RuntimeError("spawn failed")

        mock_session = MagicMock()
        mock_research = Mock()
        mock_research.status = ResearchStatus.QUEUED
        mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research

        queued = Mock()
        queued.research_id = "r-fail"
        queued.query = "q"
        queued.mode = "standard"
        queued.settings_snapshot = {}

        with pytest.raises(RuntimeError, match="spawn failed"):
            processor._start_research(mock_session, "alice", "pw", queued)

        assert mock_research.status == ResearchStatus.QUEUED
        # One commit for IN_PROGRESS before spawn, a second for the QUEUED
        # reset after spawn failed. Without the second commit the in-memory
        # attribute is rolled back but the DB still shows IN_PROGRESS, so
        # assert that both commits actually ran.
        assert mock_session.commit.call_count >= 2

    @patch(f"{MODULE}.start_research_process")
    def test_duplicate_research_error_does_not_reset_status(
        self, mock_start_research
    ):
        """DuplicateResearchError means a live thread already exists for
        this research_id (typically a retry after a post-spawn commit
        failure). Mutating status in that case would contradict the
        running thread — it must be left IN_PROGRESS."""
        from local_deep_research.constants import ResearchStatus
        from local_deep_research.exceptions import DuplicateResearchError

        processor = _make_processor()

        mock_start_research.side_effect = DuplicateResearchError(
            "thread already live"
        )

        mock_session = MagicMock()
        mock_research = Mock()
        mock_research.status = ResearchStatus.QUEUED
        mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research

        queued = Mock()
        queued.research_id = "r-dup"
        queued.query = "q"
        queued.mode = "standard"
        queued.settings_snapshot = {}

        with pytest.raises(DuplicateResearchError):
            processor._start_research(mock_session, "alice", "pw", queued)

        # Status must remain IN_PROGRESS (set before spawn, never reset).
        assert mock_research.status == ResearchStatus.IN_PROGRESS

    @patch(f"{MODULE}.start_research_process")
    def test_non_queued_status_raises_duplicate_without_spawning(
        self, mock_start_research
    ):
        """If _start_research is re-entered on a retry and finds the row
        already in a non-QUEUED state (IN_PROGRESS from a prior attempt
        whose post-spawn commit failed, or terminal COMPLETED/FAILED
        because the prior thread already finished and cleaned up), it
        must raise DuplicateResearchError *without* mutating status and
        *without* calling start_research_process. Otherwise we would
        overwrite a terminal status with IN_PROGRESS and spawn a second
        thread that re-runs the whole research."""
        from local_deep_research.constants import ResearchStatus
        from local_deep_research.exceptions import DuplicateResearchError

        for starting_status in (
            ResearchStatus.IN_PROGRESS,
            ResearchStatus.COMPLETED,
            ResearchStatus.FAILED,
            ResearchStatus.SUSPENDED,
        ):
            processor = _make_processor()
            mock_start_research.reset_mock()

            mock_session = MagicMock()
            mock_research = Mock()
            mock_research.status = starting_status
            mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research

            queued = Mock()
            queued.research_id = f"r-stale-{starting_status}"
            queued.query = "q"
            queued.mode = "standard"
            queued.settings_snapshot = {}

            with pytest.raises(DuplicateResearchError):
                processor._start_research(mock_session, "alice", "pw", queued)

            # Status unchanged — we did not overwrite a terminal state.
            assert mock_research.status == starting_status
            # Spawn never attempted — no risk of a second thread.
            mock_start_research.assert_not_called()
            # No commit for a status we did not change.
            mock_session.commit.assert_not_called()

    @patch(f"{MODULE}.UserActiveResearch")
    @patch(f"{MODULE}.start_research_process")
    def test_user_active_research_commit_failure_raises_duplicate(
        self, mock_start_research, mock_user_active_research
    ):
        """If the post-spawn UserActiveResearch commit fails, the thread
        is already running. We must raise DuplicateResearchError so the
        caller's dup branch deletes the queue row without bumping the
        spawn-retry counter — otherwise a commit failure at the retry
        limit would mark a LIVE thread as terminal FAILED."""
        from local_deep_research.constants import ResearchStatus
        from local_deep_research.exceptions import DuplicateResearchError

        processor = _make_processor()

        mock_thread = Mock()
        mock_thread.ident = 42
        mock_start_research.return_value = mock_thread

        mock_session = MagicMock()
        mock_research = Mock()
        mock_research.status = ResearchStatus.QUEUED
        mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research

        # Two commits: one for IN_PROGRESS (succeeds), one for
        # UserActiveResearch (fails).
        commit_results = [None, RuntimeError("UAR commit failed")]

        def _commit():
            r = commit_results.pop(0)
            if isinstance(r, Exception):
                raise r

        mock_session.commit.side_effect = _commit

        queued = Mock()
        queued.research_id = "r-uar-fail"
        queued.query = "q"
        queued.mode = "standard"
        queued.settings_snapshot = {}

        with pytest.raises(DuplicateResearchError):
            processor._start_research(mock_session, "alice", "pw", queued)

        # Spawn *was* called (that's how we get into the post-spawn commit).
        mock_start_research.assert_called_once()
        # Rollback happened after the commit failure.
        assert mock_session.rollback.called
        # Status must remain IN_PROGRESS — the thread is live, resetting
        # would contradict its own writes. This is the key invariant
        # that distinguishes this branch from the generic spawn-failure
        # branch (which DOES reset to QUEUED).
        assert mock_research.status == ResearchStatus.IN_PROGRESS
        # Regression guard: _start_research itself must not touch the
        # spawn-retry counter on the UAR-commit-failure path. The
        # counter is the caller's responsibility (the dup branch in
        # _start_queued_researches clears it); if anyone moves bump
        # logic into _start_research, this assertion catches it and
        # prevents a live thread from being marked terminal FAILED at
        # SPAWN_RETRY_LIMIT.
        assert "r-uar-fail" not in processor._spawn_retry_counts


# ---------------------------------------------------------------------------
# process_pending_operations_for_user — progress/error updates, rollback
# ---------------------------------------------------------------------------


class TestProcessPendingOperationsForUser:
    """Tests for process_pending_operations_for_user."""

    def test_progress_update_sets_progress(self):
        """Progress update operation sets research.progress."""
        processor = _make_processor()

        # Queue a progress update
        processor.queue_progress_update("alice", "r-200", 42.5)

        mock_session = MagicMock()
        mock_research = Mock()
        mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research

        count = processor.process_pending_operations_for_user(
            "alice", mock_session
        )

        assert count == 1
        assert mock_research.progress == 42.5
        mock_session.commit.assert_called()

    def test_error_update_sets_all_fields(self):
        """Error update sets status, error_message, metadata, completed_at."""
        processor = _make_processor()

        processor.queue_error_update(
            username="alice",
            research_id="r-201",
            status="failed",
            error_message="OOM killed",
            metadata={"retries": 3},
            completed_at="2026-01-01T00:00:00",
            report_path="/tmp/error.html",
        )

        mock_session = MagicMock()
        mock_research = Mock()
        mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research

        count = processor.process_pending_operations_for_user(
            "alice", mock_session
        )

        assert count == 1
        assert mock_research.status == "failed"
        assert mock_research.error_message == "OOM killed"
        assert mock_research.research_meta == {"retries": 3}
        assert mock_research.completed_at == "2026-01-01T00:00:00"
        assert mock_research.report_path == "/tmp/error.html"

    def test_no_operations_returns_zero(self):
        """No pending operations for user -> returns 0."""
        processor = _make_processor()

        mock_session = MagicMock()
        count = processor.process_pending_operations_for_user(
            "alice", mock_session
        )

        assert count == 0

    def test_operations_for_other_users_not_processed(self):
        """Only processes operations for the specified user."""
        processor = _make_processor()

        processor.queue_progress_update("bob", "r-300", 50.0)
        processor.queue_progress_update("alice", "r-301", 75.0)

        mock_session = MagicMock()
        mock_research = Mock()
        mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research

        count = processor.process_pending_operations_for_user(
            "alice", mock_session
        )

        assert count == 1
        assert mock_research.progress == 75.0
        # Bob's operation should still be pending
        assert len(processor.pending_operations) == 1

    def test_exception_triggers_rollback(self):
        """Exception during processing triggers session rollback."""
        processor = _make_processor()

        processor.queue_progress_update("alice", "r-400", 50.0)

        mock_session = MagicMock()
        mock_session.query.side_effect = RuntimeError("DB error")

        count = processor.process_pending_operations_for_user(
            "alice", mock_session
        )

        assert count == 0
        mock_session.rollback.assert_called_once()

    def test_error_update_without_report_path(self):
        """Error update without report_path doesn't set report_path."""
        processor = _make_processor()

        processor.queue_error_update(
            username="alice",
            research_id="r-401",
            status="suspended",
            error_message="Timeout",
            metadata={},
            completed_at="2026-01-01",
        )

        mock_session = MagicMock()
        mock_research = Mock(
            spec=[
                "status",
                "error_message",
                "research_meta",
                "completed_at",
                "report_path",
            ]
        )
        mock_session.query.return_value.filter_by.return_value.first.return_value = mock_research

        count = processor.process_pending_operations_for_user(
            "alice", mock_session
        )
        assert count == 1
        assert mock_research.status == "suspended"