chore: import upstream snapshot with attribution

tests/worker/conftest.py

@@ -0,0 +1,51 @@
+"""Fixtures for Celery worker smoke tests.
+
+These tests exercise the task *bodies* in ``application.worker`` against a
+real Postgres schema (via the ephemeral ``pg_conn`` fixture from the root
+``tests/conftest.py``). External I/O — storage, the embedding pipeline,
+the retriever, the LLM, the backend HTTP callback — is mocked, but every
+PG write is allowed to hit the real ephemeral DB so the assertions can
+read the resulting rows back.
+"""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import Iterator
+from unittest.mock import MagicMock
+
+import pytest
+from sqlalchemy import Connection
+
+
+@pytest.fixture
+def patch_worker_db(pg_conn, monkeypatch):
+    """Redirect ``db_session`` / ``db_readonly`` in ``application.worker``.
+
+    Both helpers yield the per-test transactional ``pg_conn``, so any
+    writes a task performs are visible to the test and roll back on
+    teardown. Without this patch the worker would open its own pooled
+    engine and punch past the per-test transaction.
+    """
+
+    @contextmanager
+    def _use_pg_conn() -> Iterator[Connection]:
+        yield pg_conn
+
+    monkeypatch.setattr("application.worker.db_session", _use_pg_conn)
+    monkeypatch.setattr("application.worker.db_readonly", _use_pg_conn)
+
+
+@pytest.fixture
+def task_self():
+    """Minimal stand-in for the Celery task ``self`` passed to workers.
+
+    ``update_state`` and ``request.retries`` are the only attributes the
+    worker bodies touch in our tests. Defaulting ``retries`` to 0 makes
+    every fixture instance behave like a fresh first attempt, which is
+    what the queued-event publishes are gated on. Tests covering the
+    retry path override this to a positive int.
+    """
+    task = MagicMock(name="celery_task_self")
+    task.request.retries = 0
+    return task

tests/worker/test_agent_workers.py

@@ -0,0 +1,348 @@
+"""Smoke tests for ``agent_webhook_worker`` (the shared headless runner).
+
+``agent_webhook_worker`` doesn't write to Postgres directly — it only
+*reads* the agent row. The concrete PG side-effect we assert is
+therefore a read: the task has to resolve the row from the ephemeral
+DB to proceed at all; if the lookup returned ``None`` the task would
+short-circuit with a "not found" error.
+
+The LLM, retriever, and agent factory are all stubbed — we only care
+that the PG read path wires up correctly.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from application.storage.db.repositories.agents import AgentsRepository
+
+
+@pytest.mark.unit
+class TestAgentWebhookWorker:
+    def test_resolves_agent_by_uuid_and_runs_logic(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+        from application.agents import headless_runner
+
+        agent = AgentsRepository(pg_conn).create(
+            user_id="alice",
+            name="hook-agent",
+            status="active",
+            agent_type="classic",
+            retriever="classic",
+            chunks=2,
+            key="sk-test-123",
+        )
+        agent_id = str(agent["id"])
+
+        # Capture the resolved agent_config + input; return a fake result.
+        captured: dict = {}
+
+        def _fake_run_agent_headless(agent_config, query, **kwargs):
+            captured["config"] = agent_config
+            captured["input"] = query
+            captured["kwargs"] = kwargs
+            return {
+                "answer": "ok",
+                "sources": [],
+                "tool_calls": [],
+                "thought": "",
+                "prompt_tokens": 0,
+                "generated_tokens": 0,
+                "denied": [],
+                "error_type": None,
+                "model_id": "fake",
+            }
+
+        monkeypatch.setattr(
+            headless_runner, "run_agent_headless", _fake_run_agent_headless,
+        )
+
+        result = worker.agent_webhook_worker(
+            task_self, agent_id, {"event": "ping"}
+        )
+
+        assert result == {
+            "status": "success",
+            "result": {"answer": "ok", "sources": [], "tool_calls": [], "thought": ""},
+        }
+        # The row pulled from PG is the one we seeded.
+        assert captured["config"]["name"] == "hook-agent"
+        assert str(captured["config"]["id"]) == agent_id
+        assert captured["input"] == '{"event": "ping"}'
+        # Webhook caller should pass endpoint='webhook'.
+        assert captured["kwargs"].get("endpoint") == "webhook"
+
+    def test_missing_agent_raises(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+        from application.agents import headless_runner
+
+        monkeypatch.setattr(
+            headless_runner, "run_agent_headless", lambda *a, **k: {},
+        )
+        with pytest.raises(ValueError, match="not found"):
+            worker.agent_webhook_worker(task_self, "no-such-agent", {})
+
+    def test_agent_webhook_worker_propagates_runtime_errors(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        """Headless runner errors must raise — a returned dict reads as success."""
+        from application import worker
+        from application.agents import headless_runner
+        from application.storage.db.repositories.agents import AgentsRepository
+
+        agent = AgentsRepository(pg_conn).create(
+            user_id="alice", name="hook-agent", status="active",
+            agent_type="classic", retriever="classic", chunks=2, key="sk-test-123",
+        )
+        agent_id = str(agent["id"])
+
+        def _boom(*a, **k):
+            raise RuntimeError("LLM exploded")
+
+        monkeypatch.setattr(headless_runner, "run_agent_headless", _boom)
+
+        with pytest.raises(RuntimeError, match="LLM exploded"):
+            worker.agent_webhook_worker(task_self, agent_id, {"event": "ping"})
+
+    def test_webhook_journals_headless_denial_for_approval_gated_tool(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        """Wire-through: approval-gated denial journals to tool_call_attempts."""
+        from contextlib import contextmanager
+        from types import SimpleNamespace
+
+        from application import worker
+        from application.agents import headless_runner
+        from application.agents.tool_executor import ToolExecutor
+        from application.llm.handlers.base import (
+            LLMHandler,
+            ToolCall,
+        )
+        from application.storage.db.repositories.agents import AgentsRepository
+
+        agent = AgentsRepository(pg_conn).create(
+            user_id="alice", name="hook-agent", status="active",
+            agent_type="classic", retriever="classic", chunks=2,
+            key="sk-deny-test",
+        )
+        agent_id = str(agent["id"])
+
+        @contextmanager
+        def _use_pg_conn():
+            yield pg_conn
+
+        monkeypatch.setattr(
+            "application.agents.tool_executor.db_session", _use_pg_conn,
+        )
+
+        # Stub model resolution + retriever so the call threads through.
+        monkeypatch.setattr(
+            "application.core.model_utils.get_default_model_id",
+            lambda: "gpt-4",
+        )
+        monkeypatch.setattr(
+            "application.core.model_utils.validate_model_id",
+            lambda m, **_kwargs: True,
+        )
+        monkeypatch.setattr(
+            "application.core.model_utils.get_provider_from_model_id",
+            lambda m, **_kwargs: "openai",
+        )
+        monkeypatch.setattr(
+            "application.core.model_utils.get_api_key_for_provider",
+            lambda p: "sk-test",
+        )
+        monkeypatch.setattr(
+            "application.utils.calculate_doc_token_budget",
+            lambda model_id=None, **_kwargs: 1000,
+        )
+        monkeypatch.setattr(
+            "application.api.answer.services.stream_processor.get_prompt",
+            lambda prompt_id: "prompt text",
+        )
+        monkeypatch.setattr(
+            "application.retriever.retriever_creator.RetrieverCreator.create_retriever",
+            lambda *a, **kw: SimpleNamespace(search=lambda q: []),
+        )
+
+        # Approval-gated tool + an agent that funnels one call through handle_tool_calls.
+        approval_tool_id = "tool-approval-gated"
+        approval_tool_row = {
+            "id": approval_tool_id,
+            "name": "telegram",
+            "actions": [
+                {
+                    "name": "send_message",
+                    "active": True,
+                    "require_approval": True,
+                    "parameters": {"type": "object", "properties": {}},
+                },
+            ],
+        }
+
+        def _fake_agent_factory(*a, **kw):
+            executor: ToolExecutor = kw["tool_executor"]
+            tools_dict = {approval_tool_id: approval_tool_row}
+            executor._name_to_tool = {
+                "send_message": (approval_tool_id, "send_message"),
+            }
+
+            class _MockLLM:
+                token_usage: dict = {}
+
+            class _FakeAgent:
+                def __init__(self):
+                    self.tool_executor = executor
+                    self.llm = _MockLLM()
+                    self.conversation_id = None
+
+                def gen(self, query):
+                    # arguments must be a JSON string for the default OpenAI-shaped parser.
+                    import json as _json
+                    call = ToolCall(
+                        id="webhook-denial-call",
+                        name="send_message",
+                        arguments=_json.dumps({"to": "x"}),
+                    )
+
+                    class _Handler(LLMHandler):
+                        def parse_response(self, response):
+                            return None
+
+                        def create_tool_message(self, tool_call, result):
+                            return {
+                                "role": "tool",
+                                "tool_call_id": tool_call.id,
+                                "content": result,
+                            }
+
+                        def _iterate_stream(self, response):
+                            yield from ()
+
+                    handler = _Handler()
+                    for evt in handler.handle_tool_calls(
+                        self, [call], tools_dict, [],
+                    ):
+                        yield evt
+                    yield {"answer": "ack"}
+
+            return _FakeAgent()
+
+        monkeypatch.setattr(
+            "application.agents.agent_creator.AgentCreator.create_agent",
+            _fake_agent_factory,
+        )
+        monkeypatch.setattr(headless_runner, "db_readonly", _use_pg_conn)
+
+        result = worker.agent_webhook_worker(
+            task_self, agent_id, {"event": "ping"}
+        )
+
+        assert result["status"] == "success"
+
+        from sqlalchemy import text as sql_text
+
+        row = pg_conn.execute(
+            sql_text(
+                "SELECT status, error FROM tool_call_attempts "
+                "WHERE call_id = :cid"
+            ),
+            {"cid": "webhook-denial-call"},
+        ).fetchone()
+        assert row is not None, "denial must be journaled"
+        assert row.status == "failed"
+        assert (row.error or "").startswith("headless: ")
+
+
+@pytest.mark.unit
+class TestRunAgentHeadlessFromWebhook:
+    def test_reads_source_row_from_pg(
+        self, pg_conn, patch_worker_db, monkeypatch
+    ):
+        """Smoke-test that run_agent_headless reads the source row from PG."""
+        from contextlib import contextmanager
+
+        from application.agents import headless_runner
+        from application.storage.db.repositories.sources import SourcesRepository
+
+        @contextmanager
+        def _use_pg_conn():
+            yield pg_conn
+        monkeypatch.setattr(headless_runner, "db_readonly", _use_pg_conn)
+
+        src = SourcesRepository(pg_conn).create(
+            "src",
+            user_id="alice",
+            type="local",
+            retriever="hybrid",
+        )
+        source_id = str(src["id"])
+
+        # Silence model/provider resolution so we don't need a real key.
+        monkeypatch.setattr(
+            "application.core.model_utils.get_default_model_id", lambda: "gpt-4"
+        )
+        monkeypatch.setattr(
+            "application.core.model_utils.validate_model_id", lambda m, **_kwargs: True
+        )
+        monkeypatch.setattr(
+            "application.core.model_utils.get_provider_from_model_id",
+            lambda m, **_kwargs: "openai",
+        )
+        monkeypatch.setattr(
+            "application.core.model_utils.get_api_key_for_provider",
+            lambda p: "sk-test",
+        )
+        monkeypatch.setattr(
+            "application.utils.calculate_doc_token_budget",
+            lambda model_id=None, **_kwargs: 1000,
+        )
+        monkeypatch.setattr(
+            "application.api.answer.services.stream_processor.get_prompt",
+            lambda prompt_id: "prompt text",
+        )
+
+        # Retriever search returns no docs; agent gen yields a single answer
+        # line so the aggregation loop runs through.
+        captured_source: dict = {}
+
+        class _FakeRetriever:
+            def __init__(self, *args, **kwargs):
+                captured_source.update(kwargs.get("source", {}))
+
+            def search(self, query):
+                return []
+
+        monkeypatch.setattr(
+            "application.retriever.retriever_creator.RetrieverCreator.create_retriever",
+            lambda *a, **kw: _FakeRetriever(**kw),
+        )
+
+        fake_agent = MagicMock(name="agent")
+        fake_agent.gen.return_value = iter([{"answer": "done"}])
+        fake_agent.current_token_count = 0
+        monkeypatch.setattr(
+            "application.agents.agent_creator.AgentCreator.create_agent",
+            lambda *a, **kw: fake_agent,
+        )
+
+        agent_config = {
+            "id": "agent-uuid",
+            "source_id": source_id,
+            "user_id": "alice",
+            "key": "sk-user",
+            "agent_type": "classic",
+            "chunks": 2,
+            "prompt_id": "default",
+        }
+
+        outcome = headless_runner.run_agent_headless(agent_config, "hello")
+
+        assert outcome["answer"] == "done"
+        assert captured_source.get("active_docs") == source_id

tests/worker/test_attachment_worker.py

@@ -0,0 +1,67 @@
+"""Smoke test for ``application.worker.attachment_worker``.
+
+The happy path parses an uploaded file and inserts a row into
+``attachments``. We mock the parser boundary (``StorageCreator.get_storage``
+returns a storage whose ``process_file`` produces a pre-built Document)
+but let the PG insert run against the ephemeral ``pg_conn`` so we can
+assert one concrete row is visible after the task returns.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from application.parser.schema.base import Document
+from application.storage.db.repositories.attachments import AttachmentsRepository
+
+
+@pytest.mark.unit
+class TestAttachmentWorker:
+    def test_inserts_row_in_attachments(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        fake_doc = Document(
+            text="hello world",
+            extra_info={"transcript_language": "en"},
+        )
+
+        fake_storage = MagicMock(name="storage")
+        fake_storage.process_file.return_value = fake_doc
+        monkeypatch.setattr(
+            worker.StorageCreator, "get_storage", lambda: fake_storage
+        )
+
+        # Stub the parser selection so the docling import path isn't taken.
+        monkeypatch.setattr(
+            worker, "get_default_file_extractor", lambda ocr_enabled=False: {}
+        )
+
+        file_info = {
+            "filename": "notes.txt",
+            "attachment_id": "507f1f77bcf86cd799439011",
+            "path": "uploads/user1/notes.txt",
+            "metadata": {"source": "chat"},
+        }
+
+        result = worker.attachment_worker(task_self, file_info, "user1")
+
+        assert result["filename"] == "notes.txt"
+        assert result["token_count"] > 0
+        # Parser metadata (``transcript_*``) should have been merged in.
+        assert result["metadata"]["transcript_language"] == "en"
+        assert result["metadata"]["source"] == "chat"
+
+        # Row should be resolvable by the caller-visible handle stored in
+        # ``legacy_mongo_id``.
+        row = AttachmentsRepository(pg_conn).get_by_legacy_id(
+            file_info["attachment_id"], "user1"
+        )
+        assert row is not None, "attachment_worker should insert a row"
+        assert row["filename"] == "notes.txt"
+        assert row["upload_path"] == "uploads/user1/notes.txt"
+        assert row["content"] == "hello world"
+        assert row["user_id"] == "user1"

tests/worker/test_convert_source_to_wiki.py

@@ -0,0 +1,440 @@
+"""Tests for ``application.worker.convert_source_to_wiki_worker``.
+
+The worker reassembles wiki pages from a source's existing vector-store
+chunks (grouped by ``metadata.source``) and enqueues a per-page re-embed.
+``VectorCreator.create_vectorstore`` and the re-embed task are mocked so no
+real store access or embeddings run; the ``sources`` / ``wiki_pages`` rows
+are real so the assertions read them back.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from application.storage.db.repositories.sources import SourcesRepository
+from application.storage.db.repositories.wiki_pages import WikiPagesRepository
+
+
+def _seed_source(pg_conn, config=None, file_path=""):
+    src = SourcesRepository(pg_conn).create(
+        "doc-set",
+        user_id="alice",
+        type="crawler",
+        retriever="classic",
+        file_path=file_path,
+        config=config or {},
+        directory_structure={"a.md": {"type": "text/markdown", "size_bytes": 1}},
+    )
+    return str(src["id"])
+
+
+def _chunk(text, source=None, doc_id="d", **extra_meta):
+    metadata = dict(extra_meta)
+    if source is not None:
+        metadata["source"] = source
+    return {"doc_id": doc_id, "text": text, "metadata": metadata}
+
+
+def _patch_store(monkeypatch, chunks):
+    store = MagicMock(name="vectorstore")
+    store.get_chunks.return_value = chunks
+    monkeypatch.setattr(
+        "application.vectorstore.vector_creator.VectorCreator.create_vectorstore",
+        lambda *a, **kw: store,
+    )
+    return store
+
+
+def _patch_reembed(monkeypatch):
+    delay = MagicMock(name="reembed_delay")
+    monkeypatch.setattr("application.api.user.tasks.reembed_wiki_page.delay", delay)
+    return delay
+
+
+@pytest.mark.unit
+class TestConvertSourceToWikiWorker:
+    def test_two_pages_reassembled_from_chunks(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        _patch_store(
+            monkeypatch,
+            [
+                _chunk("guide part one.", source="guide.md", doc_id="g1"),
+                _chunk("guide part two.", source="guide.md", doc_id="g2"),
+                _chunk("intro body.", source="notes/intro.md", doc_id="i1"),
+            ],
+        )
+        delay = _patch_reembed(monkeypatch)
+
+        result = worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        assert result["status"] == "converted"
+        assert result["pages_created"] == 2
+
+        pages = WikiPagesRepository(pg_conn).list_for_source(source_id)
+        by_path = {p["path"]: p for p in pages}
+        assert set(by_path) == {"/guide.md", "/notes/intro.md"}
+        assert by_path["/guide.md"]["content"] == "guide part one.\n\nguide part two."
+        assert by_path["/guide.md"]["title"] == "guide.md"
+        assert by_path["/notes/intro.md"]["content"] == "intro body."
+        assert by_path["/notes/intro.md"]["title"] == "intro.md"
+
+        assert delay.call_count == 2
+        paths = {c.args[1] for c in delay.call_args_list}
+        assert paths == {"/guide.md", "/notes/intro.md"}
+        for call in delay.call_args_list:
+            assert call.args[0] == source_id
+            assert call.kwargs["user"] == "alice"
+            assert call.kwargs["idempotency_key"].startswith(
+                f"reembed-wiki:{source_id}:"
+            )
+
+    def test_original_chunks_deleted_after_convert(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        store = _patch_store(
+            monkeypatch,
+            [
+                _chunk("guide part one.", source="guide.md", doc_id="g1"),
+                _chunk("guide part two.", source="guide.md", doc_id="g2"),
+                _chunk("intro body.", source="notes/intro.md", doc_id="i1"),
+            ],
+        )
+        delay = _patch_reembed(monkeypatch)
+
+        result = worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        assert result["status"] == "converted"
+        deleted = {c.args[0] for c in store.delete_chunk.call_args_list}
+        assert deleted == {"g1", "g2", "i1"}
+        assert delay.call_count == 2
+
+    def test_chunk_without_doc_id_not_deleted(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        chunk = {"text": "body", "metadata": {"source": "a.md"}}
+        store = _patch_store(monkeypatch, [chunk])
+        _patch_reembed(monkeypatch)
+
+        result = worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        assert result["status"] == "converted"
+        store.delete_chunk.assert_not_called()
+
+    def test_skipped_chunk_with_doc_id_still_deleted(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        # A chunk skipped for an invalid path must still have its original
+        # vector chunk purged, not left orphaned after the source flips to wiki.
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        store = _patch_store(
+            monkeypatch,
+            [
+                _chunk("good", source="ok.md", doc_id="ok"),
+                _chunk("bad", source="../evil.md", doc_id="evil"),
+            ],
+        )
+        _patch_reembed(monkeypatch)
+
+        result = worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        assert result["pages_created"] == 1
+        deleted = {c.args[0] for c in store.delete_chunk.call_args_list}
+        assert deleted == {"ok", "evil"}
+
+    def test_no_pages_path_deletes_nothing(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        store = _patch_store(monkeypatch, [])
+        _patch_reembed(monkeypatch)
+
+        result = worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        assert result["status"] == "no_pages"
+        store.delete_chunk.assert_not_called()
+
+    def test_kind_flipped_and_exposure_defaulted(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+        from application.storage.db.source_config import SourceConfig
+
+        source_id = _seed_source(pg_conn)
+        _patch_store(monkeypatch, [_chunk("body", source="a.md")])
+        _patch_reembed(monkeypatch)
+
+        worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        refreshed = SourcesRepository(pg_conn).get_any(source_id, "alice")
+        cfg = SourceConfig.parse(refreshed.get("config"))
+        assert cfg.kind == "wiki"
+        assert cfg.retrieval.exposure == "agentic_tool"
+
+    def test_preserves_non_default_exposure(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+        from application.storage.db.source_config import SourceConfig
+
+        source_id = _seed_source(
+            pg_conn, config={"retrieval": {"exposure": "agentic_tool"}}
+        )
+        _patch_store(monkeypatch, [_chunk("body", source="a.md")])
+        _patch_reembed(monkeypatch)
+
+        worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        refreshed = SourcesRepository(pg_conn).get_any(source_id, "alice")
+        cfg = SourceConfig.parse(refreshed.get("config"))
+        assert cfg.kind == "wiki"
+        assert cfg.retrieval.exposure == "agentic_tool"
+
+    def test_chunk_order_hint_respected(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        _patch_store(
+            monkeypatch,
+            [
+                _chunk("second", source="a.md", doc_id="a2", chunk_index=1),
+                _chunk("first", source="a.md", doc_id="a1", chunk_index=0),
+            ],
+        )
+        _patch_reembed(monkeypatch)
+
+        worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        pages = WikiPagesRepository(pg_conn).list_for_source(source_id)
+        by_path = {p["path"]: p for p in pages}
+        assert by_path["/a.md"]["content"] == "first\n\nsecond"
+
+    def test_short_overlap_not_trimmed(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        _patch_store(
+            monkeypatch,
+            [
+                _chunk("The end.", source="a.md", doc_id="a1"),
+                _chunk("The end. More", source="a.md", doc_id="a2"),
+            ],
+        )
+        _patch_reembed(monkeypatch)
+
+        worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        pages = WikiPagesRepository(pg_conn).list_for_source(source_id)
+        by_path = {p["path"]: p for p in pages}
+        assert by_path["/a.md"]["content"] == "The end.\n\nThe end. More"
+
+    def test_long_overlap_trimmed_between_chunks(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        overlap = "x" * 40
+        source_id = _seed_source(pg_conn)
+        _patch_store(
+            monkeypatch,
+            [
+                _chunk("head " + overlap, source="a.md", doc_id="a1"),
+                _chunk(overlap + " tail", source="a.md", doc_id="a2"),
+            ],
+        )
+        _patch_reembed(monkeypatch)
+
+        worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        pages = WikiPagesRepository(pg_conn).list_for_source(source_id)
+        by_path = {p["path"]: p for p in pages}
+        assert by_path["/a.md"]["content"] == "head " + overlap + "\n\n tail"
+
+    def test_no_chunks_leaves_kind_and_structure(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+        from application.storage.db.source_config import SourceConfig
+
+        original_structure = {"a.md": {"type": "text/markdown", "size_bytes": 1}}
+        source_id = _seed_source(pg_conn)
+        _patch_store(monkeypatch, [])
+        rebuild = MagicMock(name="rebuild")
+        monkeypatch.setattr(worker, "rebuild_wiki_directory_structure", rebuild)
+        delay = _patch_reembed(monkeypatch)
+
+        result = worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        assert result["status"] == "no_pages"
+        assert result["pages_created"] == 0
+        rebuild.assert_not_called()
+        delay.assert_not_called()
+
+        refreshed = SourcesRepository(pg_conn).get_any(source_id, "alice")
+        assert SourceConfig.parse(refreshed.get("config")).kind != "wiki"
+        assert refreshed.get("directory_structure") == original_structure
+
+    def test_missing_path_chunk_skipped(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        _patch_store(
+            monkeypatch,
+            [
+                _chunk("good", source="ok.md"),
+                _chunk("orphan", source=None),
+            ],
+        )
+        _patch_reembed(monkeypatch)
+
+        result = worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        assert result["pages_created"] == 1
+        pages = WikiPagesRepository(pg_conn).list_for_source(source_id)
+        assert {p["path"] for p in pages} == {"/ok.md"}
+        assert any(s["reason"] == "missing path" for s in result["skipped"])
+
+    def test_invalid_path_chunk_skipped(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        _patch_store(
+            monkeypatch,
+            [
+                _chunk("good", source="ok.md"),
+                _chunk("bad", source="../evil.md"),
+            ],
+        )
+        _patch_reembed(monkeypatch)
+
+        result = worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        assert result["pages_created"] == 1
+        pages = WikiPagesRepository(pg_conn).list_for_source(source_id)
+        assert {p["path"] for p in pages} == {"/ok.md"}
+        skipped = {(s["file"], s["reason"]) for s in result["skipped"]}
+        assert ("../evil.md", "invalid path") in skipped
+
+    def test_path_falls_back_to_filename_then_title(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        _patch_store(
+            monkeypatch,
+            [
+                _chunk("from filename", source=None, filename="byname.md"),
+                _chunk("from title", source=None, title="bytitle.md"),
+            ],
+        )
+        _patch_reembed(monkeypatch)
+
+        result = worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        assert result["pages_created"] == 2
+        pages = WikiPagesRepository(pg_conn).list_for_source(source_id)
+        assert {p["path"] for p in pages} == {"/byname.md", "/bytitle.md"}
+
+    def test_crawler_chunk_uses_file_path_not_url(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        _patch_store(
+            monkeypatch,
+            [
+                _chunk(
+                    "setup body",
+                    source="https://docs.x.com/guides/setup",
+                    file_path="guides/setup.md",
+                ),
+            ],
+        )
+        _patch_reembed(monkeypatch)
+
+        result = worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        assert result["status"] == "converted"
+        assert result["skipped"] == []
+        pages = WikiPagesRepository(pg_conn).list_for_source(source_id)
+        assert {p["path"] for p in pages} == {"/guides/setup.md"}
+
+    def test_connector_chunks_kept_separate_by_file_name(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        _patch_store(
+            monkeypatch,
+            [
+                _chunk("page a", source="confluence", file_name="Page A", doc_id="a"),
+                _chunk("page b", source="confluence", file_name="Page B", doc_id="b"),
+            ],
+        )
+        _patch_reembed(monkeypatch)
+
+        result = worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        assert result["pages_created"] == 2
+        pages = WikiPagesRepository(pg_conn).list_for_source(source_id)
+        assert {p["path"] for p in pages} == {"/Page A", "/Page B"}
+
+    def test_url_only_chunk_normalized_not_skipped(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        _patch_store(
+            monkeypatch,
+            [_chunk("body", source="https://x.com/a/b")],
+        )
+        _patch_reembed(monkeypatch)
+
+        result = worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        assert result["status"] == "converted"
+        assert result["skipped"] == []
+        pages = WikiPagesRepository(pg_conn).list_for_source(source_id)
+        assert {p["path"] for p in pages} == {"/a/b"}
+
+    def test_already_wiki_returns_early(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn, config={"kind": "wiki"})
+        store = _patch_store(monkeypatch, [_chunk("body", source="a.md")])
+        delay = _patch_reembed(monkeypatch)
+
+        result = worker.convert_source_to_wiki_worker(task_self, source_id, "alice")
+
+        assert result["status"] == "already_wiki"
+        store.get_chunks.assert_not_called()
+        delay.assert_not_called()

tests/worker/test_extract_graph.py

@@ -0,0 +1,172 @@
+"""Tests for ``application.worker.extract_graph_worker``.
+
+The worker loads the source row, fetches its chunks from the vector store, and
+delegates to ``extract_graph_for_source``. ``graphrag_available``,
+``VectorCreator.create_vectorstore``, and the extraction pipeline are mocked so
+no live store access or LLM/model calls run; the ``sources`` row is real so
+``SourcesRepository.get_any`` resolves.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from application.storage.db.repositories.sources import SourcesRepository
+
+
+def _seed_source(pg_conn, config=None):
+    src = SourcesRepository(pg_conn).create(
+        "graph-set",
+        user_id="alice",
+        type="file",
+        retriever="classic",
+        config=config or {"kind": "graphrag", "retrieval": {"retriever": "graphrag"}},
+    )
+    return str(src["id"])
+
+
+def _patch_store(monkeypatch, chunks):
+    store = MagicMock(name="vectorstore")
+    store.get_chunks.return_value = chunks
+    monkeypatch.setattr(
+        "application.vectorstore.vector_creator.VectorCreator.create_vectorstore",
+        lambda *a, **kw: store,
+    )
+    return store
+
+
+@pytest.mark.unit
+class TestExtractGraphWorker:
+    def test_fetches_chunks_and_calls_extraction(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        chunks = [
+            {"doc_id": "c1", "text": "alpha"},
+            {"doc_id": "c2", "text": "beta"},
+        ]
+        store = _patch_store(monkeypatch, chunks)
+
+        monkeypatch.setattr(
+            "application.graphrag.graphrag_available", lambda: True
+        )
+        extract = MagicMock(
+            name="extract_graph_for_source",
+            return_value={"nodes": 3, "edges": 2, "chunks_processed": 2},
+        )
+        monkeypatch.setattr(
+            "application.graphrag.extraction.extract_graph_for_source", extract
+        )
+
+        result = worker.extract_graph_worker(task_self, source_id, "alice")
+
+        store.get_chunks.assert_called_once()
+        extract.assert_called_once()
+        assert extract.call_args.args[0] == source_id
+        assert extract.call_args.args[1] == "alice"
+        assert extract.call_args.args[2] == chunks
+        assert extract.call_args.kwargs["config"].kind == "graphrag"
+        assert result == {"nodes": 3, "edges": 2, "chunks_processed": 2}
+
+    def test_unavailable_returns_status_no_extraction(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        store = _patch_store(monkeypatch, [])
+        monkeypatch.setattr(
+            "application.graphrag.graphrag_available", lambda: False
+        )
+        extract = MagicMock(name="extract_graph_for_source")
+        monkeypatch.setattr(
+            "application.graphrag.extraction.extract_graph_for_source", extract
+        )
+
+        result = worker.extract_graph_worker(task_self, "src-x", "alice")
+
+        assert result == {"status": "unavailable"}
+        store.get_chunks.assert_not_called()
+        extract.assert_not_called()
+
+    def test_empty_chunks_still_calls_extraction(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        _patch_store(monkeypatch, [])
+        monkeypatch.setattr(
+            "application.graphrag.graphrag_available", lambda: True
+        )
+        extract = MagicMock(
+            name="extract_graph_for_source",
+            return_value={"nodes": 0, "edges": 0, "chunks_processed": 0},
+        )
+        monkeypatch.setattr(
+            "application.graphrag.extraction.extract_graph_for_source", extract
+        )
+
+        result = worker.extract_graph_worker(task_self, source_id, "alice")
+
+        extract.assert_called_once()
+        assert extract.call_args.args[2] == []
+        assert result["chunks_processed"] == 0
+
+    def test_publishes_completed_event(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        _patch_store(monkeypatch, [{"doc_id": "c1", "text": "alpha"}])
+        monkeypatch.setattr(
+            "application.graphrag.graphrag_available", lambda: True
+        )
+        monkeypatch.setattr(
+            "application.graphrag.extraction.extract_graph_for_source",
+            MagicMock(return_value={"nodes": 1, "edges": 0, "chunks_processed": 1}),
+        )
+        events = []
+        monkeypatch.setattr(
+            worker, "publish_user_event",
+            lambda user, etype, payload, **kw: events.append((etype, payload)),
+        )
+
+        worker.extract_graph_worker(task_self, source_id, "alice")
+
+        types = [e[0] for e in events]
+        assert "graph.extract.progress" in types
+        assert types[-1] == "graph.extract.completed"
+        assert events[-1][1]["nodes"] == 1
+
+    def test_publishes_failed_event_on_error(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        _patch_store(monkeypatch, [{"doc_id": "c1", "text": "alpha"}])
+        monkeypatch.setattr(
+            "application.graphrag.graphrag_available", lambda: True
+        )
+
+        def _boom(*a, **kw):
+            raise RuntimeError("extraction blew up")
+
+        monkeypatch.setattr(
+            "application.graphrag.extraction.extract_graph_for_source", _boom
+        )
+        events = []
+        monkeypatch.setattr(
+            worker, "publish_user_event",
+            lambda user, etype, payload, **kw: events.append((etype, payload)),
+        )
+
+        with pytest.raises(RuntimeError):
+            worker.extract_graph_worker(task_self, source_id, "alice")
+
+        assert "graph.extract.failed" in [e[0] for e in events]

tests/worker/test_graph_extraction_enqueue.py

@@ -0,0 +1,315 @@
+"""Ingest paths enqueue graph extraction after embed for graphrag sources.
+
+Exercises ``remote_worker`` as the representative ingest path: the remote
+loader, embedding pipeline, and ``upload_index`` are mocked, so the only thing
+under test is the post-embed ``extract_graph.delay`` hook. ``graphrag_available``
+is forced on so the gate doesn't depend on the test env's vector store. The
+``sources`` row is real (seeded via ``pg_conn``) so the hook can read the
+source's ``updated_at`` for the idempotency key.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from application.parser.schema.base import Document
+from application.storage.db.repositories.sources import SourcesRepository
+
+
+@pytest.fixture
+def _mock_remote_pipeline(monkeypatch):
+    from application import worker
+
+    fake_loader = MagicMock(name="remote_loader")
+    fake_loader.load_data.return_value = [
+        Document(
+            text="page body",
+            extra_info={"file_path": "guides/setup.md", "title": "setup"},
+            doc_id="d1",
+        )
+    ]
+    monkeypatch.setattr(
+        worker.RemoteCreator, "create_loader", lambda loader: fake_loader
+    )
+    monkeypatch.setattr(
+        worker,
+        "embed_and_store_documents",
+        lambda docs, full_path, source_id, task, **kw: None,
+    )
+    monkeypatch.setattr(
+        worker, "assert_index_complete", lambda *a, **kw: None
+    )
+    monkeypatch.setattr(
+        worker, "upload_index", lambda full_path, file_data: None
+    )
+    # Reset constructs a real GraphStore (pgvector); not under test here.
+    monkeypatch.setattr(
+        worker, "_reset_graph_for_source", lambda *a, **kw: None
+    )
+
+
+def _patch_delay(monkeypatch):
+    delay = MagicMock(name="extract_graph_delay")
+    monkeypatch.setattr(
+        "application.api.user.tasks.extract_graph.delay", delay
+    )
+    return delay
+
+
+def _seed_source(pg_conn, user, config):
+    src = SourcesRepository(pg_conn).create(
+        "graph-remote", user_id=user, type="crawler", config=config,
+    )
+    return str(src["id"])
+
+
+@pytest.mark.unit
+class TestGraphExtractionKey:
+    def test_shape_and_state_sensitivity(self):
+        from application.worker import _source_updated_at, graph_extraction_key
+
+        sid = "11111111-1111-1111-1111-111111111111"
+        key_a = graph_extraction_key(
+            sid, _source_updated_at({"updated_at": "2026-06-23T00:00:00+00:00"})
+        )
+        key_b = graph_extraction_key(
+            sid, _source_updated_at({"updated_at": "2026-06-23T00:00:01+00:00"})
+        )
+        key_a_again = graph_extraction_key(
+            sid, _source_updated_at({"updated_at": "2026-06-23T00:00:00+00:00"})
+        )
+
+        assert key_a.startswith(f"extract-graph:{sid}:")
+        # A changed ``updated_at`` (re-ingest/re-enable) yields a fresh key.
+        assert key_a != key_b
+        # The same state collapses to one key (concurrent dups dedupe).
+        assert key_a == key_a_again
+
+    def test_falls_back_to_date(self):
+        from application.worker import _source_updated_at
+
+        assert _source_updated_at({"date": "2026-06-23T00:00:00+00:00"}) == (
+            "2026-06-23T00:00:00+00:00"
+        )
+        assert _source_updated_at({"updated_at": None, "date": "x"}) == "x"
+        assert _source_updated_at(None) == ""
+
+
+@pytest.mark.unit
+class TestRemoteWorkerEnqueuesGraphExtraction:
+    def test_graphrag_source_enqueues_after_embed(
+        self, task_self, pg_conn, patch_worker_db, monkeypatch,
+        _mock_remote_pipeline,
+    ):
+        from application import worker
+
+        monkeypatch.setattr(
+            "application.graphrag.graphrag_available", lambda: True
+        )
+        delay = _patch_delay(monkeypatch)
+
+        config = {"kind": "graphrag", "retrieval": {"retriever": "graphrag"}}
+        sid = _seed_source(pg_conn, "bob", config)
+
+        worker.remote_worker(
+            task_self,
+            {"urls": ["http://example.com"]},
+            "graph-remote",
+            "bob",
+            "crawler",
+            directory="temp",
+            retriever="classic",
+            operation_mode="upload",
+            config=config,
+            source_id=sid,
+        )
+
+        delay.assert_called_once()
+        assert delay.call_args.args[0] == sid
+        assert delay.call_args.args[1] == "bob"
+        key = delay.call_args.kwargs["idempotency_key"]
+        assert key.startswith(f"extract-graph:{sid}:")
+
+    def test_same_state_dedupes_to_one_key(
+        self, task_self, pg_conn, patch_worker_db, monkeypatch,
+        _mock_remote_pipeline,
+    ):
+        """Two enqueues for the same source state share a key (concurrent dups)."""
+        from application import worker
+
+        monkeypatch.setattr(
+            "application.graphrag.graphrag_available", lambda: True
+        )
+        delay = _patch_delay(monkeypatch)
+
+        config = {"kind": "graphrag", "retrieval": {"retriever": "graphrag"}}
+        sid = _seed_source(pg_conn, "bob", config)
+
+        def _run():
+            worker.remote_worker(
+                task_self,
+                {"urls": ["http://example.com"]},
+                "graph-remote",
+                "bob",
+                "crawler",
+                directory="temp",
+                retriever="classic",
+                operation_mode="upload",
+                config=config,
+                source_id=sid,
+            )
+            return delay.call_args.kwargs["idempotency_key"]
+
+        assert _run() == _run()
+
+    def test_resets_graph_before_enqueue(
+        self, task_self, pg_conn, patch_worker_db, monkeypatch,
+        _mock_remote_pipeline,
+    ):
+        """A re-ingest clears the prior graph before re-enqueuing extraction."""
+        from application import worker
+
+        monkeypatch.setattr(
+            "application.graphrag.graphrag_available", lambda: True
+        )
+        reset = MagicMock(name="reset_graph")
+        monkeypatch.setattr(worker, "_reset_graph_for_source", reset)
+        delay = _patch_delay(monkeypatch)
+
+        config = {"kind": "graphrag", "retrieval": {"retriever": "graphrag"}}
+        sid = _seed_source(pg_conn, "bob", config)
+
+        worker.remote_worker(
+            task_self,
+            {"urls": ["http://example.com"]},
+            "graph-remote",
+            "bob",
+            "crawler",
+            directory="temp",
+            retriever="classic",
+            operation_mode="upload",
+            config=config,
+            source_id=sid,
+        )
+
+        reset.assert_called_once_with(sid)
+        delay.assert_called_once()
+
+    def test_classic_source_does_not_enqueue(
+        self, task_self, pg_conn, patch_worker_db, monkeypatch,
+        _mock_remote_pipeline,
+    ):
+        from application import worker
+
+        monkeypatch.setattr(
+            "application.graphrag.graphrag_available", lambda: True
+        )
+        delay = _patch_delay(monkeypatch)
+
+        sid = _seed_source(pg_conn, "bob", {"kind": "classic"})
+
+        worker.remote_worker(
+            task_self,
+            {"urls": ["http://example.com"]},
+            "classic-remote",
+            "bob",
+            "crawler",
+            directory="temp",
+            retriever="classic",
+            operation_mode="upload",
+            config={"kind": "classic"},
+            source_id=sid,
+        )
+
+        delay.assert_not_called()
+
+    def test_graphrag_unavailable_does_not_enqueue(
+        self, task_self, pg_conn, patch_worker_db, monkeypatch,
+        _mock_remote_pipeline,
+    ):
+        from application import worker
+
+        monkeypatch.setattr(
+            "application.graphrag.graphrag_available", lambda: False
+        )
+        delay = _patch_delay(monkeypatch)
+
+        config = {"kind": "graphrag", "retrieval": {"retriever": "graphrag"}}
+        sid = _seed_source(pg_conn, "bob", config)
+
+        worker.remote_worker(
+            task_self,
+            {"urls": ["http://example.com"]},
+            "graph-remote",
+            "bob",
+            "crawler",
+            directory="temp",
+            retriever="classic",
+            operation_mode="upload",
+            config=config,
+            source_id=sid,
+        )
+
+        delay.assert_not_called()
+
+
+@pytest.mark.unit
+class TestEnqueueIsolatesBrokerFailures:
+    def test_delay_exception_does_not_propagate(
+        self, pg_conn, patch_worker_db, monkeypatch
+    ):
+        """A broker hiccup in ``.delay`` must not fail an otherwise-good ingest."""
+        from application import worker
+        from application.storage.db.source_config import SourceConfig
+
+        monkeypatch.setattr(
+            "application.graphrag.graphrag_available", lambda: True
+        )
+        monkeypatch.setattr(
+            worker, "_reset_graph_for_source", lambda *a, **kw: None
+        )
+
+        def _boom(*a, **kw):
+            raise RuntimeError("broker down")
+
+        monkeypatch.setattr(
+            "application.api.user.tasks.extract_graph.delay", _boom
+        )
+
+        config = {"kind": "graphrag", "retrieval": {"retriever": "graphrag"}}
+        sid = _seed_source(pg_conn, "bob", config)
+
+        worker._maybe_enqueue_graph_extraction(
+            SourceConfig.parse(config), sid, "bob"
+        )
+
+    def test_updated_at_read_exception_does_not_propagate(
+        self, pg_conn, patch_worker_db, monkeypatch
+    ):
+        """A DB hiccup reading ``updated_at`` must also be swallowed."""
+        from application import worker
+        from application.storage.db.source_config import SourceConfig
+
+        monkeypatch.setattr(
+            "application.graphrag.graphrag_available", lambda: True
+        )
+        monkeypatch.setattr(
+            worker, "_reset_graph_for_source", lambda *a, **kw: None
+        )
+        delay = _patch_delay(monkeypatch)
+
+        def _boom(self, source_id, user_id):
+            raise RuntimeError("pg down")
+
+        monkeypatch.setattr(SourcesRepository, "get_any", _boom)
+
+        config = {"kind": "graphrag", "retrieval": {"retriever": "graphrag"}}
+        sid = _seed_source(pg_conn, "bob", config)
+
+        worker._maybe_enqueue_graph_extraction(
+            SourceConfig.parse(config), sid, "bob"
+        )
+
+        delay.assert_not_called()

tests/worker/test_ingest_checkpoint.py

@@ -0,0 +1,531 @@
+"""Tests for the ingest chunk checkpoint and heartbeat.
+
+* ``embed_and_store_documents`` writes/reads ``ingest_chunk_progress``;
+  fresh runs end with ``embedded_chunks=N`` and resumes seeded at
+  ``last_index=k`` embed only chunks ``k+1..N-1``.
+* ``_ingest_heartbeat_loop`` is the daemon body that bumps ``last_updated``.
+"""
+
+from __future__ import annotations
+
+import threading
+import uuid
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from typing import Iterator
+from unittest.mock import MagicMock
+
+import pytest
+from sqlalchemy import Connection, text
+
+
+@dataclass
+class _LCDoc:
+    """Tiny stand-in for a LangChain document — only the fields the loop reads."""
+    page_content: str
+    metadata: dict = field(default_factory=dict)
+
+
+@pytest.fixture
+def patch_pipeline_db(pg_conn, monkeypatch):
+    """Make ``db_session`` inside the embedding pipeline yield ``pg_conn``."""
+
+    @contextmanager
+    def _use_pg_conn() -> Iterator[Connection]:
+        yield pg_conn
+
+    monkeypatch.setattr(
+        "application.parser.embedding_pipeline.db_session", _use_pg_conn
+    )
+
+
+@pytest.fixture
+def faiss_settings(monkeypatch):
+    """Force the embed pipeline down the faiss branch with a stub vector store."""
+    from application.parser import embedding_pipeline as ep
+
+    monkeypatch.setattr(ep.settings, "VECTOR_STORE", "faiss", raising=False)
+
+    fake_store = MagicMock(name="vector_store")
+    monkeypatch.setattr(
+        ep.VectorCreator, "create_vectorstore", lambda *a, **kw: fake_store
+    )
+    return fake_store
+
+
+def _seed_progress_row(
+    pg_conn, source_id: str, total: int, last_index: int,
+    attempt_id: str | None = None,
+) -> None:
+    pg_conn.execute(
+        text(
+            """
+            INSERT INTO ingest_chunk_progress
+                (source_id, total_chunks, embedded_chunks, last_index,
+                 attempt_id, last_updated)
+            VALUES (CAST(:sid AS uuid), :total, :emb, :idx, :att, now())
+            """
+        ),
+        {
+            "sid": source_id,
+            "total": total,
+            "emb": last_index + 1,
+            "idx": last_index,
+            "att": attempt_id,
+        },
+    )
+
+
+def _make_docs(n: int) -> list:
+    return [_LCDoc(page_content=f"chunk-{i}", metadata={"i": i}) for i in range(n)]
+
+
+@pytest.mark.unit
+class TestEmbedCheckpoint:
+    def test_fresh_run_records_progress_to_completion(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        source_id = str(uuid.uuid4())
+        docs = _make_docs(5)
+        captured_chunks: list[str] = []
+
+        def _fake_add(store, doc, sid):
+            captured_chunks.append(doc.page_content)
+
+        # The retry decorator wraps the real fn. Patch the module-level name
+        # so our loop calls the spy directly.
+        import application.parser.embedding_pipeline as ep_mod
+
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = _fake_add
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock()
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        # Faiss seeds the store with docs[0]; the loop picks up at idx=1.
+        assert captured_chunks == ["chunk-1", "chunk-2", "chunk-3", "chunk-4"]
+
+        row = pg_conn.execute(
+            text(
+                "SELECT total_chunks, embedded_chunks, last_index "
+                "FROM ingest_chunk_progress WHERE source_id = CAST(:sid AS uuid)"
+            ),
+            {"sid": source_id},
+        ).fetchone()
+        assert row is not None
+        m = dict(row._mapping)
+        # Loop wrote the last record at idx=4 over the full 5-doc list.
+        assert m["total_chunks"] == 5
+        assert m["embedded_chunks"] == 5
+        assert m["last_index"] == 4
+
+    def test_resume_skips_already_embedded_chunks(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        """Pre-seed progress at ``last_index=2`` and assert chunks 0..2 are not re-embedded."""
+        import application.parser.embedding_pipeline as ep_mod
+
+        source_id = str(uuid.uuid4())
+        docs = _make_docs(6)
+        # Seed: last_index=2 means chunks at original indices 0..2 are done;
+        # resume should pick up at idx=3 (chunks 3, 4, 5).
+        _seed_progress_row(pg_conn, source_id, total=6, last_index=2)
+
+        captured_chunks: list[str] = []
+
+        def _fake_add(store, doc, sid):
+            captured_chunks.append(doc.page_content)
+
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = _fake_add
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock()
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        # On resume the FAISS store is loaded from storage (no docs_init);
+        # the loop iterates the un-popped docs list starting at resume_index.
+        assert captured_chunks == ["chunk-3", "chunk-4", "chunk-5"]
+
+        row = pg_conn.execute(
+            text(
+                "SELECT embedded_chunks, last_index "
+                "FROM ingest_chunk_progress WHERE source_id = CAST(:sid AS uuid)"
+            ),
+            {"sid": source_id},
+        ).fetchone()
+        m = dict(row._mapping)
+        assert m["last_index"] == 5
+        assert m["embedded_chunks"] == 6
+
+    def test_faiss_resume_loads_existing_index_without_docs_init(
+        self, pg_conn, patch_pipeline_db, monkeypatch, tmp_path
+    ):
+        """Resuming a FAISS run must NOT pass ``docs_init`` (which would
+        overwrite the previously-saved index with a partial reset).
+        """
+        import application.parser.embedding_pipeline as ep_mod
+
+        monkeypatch.setattr(ep_mod.settings, "VECTOR_STORE", "faiss", raising=False)
+
+        captured_kwargs: list[dict] = []
+
+        def _capture_create(*args, **kwargs):
+            captured_kwargs.append(kwargs)
+            return MagicMock(name="vector_store")
+
+        monkeypatch.setattr(
+            ep_mod.VectorCreator, "create_vectorstore", _capture_create,
+        )
+
+        source_id = str(uuid.uuid4())
+        docs = _make_docs(6)
+        _seed_progress_row(pg_conn, source_id, total=6, last_index=2)
+
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = lambda store, doc, sid: None
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock()
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        assert len(captured_kwargs) == 1
+        # No ``docs_init`` on resume — this is what triggers FaissStore to
+        # load the existing index from storage rather than minting a new
+        # single-doc store.
+        assert "docs_init" not in captured_kwargs[0]
+
+    def test_resume_keeps_existing_index_for_non_faiss(
+        self, pg_conn, patch_pipeline_db, monkeypatch, tmp_path
+    ):
+        """Non-faiss stores must NOT have ``delete_index`` called on resume."""
+        import application.parser.embedding_pipeline as ep_mod
+
+        monkeypatch.setattr(ep_mod.settings, "VECTOR_STORE", "qdrant", raising=False)
+
+        fake_store = MagicMock(name="vector_store")
+        monkeypatch.setattr(
+            ep_mod.VectorCreator, "create_vectorstore", lambda *a, **kw: fake_store
+        )
+
+        source_id = str(uuid.uuid4())
+        docs = _make_docs(4)
+        _seed_progress_row(pg_conn, source_id, total=4, last_index=1)
+
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = lambda store, doc, sid: None
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock()
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        fake_store.delete_index.assert_not_called()
+
+    def test_same_attempt_id_resumes_from_checkpoint(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        """A Celery autoretry passes the same ``self.request.id`` and
+        must resume from the persisted ``last_index``.
+        """
+        import application.parser.embedding_pipeline as ep_mod
+
+        source_id = str(uuid.uuid4())
+        docs = _make_docs(6)
+        _seed_progress_row(
+            pg_conn, source_id, total=6, last_index=2, attempt_id="att-A",
+        )
+
+        captured: list[str] = []
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = (
+            lambda store, doc, sid: captured.append(doc.page_content)
+        )
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock(),
+                attempt_id="att-A",
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        # Same attempt → resume past the last persisted index.
+        assert captured == ["chunk-3", "chunk-4", "chunk-5"]
+
+    def test_different_attempt_id_resets_checkpoint(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        """A fresh sync/reingest passes a new ``attempt_id`` and must
+        reset the checkpoint so the index is rebuilt from chunk 0.
+        """
+        import application.parser.embedding_pipeline as ep_mod
+
+        source_id = str(uuid.uuid4())
+        # Prior run completed: last_index=4 (chunks 0..4 embedded over a
+        # 5-doc list). New run brings 5 fresh docs under a different
+        # attempt_id.
+        _seed_progress_row(
+            pg_conn, source_id, total=5, last_index=4, attempt_id="att-old",
+        )
+        docs = _make_docs(5)
+
+        captured: list[str] = []
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = (
+            lambda store, doc, sid: captured.append(doc.page_content)
+        )
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock(),
+                attempt_id="att-new",
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        # Fresh attempt → reset to chunk 0; FAISS branch seeds with
+        # docs[0] and the loop picks up at idx=1.
+        assert captured == ["chunk-1", "chunk-2", "chunk-3", "chunk-4"]
+
+        # Post-run state belongs to the new attempt.
+        row = pg_conn.execute(
+            text(
+                "SELECT total_chunks, embedded_chunks, last_index, attempt_id "
+                "FROM ingest_chunk_progress WHERE source_id = CAST(:sid AS uuid)"
+            ),
+            {"sid": source_id},
+        ).fetchone()
+        m = dict(row._mapping)
+        assert m["embedded_chunks"] == 5
+        assert m["last_index"] == 4
+        assert m["attempt_id"] == "att-new"
+
+    def test_completed_checkpoint_does_not_block_fresh_attempt(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        """Regression: a completed-and-cached checkpoint from an earlier
+        upload must not silently no-op a later sync. Pre-fix the embed
+        loop saw ``loop_start >= total_docs`` and embedded zero chunks,
+        leaving stale vectors in place.
+        """
+        import application.parser.embedding_pipeline as ep_mod
+
+        source_id = str(uuid.uuid4())
+        # Upload finished cleanly with 5 chunks; checkpoint reflects done.
+        _seed_progress_row(
+            pg_conn, source_id, total=5, last_index=4, attempt_id="upload-1",
+        )
+        # Sync brings the same number of docs (the dangerous case where
+        # the old code's ``loop_start >= total_docs`` branch fired).
+        docs = _make_docs(5)
+
+        captured: list[str] = []
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = (
+            lambda store, doc, sid: captured.append(doc.page_content)
+        )
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock(),
+                attempt_id="sync-2",
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        # All non-seed chunks re-embedded under the new attempt.
+        assert captured == ["chunk-1", "chunk-2", "chunk-3", "chunk-4"]
+
+    def test_legacy_null_attempt_id_resumes_against_null_caller(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        """Pre-migration rows have ``attempt_id=NULL``; legacy callers
+        (or tests) that pass no ``attempt_id`` must still resume against
+        them — IS NOT DISTINCT FROM treats NULL/NULL as equal.
+        """
+        import application.parser.embedding_pipeline as ep_mod
+
+        source_id = str(uuid.uuid4())
+        _seed_progress_row(
+            pg_conn, source_id, total=4, last_index=1, attempt_id=None,
+        )
+        docs = _make_docs(4)
+
+        captured: list[str] = []
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = (
+            lambda store, doc, sid: captured.append(doc.page_content)
+        )
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock(),
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        # Resumed past last_index=1.
+        assert captured == ["chunk-2", "chunk-3"]
+
+    def test_single_chunk_faiss_records_seeded_doc(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        """Regression: a 1-doc FAISS ingest seeds with ``docs[0]`` and
+        the loop runs zero iterations. Pre-fix, no ``_record_progress``
+        call ever ran, ``embedded_chunks`` stayed at 0, and
+        ``assert_index_complete`` raised on every retry until the
+        poison-loop guard finalised the row. Post-fix, the seed is
+        recorded immediately so ``embedded == total == 1``.
+        """
+        import application.parser.embedding_pipeline as ep_mod
+
+        source_id = str(uuid.uuid4())
+        docs = _make_docs(1)
+
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = lambda store, doc, sid: None
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock(),
+                attempt_id="att-single",
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        row = pg_conn.execute(
+            text(
+                "SELECT total_chunks, embedded_chunks, last_index, attempt_id "
+                "FROM ingest_chunk_progress WHERE source_id = CAST(:sid AS uuid)"
+            ),
+            {"sid": source_id},
+        ).fetchone()
+        assert row is not None
+        m = dict(row._mapping)
+        assert m["total_chunks"] == 1
+        assert m["embedded_chunks"] == 1
+        assert m["last_index"] == 0
+        assert m["attempt_id"] == "att-single"
+
+        # Tripwire passes — assert_index_complete reads the same row.
+        ep_mod.assert_index_complete(source_id)
+
+    def test_multi_chunk_faiss_seed_record_is_overwritten_correctly(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        """The new seed-record call must not break multi-chunk runs:
+        the loop's per-iteration record overshoots correctly (counts
+        seed + iterations) and the final state is ``embedded=total``.
+        """
+        import application.parser.embedding_pipeline as ep_mod
+
+        source_id = str(uuid.uuid4())
+        docs = _make_docs(4)
+
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = lambda store, doc, sid: None
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock(),
+                attempt_id="att-multi",
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        row = pg_conn.execute(
+            text(
+                "SELECT total_chunks, embedded_chunks, last_index "
+                "FROM ingest_chunk_progress WHERE source_id = CAST(:sid AS uuid)"
+            ),
+            {"sid": source_id},
+        ).fetchone()
+        m = dict(row._mapping)
+        assert m["total_chunks"] == 4
+        assert m["embedded_chunks"] == 4
+        assert m["last_index"] == 3
+
+
+@pytest.mark.unit
+class TestIngestHeartbeat:
+    def test_loop_bumps_last_updated_then_exits(
+        self, pg_conn, patch_worker_db, monkeypatch
+    ):
+        """One tick of the heartbeat must move ``last_updated`` forward."""
+        from application import worker
+
+        source_id = str(uuid.uuid4())
+        # Seed a row with ``last_updated`` deliberately in the past so we
+        # can assert the heartbeat tick moves it forward.
+        pg_conn.execute(
+            text(
+                """
+                INSERT INTO ingest_chunk_progress
+                    (source_id, total_chunks, embedded_chunks, last_index, last_updated)
+                VALUES (CAST(:sid AS uuid), 1, 0, -1, now() - interval '1 hour')
+                """
+            ),
+            {"sid": source_id},
+        )
+        before = pg_conn.execute(
+            text(
+                "SELECT last_updated FROM ingest_chunk_progress "
+                "WHERE source_id = CAST(:sid AS uuid)"
+            ),
+            {"sid": source_id},
+        ).fetchone()._mapping["last_updated"]
+
+        stop_event = threading.Event()
+        # First call returns False -> loop body runs once. Second returns
+        # True -> loop exits cleanly.
+        wait_returns = iter([False, True])
+        monkeypatch.setattr(stop_event, "wait", lambda interval: next(wait_returns))
+
+        worker._ingest_heartbeat_loop(source_id, stop_event, interval=0)
+
+        after = pg_conn.execute(
+            text(
+                "SELECT last_updated FROM ingest_chunk_progress "
+                "WHERE source_id = CAST(:sid AS uuid)"
+            ),
+            {"sid": source_id},
+        ).fetchone()._mapping["last_updated"]
+        assert after > before
+
+    def test_loop_swallows_db_errors(self, monkeypatch):
+        """A failing DB call must not crash the daemon — it should keep ticking."""
+        from application import worker
+
+        @contextmanager
+        def _broken_session():
+            raise RuntimeError("boom")
+            yield  # pragma: no cover
+
+        monkeypatch.setattr(worker, "db_session", _broken_session)
+
+        stop_event = threading.Event()
+        wait_returns = iter([False, False, True])
+        monkeypatch.setattr(stop_event, "wait", lambda interval: next(wait_returns))
+
+        # Should not raise — failures are logged, loop continues.
+        worker._ingest_heartbeat_loop("00000000-0000-0000-0000-000000000000", stop_event, interval=0)
+
+    def test_start_and_stop_helpers_join_quickly(self, monkeypatch):
+        """``_start_ingest_heartbeat`` + ``_stop_ingest_heartbeat`` must not hang."""
+        from application import worker
+
+        @contextmanager
+        def _noop_session():
+            yield MagicMock()
+
+        monkeypatch.setattr(worker, "db_session", _noop_session)
+
+        thread, stop_event = worker._start_ingest_heartbeat(
+            "00000000-0000-0000-0000-000000000000"
+        )
+        assert thread.is_alive()
+        worker._stop_ingest_heartbeat(thread, stop_event)
+        assert not thread.is_alive()

tests/worker/test_ingest_connector.py

@@ -0,0 +1,178 @@
+"""Smoke test for ``application.worker.ingest_connector`` in sync mode.
+
+Sync mode (``operation_mode="sync"``) bumps ``sources.date`` on the
+target source row, the same PG side-effect as ``remote_worker``. Upload
+mode does not write to PG directly — that happens indirectly via the
+``upload_index`` HTTP callback to the backend — so the happy-path smoke
+is the sync variant.
+"""
+
+from __future__ import annotations
+
+import uuid
+from unittest.mock import MagicMock
+
+import pytest
+
+from application.parser.schema.base import Document
+from application.storage.db.repositories.sources import SourcesRepository
+
+
+@pytest.fixture
+def _mock_connector_pipeline(monkeypatch):
+    """Stub the connector + pipeline so only PG writes are real."""
+    from application import worker
+
+    fake_connector = MagicMock(name="connector")
+    fake_connector.download_to_directory.return_value = {
+        "files_downloaded": 1,
+        "empty_result": False,
+    }
+    monkeypatch.setattr(
+        worker.ConnectorCreator, "is_supported", staticmethod(lambda s: True)
+    )
+    monkeypatch.setattr(
+        worker.ConnectorCreator,
+        "create_connector",
+        staticmethod(lambda source_type, session_token: fake_connector),
+    )
+
+    fake_reader = MagicMock(name="reader")
+    fake_reader.load_data.return_value = [
+        Document(
+            text="connector body",
+            extra_info={"source": "connector/file.md", "file_path": "file.md"},
+        )
+    ]
+    fake_reader.directory_structure = {
+        "file.md": {
+            "type": "text/markdown",
+            "size_bytes": 12,
+            "token_count": 3,
+        }
+    }
+    monkeypatch.setattr(
+        worker, "SimpleDirectoryReader", lambda *a, **kw: fake_reader
+    )
+
+    monkeypatch.setattr(
+        worker,
+        "embed_and_store_documents",
+        lambda docs, full_path, source_id, task, **kw: None,
+    )
+    monkeypatch.setattr(
+        worker, "upload_index", lambda full_path, file_data: None
+    )
+
+
+@pytest.mark.unit
+class TestIngestConnectorSyncUpdatesDate:
+    def test_sync_mode_bumps_source_date(
+        self,
+        pg_conn,
+        patch_worker_db,
+        task_self,
+        monkeypatch,
+        _mock_connector_pipeline,
+    ):
+        from application import worker
+        import datetime as dt
+
+        old_date = dt.datetime(2019, 6, 1, tzinfo=dt.timezone.utc)
+        src = SourcesRepository(pg_conn).create(
+            "gdrive-folder",
+            user_id="dave",
+            type="connector:file",
+            retriever="classic",
+            sync_frequency="weekly",
+            remote_data={"provider": "google_drive"},
+            date=old_date,
+        )
+        source_id = str(src["id"])
+
+        worker.ingest_connector(
+            task_self,
+            "gdrive-folder",
+            "dave",
+            "google_drive",
+            session_token="tok",
+            file_ids=["f1"],
+            folder_ids=[],
+            operation_mode="sync",
+            doc_id=source_id,
+            sync_frequency="weekly",
+        )
+
+        refreshed = SourcesRepository(pg_conn).get(source_id, "dave")
+        assert refreshed is not None
+        # row_to_dict coerces datetimes to ISO 8601 strings; UTC
+        # timezone-aware ISO strings sort chronologically.
+        assert refreshed["date"] > old_date.isoformat(), (
+            "ingest_connector(sync) should push sources.date forward"
+        )
+
+
+@pytest.mark.unit
+class TestIngestConnectorDeterministicSourceId:
+    """Upload-mode ``ingest_connector`` derives a stable ``source_id`` from the key."""
+
+    def test_uses_uuid5_when_idempotency_key_present(
+        self,
+        task_self,
+        monkeypatch,
+        _mock_connector_pipeline,
+    ):
+        from application import worker
+
+        captured: list[dict] = []
+        monkeypatch.setattr(
+            worker, "upload_index",
+            lambda full_path, file_data: captured.append(file_data),
+        )
+
+        for _ in range(2):
+            worker.ingest_connector(
+                task_self,
+                "gdrive-folder",
+                "dave",
+                "google_drive",
+                session_token="tok",
+                file_ids=["f1"],
+                folder_ids=[],
+                operation_mode="upload",
+                idempotency_key="abc",
+            )
+
+        expected = str(uuid.uuid5(worker.DOCSGPT_INGEST_NAMESPACE, "abc"))
+        assert len(captured) == 2
+        assert captured[0]["id"] == expected
+        assert captured[1]["id"] == expected
+
+    def test_falls_back_to_uuid4_without_key(
+        self,
+        task_self,
+        monkeypatch,
+        _mock_connector_pipeline,
+    ):
+        from application import worker
+
+        captured: list[dict] = []
+        monkeypatch.setattr(
+            worker, "upload_index",
+            lambda full_path, file_data: captured.append(file_data),
+        )
+
+        for _ in range(2):
+            worker.ingest_connector(
+                task_self,
+                "gdrive-folder",
+                "dave",
+                "google_drive",
+                session_token="tok",
+                file_ids=["f1"],
+                folder_ids=[],
+                operation_mode="upload",
+            )
+
+        assert len(captured) == 2
+        assert captured[0]["id"] != captured[1]["id"]

tests/worker/test_ingest_worker.py

@@ -0,0 +1,315 @@
+"""Smoke test for ``application.worker.ingest_worker``.
+
+``ingest_worker`` does **not** write to Postgres directly. Its PG
+side-effect (creating the ``sources`` row) goes through the
+``upload_index`` HTTP callback to the backend, which writes to PG in
+its own request context. That callback is intentionally out of scope
+here — we can't reach it from the worker test without spinning up the
+Flask app.
+
+What we *can* smoke: the task body runs end-to-end, the pipeline gets
+invoked with the expected job metadata, and ``upload_index`` is handed
+a ``file_data`` payload that carries the caller-provided ``user``,
+``job_name``, and ``retriever``. That's the contract the
+backend-facing write depends on.
+"""
+
+from __future__ import annotations
+
+import uuid
+from io import BytesIO
+from unittest.mock import MagicMock
+
+import pytest
+
+from application.parser.schema.base import Document
+
+
+def _patch_ingest_pipeline(monkeypatch, captured):
+    """Stub the non-PG boundaries used by ``ingest_worker``.
+
+    Storage, the directory reader, the embedding pipeline, and
+    ``upload_index`` are all replaced with cheap fakes so the worker
+    runs end-to-end inside the test process. ``captured`` is appended
+    with each ``upload_index`` payload so callers can assert on the
+    derived ``source_id``.
+    """
+    from application import worker
+
+    fake_storage = MagicMock(name="storage")
+    fake_storage.is_directory.return_value = False
+    fake_storage.get_file.return_value = BytesIO(b"hello")
+    monkeypatch.setattr(
+        worker.StorageCreator, "get_storage", lambda: fake_storage
+    )
+
+    fake_reader = MagicMock(name="reader")
+    fake_reader.load_data.return_value = [
+        Document(text="hello body", extra_info={"source": "a.txt"})
+    ]
+    fake_reader.directory_structure = {
+        "a.txt": {
+            "type": "text/plain",
+            "size_bytes": 5,
+            "token_count": 2,
+        }
+    }
+    monkeypatch.setattr(
+        worker, "SimpleDirectoryReader", lambda *a, **kw: fake_reader
+    )
+
+    monkeypatch.setattr(
+        worker,
+        "embed_and_store_documents",
+        lambda docs, full_path, source_id, task, **kw: None,
+    )
+
+    monkeypatch.setattr(
+        worker, "upload_index",
+        lambda full_path, file_data: captured.append(file_data),
+    )
+
+
+def _spy_chunker(monkeypatch):
+    """Spy on ``ChunkerCreator.create_chunker`` and return the recorded kwargs.
+
+    Replaces it with a fake that records the call's kwargs and returns a
+    chunker whose ``chunk`` passes documents through unchanged, so the
+    config the worker threads into chunking can be asserted.
+    """
+    from application import worker
+
+    calls: list[dict] = []
+
+    def _create_chunker(strategy, **kwargs):
+        calls.append({"strategy": strategy, **kwargs})
+        chunker = MagicMock(name="chunker")
+        chunker.chunk.side_effect = lambda documents: documents
+        return chunker
+
+    monkeypatch.setattr(
+        worker.ChunkerCreator, "create_chunker", staticmethod(_create_chunker)
+    )
+    return calls
+
+
+@pytest.mark.unit
+class TestIngestWorker:
+    def test_invokes_upload_index_with_expected_payload(
+        self, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        captured: list[dict] = []
+        _patch_ingest_pipeline(monkeypatch, captured)
+
+        result = worker.ingest_worker(
+            task_self,
+            directory="inputs",
+            formats=[".txt"],
+            job_name="job1",
+            file_path="inputs/eve/job1/a.txt",
+            filename="a.txt",
+            user="eve",
+            retriever="classic",
+        )
+
+        assert result["limited"] is False
+        assert result["user"] == "eve"
+        assert result["name_job"] == "job1"
+
+        assert len(captured) == 1
+        payload = captured[0]
+        assert payload["user"] == "eve"
+        assert payload["name"] == "job1"
+        assert payload["retriever"] == "classic"
+        assert payload["type"] == "local"
+        # A fresh source UUID is minted for the backend /upload_index route.
+        assert payload["id"]
+
+
+@pytest.mark.unit
+class TestIngestWorkerConfigThreading:
+    """``config.chunking`` must drive the chunker the worker builds (D1/D8)."""
+
+    def test_no_config_uses_classic_defaults(
+        self, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        captured: list[dict] = []
+        _patch_ingest_pipeline(monkeypatch, captured)
+        calls = _spy_chunker(monkeypatch)
+
+        worker.ingest_worker(
+            task_self,
+            directory="inputs",
+            formats=[".txt"],
+            job_name="job1",
+            file_path="inputs/eve/job1/a.txt",
+            filename="a.txt",
+            user="eve",
+        )
+
+        assert len(calls) == 1
+        # Byte-identical-defaults guarantee: empty config → classic 1250/150.
+        assert calls[0]["strategy"] == "classic_chunk"
+        assert calls[0]["chunking_strategy"] == "classic_chunk"
+        assert calls[0]["max_tokens"] == 1250
+        assert calls[0]["min_tokens"] == 150
+        assert calls[0]["duplicate_headers"] is False
+
+    def test_non_default_config_is_threaded(
+        self, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        captured: list[dict] = []
+        _patch_ingest_pipeline(monkeypatch, captured)
+        calls = _spy_chunker(monkeypatch)
+
+        worker.ingest_worker(
+            task_self,
+            directory="inputs",
+            formats=[".txt"],
+            job_name="job1",
+            file_path="inputs/eve/job1/a.txt",
+            filename="a.txt",
+            user="eve",
+            config={
+                "chunking": {
+                    "strategy": "recursive",
+                    "max_tokens": 800,
+                    "min_tokens": 50,
+                }
+            },
+        )
+
+        assert len(calls) == 1
+        assert calls[0]["strategy"] == "recursive"
+        assert calls[0]["chunking_strategy"] == "recursive"
+        assert calls[0]["max_tokens"] == 800
+        assert calls[0]["min_tokens"] == 50
+
+
+@pytest.mark.unit
+class TestIngestWorkerDeterministicSourceId:
+    """Retried ingests with the same key should land on the same source row."""
+
+    def test_uses_uuid5_when_idempotency_key_present(
+        self, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        captured: list[dict] = []
+        _patch_ingest_pipeline(monkeypatch, captured)
+
+        worker.ingest_worker(
+            task_self,
+            directory="inputs",
+            formats=[".txt"],
+            job_name="job1",
+            file_path="inputs/eve/job1/a.txt",
+            filename="a.txt",
+            user="eve",
+            idempotency_key="abc",
+        )
+        worker.ingest_worker(
+            task_self,
+            directory="inputs",
+            formats=[".txt"],
+            job_name="job1",
+            file_path="inputs/eve/job1/a.txt",
+            filename="a.txt",
+            user="eve",
+            idempotency_key="abc",
+        )
+
+        expected = str(
+            uuid.uuid5(worker.DOCSGPT_INGEST_NAMESPACE, "abc")
+        )
+        assert len(captured) == 2
+        assert captured[0]["id"] == expected
+        assert captured[1]["id"] == expected
+
+    def test_falls_back_to_uuid4_without_key(
+        self, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        captured: list[dict] = []
+        _patch_ingest_pipeline(monkeypatch, captured)
+
+        for _ in range(2):
+            worker.ingest_worker(
+                task_self,
+                directory="inputs",
+                formats=[".txt"],
+                job_name="job1",
+                file_path="inputs/eve/job1/a.txt",
+                filename="a.txt",
+                user="eve",
+            )
+
+        assert len(captured) == 2
+        # Random uuid4 fallback: two runs must produce different ids.
+        assert captured[0]["id"] != captured[1]["id"]
+
+    def test_double_ingest_writes_one_source_row(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        """Run the worker body twice with the same key; assert the
+        backend-side ``upload_index`` would persist one row, not two.
+
+        ``upload_index`` is the actual backend write path; the worker
+        feeds it a ``file_data`` payload keyed by the derived id. We
+        emulate that path locally (``SourcesRepository.create``) on the
+        first call and assert the second call hits the existing-row
+        branch instead of inserting again.
+        """
+        from application import worker
+        from application.storage.db.repositories.sources import SourcesRepository
+
+        captured: list[dict] = []
+        _patch_ingest_pipeline(monkeypatch, captured)
+
+        def _persist(full_path, file_data):
+            captured.append(file_data)
+            repo = SourcesRepository(pg_conn)
+            existing = repo.get(file_data["id"], file_data["user"])
+            if existing is None:
+                repo.create(
+                    file_data["name"],
+                    source_id=file_data["id"],
+                    user_id=file_data["user"],
+                    type=file_data["type"],
+                    retriever=file_data["retriever"],
+                    tokens=file_data["tokens"],
+                )
+
+        monkeypatch.setattr(worker, "upload_index", _persist)
+
+        for _ in range(2):
+            worker.ingest_worker(
+                task_self,
+                directory="inputs",
+                formats=[".txt"],
+                job_name="job1",
+                file_path="inputs/eve/job1/a.txt",
+                filename="a.txt",
+                user="eve",
+                idempotency_key="dedupe-key",
+            )
+
+        expected = str(
+            uuid.uuid5(worker.DOCSGPT_INGEST_NAMESPACE, "dedupe-key")
+        )
+        assert captured[0]["id"] == expected
+        assert captured[1]["id"] == expected
+
+        # Exactly one row in PG for that derived id.
+        result = pg_conn.exec_driver_sql(
+            "SELECT count(*) FROM sources WHERE id = %s",
+            (expected,),
+        ).fetchone()
+        assert result[0] == 1

tests/worker/test_parse_document_worker.py

@@ -0,0 +1,266 @@
+"""Unit tests for ``application.worker.parse_document_worker``.
+
+The worker re-resolves the artifact through the run-scoped gate (independent of
+the tool), reads its bytes, shapes the result, and persists when asked. The DB,
+storage, parser, and persistence boundaries are mocked so no live worker / DB is
+needed (call the underlying function directly).
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Any, Dict, Optional
+
+import pytest
+
+import application.worker as worker
+
+_ART_ID = str(uuid.uuid4())
+
+
+class _FakeFile:
+    def __init__(self, data: bytes) -> None:
+        self._data = data
+
+    def read(self) -> bytes:
+        return self._data
+
+
+class _FakeStorage:
+    def __init__(self, data: bytes = b"%PDF-1.4 fake") -> None:
+        self._data = data
+
+    def get_file(self, path):
+        return _FakeFile(self._data)
+
+
+def _patch_repo(monkeypatch, *, found: bool, run: Optional[str]):
+    class _Repo:
+        def __init__(self, conn):
+            pass
+
+        def artifact_id_at_position(self, n, *, conversation_id=None, workflow_run_id=None):
+            if not found or n != 1 or (run is not None and workflow_run_id != run):
+                return None
+            return _ART_ID
+
+        def get_artifact_in_parent(self, artifact_id, *, conversation_id=None, workflow_run_id=None):
+            if not found or (run is not None and workflow_run_id != run):
+                return None
+            return {"id": artifact_id, "current_version": 1, "title": "statement.pdf"}
+
+        def get_version(self, artifact_id, version):
+            return {"filename": "statement.pdf", "storage_path": f"inputs/u/artifacts/{artifact_id}/v1/x.pdf"}
+
+    class _Conn:
+        def __enter__(self):
+            return object()
+
+        def __exit__(self, *exc):
+            return False
+
+    monkeypatch.setattr(worker, "db_readonly", lambda: _Conn())
+    monkeypatch.setattr(worker, "ArtifactsRepository", _Repo)
+    monkeypatch.setattr(worker.StorageCreator, "get_storage", staticmethod(lambda: _FakeStorage()))
+
+
+def _patch_parse(monkeypatch, result: Dict[str, Any]):
+    import application.parser.document_reader as dr
+
+    monkeypatch.setattr(dr, "parse_document_bytes", lambda data, filename, **opts: result)
+
+
+@pytest.mark.unit
+def test_requires_parent():
+    out = worker.parse_document_worker(None, _ART_ID, {}, "u-1", {})
+    assert out["status"] == "error" and "conversation_id or workflow_run_id" in out["error"]
+
+
+@pytest.mark.unit
+def test_happy_path_shapes_result(monkeypatch):
+    _patch_repo(monkeypatch, found=True, run="run-1")
+    _patch_parse(monkeypatch, {"output": "markdown", "content": "# Hi", "truncated": False})
+
+    out = worker.parse_document_worker(
+        None, _ART_ID, {"workflow_run_id": "run-1"}, "u-1", {"output": "markdown", "persist": False}
+    )
+    assert out["status"] == "ok"
+    assert out["content"] == "# Hi"
+    assert out["output"] == "markdown"
+
+
+@pytest.mark.unit
+def test_cross_run_artifact_is_rejected(monkeypatch):
+    # The artifact only resolves for run-OTHER; the worker is asked for run-1 -> denied.
+    _patch_repo(monkeypatch, found=True, run="run-OTHER")
+    _patch_parse(monkeypatch, {"output": "markdown", "content": "x", "truncated": False})
+
+    out = worker.parse_document_worker(
+        None, _ART_ID, {"workflow_run_id": "run-1"}, "u-1", {"persist": False}
+    )
+    assert out["status"] == "error" and "not found in this conversation/run" in out["error"]
+
+
+@pytest.mark.unit
+def test_missing_artifact_is_rejected(monkeypatch):
+    _patch_repo(monkeypatch, found=False, run="run-1")
+    _patch_parse(monkeypatch, {"output": "markdown", "content": "x", "truncated": False})
+
+    out = worker.parse_document_worker(
+        None, "ghost", {"workflow_run_id": "run-1"}, "u-1", {"persist": False}
+    )
+    assert out["status"] == "error" and "not found in this conversation/run" in out["error"]
+
+
+@pytest.mark.unit
+def test_parse_error_is_surfaced(monkeypatch):
+    _patch_repo(monkeypatch, found=True, run="run-1")
+    _patch_parse(monkeypatch, {"error": "unsupported file type '.exe'."})
+
+    out = worker.parse_document_worker(
+        None, _ART_ID, {"workflow_run_id": "run-1"}, "u-1", {"persist": False}
+    )
+    assert out["status"] == "error" and "unsupported file type" in out["error"]
+
+
+@pytest.mark.unit
+def test_persist_stores_full_result_and_returns_ref(monkeypatch):
+    _patch_repo(monkeypatch, found=True, run="run-1")
+    full = {"output": "structured", "content": "# Big", "structured": {"texts": [{}]}, "truncated": False}
+    _patch_parse(monkeypatch, full)
+
+    captured: Dict[str, Any] = {}
+
+    def _fake_persist(**kwargs):
+        captured.update(kwargs)
+        return {"artifact_id": "new-art", "version": 1, "filename": "x.json",
+                "mime_type": "application/json", "size": 10}
+
+    import application.sandbox.artifacts_capture as ac
+
+    monkeypatch.setattr(ac, "persist_new_artifact", _fake_persist)
+
+    out = worker.parse_document_worker(
+        None, _ART_ID, {"workflow_run_id": "run-1"}, "u-1", {"output": "structured", "persist": True}
+    )
+    assert out["status"] == "ok"
+    assert out["artifact"]["artifact_id"] == "new-art"
+    # The FULL shaped result is persisted by reference (not just the bounded view).
+    import json
+
+    assert captured["kind"] == "data"
+    assert json.loads(captured["data"].decode("utf-8")) == full
+    assert captured["workflow_run_id"] == "run-1"
+
+
+@pytest.mark.unit
+def test_persist_quota_surfaces_as_artifact_error(monkeypatch):
+    _patch_repo(monkeypatch, found=True, run="run-1")
+    _patch_parse(monkeypatch, {"output": "markdown", "content": "x", "truncated": False})
+
+    import application.sandbox.artifacts_capture as ac
+
+    def _quota(**kwargs):
+        raise ac.QuotaExceeded("artifact storage quota reached")
+
+    monkeypatch.setattr(ac, "persist_new_artifact", _quota)
+
+    out = worker.parse_document_worker(
+        None, _ART_ID, {"workflow_run_id": "run-1"}, "u-1", {"persist": True}
+    )
+    # Parse still succeeded; quota failure is a non-fatal note.
+    assert out["status"] == "ok"
+    assert "artifact" not in out
+    assert "quota" in out["artifact_error"].lower()
+
+
+@pytest.mark.unit
+def test_result_payload_content_is_bounded(monkeypatch):
+    _patch_repo(monkeypatch, found=True, run="run-1")
+    huge = "Z" * 50000
+    _patch_parse(monkeypatch, {"output": "markdown", "content": huge, "truncated": False})
+
+    out = worker.parse_document_worker(
+        None, _ART_ID, {"workflow_run_id": "run-1"}, "u-1", {"persist": False}
+    )
+    # The bounded view ridden back to the (Redis) result backend is far smaller.
+    assert len(out["content"]) < len(huge)
+    assert "...[truncated" in out["content"]
+
+
+@pytest.mark.unit
+def test_result_payload_chunks_are_bounded(monkeypatch):
+    import application.parser.document_reader as dr
+
+    _patch_repo(monkeypatch, found=True, run="run-1")
+    # Many oversized chunks: count is capped AND each chunk is windowed.
+    huge_chunk = "Z" * 50000
+    chunks = [huge_chunk for _ in range(dr._MAX_CHUNKS_RETURNED * 3)]
+    _patch_parse(monkeypatch, {"output": "chunks", "chunks": chunks, "truncated": False})
+
+    out = worker.parse_document_worker(
+        None, _ART_ID, {"workflow_run_id": "run-1"}, "u-1", {"persist": False}
+    )
+    assert len(out["chunks"]) == dr._MAX_CHUNKS_RETURNED
+    assert out["chunks_truncated"] is True
+    assert out["total_chunks"] == dr._MAX_CHUNKS_RETURNED * 3
+    assert all("...[truncated" in c for c in out["chunks"])
+    assert all(len(c) < len(huge_chunk) for c in out["chunks"])
+
+
+def _capture_persist(monkeypatch):
+    """Patch persist_new_artifact to record what would be stored; return the capture dict."""
+    import application.sandbox.artifacts_capture as ac
+
+    captured: Dict[str, Any] = {}
+
+    def _fake_persist(**kwargs):
+        captured.update(kwargs)
+        return {"artifact_id": "new-art", "version": 1, "filename": "x.json",
+                "mime_type": "application/json", "size": len(kwargs.get("data", b""))}
+
+    monkeypatch.setattr(ac, "persist_new_artifact", _fake_persist)
+    return captured
+
+
+@pytest.mark.unit
+def test_persist_keeps_full_content_while_view_is_bounded(monkeypatch):
+    # A >8000-byte doc: the persisted artifact must keep the FULL text, while the returned
+    # (Redis/LLM) view is head+tail windowed.
+    import json
+
+    _patch_repo(monkeypatch, found=True, run="run-1")
+    big = "Z" * 12000
+    _patch_parse(monkeypatch, {"output": "markdown", "content": big, "truncated": False})
+    captured = _capture_persist(monkeypatch)
+
+    out = worker.parse_document_worker(
+        None, _ART_ID, {"workflow_run_id": "run-1"}, "u-1", {"output": "markdown", "persist": True}
+    )
+
+    persisted = json.loads(captured["data"].decode("utf-8"))
+    assert persisted["content"] == big  # FULL parse persisted
+    assert len(persisted["content"]) == 12000
+    assert len(out["content"]) < len(big)  # bounded view
+    assert "...[truncated" in out["content"]
+    assert out["truncated"] is True
+
+
+@pytest.mark.unit
+def test_persist_full_but_view_respects_max_chars(monkeypatch):
+    # max_chars bounds only the returned view; the persisted artifact stays full.
+    import json
+
+    _patch_repo(monkeypatch, found=True, run="run-1")
+    big = "Z" * 5000
+    _patch_parse(monkeypatch, {"output": "markdown", "content": big, "truncated": False})
+    captured = _capture_persist(monkeypatch)
+
+    out = worker.parse_document_worker(
+        None, _ART_ID, {"workflow_run_id": "run-1"}, "u-1",
+        {"output": "markdown", "persist": True, "max_chars": 100},
+    )
+
+    assert json.loads(captured["data"].decode("utf-8"))["content"] == big  # full persisted
+    assert len(out["content"]) == 100  # view capped by max_chars
+    assert out["truncated"] is True

tests/worker/test_reembed_wiki_page.py

@@ -0,0 +1,183 @@
+"""Tests for the per-page wiki re-embed worker and Celery task.
+
+The worker loads the source row from the ephemeral DB, then re-embeds (or
+purges) one wiki page. The vector store, ``ChunkerCreator``, and the page
+lookup are mocked so no live embeddings run; the ``sources`` row is real so
+``SourcesRepository.get_any`` resolves.
+"""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from unittest.mock import MagicMock
+
+import pytest
+
+from application.parser.schema.base import Document
+from application.storage.db.repositories.sources import SourcesRepository
+
+
+def _seed_source(pg_conn) -> str:
+    src = SourcesRepository(pg_conn).create(
+        "wiki-set",
+        user_id="alice",
+        type="wiki",
+        config={"kind": "wiki"},
+    )
+    return str(src["id"])
+
+
+def _patch_store(monkeypatch, store):
+    monkeypatch.setattr(
+        "application.vectorstore.vector_creator.VectorCreator.create_vectorstore",
+        lambda *a, **kw: store,
+    )
+
+
+def _patch_repo(monkeypatch, page):
+    """Replace ``WikiPagesRepository`` with one returning ``page`` and a stub setter."""
+    repo = MagicMock(name="wiki_repo")
+    repo.get_by_path.return_value = page
+    repo.set_embed_status.return_value = True
+    monkeypatch.setattr(
+        "application.worker.WikiPagesRepository", lambda conn: repo
+    )
+    return repo
+
+
+def _patch_chunker(monkeypatch, chunks):
+    calls: list[dict] = []
+
+    def _create_chunker(strategy, **kwargs):
+        calls.append({"strategy": strategy, **kwargs})
+        chunker = MagicMock(name="chunker")
+        chunker.chunk.return_value = chunks
+        return chunker
+
+    monkeypatch.setattr(
+        "application.worker.ChunkerCreator.create_chunker",
+        staticmethod(_create_chunker),
+    )
+    return calls
+
+
+@pytest.mark.unit
+class TestReembedWikiPageWorker:
+    def test_page_exists_reembeds(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+
+        store = MagicMock(name="vector_store")
+        store.delete_chunks_by_source_path.return_value = 3
+        _patch_store(monkeypatch, store)
+
+        repo = _patch_repo(
+            monkeypatch,
+            {"content": "page body", "title": "Page Title"},
+        )
+        _patch_chunker(
+            monkeypatch,
+            [Document(text="c1"), Document(text="c2")],
+        )
+
+        result = worker.reembed_wiki_page_worker(
+            task_self, source_id, "guide/intro.md", "hash-1", "alice"
+        )
+
+        store.delete_chunks_by_source_path.assert_called_once_with(
+            "guide/intro.md"
+        )
+        assert store.add_chunk.call_count == 2
+        for call in store.add_chunk.call_args_list:
+            assert call.kwargs["metadata"]["source"] == "guide/intro.md"
+            assert call.kwargs["metadata"]["title"] == "Page Title"
+            assert call.kwargs["metadata"]["filename"] == "guide/intro.md"
+        repo.set_embed_status.assert_called_once_with(
+            source_id, "guide/intro.md", "embedded"
+        )
+        assert result == {"status": "embedded", "added": 2, "deleted": 3}
+
+    def test_page_missing_purges(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+
+        store = MagicMock(name="vector_store")
+        store.delete_chunks_by_source_path.return_value = 4
+        _patch_store(monkeypatch, store)
+
+        repo = _patch_repo(monkeypatch, None)
+
+        result = worker.reembed_wiki_page_worker(
+            task_self, source_id, "old/page.md", "hash-x", "alice"
+        )
+
+        store.delete_chunks_by_source_path.assert_called_once_with("old/page.md")
+        store.add_chunk.assert_not_called()
+        repo.set_embed_status.assert_not_called()
+        assert result == {"status": "deleted", "deleted": 4}
+
+    def test_embed_failure_sets_failed_and_reraises(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+
+        store = MagicMock(name="vector_store")
+        store.delete_chunks_by_source_path.return_value = 0
+        store.add_chunk.side_effect = RuntimeError("embed boom")
+        _patch_store(monkeypatch, store)
+
+        repo = _patch_repo(
+            monkeypatch, {"content": "page body", "title": "T"}
+        )
+        _patch_chunker(monkeypatch, [Document(text="c1")])
+
+        with pytest.raises(RuntimeError, match="embed boom"):
+            worker.reembed_wiki_page_worker(
+                task_self, source_id, "p.md", "hash-2", "alice"
+            )
+
+        repo.set_embed_status.assert_called_once_with(source_id, "p.md", "failed")
+
+
+@pytest.mark.unit
+class TestReembedWikiPageTask:
+    def test_idempotency_key_is_content_hash(self, pg_conn, monkeypatch):
+        """A redelivery with the same content_hash key short-circuits the worker."""
+        from application.api.user import tasks
+
+        @contextmanager
+        def _yield():
+            yield pg_conn
+
+        monkeypatch.setattr(
+            "application.api.user.idempotency.db_session", _yield
+        )
+        monkeypatch.setattr(
+            "application.api.user.idempotency.db_readonly", _yield
+        )
+
+        calls: list[str] = []
+
+        def _fake_worker(self, source_id, path, content_hash, user):
+            calls.append(content_hash)
+            return {"status": "embedded", "added": 1, "deleted": 0}
+
+        monkeypatch.setattr(tasks, "reembed_wiki_page_worker", _fake_worker)
+
+        first = tasks.reembed_wiki_page(
+            "src-1", "p.md", "abc123", "alice", idempotency_key="abc123",
+        )
+        second = tasks.reembed_wiki_page(
+            "src-1", "p.md", "abc123", "alice", idempotency_key="abc123",
+        )
+
+        assert first == second
+        assert len(calls) == 1

tests/worker/test_reingest_source_worker.py

@@ -0,0 +1,182 @@
+"""Smoke test for ``application.worker.reingest_source_worker``.
+
+The task reads a source row, diffs its stored ``directory_structure``
+against what's currently in storage, updates the vector store, then
+writes the refreshed ``directory_structure`` / ``date`` / ``tokens``
+back to the ``sources`` row. We assert that last PG update actually
+lands on the row for our ephemeral DB.
+"""
+
+from __future__ import annotations
+
+from io import BytesIO
+from unittest.mock import MagicMock
+
+import pytest
+
+from application.parser.schema.base import Document
+from application.storage.db.repositories.sources import SourcesRepository
+
+
+@pytest.mark.unit
+class TestReingestSourceWorker:
+    def test_updates_source_directory_structure_and_tokens(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        # Seed a source we can re-ingest.
+        src = SourcesRepository(pg_conn).create(
+            "doc-set",
+            user_id="alice",
+            type="local",
+            retriever="classic",
+            file_path="inputs/alice/doc-set",
+            directory_structure={
+                "stale.txt": {
+                    "type": "text/plain",
+                    "size_bytes": 10,
+                    "token_count": 5,
+                }
+            },
+        )
+        source_id = str(src["id"])
+
+        # Storage: pretend the source directory has one file now, a
+        # different one from what the source row last stored — that way
+        # the "added/removed" branch runs end-to-end.
+        fake_storage = MagicMock(name="storage")
+        fake_storage.is_directory.return_value = True
+        fake_storage.list_files.return_value = []
+        monkeypatch.setattr(
+            worker.StorageCreator, "get_storage", lambda: fake_storage
+        )
+
+        # Reader: return no docs but advertise the new directory shape and
+        # per-file token counts. The worker's update statement uses
+        # ``sum(reader.file_token_counts.values())`` and the ``
+        # directory_structure`` as-is.
+        fake_reader = MagicMock(name="reader")
+        fake_reader.load_data.return_value = []
+        fake_reader.directory_structure = {
+            "fresh.md": {
+                "type": "text/markdown",
+                "size_bytes": 42,
+                "token_count": 17,
+            }
+        }
+        fake_reader.file_token_counts = {"fresh.md": 17}
+        monkeypatch.setattr(
+            worker, "SimpleDirectoryReader", lambda *a, **kw: fake_reader
+        )
+
+        # Vector store: report no existing chunks, and swallow any add/
+        # delete call.
+        fake_store = MagicMock(name="vector_store")
+        fake_store.get_chunks.return_value = []
+        monkeypatch.setattr(
+            "application.vectorstore.vector_creator.VectorCreator.create_vectorstore",
+            lambda *a, **kw: fake_store,
+        )
+
+        result = worker.reingest_source_worker(task_self, source_id, "alice")
+
+        assert result["status"] == "completed"
+        # ``fresh.md`` is new vs ``stale.txt``, which was removed.
+        assert "fresh.md" in result["added_files"]
+        assert "stale.txt" in result["removed_files"]
+
+        refreshed = SourcesRepository(pg_conn).get(source_id, "alice")
+        assert refreshed is not None
+        # Token count was recomputed from the reader.
+        assert refreshed["tokens"] == "17"
+        # And the new directory_structure replaced the stale one.
+        assert "fresh.md" in refreshed["directory_structure"]
+        assert "stale.txt" not in refreshed["directory_structure"]
+
+    def test_reingest_threads_source_chunking_config(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        """The source's stored ``config.chunking`` must drive re-chunking.
+
+        Seeds a source with a non-default chunking config and an added file,
+        then asserts ``ChunkerCreator.create_chunker`` is built with that
+        config — not the 1250/150 classic defaults (D1/D8).
+        """
+        from application import worker
+
+        src = SourcesRepository(pg_conn).create(
+            "doc-set",
+            user_id="alice",
+            type="local",
+            retriever="classic",
+            file_path="inputs/alice/doc-set",
+            config={
+                "chunking": {
+                    "strategy": "recursive",
+                    "max_tokens": 800,
+                    "min_tokens": 50,
+                }
+            },
+            directory_structure={},
+        )
+        source_id = str(src["id"])
+
+        # Storage: one file in the source directory, downloaded to temp_dir so
+        # the worker's added-files branch finds it on disk and chunks it.
+        fake_storage = MagicMock(name="storage")
+        fake_storage.is_directory.side_effect = lambda p: p == "inputs/alice/doc-set"
+        fake_storage.list_files.return_value = ["inputs/alice/doc-set/fresh.md"]
+        fake_storage.get_file.return_value = BytesIO(b"fresh body")
+        monkeypatch.setattr(
+            worker.StorageCreator, "get_storage", lambda: fake_storage
+        )
+
+        # Reader: advertise the new file and hand back one doc to chunk.
+        fake_reader = MagicMock(name="reader")
+        fake_reader.load_data.return_value = [
+            Document(text="fresh body", extra_info={"source": "fresh.md"})
+        ]
+        fake_reader.directory_structure = {
+            "fresh.md": {
+                "type": "text/markdown",
+                "size_bytes": 10,
+                "token_count": 3,
+            }
+        }
+        fake_reader.file_token_counts = {"fresh.md": 3}
+        monkeypatch.setattr(
+            worker, "SimpleDirectoryReader", lambda *a, **kw: fake_reader
+        )
+
+        fake_store = MagicMock(name="vector_store")
+        fake_store.get_chunks.return_value = []
+        monkeypatch.setattr(
+            "application.vectorstore.vector_creator.VectorCreator.create_vectorstore",
+            lambda *a, **kw: fake_store,
+        )
+
+        calls: list[dict] = []
+
+        def _create_chunker(strategy, **kwargs):
+            calls.append({"strategy": strategy, **kwargs})
+            chunker = MagicMock(name="chunker")
+            chunker.chunk.side_effect = lambda documents: documents
+            return chunker
+
+        monkeypatch.setattr(
+            worker.ChunkerCreator,
+            "create_chunker",
+            staticmethod(_create_chunker),
+        )
+
+        result = worker.reingest_source_worker(task_self, source_id, "alice")
+
+        assert result["status"] == "completed"
+        assert "fresh.md" in result["added_files"]
+        assert len(calls) == 1
+        # Config-driven, not the 1250/150 classic defaults.
+        assert calls[0]["strategy"] == "recursive"
+        assert calls[0]["chunking_strategy"] == "recursive"
+        assert calls[0]["max_tokens"] == 800
+        assert calls[0]["min_tokens"] == 50

tests/worker/test_remote_worker.py

@@ -0,0 +1,405 @@
+"""Smoke tests for ``application.worker.remote_worker`` and ``sync_worker``.
+
+``remote_worker`` in ``sync`` mode does one PG write: it bumps
+``sources.date`` on the referenced source row to ``now()``. That's the
+side-effect we assert here. The remote loader, chunker, embedding
+pipeline, and the backend HTTP callback are all mocked — only the PG
+update is real.
+
+``sync_worker`` reads rows out of ``sources`` whose ``sync_frequency``
+matches and dispatches them through ``sync`` → ``remote_worker``. We
+assert one seeded row is discovered and forwarded.
+"""
+
+from __future__ import annotations
+
+import os
+import uuid
+from unittest.mock import MagicMock
+
+import pytest
+
+from application.parser.schema.base import Document
+from application.storage.db.repositories.sources import SourcesRepository
+
+
+@pytest.fixture
+def _mock_remote_pipeline(monkeypatch):
+    """Stub out the non-PG boundaries used by ``remote_worker``."""
+    from application import worker
+
+    fake_loader = MagicMock(name="remote_loader")
+    fake_loader.load_data.return_value = [
+        Document(
+            text="page body",
+            extra_info={"file_path": "guides/setup.md", "title": "setup"},
+            doc_id="d1",
+        )
+    ]
+    monkeypatch.setattr(
+        worker.RemoteCreator, "create_loader", lambda loader: fake_loader
+    )
+    monkeypatch.setattr(
+        worker,
+        "embed_and_store_documents",
+        lambda docs, full_path, source_id, task, **kw: None,
+    )
+    monkeypatch.setattr(
+        worker, "upload_index", lambda full_path, file_data: None
+    )
+
+
+@pytest.mark.unit
+class TestRemoteWorkerSyncUpdatesDate:
+    def test_sync_mode_bumps_source_date(
+        self,
+        pg_conn,
+        patch_worker_db,
+        task_self,
+        monkeypatch,
+        _mock_remote_pipeline,
+    ):
+        from application import worker
+
+        # Seed a source with a known old ``date`` we can compare against.
+        import datetime as dt
+
+        old_date = dt.datetime(2020, 1, 1, tzinfo=dt.timezone.utc)
+        src = SourcesRepository(pg_conn).create(
+            "my-remote",
+            user_id="bob",
+            type="crawler",
+            retriever="classic",
+            sync_frequency="daily",
+            remote_data={"urls": ["http://example.com"]},
+            date=old_date,
+        )
+        source_id = str(src["id"])
+
+        worker.remote_worker(
+            task_self,
+            {"urls": ["http://example.com"]},
+            "my-remote",
+            "bob",
+            "crawler",
+            directory="temp",
+            retriever="classic",
+            sync_frequency="daily",
+            operation_mode="sync",
+            doc_id=source_id,
+        )
+
+        refreshed = SourcesRepository(pg_conn).get(source_id, "bob")
+        assert refreshed is not None
+        # row_to_dict coerces datetimes to ISO 8601 strings; UTC
+        # timezone-aware ISO strings sort chronologically.
+        assert refreshed["date"] > old_date.isoformat(), (
+            "remote_worker(sync) should push sources.date forward"
+        )
+
+
+@pytest.mark.unit
+class TestSyncWorker:
+    def test_reads_sources_and_dispatches_sync(
+        self,
+        pg_conn,
+        patch_worker_db,
+        task_self,
+        monkeypatch,
+    ):
+        """``sync_worker`` selects rows by ``sync_frequency`` and passes
+        each to ``sync``. We assert the seeded row is discovered and
+        forwarded with the right doc_id — the nested ``sync`` call is
+        stubbed so we don't re-run the whole remote pipeline here."""
+        from application import worker
+
+        src = SourcesRepository(pg_conn).create(
+            "weekly-feed",
+            user_id="carol",
+            type="url",
+            retriever="classic",
+            sync_frequency="weekly",
+            remote_data={"url": "http://example.com"},
+        )
+
+        captured: list[dict] = []
+
+        def _fake_sync(self, source_data, name_job, user, loader,
+                       sync_frequency, retriever, doc_id=None, directory="temp"):
+            captured.append({
+                "name_job": name_job,
+                "user": user,
+                "loader": loader,
+                "sync_frequency": sync_frequency,
+                "retriever": retriever,
+                "doc_id": doc_id,
+            })
+            return {"status": "success"}
+
+        monkeypatch.setattr(worker, "sync", _fake_sync)
+
+        result = worker.sync_worker(task_self, "weekly")
+
+        assert result["total_sync_count"] == 1
+        assert result["sync_success"] == 1
+        assert len(captured) == 1
+        assert captured[0]["name_job"] == "weekly-feed"
+        assert captured[0]["user"] == "carol"
+        assert captured[0]["loader"] == "url"
+        assert captured[0]["doc_id"] == str(src["id"])
+
+    def test_connector_sources_are_skipped(
+        self,
+        pg_conn,
+        patch_worker_db,
+        task_self,
+        monkeypatch,
+    ):
+        """connector:* sources have no RemoteCreator loader — sync_worker
+        must skip them, not dispatch them into sync()."""
+        from application import worker
+
+        SourcesRepository(pg_conn).create(
+            "drive-folder",
+            user_id="dave",
+            type="connector:file",
+            retriever="classic",
+            sync_frequency="daily",
+            remote_data={
+                "provider": "google_drive",
+                "file_ids": ["f1"],
+                "folder_ids": [],
+                "recursive": False,
+            },
+        )
+
+        def _must_not_run(*args, **kwargs):
+            raise AssertionError("sync() must not run for connector sources")
+
+        monkeypatch.setattr(worker, "sync", _must_not_run)
+
+        result = worker.sync_worker(task_self, "daily")
+
+        assert result["total_sync_count"] == 1
+        assert result["sync_skipped"] == 1
+        assert result["sync_success"] == 0
+        assert result["sync_failure"] == 0
+
+    def test_dict_remote_data_is_normalized_before_loader(
+        self,
+        pg_conn,
+        patch_worker_db,
+        task_self,
+        monkeypatch,
+    ):
+        """Regression: remote_data reads back as a dict; sync_worker must
+        hand the loader the URL string, not the raw dict."""
+        from application import worker
+
+        SourcesRepository(pg_conn).create(
+            "docs-crawl",
+            user_id="erin",
+            type="crawler",
+            retriever="classic",
+            sync_frequency="weekly",
+            remote_data={"url": "https://example.com", "provider": "crawler"},
+        )
+
+        received: list = []
+        fake_loader = MagicMock(name="remote_loader")
+
+        def _capture(source_data):
+            received.append(source_data)
+            return [
+                Document(
+                    text="page body",
+                    extra_info={"file_path": "index.md", "title": "home"},
+                    doc_id="d1",
+                )
+            ]
+
+        fake_loader.load_data.side_effect = _capture
+        monkeypatch.setattr(
+            worker.RemoteCreator, "create_loader", lambda loader: fake_loader
+        )
+        monkeypatch.setattr(
+            worker,
+            "embed_and_store_documents",
+            lambda docs, full_path, source_id, task, **kw: None,
+        )
+        monkeypatch.setattr(
+            worker, "upload_index", lambda full_path, file_data: None
+        )
+
+        result = worker.sync_worker(task_self, "weekly")
+
+        assert result["total_sync_count"] == 1
+        assert result["sync_success"] == 1
+        assert result["sync_failure"] == 0
+        assert received == ["https://example.com"], (
+            "loader must receive the URL string, not the remote_data dict"
+        )
+
+    def test_unsyncable_remote_data_is_skipped(
+        self,
+        pg_conn,
+        patch_worker_db,
+        task_self,
+        monkeypatch,
+    ):
+        """A URL source whose remote_data dict has no URL key normalizes
+        to None — sync_worker must skip it, not dispatch a doomed sync()."""
+        from application import worker
+
+        SourcesRepository(pg_conn).create(
+            "broken-feed",
+            user_id="frank",
+            type="url",
+            retriever="classic",
+            sync_frequency="monthly",
+            remote_data={"provider": "url"},
+        )
+
+        def _must_not_run(*args, **kwargs):
+            raise AssertionError("sync() must not run for unsyncable sources")
+
+        monkeypatch.setattr(worker, "sync", _must_not_run)
+
+        result = worker.sync_worker(task_self, "monthly")
+
+        assert result["total_sync_count"] == 1
+        assert result["sync_skipped"] == 1
+        assert result["sync_failure"] == 0
+        assert result["sync_success"] == 0
+
+
+@pytest.mark.unit
+class TestRemoteWorkerPathTraversal:
+    """Regression: ``name_job`` must not be usable as a path segment.
+
+    Historically ``remote_worker`` built its workspace from
+    ``os.path.join(directory, user, name_job)`` and cleaned it up with
+    ``shutil.rmtree`` in a ``finally``. A ``name_job`` like
+    ``../../evil`` would therefore let an authenticated caller delete
+    directories outside the intended ``<directory>/<user>/`` root.
+    The fix uses a random uuid leaf; ``name_job`` is metadata only.
+    """
+
+    def test_traversal_name_job_does_not_escape_user_workspace(
+        self,
+        tmp_path,
+        task_self,
+        monkeypatch,
+        _mock_remote_pipeline,
+    ):
+        from application import worker
+
+        created_paths: list[str] = []
+        deleted_paths: list[str] = []
+        real_makedirs = os.makedirs
+        real_rmtree = worker.shutil.rmtree
+
+        def _spy_makedirs(path, *args, **kwargs):
+            created_paths.append(path)
+            return real_makedirs(path, *args, **kwargs)
+
+        def _spy_rmtree(path, *args, **kwargs):
+            deleted_paths.append(path)
+            return real_rmtree(path, *args, **kwargs)
+
+        monkeypatch.setattr(worker.os, "makedirs", _spy_makedirs)
+        monkeypatch.setattr(worker.shutil, "rmtree", _spy_rmtree)
+
+        directory = str(tmp_path / "temp")
+        user = "bob"
+        malicious_name = "../../evil"
+
+        worker.remote_worker(
+            task_self,
+            {"urls": ["http://example.com"]},
+            malicious_name,
+            user,
+            "crawler",
+            directory=directory,
+            operation_mode="upload",
+        )
+
+        directory_real = os.path.realpath(directory)
+        user_root = os.path.realpath(os.path.join(directory, user))
+
+        rmtree_targets = [
+            os.path.realpath(p)
+            for p in deleted_paths
+            if os.path.realpath(p).startswith(directory_real)
+        ]
+        assert len(rmtree_targets) == 1, rmtree_targets
+        assert rmtree_targets[0].startswith(user_root + os.sep), (
+            f"rmtree target {rmtree_targets[0]} escaped {user_root}"
+        )
+        assert malicious_name not in "".join(created_paths + deleted_paths)
+
+
+@pytest.mark.unit
+class TestRemoteWorkerDeterministicSourceId:
+    """Upload-mode ``remote_worker`` derives a stable ``source_id`` from the key."""
+
+    def test_uses_uuid5_when_idempotency_key_present(
+        self,
+        tmp_path,
+        task_self,
+        monkeypatch,
+        _mock_remote_pipeline,
+    ):
+        from application import worker
+
+        captured: list[dict] = []
+        monkeypatch.setattr(
+            worker, "upload_index",
+            lambda full_path, file_data: captured.append(file_data),
+        )
+
+        for _ in range(2):
+            worker.remote_worker(
+                task_self,
+                {"urls": ["http://example.com"]},
+                "feed",
+                "bob",
+                "crawler",
+                directory=str(tmp_path / "temp"),
+                operation_mode="upload",
+                idempotency_key="abc",
+            )
+
+        expected = str(uuid.uuid5(worker.DOCSGPT_INGEST_NAMESPACE, "abc"))
+        assert len(captured) == 2
+        assert captured[0]["id"] == expected
+        assert captured[1]["id"] == expected
+
+    def test_falls_back_to_uuid4_without_key(
+        self,
+        tmp_path,
+        task_self,
+        monkeypatch,
+        _mock_remote_pipeline,
+    ):
+        from application import worker
+
+        captured: list[dict] = []
+        monkeypatch.setattr(
+            worker, "upload_index",
+            lambda full_path, file_data: captured.append(file_data),
+        )
+
+        for _ in range(2):
+            worker.remote_worker(
+                task_self,
+                {"urls": ["http://example.com"]},
+                "feed",
+                "bob",
+                "crawler",
+                directory=str(tmp_path / "temp"),
+                operation_mode="upload",
+            )
+
+        assert len(captured) == 2
+        assert captured[0]["id"] != captured[1]["id"]