arc53--docsgpt/tests/parser/file/test_embedding_pipeline.py

import pytest
import logging
from unittest.mock import patch, MagicMock

from application.parser.embedding_pipeline import (
    EmbeddingPipelineError,
    add_text_to_store_with_retry,
    assert_index_complete,
    embed_and_store_documents,
    sanitize_content,
)



def test_sanitize_content_removes_nulls():
    content = "This\x00is\x00a\x00test"
    result = sanitize_content(content)
    assert "\x00" not in result
    assert result == "Thisisatest"


def test_sanitize_content_empty_or_none():
    assert sanitize_content("") == ""
    assert sanitize_content(None) is None



def test_add_text_to_store_with_retry_success():
    store = MagicMock()
    doc = MagicMock()
    doc.page_content = "Test content"
    doc.metadata = {}

    add_text_to_store_with_retry(store, doc, "123")

    store.add_texts.assert_called_once_with(
        ["Test content"], metadatas=[{"source_id": "123"}]
    )


@pytest.fixture
def mock_settings(monkeypatch):
    mock_settings = MagicMock()
    monkeypatch.setattr(
        "application.parser.embedding_pipeline.settings", mock_settings
    )
    return mock_settings


@pytest.fixture
def mock_vector_creator(monkeypatch):
    mock_creator = MagicMock()
    monkeypatch.setattr(
        "application.parser.embedding_pipeline.VectorCreator", mock_creator
    )
    return mock_creator



def test_embed_and_store_documents_creates_folder(tmp_path, mock_settings, mock_vector_creator):
    mock_settings.VECTOR_STORE = "faiss"

    docs = [MagicMock(page_content="doc1", metadata={}), MagicMock(page_content="doc2", metadata={})]
    folder_name = tmp_path / "test_store"
    source_id = "xyz"
    task_status = MagicMock()

    mock_store = MagicMock()
    mock_vector_creator.create_vectorstore.return_value = mock_store

    embed_and_store_documents(docs, str(folder_name), source_id, task_status)

    assert folder_name.exists()
    mock_vector_creator.create_vectorstore.assert_called_once()
    mock_store.save_local.assert_called_once_with(str(folder_name))
    task_status.update_state.assert_called()


def test_embed_and_store_documents_non_faiss(tmp_path, mock_settings, mock_vector_creator):
    mock_settings.VECTOR_STORE = "chromadb"

    docs = [MagicMock(page_content="doc1", metadata={}), MagicMock(page_content="doc2", metadata={})]
    folder_name = tmp_path / "chromadb_store"
    source_id = "test123"
    task_status = MagicMock()

    mock_store = MagicMock()
    mock_vector_creator.create_vectorstore.return_value = mock_store

    embed_and_store_documents(docs, str(folder_name), source_id, task_status)

    mock_store.delete_index.assert_called_once()
    task_status.update_state.assert_called()
    assert folder_name.exists()


def test_embed_and_store_documents_progress_band(
    tmp_path, mock_settings, mock_vector_creator
):
    """progress_start/progress_end remap the embed loop into a sub-band
    so an earlier stage (parsing) can own the lower part of the bar.
    """
    mock_settings.VECTOR_STORE = "chromadb"

    docs = [MagicMock(page_content=f"d{i}", metadata={}) for i in range(4)]
    task_status = MagicMock()
    mock_vector_creator.create_vectorstore.return_value = MagicMock()

    embed_and_store_documents(
        docs, str(tmp_path / "store"), "sid", task_status,
        progress_start=50, progress_end=100,
    )

    currents = [
        call.kwargs["meta"]["current"]
        for call in task_status.update_state.call_args_list
        if "meta" in call.kwargs and "current" in call.kwargs["meta"]
    ]
    assert currents, "expected progress updates"
    # Embedding stays in the upper band and tops out at 100.
    assert min(currents) > 50
    assert max(currents) == 100
    assert currents == sorted(currents)


@patch("application.parser.embedding_pipeline.add_text_to_store_with_retry")
def test_embed_and_store_documents_partial_failure_raises(
    mock_add_retry, tmp_path, mock_settings, mock_vector_creator, caplog
):
    """Regression: a per-chunk failure must escape the function so
    Celery's autoretry_for can fire and ``with_idempotency`` doesn't
    cache a partial index as ``completed``. Pre-fix, this branch
    swallowed and returned success.
    """
    mock_settings.VECTOR_STORE = "faiss"

    docs = [
        MagicMock(page_content="good", metadata={}),
        MagicMock(page_content="bad", metadata={}),
    ]
    folder_name = tmp_path / "partial_fail"
    source_id = "id123"
    task_status = MagicMock()

    mock_store = MagicMock()
    mock_vector_creator.create_vectorstore.return_value = mock_store

    # First document succeeds (FAISS init seeds with docs[0]; the loop
    # picks up at idx=1 and raises on the bad chunk).
    def side_effect(*args, **kwargs):
        if "bad" in args[1].page_content:
            raise RuntimeError("Embedding failed")
    mock_add_retry.side_effect = side_effect

    with caplog.at_level(logging.ERROR):
        with pytest.raises(EmbeddingPipelineError) as exc_info:
            embed_and_store_documents(
                docs, str(folder_name), source_id, task_status,
            )

    # Original cause is chained via ``raise ... from`` for diagnostics.
    assert isinstance(exc_info.value.__cause__, RuntimeError)
    assert "Error embedding document" in caplog.text
    # Partial save still ran (chunks that did embed are flushed to disk).
    mock_store.save_local.assert_called()


@patch("application.parser.embedding_pipeline.add_text_to_store_with_retry")
def test_embed_and_store_documents_all_chunks_succeed_no_raise(
    mock_add_retry, tmp_path, mock_settings, mock_vector_creator,
):
    """Happy path: no exception escapes when every chunk succeeds."""
    mock_settings.VECTOR_STORE = "faiss"

    docs = [
        MagicMock(page_content="a", metadata={}),
        MagicMock(page_content="b", metadata={}),
    ]
    mock_store = MagicMock()
    mock_vector_creator.create_vectorstore.return_value = mock_store

    embed_and_store_documents(
        docs, str(tmp_path / "ok"), "id-ok", MagicMock(),
    )
    mock_store.save_local.assert_called()


# ── assert_index_complete ──────────────────────────────────────────────────


def test_assert_index_complete_raises_on_partial(monkeypatch):
    """Worker-level tripwire: chunk-progress with embedded < total raises."""
    fake_repo = MagicMock()
    fake_repo.get_progress.return_value = {
        "embedded_chunks": 4, "total_chunks": 10,
    }
    monkeypatch.setattr(
        "application.parser.embedding_pipeline.IngestChunkProgressRepository",
        lambda conn: fake_repo,
    )
    from contextlib import contextmanager

    @contextmanager
    def _fake_session():
        yield None

    monkeypatch.setattr(
        "application.parser.embedding_pipeline.db_session", _fake_session,
    )
    with pytest.raises(EmbeddingPipelineError, match=r"4/10"):
        assert_index_complete("src-partial")


def test_assert_index_complete_passes_on_full(monkeypatch):
    fake_repo = MagicMock()
    fake_repo.get_progress.return_value = {
        "embedded_chunks": 10, "total_chunks": 10,
    }
    monkeypatch.setattr(
        "application.parser.embedding_pipeline.IngestChunkProgressRepository",
        lambda conn: fake_repo,
    )
    from contextlib import contextmanager

    @contextmanager
    def _fake_session():
        yield None

    monkeypatch.setattr(
        "application.parser.embedding_pipeline.db_session", _fake_session,
    )
    assert_index_complete("src-full")  # no raise


def test_assert_index_complete_no_op_when_no_progress_row(monkeypatch):
    """Zero-doc validation raises before init → no progress row exists."""
    fake_repo = MagicMock()
    fake_repo.get_progress.return_value = None
    monkeypatch.setattr(
        "application.parser.embedding_pipeline.IngestChunkProgressRepository",
        lambda conn: fake_repo,
    )
    from contextlib import contextmanager

    @contextmanager
    def _fake_session():
        yield None

    monkeypatch.setattr(
        "application.parser.embedding_pipeline.db_session", _fake_session,
    )
    assert_index_complete("src-missing")


def test_assert_index_complete_no_op_when_lookup_fails(monkeypatch, caplog):
    """DB outage during lookup mustn't fail the whole task — log and
    return so the embed function's own raise (Option A) remains the
    primary signal.
    """
    from contextlib import contextmanager

    @contextmanager
    def _broken_session():
        raise RuntimeError("DB unreachable")
        yield  # pragma: no cover

    monkeypatch.setattr(
        "application.parser.embedding_pipeline.db_session", _broken_session,
    )
    with caplog.at_level(logging.WARNING, logger="root"):
        assert_index_complete("src-db-down")  # no raise
    assert any(
        "progress lookup failed" in r.getMessage() for r in caplog.records
    )


def test_embed_and_store_documents_save_fails_raises_oserror(
    tmp_path, mock_settings, mock_vector_creator
):
    mock_settings.VECTOR_STORE = "faiss"

    docs = [MagicMock(page_content="good", metadata={})]
    folder_name = tmp_path / "save_fail"
    source_id = "id789"
    task_status = MagicMock()

    mock_store = MagicMock()
    mock_store.save_local.side_effect = Exception("Disk full")
    mock_vector_creator.create_vectorstore.return_value = mock_store

    with pytest.raises(OSError, match="Unable to save vector store"):
        embed_and_store_documents(docs, str(folder_name), source_id, task_status)