learningcircuit--local-deep-research/tests/domain_classifier/test_classifier_extended.py

"""Extended tests for domain_classifier/classifier.py - targeting untested paths.

Covers:
- classify_all_domains() - completely untested (lines 276-407)
- classify_domain() with force_update=True
- get_categories_summary() - subcategory aggregation
- _get_domain_samples() edge cases
"""

from unittest.mock import MagicMock, Mock, patch


# ── classify_all_domains ──────────────────────────────────────────


class TestClassifyAllDomains:
    """Tests for classify_all_domains (lines 276-407, entirely untested)."""

    def _make_classifier(self):
        from local_deep_research.domain_classifier.classifier import (
            DomainClassifier,
        )

        return DomainClassifier(username="testuser", settings_snapshot={})

    def test_empty_database_returns_zero_total(self):
        """No resources → total=0, no processing."""
        classifier = self._make_classifier()

        mock_session = MagicMock()
        mock_session.query.return_value.distinct.return_value.all.return_value = []

        with patch(
            "local_deep_research.domain_classifier.classifier.get_user_db_session"
        ) as mock_gs:
            mock_gs.return_value.__enter__ = Mock(return_value=mock_session)
            mock_gs.return_value.__exit__ = Mock(return_value=False)

            result = classifier.classify_all_domains()

        assert result["total"] == 0
        assert result["classified"] == 0
        assert result["failed"] == 0
        assert result["skipped"] == 0

    def test_extracts_domains_from_urls(self):
        """Domains are extracted and normalized from resource URLs."""
        classifier = self._make_classifier()

        mock_session = MagicMock()
        mock_session.query.return_value.distinct.return_value.all.return_value = [
            ("https://www.example.com/page1",),
            ("https://www.example.com/page2",),
            ("https://other.org/doc",),
        ]
        # For the domain query in classify_all_domains, we need to handle
        # the subsequent query for DomainClassification filter_by
        mock_session.query.return_value.filter_by.return_value.first.return_value = None

        with (
            patch(
                "local_deep_research.domain_classifier.classifier.get_user_db_session"
            ) as mock_gs,
            patch.object(classifier, "classify_domain", return_value=None),
        ):
            mock_gs.return_value.__enter__ = Mock(return_value=mock_session)
            mock_gs.return_value.__exit__ = Mock(return_value=False)

            result = classifier.classify_all_domains(force_update=True)

        # www.example.com → example.com (deduplicated), other.org → 2 unique domains
        assert result["total"] == 2

    def test_progress_callback_invoked(self):
        """Progress callback is called for each domain with percentage."""
        classifier = self._make_classifier()

        mock_session = MagicMock()
        mock_session.query.return_value.distinct.return_value.all.return_value = [
            ("https://a.com/page",),
            ("https://b.com/page",),
        ]

        callback = Mock()

        with (
            patch(
                "local_deep_research.domain_classifier.classifier.get_user_db_session"
            ) as mock_gs,
            patch.object(classifier, "classify_domain", return_value=None),
        ):
            mock_gs.return_value.__enter__ = Mock(return_value=mock_session)
            mock_gs.return_value.__exit__ = Mock(return_value=False)

            classifier.classify_all_domains(
                force_update=True, progress_callback=callback
            )

        assert callback.call_count == 2
        first_call = callback.call_args_list[0][0][0]
        assert "current" in first_call
        assert "total" in first_call
        assert "percentage" in first_call

    def test_skips_already_classified_without_force(self):
        """Without force_update, already-classified domains are skipped."""
        classifier = self._make_classifier()

        mock_session = MagicMock()
        mock_session.query.return_value.distinct.return_value.all.return_value = [
            ("https://known.com/page",),
        ]

        existing = Mock()
        existing.category = "Technology"
        existing.subcategory = "Software Development"
        mock_session.query.return_value.filter_by.return_value.first.return_value = existing

        with patch(
            "local_deep_research.domain_classifier.classifier.get_user_db_session"
        ) as mock_gs:
            mock_gs.return_value.__enter__ = Mock(return_value=mock_session)
            mock_gs.return_value.__exit__ = Mock(return_value=False)

            result = classifier.classify_all_domains(force_update=False)

        assert result["skipped"] == 1
        assert result["classified"] == 0

    def test_successful_classification_counted(self):
        """Successful classify_domain calls increment 'classified' counter."""
        classifier = self._make_classifier()

        mock_session = MagicMock()
        mock_session.query.return_value.distinct.return_value.all.return_value = [
            ("https://new-site.com/page",),
        ]

        mock_classification = Mock()
        mock_classification.category = "Technology"
        mock_classification.subcategory = "Cloud Service"
        mock_classification.confidence = 0.9

        with (
            patch(
                "local_deep_research.domain_classifier.classifier.get_user_db_session"
            ) as mock_gs,
            patch.object(
                classifier,
                "classify_domain",
                return_value=mock_classification,
            ),
        ):
            mock_gs.return_value.__enter__ = Mock(return_value=mock_session)
            mock_gs.return_value.__exit__ = Mock(return_value=False)

            result = classifier.classify_all_domains(force_update=True)

        assert result["classified"] == 1
        assert result["domains"][0]["status"] == "classified"
        assert result["domains"][0]["category"] == "Technology"

    def test_failed_classification_counted(self):
        """Failed classify_domain calls increment 'failed' counter."""
        classifier = self._make_classifier()

        mock_session = MagicMock()
        mock_session.query.return_value.distinct.return_value.all.return_value = [
            ("https://bad-site.com/page",),
        ]

        with (
            patch(
                "local_deep_research.domain_classifier.classifier.get_user_db_session"
            ) as mock_gs,
            patch.object(classifier, "classify_domain", return_value=None),
        ):
            mock_gs.return_value.__enter__ = Mock(return_value=mock_session)
            mock_gs.return_value.__exit__ = Mock(return_value=False)

            result = classifier.classify_all_domains(force_update=True)

        assert result["failed"] == 1
        assert result["domains"][0]["status"] == "failed"

    def test_exception_in_domain_increments_failed(self):
        """Exception during single domain classification is caught and counted."""
        classifier = self._make_classifier()

        mock_session = MagicMock()
        mock_session.query.return_value.distinct.return_value.all.return_value = [
            ("https://crash.com/page",),
        ]

        with (
            patch(
                "local_deep_research.domain_classifier.classifier.get_user_db_session"
            ) as mock_gs,
            patch.object(
                classifier,
                "classify_domain",
                side_effect=RuntimeError("LLM error"),
            ),
        ):
            mock_gs.return_value.__enter__ = Mock(return_value=mock_session)
            mock_gs.return_value.__exit__ = Mock(return_value=False)

            result = classifier.classify_all_domains(force_update=True)

        assert result["failed"] == 1
        assert result["domains"][0]["status"] == "failed"

    def test_invalid_url_skipped_gracefully(self):
        """URLs that fail parsing are skipped without crashing."""
        classifier = self._make_classifier()

        mock_session = MagicMock()
        mock_session.query.return_value.distinct.return_value.all.return_value = [
            ("not a valid url",),
            (None,),
            ("https://valid.com/page",),
        ]

        with (
            patch(
                "local_deep_research.domain_classifier.classifier.get_user_db_session"
            ) as mock_gs,
            patch.object(classifier, "classify_domain", return_value=None),
        ):
            mock_gs.return_value.__enter__ = Mock(return_value=mock_session)
            mock_gs.return_value.__exit__ = Mock(return_value=False)

            result = classifier.classify_all_domains(force_update=True)

        # "not a valid url" has empty netloc, None is skipped → only valid.com
        assert result["total"] == 1


# ── get_categories_summary: subcategory aggregation ───────────────


class TestGetCategoriesSummary:
    """Tests for get_categories_summary subcategory aggregation (lines 449-488)."""

    def _make_classifier(self):
        from local_deep_research.domain_classifier.classifier import (
            DomainClassifier,
        )

        return DomainClassifier(username="testuser")

    def test_multiple_domains_same_category(self):
        """Multiple domains in the same category are aggregated."""
        classifier = self._make_classifier()

        c1 = Mock(
            domain="a.com",
            category="Technology",
            subcategory="Cloud",
            confidence=0.9,
        )
        c2 = Mock(
            domain="b.com",
            category="Technology",
            subcategory="SaaS",
            confidence=0.8,
        )

        mock_session = MagicMock()
        mock_session.query.return_value.all.return_value = [c1, c2]

        with patch(
            "local_deep_research.domain_classifier.classifier.get_user_db_session"
        ) as mock_gs:
            mock_gs.return_value.__enter__ = Mock(return_value=mock_session)
            mock_gs.return_value.__exit__ = Mock(return_value=False)

            result = classifier.get_categories_summary()

        assert result["Technology"]["count"] == 2
        assert result["Technology"]["subcategories"]["Cloud"] == 1
        assert result["Technology"]["subcategories"]["SaaS"] == 1

    def test_none_subcategory_skipped(self):
        """None subcategory does not appear in subcategories dict."""
        classifier = self._make_classifier()

        c1 = Mock(
            domain="x.com", category="Other", subcategory=None, confidence=0.5
        )

        mock_session = MagicMock()
        mock_session.query.return_value.all.return_value = [c1]

        with patch(
            "local_deep_research.domain_classifier.classifier.get_user_db_session"
        ) as mock_gs:
            mock_gs.return_value.__enter__ = Mock(return_value=mock_session)
            mock_gs.return_value.__exit__ = Mock(return_value=False)

            result = classifier.get_categories_summary()

        assert result["Other"]["count"] == 1
        assert result["Other"]["subcategories"] == {}

    def test_empty_database_returns_empty(self):
        """No classifications → empty summary."""
        classifier = self._make_classifier()

        mock_session = MagicMock()
        mock_session.query.return_value.all.return_value = []

        with patch(
            "local_deep_research.domain_classifier.classifier.get_user_db_session"
        ) as mock_gs:
            mock_gs.return_value.__enter__ = Mock(return_value=mock_session)
            mock_gs.return_value.__exit__ = Mock(return_value=False)

            result = classifier.get_categories_summary()

        assert result == {}