learningcircuit--local-deep-research/tests/content_fetcher/test_url_classifier_high_value.py

"""
High-value pure logic tests for URL classifier.

Focuses on edge cases, boundary conditions, and behaviors not covered
by the basic test suite in test_url_classifier.py.
"""

import pytest

from local_deep_research.content_fetcher.url_classifier import (
    DANGEROUS_SCHEMES,
    URLClassifier,
    URLType,
)


class TestURLTypeEnum:
    """Verify URLType enum completeness and values."""

    def test_all_enum_members_exist(self):
        """All expected URLType members are defined."""
        expected = {
            "ARXIV",
            "PUBMED",
            "PMC",
            "SEMANTIC_SCHOLAR",
            "BIORXIV",
            "MEDRXIV",
            "DOI",
            "PDF",
            "HTML",
            "INVALID",
        }
        actual = {member.name for member in URLType}
        assert actual == expected

    def test_enum_values_are_lowercase_strings(self):
        """All enum values are lowercase strings."""
        for member in URLType:
            assert isinstance(member.value, str)
            assert member.value == member.value.lower()

    def test_enum_member_count(self):
        """Exactly 10 URL types are defined."""
        assert len(URLType) == 10


class TestDangerousSchemes:
    """Verify the DANGEROUS_SCHEMES constant."""

    def test_contains_all_expected_schemes(self):
        """All dangerous schemes are present."""
        expected = {"javascript", "data", "file", "vbscript", "about"}
        assert DANGEROUS_SCHEMES == expected

    def test_schemes_are_lowercase(self):
        """All scheme entries are lowercase."""
        for scheme in DANGEROUS_SCHEMES:
            assert scheme == scheme.lower()


class TestClassifyDangerousAndInvalidSchemes:
    """Edge cases for scheme-based rejection."""

    def test_javascript_with_payload(self):
        """javascript: with complex payload is INVALID."""
        url = "javascript:void(document.cookie)"
        assert URLClassifier.classify(url) == URLType.INVALID

    def test_data_base64_payload(self):
        """data: with base64 content is INVALID."""
        url = "data:text/html;base64,PHNjcmlwdD5hbGVydCgxKTwvc2NyaXB0Pg=="
        assert URLClassifier.classify(url) == URLType.INVALID

    def test_file_windows_path(self):
        """file: with Windows-style path is INVALID."""
        url = "file:///C:/Users/test/secret.txt"
        assert URLClassifier.classify(url) == URLType.INVALID

    def test_ftp_scheme_rejected(self):
        """ftp: is not http/https and should be INVALID."""
        assert (
            URLClassifier.classify("ftp://mirror.example.com/pub")
            == URLType.INVALID
        )

    def test_ssh_scheme_rejected(self):
        """ssh: scheme should be INVALID."""
        assert URLClassifier.classify("ssh://user@host") == URLType.INVALID

    def test_mailto_scheme_rejected(self):
        """mailto: scheme should be INVALID."""
        assert (
            URLClassifier.classify("mailto:user@example.com") == URLType.INVALID
        )

    def test_tel_scheme_rejected(self):
        """tel: scheme should be INVALID."""
        assert URLClassifier.classify("tel:+1234567890") == URLType.INVALID


class TestClassifyAcademicURLEdgeCases:
    """Edge cases for academic URL classification."""

    def test_arxiv_pdf_with_version(self):
        """arXiv PDF URL with version suffix classifies as ARXIV, not PDF."""
        url = "https://arxiv.org/pdf/2301.12345v3.pdf"
        assert URLClassifier.classify(url) == URLType.ARXIV

    def test_arxiv_mixed_case(self):
        """arXiv URL with mixed case in domain."""
        url = "https://ArXiv.Org/abs/2301.12345"
        assert URLClassifier.classify(url) == URLType.ARXIV

    def test_ar5iv_with_path(self):
        """ar5iv.org with any path classifies as ARXIV."""
        url = "https://ar5iv.org/html/2301.12345"
        assert URLClassifier.classify(url) == URLType.ARXIV

    def test_pubmed_trailing_slash(self):
        """PubMed URL with trailing slash."""
        url = "https://pubmed.ncbi.nlm.nih.gov/98765432/"
        assert URLClassifier.classify(url) == URLType.PUBMED

    def test_pmc_europe_articles_variant(self):
        """Europe PMC with /articles/ path variant."""
        url = "https://europepmc.org/articles/PMC7654321"
        assert URLClassifier.classify(url) == URLType.PMC

    def test_biorxiv_with_version(self):
        """bioRxiv URL with version suffix."""
        url = "https://www.biorxiv.org/content/10.1101/2023.05.01.538999v2"
        assert URLClassifier.classify(url) == URLType.BIORXIV

    def test_medrxiv_full_doi_path(self):
        """medRxiv URL with full DOI-style content path."""
        url = (
            "https://www.medrxiv.org/content/10.1101/2023.06.15.23291456v1.full"
        )
        assert URLClassifier.classify(url) == URLType.MEDRXIV

    def test_doi_with_complex_suffix(self):
        """DOI URL with complex suffix containing slashes and dots."""
        url = "https://doi.org/10.1038/s41586-021-03819-2"
        assert URLClassifier.classify(url) == URLType.DOI

    def test_dx_doi_org(self):
        """dx.doi.org is recognized as DOI."""
        url = "https://dx.doi.org/10.1000/xyz123"
        assert URLClassifier.classify(url) == URLType.DOI

    def test_semantic_scholar_api_url(self):
        """Semantic Scholar API URL."""
        url = "https://api.semanticscholar.org/graph/v1/paper/12345"
        assert URLClassifier.classify(url) == URLType.SEMANTIC_SCHOLAR


class TestClassifyPDFDetection:
    """Tests for PDF URL detection and academic domain exclusion."""

    def test_pdf_extension_generic_domain(self):
        """Generic domain with .pdf extension is PDF."""
        url = "https://example.com/reports/annual-report.pdf"
        assert URLClassifier.classify(url) == URLType.PDF

    def test_pdf_path_segment(self):
        """URL with /pdf/ in path on generic domain is PDF."""
        url = "https://example.com/pdf/document123"
        assert URLClassifier.classify(url) == URLType.PDF

    def test_arxiv_pdf_not_classified_as_pdf(self):
        """arXiv PDF URL should be ARXIV, not PDF."""
        url = "https://arxiv.org/pdf/2301.12345"
        assert URLClassifier.classify(url) == URLType.ARXIV

    def test_biorxiv_pdf_not_classified_as_pdf(self):
        """bioRxiv PDF URL should be BIORXIV, not PDF."""
        url = "https://www.biorxiv.org/content/10.1101/2023.01.01.123456v1.full.pdf"
        assert URLClassifier.classify(url) == URLType.BIORXIV

    def test_ncbi_pdf_not_classified_as_pdf(self):
        """NCBI domain PDF path should not be classified as generic PDF."""
        url = "https://ncbi.nlm.nih.gov/pmc/articles/pmc123456/pdf/main.pdf"
        assert URLClassifier.classify(url) == URLType.PMC


class TestIsPdfUrlDirectly:
    """Direct tests on the _is_pdf_url classmethod."""

    def test_pdf_extension_returns_true(self):
        """URL ending in .pdf returns True."""
        assert URLClassifier._is_pdf_url("https://example.com/file.pdf") is True

    def test_pdf_path_segment_returns_true(self):
        """/pdf/ in path returns True."""
        assert URLClassifier._is_pdf_url("https://example.com/pdf/doc1") is True

    def test_no_pdf_indicator_returns_false(self):
        """Normal URL returns False."""
        assert URLClassifier._is_pdf_url("https://example.com/page") is False

    def test_arxiv_domain_returns_false(self):
        """arXiv domain always returns False (academic exclusion)."""
        assert (
            URLClassifier._is_pdf_url("https://arxiv.org/pdf/2301.12345")
            is False
        )

    def test_semanticscholar_domain_returns_false(self):
        """Semantic Scholar domain always returns False."""
        assert (
            URLClassifier._is_pdf_url("https://semanticscholar.org/paper.pdf")
            is False
        )

    def test_europepmc_domain_returns_false(self):
        """Europe PMC domain always returns False."""
        assert (
            URLClassifier._is_pdf_url("https://europepmc.org/pdf/doc.pdf")
            is False
        )

    def test_medrxiv_domain_returns_false(self):
        """medRxiv domain always returns False."""
        assert (
            URLClassifier._is_pdf_url("https://medrxiv.org/content/file.pdf")
            is False
        )


class TestExtractIdEdgeCases:
    """Edge cases for ID extraction."""

    def test_arxiv_five_digit_id(self):
        """arXiv ID with 5-digit suffix (newer format)."""
        url = "https://arxiv.org/abs/2301.12345"
        assert URLClassifier.extract_id(url) == "2301.12345"

    def test_arxiv_old_format_with_version(self):
        """Old arXiv format with version number."""
        url = "https://arxiv.org/abs/hep-th/9901001v3"
        assert URLClassifier.extract_id(url) == "hep-th/9901001v3"

    def test_pmc_id_case_insensitive(self):
        """PMC ID extraction is case-insensitive."""
        url = "https://www.ncbi.nlm.nih.gov/pmc/articles/pmc9876543/"
        extracted = URLClassifier.extract_id(url, URLType.PMC)
        assert extracted == "PMC9876543"

    def test_doi_complex_suffix(self):
        """DOI with complex characters in suffix."""
        url = "https://doi.org/10.1038/s41586-021-03819-2"
        assert URLClassifier.extract_id(url) == "10.1038/s41586-021-03819-2"

    def test_extract_id_with_explicit_type_html(self):
        """Passing url_type=HTML returns None (no ID for HTML)."""
        url = "https://example.com/article/123"
        assert URLClassifier.extract_id(url, URLType.HTML) is None

    def test_extract_id_with_explicit_type_pdf(self):
        """Passing url_type=PDF returns None (no ID for PDF)."""
        url = "https://example.com/paper.pdf"
        assert URLClassifier.extract_id(url, URLType.PDF) is None

    def test_extract_id_with_explicit_type_invalid(self):
        """Passing url_type=INVALID returns None."""
        url = "javascript:alert(1)"
        assert URLClassifier.extract_id(url, URLType.INVALID) is None

    def test_pubmed_id_no_trailing_slash(self):
        """PubMed ID extraction without trailing slash."""
        url = "https://pubmed.ncbi.nlm.nih.gov/55555555"
        assert URLClassifier.extract_id(url) == "55555555"


class TestGetSourceNameCompleteness:
    """Verify get_source_name covers all URLType values."""

    @pytest.mark.parametrize(
        "url_type,expected_name",
        [
            (URLType.ARXIV, "arXiv"),
            (URLType.PUBMED, "PubMed"),
            (URLType.PMC, "PubMed Central"),
            (URLType.SEMANTIC_SCHOLAR, "Semantic Scholar"),
            (URLType.BIORXIV, "bioRxiv"),
            (URLType.MEDRXIV, "medRxiv"),
            (URLType.DOI, "DOI"),
            (URLType.PDF, "PDF"),
            (URLType.HTML, "Web Page"),
            (URLType.INVALID, "Invalid URL"),
        ],
    )
    def test_source_name_mapping(self, url_type, expected_name):
        """Every URLType has the correct human-readable name."""
        assert URLClassifier.get_source_name(url_type) == expected_name

    def test_all_url_types_have_source_names(self):
        """Every URLType member returns a non-'Unknown' source name."""
        for member in URLType:
            name = URLClassifier.get_source_name(member)
            assert name != "Unknown", f"{member} has no source name mapping"


class TestClassifyDefaultToHTML:
    """URLs that don't match any special pattern default to HTML."""

    def test_plain_http_url(self):
        """Plain HTTP URL defaults to HTML."""
        assert URLClassifier.classify("http://example.com") == URLType.HTML

    def test_url_without_scheme(self):
        """URL without scheme defaults to HTML (empty scheme is allowed)."""
        assert URLClassifier.classify("example.com/page") == URLType.HTML

    def test_url_with_port(self):
        """URL with port number defaults to HTML."""
        assert (
            URLClassifier.classify("https://example.com:8080/page")
            == URLType.HTML
        )

    def test_url_with_auth(self):
        """URL with authentication info defaults to HTML."""
        assert (
            URLClassifier.classify("https://user:pass@example.com/page")
            == URLType.HTML
        )