learningcircuit--local-deep-research/tests/utilities/test_citation_normalizer.py

"""Tests for citation_normalizer — engine-specific dict → CSL-JSON metadata."""

from local_deep_research.utilities.citation_normalizer import (
    detect_engine,
    normalize_citation,
    normalize_issn,
    _parse_authors_list,
    _parse_name,
    _extract_doi,
    _extract_arxiv_id,
)


class TestNormalizeIssn:
    def test_dashed_form(self):
        assert normalize_issn("1522-9645") == "15229645"

    def test_undashed_form(self):
        assert normalize_issn("15229645") == "15229645"

    def test_round_trip_equality(self):
        assert normalize_issn("1522-9645") == normalize_issn("15229645")

    def test_uppercase_x_check_digit(self):
        assert normalize_issn("1234-567x") == "1234567X"

    def test_already_uppercase_x(self):
        assert normalize_issn("1234-567X") == "1234567X"

    def test_whitespace_stripped(self):
        assert normalize_issn(" 1522-9645 ") == "15229645"

    def test_empty_returns_none(self):
        assert normalize_issn("") is None

    def test_none_returns_none(self):
        assert normalize_issn(None) is None

    def test_too_short_returns_none(self):
        assert normalize_issn("1234") is None

    def test_too_long_returns_none(self):
        assert normalize_issn("123456789") is None

    def test_non_digit_returns_none(self):
        assert normalize_issn("abcd-efgh") is None


class TestDetectEngine:
    def test_arxiv_url(self):
        assert (
            detect_engine({"link": "https://arxiv.org/abs/2401.12345"})
            == "arxiv"
        )

    def test_openalex_url(self):
        assert (
            detect_engine({"link": "https://openalex.org/W12345"}) == "openalex"
        )

    def test_pubmed_url(self):
        assert (
            detect_engine({"link": "https://pubmed.ncbi.nlm.nih.gov/12345"})
            == "pubmed"
        )

    def test_explicit_source(self):
        assert detect_engine({"source": "arXiv"}) == "arxiv"

    def test_web_returns_none(self):
        assert detect_engine({"link": "https://example.com/blog"}) is None

    def test_empty_returns_none(self):
        assert detect_engine({}) is None


class TestParseAuthors:
    def test_string_list(self):
        result = _parse_authors_list(["John Smith", "Jane Doe"])
        assert len(result) == 2
        assert result[0]["given"] == "John"
        assert result[0]["family"] == "Smith"

    def test_comma_string(self):
        result = _parse_authors_list("John Smith, Jane Doe")
        assert len(result) == 2

    def test_dict_with_name(self):
        result = _parse_authors_list([{"name": "John Smith"}])
        assert result[0]["family"] == "Smith"

    def test_already_csl(self):
        csl = [{"family": "Smith", "given": "John"}]
        result = _parse_authors_list(csl)
        assert result == csl

    def test_empty(self):
        assert _parse_authors_list(None) is None
        assert _parse_authors_list([]) is None
        assert _parse_authors_list("") is None

    def test_normalize_citation_prefers_authors_csl(self):
        """When both authors_csl (structured) and authors (display string)
        are present, the structured list wins so 'Last, First' name pairs
        survive the round-trip — the bare comma-joined string would split
        into 4 singletons instead of 2 author pairs.
        """
        from local_deep_research.utilities.citation_normalizer import (
            normalize_citation,
        )

        source = {
            "source_engine": "nasa_ads",
            "title": "Test",
            "authors": "Smith, John, Doe, Jane",  # ambiguous joined form
            "authors_csl": [
                {"family": "Smith", "given": "John"},
                {"family": "Doe", "given": "Jane"},
            ],
            "year": 2023,
        }

        result = normalize_citation(source)
        assert result is not None
        assert len(result["authors"]) == 2
        assert result["authors"][0] == {"family": "Smith", "given": "John"}
        assert result["authors"][1] == {"family": "Doe", "given": "Jane"}


class TestParseName:
    def test_first_last(self):
        assert _parse_name("John Smith") == {
            "given": "John",
            "family": "Smith",
        }

    def test_last_comma_first(self):
        assert _parse_name("Smith, John") == {
            "family": "Smith",
            "given": "John",
        }

    def test_single_name(self):
        assert _parse_name("Aristotle") == {"literal": "Aristotle"}


class TestExtractDoi:
    def test_bare_doi(self):
        assert _extract_doi({"doi": "10.1234/test"}) == "10.1234/test"

    def test_url_doi(self):
        assert (
            _extract_doi({"doi": "https://doi.org/10.1234/test"})
            == "10.1234/test"
        )

    def test_doi_list(self):
        assert (
            _extract_doi({"doi": ["10.1234/test", "10.5678/other"]})
            == "10.1234/test"
        )

    def test_none(self):
        assert _extract_doi({}) is None

    def test_external_ids_doi(self):
        # Semantic Scholar exposes DOIs through `external_ids`. After
        # consolidating `_extract_doi` into citation_normalizer, this is
        # the canonical home for DOI extraction.
        assert (
            _extract_doi({"external_ids": {"DOI": "10.1234/test"}})
            == "10.1234/test"
        )

    def test_externalIds_camelCase_doi(self):
        # Some Semantic Scholar payloads use camelCase `externalIds`.
        assert (
            _extract_doi({"externalIds": {"DOI": "10.1234/test"}})
            == "10.1234/test"
        )

    def test_lowercase_doi_key_in_external_ids(self):
        # Tolerant of lowercase variant.
        assert (
            _extract_doi({"external_ids": {"doi": "10.1234/test"}})
            == "10.1234/test"
        )

    def test_link_with_doi_org(self):
        # ArXiv-style result that embeds the DOI in the link URL.
        assert (
            _extract_doi({"link": "https://doi.org/10.1234/test"})
            == "10.1234/test"
        )

    def test_link_with_dx_doi_org(self):
        # The legacy `dx.doi.org` redirect form.
        assert (
            _extract_doi({"link": "https://dx.doi.org/10.1234/test"})
            == "10.1234/test"
        )

    def test_link_with_http_doi_org(self):
        # http (not https) variant.
        assert (
            _extract_doi({"link": "http://doi.org/10.1234/test"})
            == "10.1234/test"
        )

    def test_doi_field_takes_priority_over_external_ids(self):
        # When both are present, the explicit `doi` field wins.
        assert (
            _extract_doi(
                {
                    "doi": "10.1234/explicit",
                    "external_ids": {"DOI": "10.5678/from-external"},
                }
            )
            == "10.1234/explicit"
        )

    def test_link_without_doi_returns_none(self):
        # A link that doesn't contain a doi.org URL must NOT be matched
        # (CodeQL py/incomplete-url-substring-sanitization guard).
        assert (
            _extract_doi(
                {"link": "https://attacker.example/doi.org/10.1234/test"}
            )
            is None
        )


class TestExtractArxivId:
    def test_from_url(self):
        assert (
            _extract_arxiv_id({"link": "https://arxiv.org/abs/2401.12345"})
            == "2401.12345"
        )

    def test_explicit(self):
        assert _extract_arxiv_id({"arxiv_id": "2401.12345"}) == "2401.12345"

    def test_old_format_hyphen_archive(self):
        assert (
            _extract_arxiv_id(
                {"link": "https://arxiv.org/abs/cond-mat/0501001"}
            )
            == "cond-mat/0501001"
        )

    def test_old_format_dotted_subject_class(self):
        assert (
            _extract_arxiv_id({"link": "https://arxiv.org/abs/math.AG/0601001"})
            == "math.AG/0601001"
        )

    def test_old_format_hep_th(self):
        assert (
            _extract_arxiv_id({"link": "http://arxiv.org/abs/hep-th/9802150"})
            == "hep-th/9802150"
        )

    def test_new_format_5digit_sequence(self):
        assert (
            _extract_arxiv_id({"link": "https://arxiv.org/abs/1501.00001"})
            == "1501.00001"
        )

    def test_new_format_with_version(self):
        assert (
            _extract_arxiv_id({"link": "https://arxiv.org/abs/2501.12345v2"})
            == "2501.12345v2"
        )


class TestNormalizeCitation:
    def test_arxiv_result(self):
        source = {
            "title": "Attention Is All You Need",
            "link": "https://arxiv.org/abs/1706.03762",
            "authors": ["Ashish Vaswani", "Noam Shazeer"],
            "published": "2017-06-12",
            "journal_ref": None,
            "source": "arXiv",
            "doi": "10.5555/3295222.3295349",
        }
        result = normalize_citation(source)
        assert result is not None
        assert result["source_engine"] == "arxiv"
        assert result["arxiv_id"] == "1706.03762"
        assert result["doi"] == "10.5555/3295222.3295349"
        assert result["year"] == 2017
        assert len(result["authors"]) == 2
        assert result["item_type"] == "article"  # No journal_ref = preprint
        assert "csl_json" in result

    def test_arxiv_published_result(self):
        source = {
            "title": "Some Paper",
            "link": "https://arxiv.org/abs/2401.12345",
            "journal_ref": "Nature 612, 34-41 (2023)",
            "source": "arXiv",
        }
        result = normalize_citation(source)
        assert result["item_type"] == "article-journal"
        assert result["container_title"] == "Nature 612, 34-41 (2023)"

    def test_openalex_result(self):
        source = {
            "title": "A Study",
            "link": "https://doi.org/10.1234/test",
            "authors": "John Smith, Jane Doe et al.",
            "year": 2023,
            "journal": "Nature",
            "journal_ref": "Nature",
            "openalex_source_id": "S12345",
            "source_type": "journal",
            "doi": "https://doi.org/10.1234/test",
            "source": "openalex",
        }
        result = normalize_citation(source)
        assert result is not None
        assert result["source_engine"] == "openalex"
        assert result["doi"] == "10.1234/test"
        assert result["container_title"] == "Nature"
        assert result["year"] == 2023

    def test_web_result_returns_none(self):
        source = {
            "title": "Blog Post",
            "link": "https://example.com/blog",
        }
        assert normalize_citation(source) is None

    def test_unknown_placeholder_is_filtered(self):
        """OpenAlex / NASA ADS used to emit ``journal="unknown"`` when no
        venue was indexed. The normalizer must strip that literal so it
        doesn't become a container_title — there's an actual OpenAlex
        source named "unknown" (Q1, h_index=5) that would otherwise get
        matched by the reputation filter's name-based lookup.
        """
        source = {
            "title": "Paper",
            "link": "https://doi.org/10.1/x",
            "journal": "unknown",
            "journal_ref": None,
            "source": "openalex",
            "doi": "10.1/x",
        }
        result = normalize_citation(source)
        assert result is not None
        assert result.get("container_title") is None

    def test_unknown_placeholder_case_insensitive(self):
        """Same filter catches "Unknown", "UNKNOWN", "  unknown  " variants."""
        for placeholder in ("Unknown", "UNKNOWN", "  unknown  "):
            source = {
                "title": "Paper",
                "link": "https://doi.org/10.1/x",
                "journal": placeholder,
                "source": "openalex",
                "doi": "10.1/x",
            }
            result = normalize_citation(source)
            assert result is not None
            assert result.get("container_title") is None, (
                f"placeholder {placeholder!r} leaked through"
            )

    def test_csl_json_built(self):
        source = {
            "title": "Test Paper",
            "link": "https://arxiv.org/abs/2401.12345",
            "authors": ["John Smith"],
            "published": "2024-01-15",
            "source": "arXiv",
        }
        result = normalize_citation(source)
        csl = result["csl_json"]
        assert csl["title"] == "Test Paper"
        assert csl["type"] == "article"
        assert csl["author"][0]["family"] == "Smith"
        assert csl["issued"]["date-parts"] == [[2024, 1, 15]]
        assert csl["URL"] == "https://arxiv.org/abs/2401.12345"