"""Tests for citation_normalizer — engine-specific dict → CSL-JSON metadata.""" from local_deep_research.utilities.citation_normalizer import ( detect_engine, normalize_citation, normalize_issn, _parse_authors_list, _parse_name, _extract_doi, _extract_arxiv_id, ) class TestNormalizeIssn: def test_dashed_form(self): assert normalize_issn("1522-9645") == "15229645" def test_undashed_form(self): assert normalize_issn("15229645") == "15229645" def test_round_trip_equality(self): assert normalize_issn("1522-9645") == normalize_issn("15229645") def test_uppercase_x_check_digit(self): assert normalize_issn("1234-567x") == "1234567X" def test_already_uppercase_x(self): assert normalize_issn("1234-567X") == "1234567X" def test_whitespace_stripped(self): assert normalize_issn(" 1522-9645 ") == "15229645" def test_empty_returns_none(self): assert normalize_issn("") is None def test_none_returns_none(self): assert normalize_issn(None) is None def test_too_short_returns_none(self): assert normalize_issn("1234") is None def test_too_long_returns_none(self): assert normalize_issn("123456789") is None def test_non_digit_returns_none(self): assert normalize_issn("abcd-efgh") is None class TestDetectEngine: def test_arxiv_url(self): assert ( detect_engine({"link": "https://arxiv.org/abs/2401.12345"}) == "arxiv" ) def test_openalex_url(self): assert ( detect_engine({"link": "https://openalex.org/W12345"}) == "openalex" ) def test_pubmed_url(self): assert ( detect_engine({"link": "https://pubmed.ncbi.nlm.nih.gov/12345"}) == "pubmed" ) def test_explicit_source(self): assert detect_engine({"source": "arXiv"}) == "arxiv" def test_web_returns_none(self): assert detect_engine({"link": "https://example.com/blog"}) is None def test_empty_returns_none(self): assert detect_engine({}) is None class TestParseAuthors: def test_string_list(self): result = _parse_authors_list(["John Smith", "Jane Doe"]) assert len(result) == 2 assert result[0]["given"] == "John" assert result[0]["family"] == "Smith" def test_comma_string(self): result = _parse_authors_list("John Smith, Jane Doe") assert len(result) == 2 def test_dict_with_name(self): result = _parse_authors_list([{"name": "John Smith"}]) assert result[0]["family"] == "Smith" def test_already_csl(self): csl = [{"family": "Smith", "given": "John"}] result = _parse_authors_list(csl) assert result == csl def test_empty(self): assert _parse_authors_list(None) is None assert _parse_authors_list([]) is None assert _parse_authors_list("") is None def test_normalize_citation_prefers_authors_csl(self): """When both authors_csl (structured) and authors (display string) are present, the structured list wins so 'Last, First' name pairs survive the round-trip — the bare comma-joined string would split into 4 singletons instead of 2 author pairs. """ from local_deep_research.utilities.citation_normalizer import ( normalize_citation, ) source = { "source_engine": "nasa_ads", "title": "Test", "authors": "Smith, John, Doe, Jane", # ambiguous joined form "authors_csl": [ {"family": "Smith", "given": "John"}, {"family": "Doe", "given": "Jane"}, ], "year": 2023, } result = normalize_citation(source) assert result is not None assert len(result["authors"]) == 2 assert result["authors"][0] == {"family": "Smith", "given": "John"} assert result["authors"][1] == {"family": "Doe", "given": "Jane"} class TestParseName: def test_first_last(self): assert _parse_name("John Smith") == { "given": "John", "family": "Smith", } def test_last_comma_first(self): assert _parse_name("Smith, John") == { "family": "Smith", "given": "John", } def test_single_name(self): assert _parse_name("Aristotle") == {"literal": "Aristotle"} class TestExtractDoi: def test_bare_doi(self): assert _extract_doi({"doi": "10.1234/test"}) == "10.1234/test" def test_url_doi(self): assert ( _extract_doi({"doi": "https://doi.org/10.1234/test"}) == "10.1234/test" ) def test_doi_list(self): assert ( _extract_doi({"doi": ["10.1234/test", "10.5678/other"]}) == "10.1234/test" ) def test_none(self): assert _extract_doi({}) is None def test_external_ids_doi(self): # Semantic Scholar exposes DOIs through `external_ids`. After # consolidating `_extract_doi` into citation_normalizer, this is # the canonical home for DOI extraction. assert ( _extract_doi({"external_ids": {"DOI": "10.1234/test"}}) == "10.1234/test" ) def test_externalIds_camelCase_doi(self): # Some Semantic Scholar payloads use camelCase `externalIds`. assert ( _extract_doi({"externalIds": {"DOI": "10.1234/test"}}) == "10.1234/test" ) def test_lowercase_doi_key_in_external_ids(self): # Tolerant of lowercase variant. assert ( _extract_doi({"external_ids": {"doi": "10.1234/test"}}) == "10.1234/test" ) def test_link_with_doi_org(self): # ArXiv-style result that embeds the DOI in the link URL. assert ( _extract_doi({"link": "https://doi.org/10.1234/test"}) == "10.1234/test" ) def test_link_with_dx_doi_org(self): # The legacy `dx.doi.org` redirect form. assert ( _extract_doi({"link": "https://dx.doi.org/10.1234/test"}) == "10.1234/test" ) def test_link_with_http_doi_org(self): # http (not https) variant. assert ( _extract_doi({"link": "http://doi.org/10.1234/test"}) == "10.1234/test" ) def test_doi_field_takes_priority_over_external_ids(self): # When both are present, the explicit `doi` field wins. assert ( _extract_doi( { "doi": "10.1234/explicit", "external_ids": {"DOI": "10.5678/from-external"}, } ) == "10.1234/explicit" ) def test_link_without_doi_returns_none(self): # A link that doesn't contain a doi.org URL must NOT be matched # (CodeQL py/incomplete-url-substring-sanitization guard). assert ( _extract_doi( {"link": "https://attacker.example/doi.org/10.1234/test"} ) is None ) class TestExtractArxivId: def test_from_url(self): assert ( _extract_arxiv_id({"link": "https://arxiv.org/abs/2401.12345"}) == "2401.12345" ) def test_explicit(self): assert _extract_arxiv_id({"arxiv_id": "2401.12345"}) == "2401.12345" def test_old_format_hyphen_archive(self): assert ( _extract_arxiv_id( {"link": "https://arxiv.org/abs/cond-mat/0501001"} ) == "cond-mat/0501001" ) def test_old_format_dotted_subject_class(self): assert ( _extract_arxiv_id({"link": "https://arxiv.org/abs/math.AG/0601001"}) == "math.AG/0601001" ) def test_old_format_hep_th(self): assert ( _extract_arxiv_id({"link": "http://arxiv.org/abs/hep-th/9802150"}) == "hep-th/9802150" ) def test_new_format_5digit_sequence(self): assert ( _extract_arxiv_id({"link": "https://arxiv.org/abs/1501.00001"}) == "1501.00001" ) def test_new_format_with_version(self): assert ( _extract_arxiv_id({"link": "https://arxiv.org/abs/2501.12345v2"}) == "2501.12345v2" ) class TestNormalizeCitation: def test_arxiv_result(self): source = { "title": "Attention Is All You Need", "link": "https://arxiv.org/abs/1706.03762", "authors": ["Ashish Vaswani", "Noam Shazeer"], "published": "2017-06-12", "journal_ref": None, "source": "arXiv", "doi": "10.5555/3295222.3295349", } result = normalize_citation(source) assert result is not None assert result["source_engine"] == "arxiv" assert result["arxiv_id"] == "1706.03762" assert result["doi"] == "10.5555/3295222.3295349" assert result["year"] == 2017 assert len(result["authors"]) == 2 assert result["item_type"] == "article" # No journal_ref = preprint assert "csl_json" in result def test_arxiv_published_result(self): source = { "title": "Some Paper", "link": "https://arxiv.org/abs/2401.12345", "journal_ref": "Nature 612, 34-41 (2023)", "source": "arXiv", } result = normalize_citation(source) assert result["item_type"] == "article-journal" assert result["container_title"] == "Nature 612, 34-41 (2023)" def test_openalex_result(self): source = { "title": "A Study", "link": "https://doi.org/10.1234/test", "authors": "John Smith, Jane Doe et al.", "year": 2023, "journal": "Nature", "journal_ref": "Nature", "openalex_source_id": "S12345", "source_type": "journal", "doi": "https://doi.org/10.1234/test", "source": "openalex", } result = normalize_citation(source) assert result is not None assert result["source_engine"] == "openalex" assert result["doi"] == "10.1234/test" assert result["container_title"] == "Nature" assert result["year"] == 2023 def test_web_result_returns_none(self): source = { "title": "Blog Post", "link": "https://example.com/blog", } assert normalize_citation(source) is None def test_unknown_placeholder_is_filtered(self): """OpenAlex / NASA ADS used to emit ``journal="unknown"`` when no venue was indexed. The normalizer must strip that literal so it doesn't become a container_title — there's an actual OpenAlex source named "unknown" (Q1, h_index=5) that would otherwise get matched by the reputation filter's name-based lookup. """ source = { "title": "Paper", "link": "https://doi.org/10.1/x", "journal": "unknown", "journal_ref": None, "source": "openalex", "doi": "10.1/x", } result = normalize_citation(source) assert result is not None assert result.get("container_title") is None def test_unknown_placeholder_case_insensitive(self): """Same filter catches "Unknown", "UNKNOWN", " unknown " variants.""" for placeholder in ("Unknown", "UNKNOWN", " unknown "): source = { "title": "Paper", "link": "https://doi.org/10.1/x", "journal": placeholder, "source": "openalex", "doi": "10.1/x", } result = normalize_citation(source) assert result is not None assert result.get("container_title") is None, ( f"placeholder {placeholder!r} leaked through" ) def test_csl_json_built(self): source = { "title": "Test Paper", "link": "https://arxiv.org/abs/2401.12345", "authors": ["John Smith"], "published": "2024-01-15", "source": "arXiv", } result = normalize_citation(source) csl = result["csl_json"] assert csl["title"] == "Test Paper" assert csl["type"] == "article" assert csl["author"][0]["family"] == "Smith" assert csl["issued"]["date-parts"] == [[2024, 1, 15]] assert csl["URL"] == "https://arxiv.org/abs/2401.12345"