"""Tests for citation formatter functionality.""" import pytest from local_deep_research.text_optimization import ( CitationFormatter, CitationMode, LaTeXExporter, QuartoExporter, RISExporter, ) class TestCitationFormatter: """Test cases for CitationFormatter class.""" @pytest.fixture def sample_content(self): """Sample markdown content with citations.""" return """# Research Report This is a research report about artificial intelligence [1]. Recent advances in machine learning [2] have shown significant progress. The work by researchers at DeepMind [3] has been particularly influential. ## Key Findings The analysis reveals several important points: - Deep learning models are becoming more efficient [1] - Transfer learning enables better generalization [2] - Large language models show emergent capabilities [3] - Multiple studies confirm these findings [1][2][3] - Some research combines different approaches [2][3] - Comprehensive surveys cover all aspects [1, 2, 3] - Recent work builds on earlier findings [2, 3] ## Sources [1] Understanding Deep Learning URL: https://arxiv.org/abs/2104.12345 [2] Transfer Learning: A Survey URL: https://www.nature.com/articles/s41586-021-03819-2 [3] Emergent Abilities of Large Language Models URL: https://openai.com/research/emergent-abilities """ @pytest.fixture def content_without_urls(self): """Sample content with citations but no URLs.""" return """# Report Some findings [1] and more [2]. ## Sources [1] First Source [2] Second Source """ def test_number_hyperlinks_mode(self, sample_content): """Test citation formatting with number hyperlinks.""" formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(sample_content) # Check single citations are formatted correctly assert "[[1]](https://arxiv.org/abs/2104.12345)" in result assert ( "[[2]](https://www.nature.com/articles/s41586-021-03819-2)" in result ) assert "[[3]](https://openai.com/research/emergent-abilities)" in result # Check multiple consecutive citations assert ( "[[1]](https://arxiv.org/abs/2104.12345)[[2]](https://www.nature.com/articles/s41586-021-03819-2)[[3]](https://openai.com/research/emergent-abilities)" in result ) # Check comma-separated citations assert ( "[[1]](https://arxiv.org/abs/2104.12345)[[2]](https://www.nature.com/articles/s41586-021-03819-2)[[3]](https://openai.com/research/emergent-abilities)" in result ) # Ensure sources section is unchanged assert "## Sources" in result assert "[1] Understanding Deep Learning" in result def test_domain_hyperlinks_mode(self, sample_content): """Test citation formatting with domain hyperlinks.""" formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) result = formatter.format_document(sample_content) # Check domain formatting assert "[[arxiv.org]](https://arxiv.org/abs/2104.12345)" in result assert ( "[[nature.com]](https://www.nature.com/articles/s41586-021-03819-2)" in result ) assert ( "[[openai.com]](https://openai.com/research/emergent-abilities)" in result ) # Check multiple citations work assert ( "[[arxiv.org]](https://arxiv.org/abs/2104.12345)[[nature.com]](https://www.nature.com/articles/s41586-021-03819-2)[[openai.com]](https://openai.com/research/emergent-abilities)" in result ) def test_no_hyperlinks_mode(self, sample_content): """Test citation formatting with no hyperlinks.""" formatter = CitationFormatter(CitationMode.NO_HYPERLINKS) result = formatter.format_document(sample_content) # Content should be unchanged assert result == sample_content assert "[[1]]" not in result assert "](https://" not in result def test_citations_without_urls(self, content_without_urls): """Test handling of citations without URLs.""" formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content_without_urls) # Citations without URLs should remain unchanged assert "[1]" in result assert "[2]" in result assert "[[1]]" not in result assert "[[2]]" not in result def test_no_sources_section(self): """Test handling of content without sources section.""" content = "This is text with [1] citation but no sources." formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content) # Should return unchanged content assert result == content def test_mixed_citation_formats(self): """Test handling of mixed citation formats.""" content = """Text with various formats: - Single: [1] - Multiple: [1][2] - Comma-separated: [1, 2, 3] - With spaces: [1,2,3] and [1, 2] ## Sources [1] Source One URL: https://example1.com [2] Source Two URL: https://example2.com [3] Source Three URL: https://example3.com """ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content) # All formats should be converted assert "[[1]](https://example1.com)" in result assert ( "[[1]](https://example1.com)[[2]](https://example2.com)" in result ) assert ( "[[1]](https://example1.com)[[2]](https://example2.com)[[3]](https://example3.com)" in result ) def test_domain_extraction(self): """Test domain extraction from various URLs.""" formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) # Test various URL formats test_cases = [ ("https://arxiv.org/abs/1234", "arxiv.org"), ("https://www.nature.com/articles/123", "nature.com"), ("https://github.com/user/repo", "github.com"), ("https://subdomain.example.com/path", "example.com"), ("https://example.co.uk/path", "co.uk"), ] for url, expected_domain in test_cases: domain = formatter._extract_domain(url) assert domain == expected_domain def test_edge_cases(self): """Test edge cases and malformed content.""" formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) # Empty content assert formatter.format_document("") == "" # Content with only sources section content = """## Sources [1] Test URL: https://test.com """ result = formatter.format_document(content) assert result == content # Malformed citations content = """Text with [a] non-numeric [citation]. ## Sources [a] Invalid citation """ result = formatter.format_document(content) assert "[a]" in result # Should remain unchanged assert "[citation]" in result def test_source_word_pattern_detection(self): """Test detection and conversion of 'Source X' patterns.""" content = """# Research Report As mentioned in Source 1, the data shows improvements. According to source 2, this is confirmed. The Source 3 provides additional evidence. ## Sources [1] First Research Paper URL: https://example.com/paper1 [2] Second Study URL: https://example.com/paper2 [3] Third Analysis URL: https://example.com/paper3 """ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content) # Check that "Source X" patterns are converted to hyperlinks assert "[[1]](https://example.com/paper1)" in result assert "[[2]](https://example.com/paper2)" in result assert "[[3]](https://example.com/paper3)" in result # Original "Source X" text should be replaced assert "Source 1" not in result or "[[1]]" in result assert "source 2" not in result or "[[2]]" in result def test_source_pattern_edge_cases(self): """Test that Source pattern converts to broken references when no matching source.""" content = """# Financial Report Source 12 of income comes from investments. The power source 5 failed during testing. Open Source 100 projects were analyzed. Valid reference from Source 1 here. ## Sources [1] Financial Analysis URL: https://example.com/finance """ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content) # These get converted to broken reference markers [X] since no matching sources assert "[12] of income" in result # Converted but not hyperlinked assert "[5] failed" in result # Converted but not hyperlinked assert "[100] projects" in result # Converted but not hyperlinked # Valid Source 1 should be hyperlinked assert "[[1]](https://example.com/finance) here" in result # The broken references should NOT be hyperlinked assert "[[12]]" not in result assert "[[5]]" not in result assert "[[100]]" not in result def test_mixed_citation_formats_with_source_pattern(self): """Test document with both [X] and Source X patterns.""" content = """# Research Report Traditional citation [1] appears here. Then Source 2 is mentioned directly. Mixed with [3] and source 1 again. ## Sources [1] First Paper URL: https://example.com/paper1 [2] Second Paper URL: https://example.com/paper2 [3] Third Paper URL: https://example.com/paper3 """ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content) # All formats should be converted consistently assert ( result.count("[[1]](https://example.com/paper1)") >= 2 ) # [1] and source 1 assert "[[2]](https://example.com/paper2)" in result # Source 2 assert "[[3]](https://example.com/paper3)" in result # [3] def test_domain_id_hyperlinks_mode(self, sample_content): """Test citation formatting with domain IDs for repeated domains.""" # Add more sources from same domains to test ID functionality content_with_repeated_domains = """# Research Report Citations from GitHub [1] and more from GitHub [2]. ArXiv papers [3] and another ArXiv paper [4]. A Nature article [5] and OpenAI research [6]. ## Sources [1] First GitHub Project URL: https://github.com/user/project1 [2] Second GitHub Project URL: https://github.com/user/project2 [3] First ArXiv Paper URL: https://arxiv.org/abs/2024.1111 [4] Second ArXiv Paper URL: https://arxiv.org/abs/2024.2222 [5] Nature Article URL: https://www.nature.com/articles/s41586-024-1234 [6] OpenAI Research URL: https://openai.com/research/gpt4 """ formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS) result = formatter.format_document(content_with_repeated_domains) # Check that repeated domains get hyphen-separated IDs assert "[[github.com-1]](https://github.com/user/project1)" in result assert "[[github.com-2]](https://github.com/user/project2)" in result assert "[[arxiv.org-1]](https://arxiv.org/abs/2024.1111)" in result assert "[[arxiv.org-2]](https://arxiv.org/abs/2024.2222)" in result # Check that single domain citations don't get IDs assert ( "[[nature.com]](https://www.nature.com/articles/s41586-024-1234)" in result ) assert "[[openai.com]](https://openai.com/research/gpt4)" in result def test_real_world_example(self): """Test with real-world example from Local Deep Research output.""" content = """# Quick Research Summary Query: local deep research Local Deep Research represents a new generation of AI-powered research assistants [1][3][11]. It utilizes any LLM hosted by either Ollama or LMStudio [2]. The concept was introduced as DeepSearcher [4] and further explained on Medium [7]. Web searches are conducted using DuckDuckGo [5], and the system incorporates tools like Milvus [6]. Updates were discussed on Reddit [10] and demonstrations on YouTube [8][12]. The Ollama variant [9] provides an alternative approach. ## Sources [1] LearningCircuit/local-deep-research- GitHub URL: https://github.com/LearningCircuit/local-deep-research [2] langchain-ai/local-deep-researcher - GitHub URL: https://github.com/langchain-ai/local-deep-researcher [3] local-deep-research- PyPI URL: https://pypi.org/project/local-deep-research/ [4] Introducing DeepSearcher URL: https://zilliz.com/blog/introduce-deepsearcher [5] A local LLM research assistant URL: https://www.reddit.com/r/LocalLLaMA/comments/1ilkosp/ [6] Introducing DeepSearcher - Milvus URL: https://milvus.io/blog/introduce-deepsearcher.md [7] DeepSearcher on Medium URL: https://milvusio.medium.com/deepsearcher [8] Private AI Deep Research - YouTube URL: https://m.youtube.com/watch?v=4tDqV__jzKY [9] Ollama Deep Research URL: https://apidog.com/blog/ollama-deep-research/ [10] Local Deep Research Update URL: https://www.reddit.com/r/LocalLLaMA/comments/1j79obx/ [11] local-deep-research 0.3.3 - PyPI URL: https://pypi.org/project/local-deep-research/0.3.3/ [12] Building a local deep researcher - YouTube URL: https://www.youtube.com/watch?v=sGUjmyfof4Q """ formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS) result = formatter.format_document(content) # Check multiple citations from same domain get IDs assert ( "[[github.com-1]](https://github.com/LearningCircuit/local-deep-research)" in result ) assert ( "[[github.com-2]](https://github.com/langchain-ai/local-deep-researcher)" in result ) assert ( "[[pypi.org-1]](https://pypi.org/project/local-deep-research/)" in result ) assert ( "[[pypi.org-2]](https://pypi.org/project/local-deep-research/0.3.3/)" in result ) assert ( "[[reddit.com-1]](https://www.reddit.com/r/LocalLLaMA/comments/1ilkosp/)" in result ) assert ( "[[reddit.com-2]](https://www.reddit.com/r/LocalLLaMA/comments/1j79obx/)" in result ) assert ( "[[youtube.com-1]](https://m.youtube.com/watch?v=4tDqV__jzKY)" in result ) assert ( "[[youtube.com-2]](https://www.youtube.com/watch?v=sGUjmyfof4Q)" in result ) # Check single domain citations don't get IDs assert ( "[[zilliz.com]](https://zilliz.com/blog/introduce-deepsearcher)" in result ) assert ( "[[milvus.io]](https://milvus.io/blog/introduce-deepsearcher.md)" in result ) assert ( "[[apidog.com]](https://apidog.com/blog/ollama-deep-research/)" in result ) assert ( "[[medium.com]](https://milvusio.medium.com/deepsearcher)" in result ) def test_domain_id_always_hyperlinks_mode(self): """Test citation formatting with domain IDs always added for consistency.""" # Use the same content as test_domain_id_hyperlinks_mode content_with_repeated_domains = """# Research Report Citations from GitHub [1] and more from GitHub [2]. ArXiv papers [3] and another ArXiv paper [4]. A Nature article [5] and OpenAI research [6]. ## Sources [1] First GitHub Project URL: https://github.com/user/project1 [2] Second GitHub Project URL: https://github.com/user/project2 [3] First ArXiv Paper URL: https://arxiv.org/abs/2024.1111 [4] Second ArXiv Paper URL: https://arxiv.org/abs/2024.2222 [5] Nature Article URL: https://www.nature.com/articles/s41586-024-1234 [6] OpenAI Research URL: https://openai.com/research/gpt4 """ formatter = CitationFormatter(CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS) result = formatter.format_document(content_with_repeated_domains) # Check that ALL domains get IDs, even single ones assert "[[github.com-1]](https://github.com/user/project1)" in result assert "[[github.com-2]](https://github.com/user/project2)" in result assert "[[arxiv.org-1]](https://arxiv.org/abs/2024.1111)" in result assert "[[arxiv.org-2]](https://arxiv.org/abs/2024.2222)" in result assert ( "[[nature.com-1]](https://www.nature.com/articles/s41586-024-1234)" in result ) assert "[[openai.com-1]](https://openai.com/research/gpt4)" in result def test_source_tagged_hyperlinks_preserves_global_counter(self): """SOURCE_TAGGED_HYPERLINKS keeps the original citation number as the suffix, prefixed with the source tag. Two arxiv citations and one openai.com citation become [arxiv-1], [openai.com-2], [arxiv-3] — not the per-domain renumbering of DOMAIN_ID_*.""" content = """# Report Finding A [1]. Finding B [2]. Finding C [3]. Combined [1, 2, 3]. ## Sources [1] First ArXiv Paper URL: https://arxiv.org/abs/2024.1111 [2] OpenAI Research URL: https://openai.com/research/gpt4 [3] Second ArXiv Paper URL: https://arxiv.org/abs/2024.2222 """ formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS) result = formatter.format_document(content) # URLClassifier maps arxiv.org URLs to URLType.ARXIV → "arxiv". assert "[[arxiv-1]](https://arxiv.org/abs/2024.1111)" in result # openai.com is generic HTML → falls back to domain. assert "[[openai.com-2]](https://openai.com/research/gpt4)" in result # Second arxiv keeps the GLOBAL number 3, not a per-domain "2". assert "[[arxiv-3]](https://arxiv.org/abs/2024.2222)" in result # Comma-separated citations expand into individual tagged links. assert ( "[[arxiv-1]](https://arxiv.org/abs/2024.1111)" "[[openai.com-2]](https://openai.com/research/gpt4)" "[[arxiv-3]](https://arxiv.org/abs/2024.2222)" ) in result def test_source_tagged_hyperlinks_known_academic_sources(self): """URLClassifier-recognized academic sources use the short enum tag (arxiv, pubmed, semantic_scholar, biorxiv, ...), not the raw domain.""" content = """# Report Cite [1] and [2] and [3] and [4]. ## Sources [1] ArXiv Paper URL: https://arxiv.org/abs/2024.1234 [2] PubMed Paper URL: https://pubmed.ncbi.nlm.nih.gov/12345678/ [3] Semantic Scholar URL: https://www.semanticscholar.org/paper/Some-Paper/abc123 [4] bioRxiv Preprint URL: https://www.biorxiv.org/content/10.1101/2024.01.01.000001v1 """ formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS) result = formatter.format_document(content) assert "[[arxiv-1]]" in result assert "[[pubmed-2]]" in result assert "[[semantic_scholar-3]]" in result assert "[[biorxiv-4]]" in result def test_source_tagged_hyperlinks_uses_collection_name(self): """When the sources block carries a ``Collection:`` line for a library/RAG hit, the collection name (slugified) is used as the citation tag instead of the URL-derived ``local`` fallback.""" content = """# Report Web cite [1]. Library cite [2]. Another web [3]. Another library [4]. ## Sources [1] ArXiv Paper (source nr: 1) URL: https://arxiv.org/abs/2024.1111 [2] Local Doc (source nr: 2) URL: /library/document/abc123 Collection: My Papers [3] OpenAI Research (source nr: 3) URL: https://openai.com/research/gpt4 [4] Another local (source nr: 4) URL: /library/document/def456 Collection: team/finance """ formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS) result = formatter.format_document(content) # Web citations are unaffected — still URL-derived tags. assert "[[arxiv-1]](https://arxiv.org/abs/2024.1111)" in result assert "[[openai.com-3]](https://openai.com/research/gpt4)" in result # Library citations adopt the slugified collection name + global N. # ``My Papers`` → ``my-papers``, ``team/finance`` → ``team-finance``. # Local URLs render without a hyperlink (no http(s) scheme). assert "[my-papers-2]" in result assert "[[my-papers-2]]" not in result assert "[team-finance-4]" in result assert "[[team-finance-4]]" not in result def test_source_tagged_hyperlinks_collection_slugify_edge_cases(self): """Collection names with whitespace, slashes, unicode, or empty slugs degrade safely. The slugifier lowercases, replaces runs of non-alphanumerics with single hyphens, trims edge hyphens, and falls back to ``local`` when the result is empty.""" from local_deep_research.text_optimization.citation_formatter import ( CitationFormatter as CF, ) assert CF._slugify_collection("My Papers") == "my-papers" assert CF._slugify_collection(" trim me ") == "trim-me" assert CF._slugify_collection("team/finance") == "team-finance" assert CF._slugify_collection("ALLCAPS") == "allcaps" assert CF._slugify_collection("a/b/c") == "a-b-c" assert CF._slugify_collection("---") == "local" assert CF._slugify_collection("") == "local" # Unicode that's not [a-z0-9] collapses to hyphens too. The # behaviour is "ASCII-safe slug for inline use" — round-tripping # the original name is out of scope. assert CF._slugify_collection("résumé") == "r-sum" def test_source_tagged_hyperlinks_missing_collection_falls_back(self): """A library URL **without** a ``Collection:`` line in the sources block falls through to the existing ``local`` tag (this is what older renderers and unit tests with hand-rolled sources produce).""" content = """# Report See [1]. ## Sources [1] Local doc with no collection metadata URL: /library/document/abc123 """ formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS) result = formatter.format_document(content) # ``/library/document/...`` has no scheme → ``local`` fallback, # rendered as a plain bracketed tag (no hyperlink). assert "[local-1]" in result assert "[[local-1]]" not in result def test_source_tagged_hyperlinks_collection_line_isolation(self): """A ``Collection:`` line attached to citation [N] must not leak into citation [N+1]'s tag. Regex anchoring is the load-bearing bit here.""" content = """# Report See [1] and [2]. ## Sources [1] Local doc with collection URL: /library/document/abc Collection: project-alpha [2] ArXiv paper, NO collection URL: https://arxiv.org/abs/2024.0001 """ formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS) result = formatter.format_document(content) assert "[project-alpha-1]" in result # Citation [2] must NOT pick up citation [1]'s collection. assert "[[arxiv-2]](https://arxiv.org/abs/2024.0001)" in result assert "[project-alpha-2]" not in result def test_source_tagged_hyperlinks_local_url_falls_back(self): """Empty / non-http(s) URLs get the ``local`` tag and render without a hyperlink.""" content = """# Report See [1] and [2]. ## Sources [1] A local doc URL: file:///srv/library/mypapers/foo.pdf [2] Another local doc with no URL at all """ formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS) result = formatter.format_document(content) # file:// → "local", with no hyperlink wrapping. assert "[local-1]" in result assert "[[local-1]]" not in result # no hyperlink, not bracketed twice # No URL → also "local", and no hyperlink. assert "[local-2]" in result def test_source_tagged_hyperlinks_no_state_leak_across_calls(self): """Calling ``format_document`` twice on the same formatter instance must not leak collection metadata between calls. Doc 1 has a ``Collection:`` line for citation [1]; doc 2 has no collection line at all, so its [1] must fall back to the URL-derived tag, not pick up doc 1's collection.""" doc_with_collection = """# Doc 1 See [1]. ## Sources [1] Local doc with collection URL: /library/document/abc Collection: project-alpha """ doc_without_collection = """# Doc 2 See [1]. ## Sources [1] ArXiv paper, no collection URL: https://arxiv.org/abs/2024.9999 """ formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS) result1 = formatter.format_document(doc_with_collection) assert "[project-alpha-1]" in result1 # Second call: arxiv URL, no Collection: line in the sources block. # Must resolve to arxiv via URLClassifier, NOT to project-alpha. result2 = formatter.format_document(doc_without_collection) assert "[[arxiv-1]](https://arxiv.org/abs/2024.9999)" in result2 assert "project-alpha" not in result2 def test_unicode_lenticular_bracket_citations(self): """Test that Unicode lenticular brackets【】are recognized and converted.""" content = """# Research Report Research shows findings 【1】 and more evidence 【2】. Multiple citations 【1】【2】 are also used. ## Sources [1] First Source URL: https://example.com/1 [2] Second Source URL: https://example.com/2 """ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content) # Check lenticular brackets are converted to hyperlinks assert "[[1]](https://example.com/1)" in result assert "[[2]](https://example.com/2)" in result def test_unicode_lenticular_comma_citations(self): """Test comma-separated lenticular citations【1, 2, 3】.""" content = """# Report Multiple sources 【1, 2】 confirm this. ## Sources [1] Source One URL: https://example.com/1 [2] Source Two URL: https://example.com/2 """ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content) # Check comma-separated lenticular citations are expanded assert ( "[[1]](https://example.com/1)[[2]](https://example.com/2)" in result ) def test_mixed_bracket_styles(self): """Test documents with both standard [1] and lenticular【2】brackets.""" content = """# Report Standard citation [1] and lenticular 【2】 in same doc. ## Sources [1] First Source URL: https://example.com/1 [2] Second Source URL: https://example.com/2 """ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content) # Both bracket styles should be converted assert "[[1]](https://example.com/1)" in result assert "[[2]](https://example.com/2)" in result def test_lenticular_multi_digit_citations(self): """Test lenticular brackets with multi-digit citation numbers.""" content = """# Report Citations with higher numbers 【10】 and 【99】 work correctly. ## Sources [10] Tenth Source URL: https://example.com/10 [99] Ninety-ninth Source URL: https://example.com/99 """ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content) assert "[[10]](https://example.com/10)" in result assert "[[99]](https://example.com/99)" in result def test_lenticular_triple_comma_citations(self): """Test lenticular brackets with three or more comma-separated numbers.""" content = """# Report Many sources 【1, 2, 3】 and even more 【1,2,3,4】 confirm this. ## Sources [1] Source One URL: https://example.com/1 [2] Source Two URL: https://example.com/2 [3] Source Three URL: https://example.com/3 [4] Source Four URL: https://example.com/4 """ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content) # Three comma-separated should expand assert ( "[[1]](https://example.com/1)[[2]](https://example.com/2)" "[[3]](https://example.com/3)" in result ) # Four comma-separated (no spaces) should also expand assert ( "[[1]](https://example.com/1)[[2]](https://example.com/2)" "[[3]](https://example.com/3)[[4]](https://example.com/4)" in result ) def test_lenticular_consecutive_mixed(self): """Test alternating standard and lenticular consecutive citations.""" content = """# Report Mixed consecutive [1]【2】[3] citations work. Also reversed 【1】[2]【3】 order. ## Sources [1] First Source URL: https://example.com/1 [2] Second Source URL: https://example.com/2 [3] Third Source URL: https://example.com/3 """ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content) # All citations should be converted regardless of bracket style assert "[[1]](https://example.com/1)" in result assert "[[2]](https://example.com/2)" in result assert "[[3]](https://example.com/3)" in result def test_lenticular_domain_hyperlinks_mode(self): """Test lenticular brackets with domain hyperlinks mode.""" content = """# Report Research from 【1】 and 【2】 shows results. ## Sources [1] ArXiv Paper URL: https://arxiv.org/abs/2024.1234 [2] Nature Article URL: https://www.nature.com/articles/s41586-024-5678 """ formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) result = formatter.format_document(content) assert "[[arxiv.org]](https://arxiv.org/abs/2024.1234)" in result assert ( "[[nature.com]](https://www.nature.com/articles/s41586-024-5678)" in result ) def test_lenticular_domain_id_hyperlinks_mode(self): """Test lenticular brackets with domain ID hyperlinks mode.""" content = """# Report Multiple GitHub sources 【1】 and 【2】 referenced. ## Sources [1] First GitHub Repo URL: https://github.com/user/repo1 [2] Second GitHub Repo URL: https://github.com/user/repo2 """ formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS) result = formatter.format_document(content) assert "[[github.com-1]](https://github.com/user/repo1)" in result assert "[[github.com-2]](https://github.com/user/repo2)" in result def test_lenticular_in_bullet_list(self): """Test lenticular brackets within bullet point lists.""" content = """# Report Key findings: - First point 【1】 - Second point 【2】 - Combined evidence 【1, 2】 ## Sources [1] First Source URL: https://example.com/1 [2] Second Source URL: https://example.com/2 """ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content) assert "- First point [[1]](https://example.com/1)" in result assert "- Second point [[2]](https://example.com/2)" in result assert ( "[[1]](https://example.com/1)[[2]](https://example.com/2)" in result ) def test_lenticular_without_matching_source(self): """Test lenticular brackets referencing non-existent sources.""" content = """# Report Valid citation 【1】 and invalid 【99】 reference. ## Sources [1] Only Source URL: https://example.com/1 """ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content) # Valid citation should be hyperlinked assert "[[1]](https://example.com/1)" in result # Invalid citation should remain as plain text (not hyperlinked) assert "[[99]]" not in result assert "【99】" in result or "[99]" in result def test_lenticular_no_space_before_or_after(self): """Test lenticular brackets without spaces before or after.""" content = """# Report Text immediately before【1】and after without spaces. Also works at end of sentence【2】. And word【1】word with citations embedded. Multiple【1】【2】consecutive without spaces. Standard brackets work too: word[1]word and end[2]. ## Sources [1] First Source URL: https://example.com/1 [2] Second Source URL: https://example.com/2 """ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content) # Lenticular brackets should be converted despite no spaces assert "before[[1]](https://example.com/1)and" in result assert "sentence[[2]](https://example.com/2)." in result assert "word[[1]](https://example.com/1)word" in result assert ( "[[1]](https://example.com/1)[[2]](https://example.com/2)consecutive" in result ) # Standard brackets should work the same way assert "word[[1]](https://example.com/1)word" in result assert "end[[2]](https://example.com/2)." in result def test_lenticular_real_world_mixed_example(self): """Test real-world scenario with mixed bracket styles throughout.""" content = """# AI Safety Research Summary Query: AI alignment techniques Recent research【1】has explored various approaches to AI safety. The RLHF method [2] has shown promising results. Constitutional AI【3】builds on these foundations [1, 2]. Multiple studies【1】【2】【3】confirm the effectiveness of these techniques. A comprehensive survey [1, 2, 3] covers all major approaches, while recent work【2, 3】focuses on scalability. ## Sources [1] RLHF: Training Language Models URL: https://arxiv.org/abs/2024.rlhf [2] Constitutional AI: Harmlessness from Feedback URL: https://anthropic.com/constitutional-ai [3] Scalable Oversight of AI Systems URL: https://openai.com/research/scalable-oversight """ formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) result = formatter.format_document(content) # All citations should be converted assert "[[1]](https://arxiv.org/abs/2024.rlhf)" in result assert "[[2]](https://anthropic.com/constitutional-ai)" in result assert "[[3]](https://openai.com/research/scalable-oversight)" in result # Consecutive lenticular citations should work assert ( "[[1]](https://arxiv.org/abs/2024.rlhf)" "[[2]](https://anthropic.com/constitutional-ai)" "[[3]](https://openai.com/research/scalable-oversight)" in result ) class TestLaTeXExporter: """Test cases for LaTeX export functionality.""" @pytest.fixture def sample_markdown(self): """Sample markdown for LaTeX conversion.""" return """# Main Title ## Introduction This is a **bold** statement with *italic* text and `code`. ### Subsection - First item - Second item - Third item Text with citations [1] and [2]. ## Sources [1] First Source URL: https://example.com [2] Second Source URL: https://example2.com """ def test_latex_export_basic(self, sample_markdown): """Test basic LaTeX export functionality.""" exporter = LaTeXExporter() result = exporter.export_to_latex(sample_markdown) # Check header assert r"\documentclass[12pt]{article}" in result assert r"\begin{document}" in result assert r"\end{document}" in result # Check heading conversion assert r"\section{Main Title}" in result assert r"\subsection{Introduction}" in result assert r"\subsubsection{Subsection}" in result # Check emphasis conversion assert r"\textbf{bold}" in result assert r"\textit{italic}" in result assert r"\texttt{code}" in result # Check citation conversion assert r"\cite{1}" in result assert r"\cite{2}" in result # Check bibliography assert r"\begin{thebibliography}" in result assert r"\bibitem{1}" in result def test_latex_list_conversion(self, sample_markdown): """Test list conversion to LaTeX.""" exporter = LaTeXExporter() result = exporter.export_to_latex(sample_markdown) # Check list conversion assert r"\begin{itemize}" in result assert r"\item First item" in result assert r"\item Second item" in result assert r"\item Third item" in result assert r"\end{itemize}" in result def test_latex_empty_content(self): """Test LaTeX export with empty content.""" exporter = LaTeXExporter() result = exporter.export_to_latex("") assert r"\documentclass[12pt]{article}" in result assert r"\begin{document}" in result assert r"\end{document}" in result def test_latex_no_sources(self): """Test LaTeX export without sources section.""" content = "# Title\n\nSome text without citations." exporter = LaTeXExporter() result = exporter.export_to_latex(content) # Should not include bibliography assert r"\begin{thebibliography}" not in result def test_unicode_lenticular_citations(self): """Test LaTeX export converts lenticular citations to \\cite{N} format.""" content = """# Report Research with lenticular citations 【1】 and 【2】. ## Sources [1] First Source URL: https://example.com/1 [2] Second Source URL: https://example.com/2 """ exporter = LaTeXExporter() result = exporter.export_to_latex(content) # Check lenticular citations are converted to LaTeX cite format assert r"\cite{1}" in result assert r"\cite{2}" in result def test_unicode_lenticular_mixed_with_standard(self): """Test LaTeX with mixed standard and lenticular brackets.""" content = """# Report Standard [1] and lenticular 【2】 and consecutive【3】[4] citations. ## Sources [1] First URL: https://example.com/1 [2] Second URL: https://example.com/2 [3] Third URL: https://example.com/3 [4] Fourth URL: https://example.com/4 """ exporter = LaTeXExporter() result = exporter.export_to_latex(content) assert r"\cite{1}" in result assert r"\cite{2}" in result assert r"\cite{3}" in result assert r"\cite{4}" in result def test_unicode_lenticular_multi_digit(self): """Test LaTeX export with multi-digit lenticular citations.""" content = """# Report Higher numbers 【10】 and 【25】 work too. ## Sources [10] Tenth Source URL: https://example.com/10 [25] Twenty-fifth Source URL: https://example.com/25 """ exporter = LaTeXExporter() result = exporter.export_to_latex(content) assert r"\cite{10}" in result assert r"\cite{25}" in result class TestRISExporter: """Test cases for RIS export functionality.""" @pytest.fixture def sample_markdown(self): """Sample markdown for RIS conversion.""" return """# Research Report This research explores various topics [1][2][3]. ## Sources [1] Understanding Deep Learning URL: https://arxiv.org/abs/2104.12345 [2] GitHub Project Documentation URL: https://github.com/example/project [3] Reddit Discussion Thread URL: https://www.reddit.com/r/example/comments/abc123 [4] Local Resource Without URL """ def test_ris_export_basic(self, sample_markdown): """Test basic RIS export functionality.""" exporter = RISExporter() result = exporter.export_to_ris(sample_markdown) # Check that we have RIS entries assert "TY - ELEC" in result assert "ER - " in result # Check specific entries assert "ID - ref1" in result assert "TI - Understanding Deep Learning" in result assert "UR - https://arxiv.org/abs/2104.12345" in result assert "PB - arXiv" in result assert "ID - ref2" in result assert "TI - GitHub Project Documentation" in result assert "PB - GitHub" in result assert "ID - ref3" in result assert "TI - Reddit Discussion Thread" in result assert "PB - Reddit" in result # Check entry without URL assert "ID - ref4" in result assert "TI - Local Resource Without URL" in result # Check date fields assert "Y1 - " in result assert "DA - " in result assert "LA - en" in result def test_ris_export_empty_content(self): """Test RIS export with empty content.""" exporter = RISExporter() result = exporter.export_to_ris("") assert result == "" def test_ris_export_no_sources(self): """Test RIS export without sources section.""" content = "# Title\n\nSome text without sources." exporter = RISExporter() result = exporter.export_to_ris(content) assert result == "" def test_ris_publisher_extraction(self, sample_markdown): """Test that publishers are correctly extracted from domains.""" exporter = RISExporter() result = exporter.export_to_ris(sample_markdown) # Check known publishers assert "PB - arXiv" in result assert "PB - GitHub" in result assert "PB - Reddit" in result class TestQuartoExporter: """Test cases for Quarto export functionality.""" @pytest.fixture def sample_markdown(self): """Sample markdown for Quarto conversion.""" return """# Research on AI Safety ## Introduction This research explores **important** aspects of *AI safety* and `alignment`. ### Current Approaches - Reinforcement learning from human feedback [1] - Constitutional AI [2] - Interpretability research [3] Multiple studies [1, 2, 3] have shown promising results. ## Sources [1] RLHF: A Survey URL: https://arxiv.org/abs/2023.1234 [2] Constitutional AI URL: https://anthropic.com/constitutional-ai [3] Mechanistic Interpretability URL: https://distill.pub/2020/circuits """ def test_quarto_export_basic(self, sample_markdown): """Test basic Quarto export functionality.""" exporter = QuartoExporter() result = exporter.export_to_quarto( sample_markdown, "AI Safety Research" ) # Check YAML header assert "---" in result assert 'title: "AI Safety Research"' in result assert 'author: "Local Deep Research"' in result # Check that date is in YYYY-MM-DD format import re assert re.search(r'date: "\d{4}-\d{2}-\d{2}"', result) assert "format:" in result assert "bibliography: references.bib" in result # Check citation conversion assert "[@ref1]" in result assert "[@ref2]" in result assert "[@ref3]" in result assert "[@ref1, @ref2, @ref3]" in result # Check bibliography note assert "Bibliography File Required" in result assert "references.bib" in result assert "@misc{ref1," in result def test_quarto_title_extraction(self, sample_markdown): """Test automatic title extraction from markdown.""" exporter = QuartoExporter() # Don't provide title, should extract from markdown result = exporter.export_to_quarto(sample_markdown) assert 'title: "Research on AI Safety"' in result def test_quarto_bibliography_generation(self, sample_markdown): """Test BibTeX bibliography generation.""" exporter = QuartoExporter() bib_content = exporter._generate_bibliography(sample_markdown) # Check BibTeX entries assert "@misc{ref1," in bib_content assert 'title = "{RLHF: A Survey}"' in bib_content assert "url = {https://arxiv.org/abs/2023.1234}" in bib_content assert ( 'howpublished = "\\url{https://arxiv.org/abs/2023.1234}"' in bib_content ) assert "year = {2024}" in bib_content assert 'note = "Accessed: \\today"' in bib_content # Check all three entries assert "@misc{ref2," in bib_content assert "@misc{ref3," in bib_content def test_quarto_empty_content(self): """Test Quarto export with empty content.""" exporter = QuartoExporter() result = exporter.export_to_quarto("", "Empty Report") # Should still have YAML header assert "---" in result assert 'title: "Empty Report"' in result def test_quarto_no_sources(self): """Test Quarto export without sources section.""" content = "# Title\n\nSome text with citations [1] and [2]." exporter = QuartoExporter() result = exporter.export_to_quarto(content) # Citations should still be converted assert "[@ref1]" in result assert "[@ref2]" in result # Bibliography note should still appear but be empty assert "Bibliography File Required" in result def test_unicode_lenticular_citations(self): """Test Quarto export converts lenticular citations to [@refN] format.""" content = """# Report Single lenticular 【1】 and comma-separated 【1, 2】 citations. ## Sources [1] First Source URL: https://example.com/1 [2] Second Source URL: https://example.com/2 """ exporter = QuartoExporter() result = exporter.export_to_quarto(content) # Check lenticular citations are converted to Quarto format assert "[@ref1]" in result assert "[@ref1, @ref2]" in result def test_unicode_lenticular_triple_comma(self): """Test Quarto export with three comma-separated lenticular citations.""" content = """# Report Multiple sources 【1, 2, 3】 referenced together. ## Sources [1] First URL: https://example.com/1 [2] Second URL: https://example.com/2 [3] Third URL: https://example.com/3 """ exporter = QuartoExporter() result = exporter.export_to_quarto(content) assert "[@ref1, @ref2, @ref3]" in result def test_unicode_lenticular_mixed_with_standard(self): """Test Quarto export with mixed bracket styles.""" content = """# Report Standard [1] and lenticular 【2】 and mixed comma [1, 2] and 【2, 3】. ## Sources [1] First URL: https://example.com/1 [2] Second URL: https://example.com/2 [3] Third URL: https://example.com/3 """ exporter = QuartoExporter() result = exporter.export_to_quarto(content) assert "[@ref1]" in result assert "[@ref2]" in result assert "[@ref1, @ref2]" in result assert "[@ref2, @ref3]" in result def test_unicode_lenticular_consecutive(self): """Test Quarto export with consecutive lenticular citations.""" content = """# Report Consecutive lenticular【1】【2】【3】citations. ## Sources [1] First URL: https://example.com/1 [2] Second URL: https://example.com/2 [3] Third URL: https://example.com/3 """ exporter = QuartoExporter() result = exporter.export_to_quarto(content) # Each should be converted individually assert "[@ref1]" in result assert "[@ref2]" in result assert "[@ref3]" in result class TestExtractDomainExceptionHandling: """Tests for _extract_domain handling malformed URLs (PR #2016).""" def test_extract_domain_with_completely_invalid_url(self): """Test _extract_domain returns 'source' for completely invalid URL.""" formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) # The method catches Exception and returns "source" result = formatter._extract_domain("") # Empty URL should still produce a result (may be "source" or empty domain) assert isinstance(result, str) def test_extract_domain_with_valid_url(self): """Test _extract_domain works correctly with valid URL.""" formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) result = formatter._extract_domain("https://arxiv.org/abs/2024.1234") assert result == "arxiv.org" def test_extract_domain_with_www_prefix(self): """Test _extract_domain strips www prefix.""" formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) result = formatter._extract_domain( "https://www.nature.com/articles/s41586" ) assert result == "nature.com" def test_extract_domain_with_subdomain(self): """Test _extract_domain handles subdomains.""" formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) result = formatter._extract_domain( "https://milvusio.medium.com/article" ) assert result == "medium.com" def test_extract_domain_with_none_like_input(self): """Test _extract_domain handles edge case inputs gracefully.""" formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) # urlparse handles most strings without raising, but the except # catches any Exception that occurs during domain extraction result = formatter._extract_domain("not-a-url") assert isinstance(result, str) class TestRISExporterExceptionHandling: """Tests for RIS exporter handling malformed URLs (PR #2016).""" def test_ris_export_with_malformed_url(self): """Test RIS export handles malformed URL in source gracefully.""" content = """# Report Test content [1]. ## Sources [1] Source With Bad URL URL: ://malformed-url-no-scheme """ exporter = RISExporter() result = exporter.export_to_ris(content) # Should still produce RIS output without crashing assert "TY - ELEC" in result assert "TI - Source With Bad URL" in result assert "ER - " in result def test_ris_export_with_valid_and_invalid_urls(self): """Test RIS export when mixing valid and invalid URLs.""" content = """# Report Content [1] and [2]. ## Sources [1] Valid Source URL: https://arxiv.org/abs/2024.1234 [2] Invalid URL Source URL: not_a_valid_url_at_all """ exporter = RISExporter() result = exporter.export_to_ris(content) # Both entries should be present assert "TI - Valid Source" in result assert "TI - Invalid URL Source" in result # Valid source should have publisher assert "PB - arXiv" in result class TestFindSourcesSection: """Test cases for the find_sources_section module-level function.""" def test_find_h2_sources_header(self): """Test finding ## Sources header.""" from local_deep_research.text_optimization.citation_formatter import ( find_sources_section, ) content = """# Report Some content here. ## Sources [1] First source """ result = find_sources_section(content) assert result == content.index("## Sources") def test_find_h2_references_header(self): """Test finding ## References header.""" from local_deep_research.text_optimization.citation_formatter import ( find_sources_section, ) content = """# Report Some content here. ## References [1] First reference """ result = find_sources_section(content) assert result == content.index("## References") def test_find_h2_bibliography_header(self): """Test finding ## Bibliography header.""" from local_deep_research.text_optimization.citation_formatter import ( find_sources_section, ) content = """# Report Some content here. ## Bibliography [1] First entry """ result = find_sources_section(content) assert result == content.index("## Bibliography") def test_find_h2_citations_header(self): """Test finding ## Citations header.""" from local_deep_research.text_optimization.citation_formatter import ( find_sources_section, ) content = """# Report Some content here. ## Citations [1] First citation """ result = find_sources_section(content) assert result == content.index("## Citations") def test_find_h3_sources_header(self): """Test finding ### Sources header.""" from local_deep_research.text_optimization.citation_formatter import ( find_sources_section, ) content = """# Report ## Section ### Sources [1] First source """ result = find_sources_section(content) assert result == content.index("### Sources") def test_find_h1_sources_header(self): """Test finding # Sources header.""" from local_deep_research.text_optimization.citation_formatter import ( find_sources_section, ) content = """# Report Some content. # Sources [1] First source """ result = find_sources_section(content) assert result == content.index("# Sources") def test_case_insensitivity(self): """Test that header detection is case insensitive.""" from local_deep_research.text_optimization.citation_formatter import ( find_sources_section, ) content_lower = """## sources [1] First """ content_upper = """## SOURCES [1] First """ content_mixed = """## SoUrCeS [1] First """ assert find_sources_section(content_lower) == 0 assert find_sources_section(content_upper) == 0 assert find_sources_section(content_mixed) == 0 def test_no_sources_section_returns_negative_one(self): """Test that -1 is returned when no sources section exists.""" from local_deep_research.text_optimization.citation_formatter import ( find_sources_section, ) content = """# Report Some content with [1] citation but no sources section. ## Conclusion The end. """ result = find_sources_section(content) assert result == -1 def test_empty_content_returns_negative_one(self): """Test that empty content returns -1.""" from local_deep_research.text_optimization.citation_formatter import ( find_sources_section, ) assert find_sources_section("") == -1 def test_plain_sources_colon_format(self): """Test finding 'Sources:' format without markdown headers.""" from local_deep_research.text_optimization.citation_formatter import ( find_sources_section, ) content = """Report content. Sources: [1] First source """ result = find_sources_section(content) assert result == content.index("Sources:") def test_references_without_colon(self): """Test finding 'References' without colon.""" from local_deep_research.text_optimization.citation_formatter import ( find_sources_section, ) content = """Report content. References [1] First reference """ result = find_sources_section(content) assert result == content.index("References") class TestReplaceCommaCitations: """Test cases for the _replace_comma_citations method.""" def test_basic_comma_separated_citations(self): """Test replacement of basic comma-separated citations.""" formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) content = "See citations [1, 2, 3] for more info." lookup = { "1": ("Title 1", "https://example.com/1"), "2": ("Title 2", "https://example.com/2"), "3": ("Title 3", "https://example.com/3"), } def format_one(num, data): _, url = data return f"[[{num}]]({url})" result = formatter._replace_comma_citations(content, lookup, format_one) assert "[[1]](https://example.com/1)" in result assert "[[2]](https://example.com/2)" in result assert "[[3]](https://example.com/3)" in result assert "[1, 2, 3]" not in result def test_missing_citation_in_lookup(self): """Test that missing citations fall back to plain format.""" formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) content = "See citations [1, 2, 3] for more info." lookup = { "1": ("Title 1", "https://example.com/1"), # "2" is missing "3": ("Title 3", "https://example.com/3"), } def format_one(num, data): _, url = data return f"[[{num}]]({url})" result = formatter._replace_comma_citations(content, lookup, format_one) assert "[[1]](https://example.com/1)" in result assert "[2]" in result # Falls back to plain format assert "[[3]](https://example.com/3)" in result def test_no_spaces_after_commas(self): """Test citations without spaces after commas.""" formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) content = "See citations [1,2,3] for more info." lookup = { "1": ("Title 1", "https://example.com/1"), "2": ("Title 2", "https://example.com/2"), "3": ("Title 3", "https://example.com/3"), } def format_one(num, data): _, url = data return f"[[{num}]]({url})" result = formatter._replace_comma_citations(content, lookup, format_one) assert "[[1]](https://example.com/1)" in result assert "[[2]](https://example.com/2)" in result assert "[[3]](https://example.com/3)" in result def test_two_citations_only(self): """Test with only two comma-separated citations.""" formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) content = "See [1, 2] for details." lookup = { "1": ("Title 1", "https://example.com/1"), "2": ("Title 2", "https://example.com/2"), } def format_one(num, data): _, url = data return f"[[{num}]]({url})" result = formatter._replace_comma_citations(content, lookup, format_one) assert ( "[[1]](https://example.com/1)[[2]](https://example.com/2)" in result ) def test_domain_format_callback(self): """Test with domain-style formatting callback.""" formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) content = "See citations [1, 2] for more info." lookup = { "1": ("arxiv.org", "https://arxiv.org/abs/123"), "2": ("github.com", "https://github.com/repo"), } def format_one(num, data): domain, url = data return f"[[{domain}]]({url})" result = formatter._replace_comma_citations(content, lookup, format_one) assert "[[arxiv.org]](https://arxiv.org/abs/123)" in result assert "[[github.com]](https://github.com/repo)" in result def test_multiple_comma_groups_in_content(self): """Test multiple comma-separated citation groups in content.""" formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) content = "First group [1, 2] and second group [3, 4]." lookup = { "1": ("T1", "https://example.com/1"), "2": ("T2", "https://example.com/2"), "3": ("T3", "https://example.com/3"), "4": ("T4", "https://example.com/4"), } def format_one(num, data): _, url = data return f"[[{num}]]({url})" result = formatter._replace_comma_citations(content, lookup, format_one) assert ( "[[1]](https://example.com/1)[[2]](https://example.com/2)" in result ) assert ( "[[3]](https://example.com/3)[[4]](https://example.com/4)" in result ) def test_unicode_lenticular_brackets(self): """Test comma-separated citations with Unicode lenticular brackets.""" formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) content = "See citations【1, 2, 3】for more info." lookup = { "1": ("Title 1", "https://example.com/1"), "2": ("Title 2", "https://example.com/2"), "3": ("Title 3", "https://example.com/3"), } def format_one(num, data): _, url = data return f"[[{num}]]({url})" result = formatter._replace_comma_citations(content, lookup, format_one) assert "[[1]](https://example.com/1)" in result assert "[[2]](https://example.com/2)" in result assert "[[3]](https://example.com/3)" in result def test_empty_lookup_preserves_original(self): """Test that empty lookup preserves original citation numbers.""" formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS) content = "See citations [1, 2, 3] for more info." lookup = {} def format_one(num, data): _, url = data return f"[[{num}]]({url})" result = formatter._replace_comma_citations(content, lookup, format_one) # All citations should fall back to plain format assert "[1]" in result assert "[2]" in result assert "[3]" in result # --------------------------------------------------------------------------- # Regression tests for the empty-citation-label bug. # # Bug: _extract_domain() returns "" for relative URLs (e.g. # /library/document/ from RAG / library sources). The three # domain-hyperlink formatters used to embed that empty string directly, # producing "[[]](url)" (rendered as a bare "[]" link) and "[[-1]](url)" # for multi-citation results. _citation_label / _slugify_title close the # gap. These tests guard against regression. # --------------------------------------------------------------------------- def _library_url(suffix: str = "abc") -> str: """Return a relative /library/document/ URL like RAG emits.""" return f"/library/document/{suffix}" class TestRelativeUrlCitationLabels: """Empty-domain bug regression. All three fixed modes must produce a non-empty label for relative URLs (e.g. /library/document/... from RAG / library sources) by falling back to a slugified title.""" # ---- DOMAIN_HYPERLINKS ----------------------------------------------- def test_domain_hyperlinks_single_relative_url_uses_title_slug(self): """Before fix: [[]](url). After: [[]](url).""" content = """# Report See [1] for details. ## Sources [1] Sumerian King List URL: /library/document/abc-123 """ formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) result = formatter.format_document(content) assert "[[sumerian-king-list]](/library/document/abc-123)" in result assert "[[]]" not in result assert "[[-1]]" not in result def test_domain_hyperlinks_multi_relative_urls_use_distinct_slugs(self): """Before fix: [[-1]](url1)[[-2]](url2) (orphans). After: distinct title-based slugs.""" content = """# Report Findings [1] and [2]. ## Sources [1] Sumerian King List URL: /library/document/a [2] Ancient Mesopotamia URL: /library/document/b """ formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) result = formatter.format_document(content) assert "[[sumerian-king-list]](/library/document/a)" in result assert "[[ancient-mesopotamia]](/library/document/b)" in result assert "[[]]" not in result assert "[[-1]]" not in result assert "[[-2]]" not in result def test_domain_hyperlinks_empty_title_falls_back_to_citation_number( self, ): """Last-resort: when both domain and title slug are empty, fall back to the citation number so the label is never empty. Tested via the helper directly because the sources-block parser requires a non-empty title line (e.g. ``[1] Title\n URL: ...``).""" formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) result = formatter._format_domain_hyperlinks( "see [1]", {"1": ("", "/library/document/abc")} ) assert "[[1]](/library/document/abc)" in result assert "[[]]" not in result def test_domain_hyperlinks_comma_separated_relative_urls(self): """Comma-separated citations with relative URLs must produce non-empty labels. Before fix every label collapsed to "".""" content = """# Report Sources [1, 2, 3] confirm this. ## Sources [1] Doc A URL: /library/document/a [2] Doc B URL: /library/document/b [3] Doc C URL: /library/document/c """ formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) result = formatter.format_document(content) assert ( "[[doc-a]](/library/document/a)" "[[doc-b]](/library/document/b)" "[[doc-c]](/library/document/c)" ) in result assert "[[]]" not in result # ---- DOMAIN_ID_HYPERLINKS (default) ---------------------------------- def test_domain_id_hyperlinks_single_relative_url_uses_title_slug(self): """The default mode. Before fix: [[]](url). After: [[slug]](url).""" content = """# Report See [1]. ## Sources [1] Sumerian King List URL: /library/document/abc-123 """ formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS) result = formatter.format_document(content) assert "[[sumerian-king-list]](/library/document/abc-123)" in result assert "[[]]" not in result assert "[[-1]]" not in result def test_domain_id_hyperlinks_multi_relative_urls_use_distinct_slugs( self, ): content = """# Report Findings [1] and [2]. ## Sources [1] Sumerian King List URL: /library/document/a [2] Ancient Mesopotamia URL: /library/document/b """ formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS) result = formatter.format_document(content) assert "[[sumerian-king-list]](/library/document/a)" in result assert "[[ancient-mesopotamia]](/library/document/b)" in result assert "[[]]" not in result assert "[[-1]]" not in result assert "[[-2]]" not in result def test_domain_id_hyperlinks_same_title_relative_urls_group_with_id( self, ): """Multiple relative-URL citations of the SAME title group under one slug and append -N suffixes. Before fix this produced indistinguishable [[-1]](url1)[[-2]](url2) orphans.""" content = """# Report Findings [1] and [2]. ## Sources [1] Same Doc URL: /library/document/a [2] Same Doc URL: /library/document/b """ formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS) result = formatter.format_document(content) assert "[[same-doc-1]](/library/document/a)" in result assert "[[same-doc-2]](/library/document/b)" in result assert "[[same-doc]/" not in result # not the no-suffix form assert "[[]]" not in result def test_domain_id_hyperlinks_empty_title_falls_back_to_citation_number( self, ): """Same last-resort guarantee for DOMAIN_ID_HYPERLINKS. Tested via the helper directly (parser requires non-empty title line).""" formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS) result = formatter._format_domain_id_hyperlinks( "see [1]", {"1": ("", "/library/document/abc")} ) assert "[[1]](/library/document/abc)" in result assert "[[]]" not in result def test_domain_id_hyperlinks_comma_separated_relative_urls(self): content = """# Report Sources [1, 2, 3] confirm this. ## Sources [1] Doc A URL: /library/document/a [2] Doc B URL: /library/document/b [3] Doc C URL: /library/document/c """ formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS) result = formatter.format_document(content) # "Doc A" / "Doc B" / "Doc C" slugify to "doc-a" / "doc-b" / # "doc-c" -- single docs in DOMAIN_ID mode keep no -N suffix. assert ( "[[doc-a]](/library/document/a)" "[[doc-b]](/library/document/b)" "[[doc-c]](/library/document/c)" ) in result assert "[[]]" not in result def test_domain_id_hyperlinks_mixed_real_and_relative(self): """Real URLs keep their domain labels; relative URLs use title slugs. No regression on the real-URL side.""" content = """# Report Web source [1] and library source [2]. ## Sources [1] Web Doc URL: https://example.com/x [2] Local Doc URL: /library/document/a """ formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS) result = formatter.format_document(content) assert "[[example.com]](https://example.com/x)" in result assert "[[local-doc]](/library/document/a)" in result assert "[[]]" not in result # ---- DOMAIN_ID_ALWAYS_HYPERLINKS ------------------------------------- def test_domain_id_always_hyperlinks_single_relative_url_uses_title_slug( self, ): """Before fix: [[-1]](url). After: [[slug-1]](url).""" content = """# Report See [1]. ## Sources [1] Sumerian King List URL: /library/document/abc-123 """ formatter = CitationFormatter(CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS) result = formatter.format_document(content) assert "[[sumerian-king-list-1]](/library/document/abc-123)" in result assert "[[-1]]" not in result assert "[[]]" not in result def test_domain_id_always_hyperlinks_multi_relative_urls_use_distinct_slugs( self, ): content = """# Report Findings [1] and [2]. ## Sources [1] Sumerian King List URL: /library/document/a [2] Ancient Mesopotamia URL: /library/document/b """ formatter = CitationFormatter(CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS) result = formatter.format_document(content) assert "[[sumerian-king-list-1]](/library/document/a)" in result assert "[[ancient-mesopotamia-1]](/library/document/b)" in result assert "[[-1]]" not in result assert "[[-2]]" not in result def test_domain_id_always_hyperlinks_same_title_relative_urls_group( self, ): content = """# Report Findings [1] and [2]. ## Sources [1] Same Doc URL: /library/document/a [2] Same Doc URL: /library/document/b """ formatter = CitationFormatter(CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS) result = formatter.format_document(content) assert "[[same-doc-1]](/library/document/a)" in result assert "[[same-doc-2]](/library/document/b)" in result assert "[[-1]]" not in result assert "[[]]" not in result # ---- Real-URL regression (no behavior change) ------------------------ def test_domain_hyperlinks_real_url_unchanged(self): """Guard against accidentally breaking the real-URL path.""" formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) result = formatter._format_domain_hyperlinks( "see [1]", {"1": ("Paper", "https://arxiv.org/abs/1234")} ) assert "[[arxiv.org]](https://arxiv.org/abs/1234)" in result def test_domain_id_hyperlinks_real_url_unchanged(self): formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS) result = formatter._format_domain_id_hyperlinks( "see [1] and [2]", { "1": ("A", "https://arxiv.org/abs/1"), "2": ("B", "https://arxiv.org/abs/2"), }, ) assert "[[arxiv.org-1]](https://arxiv.org/abs/1)" in result assert "[[arxiv.org-2]](https://arxiv.org/abs/2)" in result def test_domain_id_always_hyperlinks_real_url_unchanged(self): formatter = CitationFormatter(CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS) result = formatter._format_domain_id_always_hyperlinks( "see [1]", {"1": ("Paper", "https://example.com/x")}, ) assert "[[example.com-1]](https://example.com/x)" in result class TestApplyInlineHyperlinks: """Streaming-path coverage. apply_inline_hyperlinks is the only formatter that takes structured dicts of sources (not markdown sources blocks); it shares _format_domain_id_hyperlinks so it picks up the relative-URL fix transitively.""" def test_real_url(self): fmt = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS) result = fmt.apply_inline_hyperlinks( "see [1]", [{"index": "1", "title": "T", "url": "https://example.com/x"}], ) assert "[[example.com]](https://example.com/x)" in result def test_single_relative_url_uses_title_slug(self): """Before fix: [[]](url). After: [[slug]](url).""" fmt = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS) result = fmt.apply_inline_hyperlinks( "see [1]", [ { "index": "1", "title": "Sumerian King List", "url": "/library/document/abc", } ], ) assert "[[sumerian-king-list]](/library/document/abc)" in result assert "[[]]" not in result assert "[[-1]]" not in result def test_multi_relative_urls_use_distinct_slugs(self): fmt = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS) result = fmt.apply_inline_hyperlinks( "see [1] and [2]", [ { "index": "1", "title": "Sumerian King List", "url": "/library/document/a", }, { "index": "2", "title": "Ancient Mesopotamia", "url": "/library/document/b", }, ], ) assert "[[sumerian-king-list]]" in result assert "[[ancient-mesopotamia]]" in result assert "[[]]" not in result assert "[[-1]]" not in result assert "[[-2]]" not in result def test_no_url_emits_no_link(self): """Sources with no URL must not produce empty [[]] links.""" fmt = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS) result = fmt.apply_inline_hyperlinks( "see [1]", [{"index": "1", "title": "T", "url": ""}] ) assert "[1]" in result assert "[[]]" not in result assert "[[1]]" not in result class TestCitationLabel: """Direct unit tests for the _citation_label helper.""" def test_prefers_domain_when_present(self): fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) assert ( fmt._citation_label("1", "Some Title", "https://example.com/x") == "example.com" ) def test_prefers_domain_for_arxiv(self): fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) assert ( fmt._citation_label("1", "Some Title", "https://arxiv.org/abs/1") == "arxiv.org" ) def test_falls_back_to_title_slug_for_relative_url(self): fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) assert ( fmt._citation_label( "1", "Sumerian King List", "/library/document/abc" ) == "sumerian-king-list" ) def test_falls_back_to_citation_number_when_title_empty(self): fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) assert fmt._citation_label("1", "", "/library/document/abc") == "1" def test_falls_back_to_citation_number_when_title_whitespace(self): fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) assert fmt._citation_label("1", " ", "/library/document/abc") == "1" def test_falls_back_to_title_slug_when_url_empty(self): """URL empty but title present -> use slug.""" fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) assert fmt._citation_label("1", "Title", "") == "title" def test_falls_back_to_citation_number_when_both_empty(self): fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) assert fmt._citation_label("1", "", "") == "1" def test_sanitizes_special_chars_in_title(self): fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) assert ( fmt._citation_label( "1", "Title with: weird! chars", "/library/document/a" ) == "title-with-weird-chars" ) def test_legitimate_title_doc_keeps_doc_label(self): """A document literally titled ``"Doc"`` slugifies to ``"doc"``; the label must keep that slug rather than fall through to the citation-number fallback. Regression guard for the case where ``_slugify_title`` returned ``"doc"`` as a sentinel.""" fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS) assert fmt._citation_label("1", "Doc", "/library/document/a") == "doc" assert fmt._citation_label("1", "DOC", "/library/document/a") == "doc" class TestSlugifyTitle: """Direct unit tests for the _slugify_title helper.""" def test_empty_returns_none(self): assert CitationFormatter._slugify_title("") is None def test_whitespace_only_returns_none(self): assert CitationFormatter._slugify_title(" ") is None def test_punctuation_only_returns_none(self): assert CitationFormatter._slugify_title("!!!") is None def test_basic_slugification(self): assert CitationFormatter._slugify_title("Hello World") == "hello-world" def test_special_chars_replaced_with_single_hyphen(self): assert ( CitationFormatter._slugify_title("Some/Weird: Name (with stuff)") == "some-weird-name-with-stuff" ) def test_truncates_long_titles(self): long = "x" * 100 slug = CitationFormatter._slugify_title(long) assert len(slug) == 32 assert slug == "x" * 32 def test_truncation_strips_trailing_hyphen(self): """If max_len lands on a hyphen, strip it so callers can append -N without producing a malformed --N label.""" slug = CitationFormatter._slugify_title("x" * 32 + "-tail") assert len(slug) == 32 assert not slug.endswith("-") assert slug == "x" * 32 def test_strips_leading_and_trailing_whitespace(self): assert ( CitationFormatter._slugify_title(" Hello World ") == "hello-world" ) def test_legitimate_title_doc_is_not_treated_as_missing(self): """A document literally titled ``"Doc"`` slugifies to ``"doc"``; that is a real label, not a sentinel for "no meaningful slug".""" assert CitationFormatter._slugify_title("Doc") == "doc" assert CitationFormatter._slugify_title("DOC") == "doc" class TestSourceTagCacheLookup: """_format_source_tagged_hyperlinks must resolve labels from the pre-computed tag_cache without re-running _extract_source_label on cache hits — evaluating it per citation match is pure dead-weight work.""" def test_label_resolution_runs_once_per_source(self, monkeypatch): """_extract_source_label runs once per URL-having source (while building tag_cache), not once per citation occurrence.""" formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS) calls = [] original = CitationFormatter._extract_source_label def counting(self, url, collection=None): calls.append(url) return original(self, url, collection=collection) monkeypatch.setattr( CitationFormatter, "_extract_source_label", counting ) content = "First [1], again [1], list [1, 2], and [2]." sources = { "1": ("Paper A", "https://arxiv.org/abs/2024.1111"), "2": ("Paper B", "https://example.com/x"), } result = formatter._format_source_tagged_hyperlinks( content, sources, {} ) assert "[[arxiv-1]](https://arxiv.org/abs/2024.1111)" in result assert "[[example.com-2]](https://example.com/x)" in result assert len(calls) == len(sources) def test_url_less_source_uses_cache_miss_fallback(self): """Sources without a URL are absent from tag_cache and must still resolve via the cache-miss fallback path.""" formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS) result = formatter._format_source_tagged_hyperlinks( "see [1]", {"1": ("Title", "")}, {} ) assert "[local-1]" in result assert "[[local-1]]" not in result # no hyperlink for empty URL class TestUrlClassifierIsNormalImport: """#4992 follow-up: the disk-load workaround (PR #4880) is gone. ``citation_formatter`` now imports ``URLClassifier``/``URLType`` through the package like any other module; the classes it uses must be the canonical ones, not a shadow copy exec'd from disk (the old loader produced a distinct module object whose enum members compared unequal to the real ones).""" def test_uses_canonical_url_classifier_classes(self): from local_deep_research.content_fetcher.url_classifier import ( URLClassifier, URLType, ) from local_deep_research.text_optimization import ( citation_formatter as cf, ) assert cf.URLClassifier is URLClassifier assert cf.URLType is URLType