learningcircuit--local-deep-research/tests/text_optimization/test_citation_formatter.py

"""Tests for citation formatter functionality."""

import pytest

from local_deep_research.text_optimization import (
    CitationFormatter,
    CitationMode,
    LaTeXExporter,
    QuartoExporter,
    RISExporter,
)


class TestCitationFormatter:
    """Test cases for CitationFormatter class."""

    @pytest.fixture
    def sample_content(self):
        """Sample markdown content with citations."""
        return """# Research Report

This is a research report about artificial intelligence [1]. Recent advances in
machine learning [2] have shown significant progress. The work by researchers
at DeepMind [3] has been particularly influential.

## Key Findings

The analysis reveals several important points:
- Deep learning models are becoming more efficient [1]
- Transfer learning enables better generalization [2]
- Large language models show emergent capabilities [3]
- Multiple studies confirm these findings [1][2][3]
- Some research combines different approaches [2][3]
- Comprehensive surveys cover all aspects [1, 2, 3]
- Recent work builds on earlier findings [2, 3]

## Sources

[1] Understanding Deep Learning
    URL: https://arxiv.org/abs/2104.12345

[2] Transfer Learning: A Survey
    URL: https://www.nature.com/articles/s41586-021-03819-2

[3] Emergent Abilities of Large Language Models
    URL: https://openai.com/research/emergent-abilities
"""

    @pytest.fixture
    def content_without_urls(self):
        """Sample content with citations but no URLs."""
        return """# Report

Some findings [1] and more [2].

## Sources

[1] First Source
[2] Second Source
"""

    def test_number_hyperlinks_mode(self, sample_content):
        """Test citation formatting with number hyperlinks."""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(sample_content)

        # Check single citations are formatted correctly
        assert "[[1]](https://arxiv.org/abs/2104.12345)" in result
        assert (
            "[[2]](https://www.nature.com/articles/s41586-021-03819-2)"
            in result
        )
        assert "[[3]](https://openai.com/research/emergent-abilities)" in result

        # Check multiple consecutive citations
        assert (
            "[[1]](https://arxiv.org/abs/2104.12345)[[2]](https://www.nature.com/articles/s41586-021-03819-2)[[3]](https://openai.com/research/emergent-abilities)"
            in result
        )

        # Check comma-separated citations
        assert (
            "[[1]](https://arxiv.org/abs/2104.12345)[[2]](https://www.nature.com/articles/s41586-021-03819-2)[[3]](https://openai.com/research/emergent-abilities)"
            in result
        )

        # Ensure sources section is unchanged
        assert "## Sources" in result
        assert "[1] Understanding Deep Learning" in result

    def test_domain_hyperlinks_mode(self, sample_content):
        """Test citation formatting with domain hyperlinks."""
        formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        result = formatter.format_document(sample_content)

        # Check domain formatting
        assert "[[arxiv.org]](https://arxiv.org/abs/2104.12345)" in result
        assert (
            "[[nature.com]](https://www.nature.com/articles/s41586-021-03819-2)"
            in result
        )
        assert (
            "[[openai.com]](https://openai.com/research/emergent-abilities)"
            in result
        )

        # Check multiple citations work
        assert (
            "[[arxiv.org]](https://arxiv.org/abs/2104.12345)[[nature.com]](https://www.nature.com/articles/s41586-021-03819-2)[[openai.com]](https://openai.com/research/emergent-abilities)"
            in result
        )

    def test_no_hyperlinks_mode(self, sample_content):
        """Test citation formatting with no hyperlinks."""
        formatter = CitationFormatter(CitationMode.NO_HYPERLINKS)
        result = formatter.format_document(sample_content)

        # Content should be unchanged
        assert result == sample_content
        assert "[[1]]" not in result
        assert "](https://" not in result

    def test_citations_without_urls(self, content_without_urls):
        """Test handling of citations without URLs."""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content_without_urls)

        # Citations without URLs should remain unchanged
        assert "[1]" in result
        assert "[2]" in result
        assert "[[1]]" not in result
        assert "[[2]]" not in result

    def test_no_sources_section(self):
        """Test handling of content without sources section."""
        content = "This is text with [1] citation but no sources."
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content)

        # Should return unchanged content
        assert result == content

    def test_mixed_citation_formats(self):
        """Test handling of mixed citation formats."""
        content = """Text with various formats:
- Single: [1]
- Multiple: [1][2]
- Comma-separated: [1, 2, 3]
- With spaces: [1,2,3] and [1, 2]

## Sources

[1] Source One
    URL: https://example1.com
[2] Source Two
    URL: https://example2.com
[3] Source Three
    URL: https://example3.com
"""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content)

        # All formats should be converted
        assert "[[1]](https://example1.com)" in result
        assert (
            "[[1]](https://example1.com)[[2]](https://example2.com)" in result
        )
        assert (
            "[[1]](https://example1.com)[[2]](https://example2.com)[[3]](https://example3.com)"
            in result
        )

    def test_domain_extraction(self):
        """Test domain extraction from various URLs."""
        formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)

        # Test various URL formats
        test_cases = [
            ("https://arxiv.org/abs/1234", "arxiv.org"),
            ("https://www.nature.com/articles/123", "nature.com"),
            ("https://github.com/user/repo", "github.com"),
            ("https://subdomain.example.com/path", "example.com"),
            ("https://example.co.uk/path", "co.uk"),
        ]

        for url, expected_domain in test_cases:
            domain = formatter._extract_domain(url)
            assert domain == expected_domain

    def test_edge_cases(self):
        """Test edge cases and malformed content."""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)

        # Empty content
        assert formatter.format_document("") == ""

        # Content with only sources section
        content = """## Sources
[1] Test
    URL: https://test.com
"""
        result = formatter.format_document(content)
        assert result == content

        # Malformed citations
        content = """Text with [a] non-numeric [citation].

## Sources
[a] Invalid citation
"""
        result = formatter.format_document(content)
        assert "[a]" in result  # Should remain unchanged
        assert "[citation]" in result

    def test_source_word_pattern_detection(self):
        """Test detection and conversion of 'Source X' patterns."""
        content = """# Research Report

As mentioned in Source 1, the data shows improvements.
According to source 2, this is confirmed.
The Source 3 provides additional evidence.

## Sources

[1] First Research Paper
    URL: https://example.com/paper1

[2] Second Study
    URL: https://example.com/paper2

[3] Third Analysis
    URL: https://example.com/paper3
"""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content)

        # Check that "Source X" patterns are converted to hyperlinks
        assert "[[1]](https://example.com/paper1)" in result
        assert "[[2]](https://example.com/paper2)" in result
        assert "[[3]](https://example.com/paper3)" in result

        # Original "Source X" text should be replaced
        assert "Source 1" not in result or "[[1]]" in result
        assert "source 2" not in result or "[[2]]" in result

    def test_source_pattern_edge_cases(self):
        """Test that Source pattern converts to broken references when no matching source."""
        content = """# Financial Report

Source 12 of income comes from investments.
The power source 5 failed during testing.
Open Source 100 projects were analyzed.
Valid reference from Source 1 here.

## Sources

[1] Financial Analysis
    URL: https://example.com/finance
"""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content)

        # These get converted to broken reference markers [X] since no matching sources
        assert "[12] of income" in result  # Converted but not hyperlinked
        assert "[5] failed" in result  # Converted but not hyperlinked
        assert "[100] projects" in result  # Converted but not hyperlinked

        # Valid Source 1 should be hyperlinked
        assert "[[1]](https://example.com/finance) here" in result

        # The broken references should NOT be hyperlinked
        assert "[[12]]" not in result
        assert "[[5]]" not in result
        assert "[[100]]" not in result

    def test_mixed_citation_formats_with_source_pattern(self):
        """Test document with both [X] and Source X patterns."""
        content = """# Research Report

Traditional citation [1] appears here.
Then Source 2 is mentioned directly.
Mixed with [3] and source 1 again.

## Sources

[1] First Paper
    URL: https://example.com/paper1

[2] Second Paper
    URL: https://example.com/paper2

[3] Third Paper
    URL: https://example.com/paper3
"""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content)

        # All formats should be converted consistently
        assert (
            result.count("[[1]](https://example.com/paper1)") >= 2
        )  # [1] and source 1
        assert "[[2]](https://example.com/paper2)" in result  # Source 2
        assert "[[3]](https://example.com/paper3)" in result  # [3]

    def test_domain_id_hyperlinks_mode(self, sample_content):
        """Test citation formatting with domain IDs for repeated domains."""
        # Add more sources from same domains to test ID functionality
        content_with_repeated_domains = """# Research Report

Citations from GitHub [1] and more from GitHub [2].
ArXiv papers [3] and another ArXiv paper [4].
A Nature article [5] and OpenAI research [6].

## Sources

[1] First GitHub Project
    URL: https://github.com/user/project1

[2] Second GitHub Project
    URL: https://github.com/user/project2

[3] First ArXiv Paper
    URL: https://arxiv.org/abs/2024.1111

[4] Second ArXiv Paper
    URL: https://arxiv.org/abs/2024.2222

[5] Nature Article
    URL: https://www.nature.com/articles/s41586-024-1234

[6] OpenAI Research
    URL: https://openai.com/research/gpt4
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS)
        result = formatter.format_document(content_with_repeated_domains)

        # Check that repeated domains get hyphen-separated IDs
        assert "[[github.com-1]](https://github.com/user/project1)" in result
        assert "[[github.com-2]](https://github.com/user/project2)" in result
        assert "[[arxiv.org-1]](https://arxiv.org/abs/2024.1111)" in result
        assert "[[arxiv.org-2]](https://arxiv.org/abs/2024.2222)" in result

        # Check that single domain citations don't get IDs
        assert (
            "[[nature.com]](https://www.nature.com/articles/s41586-024-1234)"
            in result
        )
        assert "[[openai.com]](https://openai.com/research/gpt4)" in result

    def test_real_world_example(self):
        """Test with real-world example from Local Deep Research output."""
        content = """# Quick Research Summary

Query: local deep research

Local Deep Research represents a new generation of AI-powered research assistants [1][3][11]. It utilizes any LLM hosted by either Ollama or LMStudio [2]. The concept was introduced as DeepSearcher [4] and further explained on Medium [7]. Web searches are conducted using DuckDuckGo [5], and the system incorporates tools like Milvus [6]. Updates were discussed on Reddit [10] and demonstrations on YouTube [8][12]. The Ollama variant [9] provides an alternative approach.

## Sources

[1] LearningCircuit/local-deep-research- GitHub
    URL: https://github.com/LearningCircuit/local-deep-research

[2] langchain-ai/local-deep-researcher - GitHub
    URL: https://github.com/langchain-ai/local-deep-researcher

[3] local-deep-research- PyPI
    URL: https://pypi.org/project/local-deep-research/

[4] Introducing DeepSearcher
    URL: https://zilliz.com/blog/introduce-deepsearcher

[5] A local LLM research assistant
    URL: https://www.reddit.com/r/LocalLLaMA/comments/1ilkosp/

[6] Introducing DeepSearcher - Milvus
    URL: https://milvus.io/blog/introduce-deepsearcher.md

[7] DeepSearcher on Medium
    URL: https://milvusio.medium.com/deepsearcher

[8] Private AI Deep Research - YouTube
    URL: https://m.youtube.com/watch?v=4tDqV__jzKY

[9] Ollama Deep Research
    URL: https://apidog.com/blog/ollama-deep-research/

[10] Local Deep Research Update
    URL: https://www.reddit.com/r/LocalLLaMA/comments/1j79obx/

[11] local-deep-research 0.3.3 - PyPI
    URL: https://pypi.org/project/local-deep-research/0.3.3/

[12] Building a local deep researcher - YouTube
    URL: https://www.youtube.com/watch?v=sGUjmyfof4Q
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS)
        result = formatter.format_document(content)

        # Check multiple citations from same domain get IDs
        assert (
            "[[github.com-1]](https://github.com/LearningCircuit/local-deep-research)"
            in result
        )
        assert (
            "[[github.com-2]](https://github.com/langchain-ai/local-deep-researcher)"
            in result
        )
        assert (
            "[[pypi.org-1]](https://pypi.org/project/local-deep-research/)"
            in result
        )
        assert (
            "[[pypi.org-2]](https://pypi.org/project/local-deep-research/0.3.3/)"
            in result
        )
        assert (
            "[[reddit.com-1]](https://www.reddit.com/r/LocalLLaMA/comments/1ilkosp/)"
            in result
        )
        assert (
            "[[reddit.com-2]](https://www.reddit.com/r/LocalLLaMA/comments/1j79obx/)"
            in result
        )
        assert (
            "[[youtube.com-1]](https://m.youtube.com/watch?v=4tDqV__jzKY)"
            in result
        )
        assert (
            "[[youtube.com-2]](https://www.youtube.com/watch?v=sGUjmyfof4Q)"
            in result
        )

        # Check single domain citations don't get IDs
        assert (
            "[[zilliz.com]](https://zilliz.com/blog/introduce-deepsearcher)"
            in result
        )
        assert (
            "[[milvus.io]](https://milvus.io/blog/introduce-deepsearcher.md)"
            in result
        )
        assert (
            "[[apidog.com]](https://apidog.com/blog/ollama-deep-research/)"
            in result
        )
        assert (
            "[[medium.com]](https://milvusio.medium.com/deepsearcher)" in result
        )

    def test_domain_id_always_hyperlinks_mode(self):
        """Test citation formatting with domain IDs always added for consistency."""
        # Use the same content as test_domain_id_hyperlinks_mode
        content_with_repeated_domains = """# Research Report

Citations from GitHub [1] and more from GitHub [2].
ArXiv papers [3] and another ArXiv paper [4].
A Nature article [5] and OpenAI research [6].

## Sources

[1] First GitHub Project
    URL: https://github.com/user/project1

[2] Second GitHub Project
    URL: https://github.com/user/project2

[3] First ArXiv Paper
    URL: https://arxiv.org/abs/2024.1111

[4] Second ArXiv Paper
    URL: https://arxiv.org/abs/2024.2222

[5] Nature Article
    URL: https://www.nature.com/articles/s41586-024-1234

[6] OpenAI Research
    URL: https://openai.com/research/gpt4
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS)
        result = formatter.format_document(content_with_repeated_domains)

        # Check that ALL domains get IDs, even single ones
        assert "[[github.com-1]](https://github.com/user/project1)" in result
        assert "[[github.com-2]](https://github.com/user/project2)" in result
        assert "[[arxiv.org-1]](https://arxiv.org/abs/2024.1111)" in result
        assert "[[arxiv.org-2]](https://arxiv.org/abs/2024.2222)" in result
        assert (
            "[[nature.com-1]](https://www.nature.com/articles/s41586-024-1234)"
            in result
        )
        assert "[[openai.com-1]](https://openai.com/research/gpt4)" in result

    def test_source_tagged_hyperlinks_preserves_global_counter(self):
        """SOURCE_TAGGED_HYPERLINKS keeps the original citation number as
        the suffix, prefixed with the source tag. Two arxiv citations and
        one openai.com citation become [arxiv-1], [openai.com-2],
        [arxiv-3] — not the per-domain renumbering of DOMAIN_ID_*."""
        content = """# Report

Finding A [1]. Finding B [2]. Finding C [3]. Combined [1, 2, 3].

## Sources

[1] First ArXiv Paper
    URL: https://arxiv.org/abs/2024.1111

[2] OpenAI Research
    URL: https://openai.com/research/gpt4

[3] Second ArXiv Paper
    URL: https://arxiv.org/abs/2024.2222
"""
        formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS)
        result = formatter.format_document(content)

        # URLClassifier maps arxiv.org URLs to URLType.ARXIV → "arxiv".
        assert "[[arxiv-1]](https://arxiv.org/abs/2024.1111)" in result
        # openai.com is generic HTML → falls back to domain.
        assert "[[openai.com-2]](https://openai.com/research/gpt4)" in result
        # Second arxiv keeps the GLOBAL number 3, not a per-domain "2".
        assert "[[arxiv-3]](https://arxiv.org/abs/2024.2222)" in result
        # Comma-separated citations expand into individual tagged links.
        assert (
            "[[arxiv-1]](https://arxiv.org/abs/2024.1111)"
            "[[openai.com-2]](https://openai.com/research/gpt4)"
            "[[arxiv-3]](https://arxiv.org/abs/2024.2222)"
        ) in result

    def test_source_tagged_hyperlinks_known_academic_sources(self):
        """URLClassifier-recognized academic sources use the short enum
        tag (arxiv, pubmed, semantic_scholar, biorxiv, ...), not the raw
        domain."""
        content = """# Report

Cite [1] and [2] and [3] and [4].

## Sources

[1] ArXiv Paper
    URL: https://arxiv.org/abs/2024.1234

[2] PubMed Paper
    URL: https://pubmed.ncbi.nlm.nih.gov/12345678/

[3] Semantic Scholar
    URL: https://www.semanticscholar.org/paper/Some-Paper/abc123

[4] bioRxiv Preprint
    URL: https://www.biorxiv.org/content/10.1101/2024.01.01.000001v1
"""
        formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS)
        result = formatter.format_document(content)
        assert "[[arxiv-1]]" in result
        assert "[[pubmed-2]]" in result
        assert "[[semantic_scholar-3]]" in result
        assert "[[biorxiv-4]]" in result

    def test_source_tagged_hyperlinks_uses_collection_name(self):
        """When the sources block carries a ``Collection:`` line for a
        library/RAG hit, the collection name (slugified) is used as the
        citation tag instead of the URL-derived ``local`` fallback."""
        content = """# Report

Web cite [1]. Library cite [2]. Another web [3]. Another library [4].

## Sources

[1] ArXiv Paper (source nr: 1)
   URL: https://arxiv.org/abs/2024.1111

[2] Local Doc (source nr: 2)
   URL: /library/document/abc123
   Collection: My Papers

[3] OpenAI Research (source nr: 3)
   URL: https://openai.com/research/gpt4

[4] Another local (source nr: 4)
   URL: /library/document/def456
   Collection: team/finance
"""
        formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS)
        result = formatter.format_document(content)

        # Web citations are unaffected — still URL-derived tags.
        assert "[[arxiv-1]](https://arxiv.org/abs/2024.1111)" in result
        assert "[[openai.com-3]](https://openai.com/research/gpt4)" in result
        # Library citations adopt the slugified collection name + global N.
        # ``My Papers`` → ``my-papers``, ``team/finance`` → ``team-finance``.
        # Local URLs render without a hyperlink (no http(s) scheme).
        assert "[my-papers-2]" in result
        assert "[[my-papers-2]]" not in result
        assert "[team-finance-4]" in result
        assert "[[team-finance-4]]" not in result

    def test_source_tagged_hyperlinks_collection_slugify_edge_cases(self):
        """Collection names with whitespace, slashes, unicode, or empty
        slugs degrade safely. The slugifier lowercases, replaces runs of
        non-alphanumerics with single hyphens, trims edge hyphens, and
        falls back to ``local`` when the result is empty."""
        from local_deep_research.text_optimization.citation_formatter import (
            CitationFormatter as CF,
        )

        assert CF._slugify_collection("My Papers") == "my-papers"
        assert CF._slugify_collection("  trim me  ") == "trim-me"
        assert CF._slugify_collection("team/finance") == "team-finance"
        assert CF._slugify_collection("ALLCAPS") == "allcaps"
        assert CF._slugify_collection("a/b/c") == "a-b-c"
        assert CF._slugify_collection("---") == "local"
        assert CF._slugify_collection("") == "local"
        # Unicode that's not [a-z0-9] collapses to hyphens too. The
        # behaviour is "ASCII-safe slug for inline use" — round-tripping
        # the original name is out of scope.
        assert CF._slugify_collection("résumé") == "r-sum"

    def test_source_tagged_hyperlinks_missing_collection_falls_back(self):
        """A library URL **without** a ``Collection:`` line in the
        sources block falls through to the existing ``local`` tag (this
        is what older renderers and unit tests with hand-rolled sources
        produce)."""
        content = """# Report

See [1].

## Sources

[1] Local doc with no collection metadata
    URL: /library/document/abc123
"""
        formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS)
        result = formatter.format_document(content)
        # ``/library/document/...`` has no scheme → ``local`` fallback,
        # rendered as a plain bracketed tag (no hyperlink).
        assert "[local-1]" in result
        assert "[[local-1]]" not in result

    def test_source_tagged_hyperlinks_collection_line_isolation(self):
        """A ``Collection:`` line attached to citation [N] must not leak
        into citation [N+1]'s tag. Regex anchoring is the load-bearing
        bit here."""
        content = """# Report

See [1] and [2].

## Sources

[1] Local doc with collection
    URL: /library/document/abc
    Collection: project-alpha

[2] ArXiv paper, NO collection
    URL: https://arxiv.org/abs/2024.0001
"""
        formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS)
        result = formatter.format_document(content)
        assert "[project-alpha-1]" in result
        # Citation [2] must NOT pick up citation [1]'s collection.
        assert "[[arxiv-2]](https://arxiv.org/abs/2024.0001)" in result
        assert "[project-alpha-2]" not in result

    def test_source_tagged_hyperlinks_local_url_falls_back(self):
        """Empty / non-http(s) URLs get the ``local`` tag and render
        without a hyperlink."""
        content = """# Report

See [1] and [2].

## Sources

[1] A local doc
    URL: file:///srv/library/mypapers/foo.pdf

[2] Another local doc with no URL at all
"""
        formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS)
        result = formatter.format_document(content)
        # file:// → "local", with no hyperlink wrapping.
        assert "[local-1]" in result
        assert "[[local-1]]" not in result  # no hyperlink, not bracketed twice
        # No URL → also "local", and no hyperlink.
        assert "[local-2]" in result

    def test_source_tagged_hyperlinks_no_state_leak_across_calls(self):
        """Calling ``format_document`` twice on the same formatter
        instance must not leak collection metadata between calls. Doc 1
        has a ``Collection:`` line for citation [1]; doc 2 has no
        collection line at all, so its [1] must fall back to the
        URL-derived tag, not pick up doc 1's collection."""
        doc_with_collection = """# Doc 1

See [1].

## Sources

[1] Local doc with collection
    URL: /library/document/abc
    Collection: project-alpha
"""
        doc_without_collection = """# Doc 2

See [1].

## Sources

[1] ArXiv paper, no collection
    URL: https://arxiv.org/abs/2024.9999
"""
        formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS)

        result1 = formatter.format_document(doc_with_collection)
        assert "[project-alpha-1]" in result1

        # Second call: arxiv URL, no Collection: line in the sources block.
        # Must resolve to arxiv via URLClassifier, NOT to project-alpha.
        result2 = formatter.format_document(doc_without_collection)
        assert "[[arxiv-1]](https://arxiv.org/abs/2024.9999)" in result2
        assert "project-alpha" not in result2

    def test_unicode_lenticular_bracket_citations(self):
        """Test that Unicode lenticular brackets【】are recognized and converted."""
        content = """# Research Report

Research shows findings 【1】 and more evidence 【2】.
Multiple citations 【1】【2】 are also used.

## Sources

[1] First Source
    URL: https://example.com/1

[2] Second Source
    URL: https://example.com/2
"""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content)

        # Check lenticular brackets are converted to hyperlinks
        assert "[[1]](https://example.com/1)" in result
        assert "[[2]](https://example.com/2)" in result

    def test_unicode_lenticular_comma_citations(self):
        """Test comma-separated lenticular citations【1, 2, 3】."""
        content = """# Report

Multiple sources 【1, 2】 confirm this.

## Sources

[1] Source One
    URL: https://example.com/1

[2] Source Two
    URL: https://example.com/2
"""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content)

        # Check comma-separated lenticular citations are expanded
        assert (
            "[[1]](https://example.com/1)[[2]](https://example.com/2)" in result
        )

    def test_mixed_bracket_styles(self):
        """Test documents with both standard [1] and lenticular【2】brackets."""
        content = """# Report

Standard citation [1] and lenticular 【2】 in same doc.

## Sources

[1] First Source
    URL: https://example.com/1

[2] Second Source
    URL: https://example.com/2
"""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content)

        # Both bracket styles should be converted
        assert "[[1]](https://example.com/1)" in result
        assert "[[2]](https://example.com/2)" in result

    def test_lenticular_multi_digit_citations(self):
        """Test lenticular brackets with multi-digit citation numbers."""
        content = """# Report

Citations with higher numbers 【10】 and 【99】 work correctly.

## Sources

[10] Tenth Source
    URL: https://example.com/10

[99] Ninety-ninth Source
    URL: https://example.com/99
"""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content)

        assert "[[10]](https://example.com/10)" in result
        assert "[[99]](https://example.com/99)" in result

    def test_lenticular_triple_comma_citations(self):
        """Test lenticular brackets with three or more comma-separated numbers."""
        content = """# Report

Many sources 【1, 2, 3】 and even more 【1,2,3,4】 confirm this.

## Sources

[1] Source One
    URL: https://example.com/1

[2] Source Two
    URL: https://example.com/2

[3] Source Three
    URL: https://example.com/3

[4] Source Four
    URL: https://example.com/4
"""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content)

        # Three comma-separated should expand
        assert (
            "[[1]](https://example.com/1)[[2]](https://example.com/2)"
            "[[3]](https://example.com/3)" in result
        )
        # Four comma-separated (no spaces) should also expand
        assert (
            "[[1]](https://example.com/1)[[2]](https://example.com/2)"
            "[[3]](https://example.com/3)[[4]](https://example.com/4)" in result
        )

    def test_lenticular_consecutive_mixed(self):
        """Test alternating standard and lenticular consecutive citations."""
        content = """# Report

Mixed consecutive [1]【2】[3] citations work.
Also reversed 【1】[2]【3】 order.

## Sources

[1] First Source
    URL: https://example.com/1

[2] Second Source
    URL: https://example.com/2

[3] Third Source
    URL: https://example.com/3
"""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content)

        # All citations should be converted regardless of bracket style
        assert "[[1]](https://example.com/1)" in result
        assert "[[2]](https://example.com/2)" in result
        assert "[[3]](https://example.com/3)" in result

    def test_lenticular_domain_hyperlinks_mode(self):
        """Test lenticular brackets with domain hyperlinks mode."""
        content = """# Report

Research from 【1】 and 【2】 shows results.

## Sources

[1] ArXiv Paper
    URL: https://arxiv.org/abs/2024.1234

[2] Nature Article
    URL: https://www.nature.com/articles/s41586-024-5678
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        result = formatter.format_document(content)

        assert "[[arxiv.org]](https://arxiv.org/abs/2024.1234)" in result
        assert (
            "[[nature.com]](https://www.nature.com/articles/s41586-024-5678)"
            in result
        )

    def test_lenticular_domain_id_hyperlinks_mode(self):
        """Test lenticular brackets with domain ID hyperlinks mode."""
        content = """# Report

Multiple GitHub sources 【1】 and 【2】 referenced.

## Sources

[1] First GitHub Repo
    URL: https://github.com/user/repo1

[2] Second GitHub Repo
    URL: https://github.com/user/repo2
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS)
        result = formatter.format_document(content)

        assert "[[github.com-1]](https://github.com/user/repo1)" in result
        assert "[[github.com-2]](https://github.com/user/repo2)" in result

    def test_lenticular_in_bullet_list(self):
        """Test lenticular brackets within bullet point lists."""
        content = """# Report

Key findings:
- First point 【1】
- Second point 【2】
- Combined evidence 【1, 2】

## Sources

[1] First Source
    URL: https://example.com/1

[2] Second Source
    URL: https://example.com/2
"""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content)

        assert "- First point [[1]](https://example.com/1)" in result
        assert "- Second point [[2]](https://example.com/2)" in result
        assert (
            "[[1]](https://example.com/1)[[2]](https://example.com/2)" in result
        )

    def test_lenticular_without_matching_source(self):
        """Test lenticular brackets referencing non-existent sources."""
        content = """# Report

Valid citation 【1】 and invalid 【99】 reference.

## Sources

[1] Only Source
    URL: https://example.com/1
"""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content)

        # Valid citation should be hyperlinked
        assert "[[1]](https://example.com/1)" in result
        # Invalid citation should remain as plain text (not hyperlinked)
        assert "[[99]]" not in result
        assert "【99】" in result or "[99]" in result

    def test_lenticular_no_space_before_or_after(self):
        """Test lenticular brackets without spaces before or after."""
        content = """# Report

Text immediately before【1】and after without spaces.
Also works at end of sentence【2】.
And word【1】word with citations embedded.
Multiple【1】【2】consecutive without spaces.
Standard brackets work too: word[1]word and end[2].

## Sources

[1] First Source
    URL: https://example.com/1

[2] Second Source
    URL: https://example.com/2
"""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content)

        # Lenticular brackets should be converted despite no spaces
        assert "before[[1]](https://example.com/1)and" in result
        assert "sentence[[2]](https://example.com/2)." in result
        assert "word[[1]](https://example.com/1)word" in result
        assert (
            "[[1]](https://example.com/1)[[2]](https://example.com/2)consecutive"
            in result
        )
        # Standard brackets should work the same way
        assert "word[[1]](https://example.com/1)word" in result
        assert "end[[2]](https://example.com/2)." in result

    def test_lenticular_real_world_mixed_example(self):
        """Test real-world scenario with mixed bracket styles throughout."""
        content = """# AI Safety Research Summary

Query: AI alignment techniques

Recent research【1】has explored various approaches to AI safety. The RLHF
method [2] has shown promising results. Constitutional AI【3】builds on
these foundations [1, 2]. Multiple studies【1】【2】【3】confirm the
effectiveness of these techniques. A comprehensive survey [1, 2, 3] covers
all major approaches, while recent work【2, 3】focuses on scalability.

## Sources

[1] RLHF: Training Language Models
    URL: https://arxiv.org/abs/2024.rlhf

[2] Constitutional AI: Harmlessness from Feedback
    URL: https://anthropic.com/constitutional-ai

[3] Scalable Oversight of AI Systems
    URL: https://openai.com/research/scalable-oversight
"""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        result = formatter.format_document(content)

        # All citations should be converted
        assert "[[1]](https://arxiv.org/abs/2024.rlhf)" in result
        assert "[[2]](https://anthropic.com/constitutional-ai)" in result
        assert "[[3]](https://openai.com/research/scalable-oversight)" in result

        # Consecutive lenticular citations should work
        assert (
            "[[1]](https://arxiv.org/abs/2024.rlhf)"
            "[[2]](https://anthropic.com/constitutional-ai)"
            "[[3]](https://openai.com/research/scalable-oversight)" in result
        )


class TestLaTeXExporter:
    """Test cases for LaTeX export functionality."""

    @pytest.fixture
    def sample_markdown(self):
        """Sample markdown for LaTeX conversion."""
        return """# Main Title

## Introduction

This is a **bold** statement with *italic* text and `code`.

### Subsection

- First item
- Second item
- Third item

Text with citations [1] and [2].

## Sources

[1] First Source
    URL: https://example.com
[2] Second Source
    URL: https://example2.com
"""

    def test_latex_export_basic(self, sample_markdown):
        """Test basic LaTeX export functionality."""
        exporter = LaTeXExporter()
        result = exporter.export_to_latex(sample_markdown)

        # Check header
        assert r"\documentclass[12pt]{article}" in result
        assert r"\begin{document}" in result
        assert r"\end{document}" in result

        # Check heading conversion
        assert r"\section{Main Title}" in result
        assert r"\subsection{Introduction}" in result
        assert r"\subsubsection{Subsection}" in result

        # Check emphasis conversion
        assert r"\textbf{bold}" in result
        assert r"\textit{italic}" in result
        assert r"\texttt{code}" in result

        # Check citation conversion
        assert r"\cite{1}" in result
        assert r"\cite{2}" in result

        # Check bibliography
        assert r"\begin{thebibliography}" in result
        assert r"\bibitem{1}" in result

    def test_latex_list_conversion(self, sample_markdown):
        """Test list conversion to LaTeX."""
        exporter = LaTeXExporter()
        result = exporter.export_to_latex(sample_markdown)

        # Check list conversion
        assert r"\begin{itemize}" in result
        assert r"\item First item" in result
        assert r"\item Second item" in result
        assert r"\item Third item" in result
        assert r"\end{itemize}" in result

    def test_latex_empty_content(self):
        """Test LaTeX export with empty content."""
        exporter = LaTeXExporter()
        result = exporter.export_to_latex("")

        assert r"\documentclass[12pt]{article}" in result
        assert r"\begin{document}" in result
        assert r"\end{document}" in result

    def test_latex_no_sources(self):
        """Test LaTeX export without sources section."""
        content = "# Title\n\nSome text without citations."
        exporter = LaTeXExporter()
        result = exporter.export_to_latex(content)

        # Should not include bibliography
        assert r"\begin{thebibliography}" not in result

    def test_unicode_lenticular_citations(self):
        """Test LaTeX export converts lenticular citations to \\cite{N} format."""
        content = """# Report

Research with lenticular citations 【1】 and 【2】.

## Sources

[1] First Source
    URL: https://example.com/1

[2] Second Source
    URL: https://example.com/2
"""
        exporter = LaTeXExporter()
        result = exporter.export_to_latex(content)

        # Check lenticular citations are converted to LaTeX cite format
        assert r"\cite{1}" in result
        assert r"\cite{2}" in result

    def test_unicode_lenticular_mixed_with_standard(self):
        """Test LaTeX with mixed standard and lenticular brackets."""
        content = """# Report

Standard [1] and lenticular 【2】 and consecutive【3】[4] citations.

## Sources

[1] First
    URL: https://example.com/1
[2] Second
    URL: https://example.com/2
[3] Third
    URL: https://example.com/3
[4] Fourth
    URL: https://example.com/4
"""
        exporter = LaTeXExporter()
        result = exporter.export_to_latex(content)

        assert r"\cite{1}" in result
        assert r"\cite{2}" in result
        assert r"\cite{3}" in result
        assert r"\cite{4}" in result

    def test_unicode_lenticular_multi_digit(self):
        """Test LaTeX export with multi-digit lenticular citations."""
        content = """# Report

Higher numbers 【10】 and 【25】 work too.

## Sources

[10] Tenth Source
    URL: https://example.com/10
[25] Twenty-fifth Source
    URL: https://example.com/25
"""
        exporter = LaTeXExporter()
        result = exporter.export_to_latex(content)

        assert r"\cite{10}" in result
        assert r"\cite{25}" in result


class TestRISExporter:
    """Test cases for RIS export functionality."""

    @pytest.fixture
    def sample_markdown(self):
        """Sample markdown for RIS conversion."""
        return """# Research Report

This research explores various topics [1][2][3].

## Sources

[1] Understanding Deep Learning
    URL: https://arxiv.org/abs/2104.12345

[2] GitHub Project Documentation
    URL: https://github.com/example/project

[3] Reddit Discussion Thread
    URL: https://www.reddit.com/r/example/comments/abc123

[4] Local Resource Without URL
"""

    def test_ris_export_basic(self, sample_markdown):
        """Test basic RIS export functionality."""
        exporter = RISExporter()
        result = exporter.export_to_ris(sample_markdown)

        # Check that we have RIS entries
        assert "TY  - ELEC" in result
        assert "ER  - " in result

        # Check specific entries
        assert "ID  - ref1" in result
        assert "TI  - Understanding Deep Learning" in result
        assert "UR  - https://arxiv.org/abs/2104.12345" in result
        assert "PB  - arXiv" in result

        assert "ID  - ref2" in result
        assert "TI  - GitHub Project Documentation" in result
        assert "PB  - GitHub" in result

        assert "ID  - ref3" in result
        assert "TI  - Reddit Discussion Thread" in result
        assert "PB  - Reddit" in result

        # Check entry without URL
        assert "ID  - ref4" in result
        assert "TI  - Local Resource Without URL" in result

        # Check date fields
        assert "Y1  - " in result
        assert "DA  - " in result
        assert "LA  - en" in result

    def test_ris_export_empty_content(self):
        """Test RIS export with empty content."""
        exporter = RISExporter()
        result = exporter.export_to_ris("")
        assert result == ""

    def test_ris_export_no_sources(self):
        """Test RIS export without sources section."""
        content = "# Title\n\nSome text without sources."
        exporter = RISExporter()
        result = exporter.export_to_ris(content)
        assert result == ""

    def test_ris_publisher_extraction(self, sample_markdown):
        """Test that publishers are correctly extracted from domains."""
        exporter = RISExporter()
        result = exporter.export_to_ris(sample_markdown)

        # Check known publishers
        assert "PB  - arXiv" in result
        assert "PB  - GitHub" in result
        assert "PB  - Reddit" in result


class TestQuartoExporter:
    """Test cases for Quarto export functionality."""

    @pytest.fixture
    def sample_markdown(self):
        """Sample markdown for Quarto conversion."""
        return """# Research on AI Safety

## Introduction

This research explores **important** aspects of *AI safety* and `alignment`.

### Current Approaches

- Reinforcement learning from human feedback [1]
- Constitutional AI [2]
- Interpretability research [3]

Multiple studies [1, 2, 3] have shown promising results.

## Sources

[1] RLHF: A Survey
    URL: https://arxiv.org/abs/2023.1234
[2] Constitutional AI
    URL: https://anthropic.com/constitutional-ai
[3] Mechanistic Interpretability
    URL: https://distill.pub/2020/circuits
"""

    def test_quarto_export_basic(self, sample_markdown):
        """Test basic Quarto export functionality."""
        exporter = QuartoExporter()
        result = exporter.export_to_quarto(
            sample_markdown, "AI Safety Research"
        )

        # Check YAML header
        assert "---" in result
        assert 'title: "AI Safety Research"' in result
        assert 'author: "Local Deep Research"' in result
        # Check that date is in YYYY-MM-DD format
        import re

        assert re.search(r'date: "\d{4}-\d{2}-\d{2}"', result)
        assert "format:" in result
        assert "bibliography: references.bib" in result

        # Check citation conversion
        assert "[@ref1]" in result
        assert "[@ref2]" in result
        assert "[@ref3]" in result
        assert "[@ref1, @ref2, @ref3]" in result

        # Check bibliography note
        assert "Bibliography File Required" in result
        assert "references.bib" in result
        assert "@misc{ref1," in result

    def test_quarto_title_extraction(self, sample_markdown):
        """Test automatic title extraction from markdown."""
        exporter = QuartoExporter()
        # Don't provide title, should extract from markdown
        result = exporter.export_to_quarto(sample_markdown)

        assert 'title: "Research on AI Safety"' in result

    def test_quarto_bibliography_generation(self, sample_markdown):
        """Test BibTeX bibliography generation."""
        exporter = QuartoExporter()
        bib_content = exporter._generate_bibliography(sample_markdown)

        # Check BibTeX entries
        assert "@misc{ref1," in bib_content
        assert 'title = "{RLHF: A Survey}"' in bib_content
        assert "url = {https://arxiv.org/abs/2023.1234}" in bib_content
        assert (
            'howpublished = "\\url{https://arxiv.org/abs/2023.1234}"'
            in bib_content
        )
        assert "year = {2024}" in bib_content
        assert 'note = "Accessed: \\today"' in bib_content

        # Check all three entries
        assert "@misc{ref2," in bib_content
        assert "@misc{ref3," in bib_content

    def test_quarto_empty_content(self):
        """Test Quarto export with empty content."""
        exporter = QuartoExporter()
        result = exporter.export_to_quarto("", "Empty Report")

        # Should still have YAML header
        assert "---" in result
        assert 'title: "Empty Report"' in result

    def test_quarto_no_sources(self):
        """Test Quarto export without sources section."""
        content = "# Title\n\nSome text with citations [1] and [2]."
        exporter = QuartoExporter()
        result = exporter.export_to_quarto(content)

        # Citations should still be converted
        assert "[@ref1]" in result
        assert "[@ref2]" in result

        # Bibliography note should still appear but be empty
        assert "Bibliography File Required" in result

    def test_unicode_lenticular_citations(self):
        """Test Quarto export converts lenticular citations to [@refN] format."""
        content = """# Report

Single lenticular 【1】 and comma-separated 【1, 2】 citations.

## Sources

[1] First Source
    URL: https://example.com/1

[2] Second Source
    URL: https://example.com/2
"""
        exporter = QuartoExporter()
        result = exporter.export_to_quarto(content)

        # Check lenticular citations are converted to Quarto format
        assert "[@ref1]" in result
        assert "[@ref1, @ref2]" in result

    def test_unicode_lenticular_triple_comma(self):
        """Test Quarto export with three comma-separated lenticular citations."""
        content = """# Report

Multiple sources 【1, 2, 3】 referenced together.

## Sources

[1] First
    URL: https://example.com/1
[2] Second
    URL: https://example.com/2
[3] Third
    URL: https://example.com/3
"""
        exporter = QuartoExporter()
        result = exporter.export_to_quarto(content)

        assert "[@ref1, @ref2, @ref3]" in result

    def test_unicode_lenticular_mixed_with_standard(self):
        """Test Quarto export with mixed bracket styles."""
        content = """# Report

Standard [1] and lenticular 【2】 and mixed comma [1, 2] and 【2, 3】.

## Sources

[1] First
    URL: https://example.com/1
[2] Second
    URL: https://example.com/2
[3] Third
    URL: https://example.com/3
"""
        exporter = QuartoExporter()
        result = exporter.export_to_quarto(content)

        assert "[@ref1]" in result
        assert "[@ref2]" in result
        assert "[@ref1, @ref2]" in result
        assert "[@ref2, @ref3]" in result

    def test_unicode_lenticular_consecutive(self):
        """Test Quarto export with consecutive lenticular citations."""
        content = """# Report

Consecutive lenticular【1】【2】【3】citations.

## Sources

[1] First
    URL: https://example.com/1
[2] Second
    URL: https://example.com/2
[3] Third
    URL: https://example.com/3
"""
        exporter = QuartoExporter()
        result = exporter.export_to_quarto(content)

        # Each should be converted individually
        assert "[@ref1]" in result
        assert "[@ref2]" in result
        assert "[@ref3]" in result


class TestExtractDomainExceptionHandling:
    """Tests for _extract_domain handling malformed URLs (PR #2016)."""

    def test_extract_domain_with_completely_invalid_url(self):
        """Test _extract_domain returns 'source' for completely invalid URL."""
        formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        # The method catches Exception and returns "source"
        result = formatter._extract_domain("")
        # Empty URL should still produce a result (may be "source" or empty domain)
        assert isinstance(result, str)

    def test_extract_domain_with_valid_url(self):
        """Test _extract_domain works correctly with valid URL."""
        formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        result = formatter._extract_domain("https://arxiv.org/abs/2024.1234")
        assert result == "arxiv.org"

    def test_extract_domain_with_www_prefix(self):
        """Test _extract_domain strips www prefix."""
        formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        result = formatter._extract_domain(
            "https://www.nature.com/articles/s41586"
        )
        assert result == "nature.com"

    def test_extract_domain_with_subdomain(self):
        """Test _extract_domain handles subdomains."""
        formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        result = formatter._extract_domain(
            "https://milvusio.medium.com/article"
        )
        assert result == "medium.com"

    def test_extract_domain_with_none_like_input(self):
        """Test _extract_domain handles edge case inputs gracefully."""
        formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        # urlparse handles most strings without raising, but the except
        # catches any Exception that occurs during domain extraction
        result = formatter._extract_domain("not-a-url")
        assert isinstance(result, str)


class TestRISExporterExceptionHandling:
    """Tests for RIS exporter handling malformed URLs (PR #2016)."""

    def test_ris_export_with_malformed_url(self):
        """Test RIS export handles malformed URL in source gracefully."""
        content = """# Report

Test content [1].

## Sources

[1] Source With Bad URL
    URL: ://malformed-url-no-scheme
"""
        exporter = RISExporter()
        result = exporter.export_to_ris(content)

        # Should still produce RIS output without crashing
        assert "TY  - ELEC" in result
        assert "TI  - Source With Bad URL" in result
        assert "ER  - " in result

    def test_ris_export_with_valid_and_invalid_urls(self):
        """Test RIS export when mixing valid and invalid URLs."""
        content = """# Report

Content [1] and [2].

## Sources

[1] Valid Source
    URL: https://arxiv.org/abs/2024.1234

[2] Invalid URL Source
    URL: not_a_valid_url_at_all
"""
        exporter = RISExporter()
        result = exporter.export_to_ris(content)

        # Both entries should be present
        assert "TI  - Valid Source" in result
        assert "TI  - Invalid URL Source" in result
        # Valid source should have publisher
        assert "PB  - arXiv" in result


class TestFindSourcesSection:
    """Test cases for the find_sources_section module-level function."""

    def test_find_h2_sources_header(self):
        """Test finding ## Sources header."""
        from local_deep_research.text_optimization.citation_formatter import (
            find_sources_section,
        )

        content = """# Report

Some content here.

## Sources

[1] First source
"""
        result = find_sources_section(content)
        assert result == content.index("## Sources")

    def test_find_h2_references_header(self):
        """Test finding ## References header."""
        from local_deep_research.text_optimization.citation_formatter import (
            find_sources_section,
        )

        content = """# Report

Some content here.

## References

[1] First reference
"""
        result = find_sources_section(content)
        assert result == content.index("## References")

    def test_find_h2_bibliography_header(self):
        """Test finding ## Bibliography header."""
        from local_deep_research.text_optimization.citation_formatter import (
            find_sources_section,
        )

        content = """# Report

Some content here.

## Bibliography

[1] First entry
"""
        result = find_sources_section(content)
        assert result == content.index("## Bibliography")

    def test_find_h2_citations_header(self):
        """Test finding ## Citations header."""
        from local_deep_research.text_optimization.citation_formatter import (
            find_sources_section,
        )

        content = """# Report

Some content here.

## Citations

[1] First citation
"""
        result = find_sources_section(content)
        assert result == content.index("## Citations")

    def test_find_h3_sources_header(self):
        """Test finding ### Sources header."""
        from local_deep_research.text_optimization.citation_formatter import (
            find_sources_section,
        )

        content = """# Report

## Section

### Sources

[1] First source
"""
        result = find_sources_section(content)
        assert result == content.index("### Sources")

    def test_find_h1_sources_header(self):
        """Test finding # Sources header."""
        from local_deep_research.text_optimization.citation_formatter import (
            find_sources_section,
        )

        content = """# Report

Some content.

# Sources

[1] First source
"""
        result = find_sources_section(content)
        assert result == content.index("# Sources")

    def test_case_insensitivity(self):
        """Test that header detection is case insensitive."""
        from local_deep_research.text_optimization.citation_formatter import (
            find_sources_section,
        )

        content_lower = """## sources

[1] First
"""
        content_upper = """## SOURCES

[1] First
"""
        content_mixed = """## SoUrCeS

[1] First
"""
        assert find_sources_section(content_lower) == 0
        assert find_sources_section(content_upper) == 0
        assert find_sources_section(content_mixed) == 0

    def test_no_sources_section_returns_negative_one(self):
        """Test that -1 is returned when no sources section exists."""
        from local_deep_research.text_optimization.citation_formatter import (
            find_sources_section,
        )

        content = """# Report

Some content with [1] citation but no sources section.

## Conclusion

The end.
"""
        result = find_sources_section(content)
        assert result == -1

    def test_empty_content_returns_negative_one(self):
        """Test that empty content returns -1."""
        from local_deep_research.text_optimization.citation_formatter import (
            find_sources_section,
        )

        assert find_sources_section("") == -1

    def test_plain_sources_colon_format(self):
        """Test finding 'Sources:' format without markdown headers."""
        from local_deep_research.text_optimization.citation_formatter import (
            find_sources_section,
        )

        content = """Report content.

Sources:
[1] First source
"""
        result = find_sources_section(content)
        assert result == content.index("Sources:")

    def test_references_without_colon(self):
        """Test finding 'References' without colon."""
        from local_deep_research.text_optimization.citation_formatter import (
            find_sources_section,
        )

        content = """Report content.

References
[1] First reference
"""
        result = find_sources_section(content)
        assert result == content.index("References")


class TestReplaceCommaCitations:
    """Test cases for the _replace_comma_citations method."""

    def test_basic_comma_separated_citations(self):
        """Test replacement of basic comma-separated citations."""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)

        content = "See citations [1, 2, 3] for more info."
        lookup = {
            "1": ("Title 1", "https://example.com/1"),
            "2": ("Title 2", "https://example.com/2"),
            "3": ("Title 3", "https://example.com/3"),
        }

        def format_one(num, data):
            _, url = data
            return f"[[{num}]]({url})"

        result = formatter._replace_comma_citations(content, lookup, format_one)
        assert "[[1]](https://example.com/1)" in result
        assert "[[2]](https://example.com/2)" in result
        assert "[[3]](https://example.com/3)" in result
        assert "[1, 2, 3]" not in result

    def test_missing_citation_in_lookup(self):
        """Test that missing citations fall back to plain format."""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)

        content = "See citations [1, 2, 3] for more info."
        lookup = {
            "1": ("Title 1", "https://example.com/1"),
            # "2" is missing
            "3": ("Title 3", "https://example.com/3"),
        }

        def format_one(num, data):
            _, url = data
            return f"[[{num}]]({url})"

        result = formatter._replace_comma_citations(content, lookup, format_one)
        assert "[[1]](https://example.com/1)" in result
        assert "[2]" in result  # Falls back to plain format
        assert "[[3]](https://example.com/3)" in result

    def test_no_spaces_after_commas(self):
        """Test citations without spaces after commas."""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)

        content = "See citations [1,2,3] for more info."
        lookup = {
            "1": ("Title 1", "https://example.com/1"),
            "2": ("Title 2", "https://example.com/2"),
            "3": ("Title 3", "https://example.com/3"),
        }

        def format_one(num, data):
            _, url = data
            return f"[[{num}]]({url})"

        result = formatter._replace_comma_citations(content, lookup, format_one)
        assert "[[1]](https://example.com/1)" in result
        assert "[[2]](https://example.com/2)" in result
        assert "[[3]](https://example.com/3)" in result

    def test_two_citations_only(self):
        """Test with only two comma-separated citations."""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)

        content = "See [1, 2] for details."
        lookup = {
            "1": ("Title 1", "https://example.com/1"),
            "2": ("Title 2", "https://example.com/2"),
        }

        def format_one(num, data):
            _, url = data
            return f"[[{num}]]({url})"

        result = formatter._replace_comma_citations(content, lookup, format_one)
        assert (
            "[[1]](https://example.com/1)[[2]](https://example.com/2)" in result
        )

    def test_domain_format_callback(self):
        """Test with domain-style formatting callback."""
        formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)

        content = "See citations [1, 2] for more info."
        lookup = {
            "1": ("arxiv.org", "https://arxiv.org/abs/123"),
            "2": ("github.com", "https://github.com/repo"),
        }

        def format_one(num, data):
            domain, url = data
            return f"[[{domain}]]({url})"

        result = formatter._replace_comma_citations(content, lookup, format_one)
        assert "[[arxiv.org]](https://arxiv.org/abs/123)" in result
        assert "[[github.com]](https://github.com/repo)" in result

    def test_multiple_comma_groups_in_content(self):
        """Test multiple comma-separated citation groups in content."""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)

        content = "First group [1, 2] and second group [3, 4]."
        lookup = {
            "1": ("T1", "https://example.com/1"),
            "2": ("T2", "https://example.com/2"),
            "3": ("T3", "https://example.com/3"),
            "4": ("T4", "https://example.com/4"),
        }

        def format_one(num, data):
            _, url = data
            return f"[[{num}]]({url})"

        result = formatter._replace_comma_citations(content, lookup, format_one)
        assert (
            "[[1]](https://example.com/1)[[2]](https://example.com/2)" in result
        )
        assert (
            "[[3]](https://example.com/3)[[4]](https://example.com/4)" in result
        )

    def test_unicode_lenticular_brackets(self):
        """Test comma-separated citations with Unicode lenticular brackets."""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)

        content = "See citations【1, 2, 3】for more info."
        lookup = {
            "1": ("Title 1", "https://example.com/1"),
            "2": ("Title 2", "https://example.com/2"),
            "3": ("Title 3", "https://example.com/3"),
        }

        def format_one(num, data):
            _, url = data
            return f"[[{num}]]({url})"

        result = formatter._replace_comma_citations(content, lookup, format_one)
        assert "[[1]](https://example.com/1)" in result
        assert "[[2]](https://example.com/2)" in result
        assert "[[3]](https://example.com/3)" in result

    def test_empty_lookup_preserves_original(self):
        """Test that empty lookup preserves original citation numbers."""
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)

        content = "See citations [1, 2, 3] for more info."
        lookup = {}

        def format_one(num, data):
            _, url = data
            return f"[[{num}]]({url})"

        result = formatter._replace_comma_citations(content, lookup, format_one)
        # All citations should fall back to plain format
        assert "[1]" in result
        assert "[2]" in result
        assert "[3]" in result


# ---------------------------------------------------------------------------
# Regression tests for the empty-citation-label bug.
#
# Bug: _extract_domain() returns "" for relative URLs (e.g.
# /library/document/<uuid> from RAG / library sources). The three
# domain-hyperlink formatters used to embed that empty string directly,
# producing "[[]](url)" (rendered as a bare "[]" link) and "[[-1]](url)"
# for multi-citation results. _citation_label / _slugify_title close the
# gap. These tests guard against regression.
# ---------------------------------------------------------------------------


def _library_url(suffix: str = "abc") -> str:
    """Return a relative /library/document/<suffix> URL like RAG emits."""
    return f"/library/document/{suffix}"


class TestRelativeUrlCitationLabels:
    """Empty-domain bug regression. All three fixed modes must produce
    a non-empty label for relative URLs (e.g. /library/document/... from
    RAG / library sources) by falling back to a slugified title."""

    # ---- DOMAIN_HYPERLINKS -----------------------------------------------

    def test_domain_hyperlinks_single_relative_url_uses_title_slug(self):
        """Before fix: [[]](url). After: [[<title-slug>]](url)."""
        content = """# Report

See [1] for details.

## Sources

[1] Sumerian King List
    URL: /library/document/abc-123
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        result = formatter.format_document(content)

        assert "[[sumerian-king-list]](/library/document/abc-123)" in result
        assert "[[]]" not in result
        assert "[[-1]]" not in result

    def test_domain_hyperlinks_multi_relative_urls_use_distinct_slugs(self):
        """Before fix: [[-1]](url1)[[-2]](url2) (orphans).
        After: distinct title-based slugs."""
        content = """# Report

Findings [1] and [2].

## Sources

[1] Sumerian King List
    URL: /library/document/a

[2] Ancient Mesopotamia
    URL: /library/document/b
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        result = formatter.format_document(content)

        assert "[[sumerian-king-list]](/library/document/a)" in result
        assert "[[ancient-mesopotamia]](/library/document/b)" in result
        assert "[[]]" not in result
        assert "[[-1]]" not in result
        assert "[[-2]]" not in result

    def test_domain_hyperlinks_empty_title_falls_back_to_citation_number(
        self,
    ):
        """Last-resort: when both domain and title slug are empty,
        fall back to the citation number so the label is never empty.
        Tested via the helper directly because the sources-block parser
        requires a non-empty title line (e.g. ``[1] Title\n    URL: ...``)."""
        formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        result = formatter._format_domain_hyperlinks(
            "see [1]", {"1": ("", "/library/document/abc")}
        )
        assert "[[1]](/library/document/abc)" in result
        assert "[[]]" not in result

    def test_domain_hyperlinks_comma_separated_relative_urls(self):
        """Comma-separated citations with relative URLs must produce
        non-empty labels. Before fix every label collapsed to ""."""
        content = """# Report

Sources [1, 2, 3] confirm this.

## Sources

[1] Doc A
    URL: /library/document/a

[2] Doc B
    URL: /library/document/b

[3] Doc C
    URL: /library/document/c
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        result = formatter.format_document(content)

        assert (
            "[[doc-a]](/library/document/a)"
            "[[doc-b]](/library/document/b)"
            "[[doc-c]](/library/document/c)"
        ) in result
        assert "[[]]" not in result

    # ---- DOMAIN_ID_HYPERLINKS (default) ----------------------------------

    def test_domain_id_hyperlinks_single_relative_url_uses_title_slug(self):
        """The default mode. Before fix: [[]](url). After: [[slug]](url)."""
        content = """# Report

See [1].

## Sources

[1] Sumerian King List
    URL: /library/document/abc-123
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS)
        result = formatter.format_document(content)

        assert "[[sumerian-king-list]](/library/document/abc-123)" in result
        assert "[[]]" not in result
        assert "[[-1]]" not in result

    def test_domain_id_hyperlinks_multi_relative_urls_use_distinct_slugs(
        self,
    ):
        content = """# Report

Findings [1] and [2].

## Sources

[1] Sumerian King List
    URL: /library/document/a

[2] Ancient Mesopotamia
    URL: /library/document/b
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS)
        result = formatter.format_document(content)

        assert "[[sumerian-king-list]](/library/document/a)" in result
        assert "[[ancient-mesopotamia]](/library/document/b)" in result
        assert "[[]]" not in result
        assert "[[-1]]" not in result
        assert "[[-2]]" not in result

    def test_domain_id_hyperlinks_same_title_relative_urls_group_with_id(
        self,
    ):
        """Multiple relative-URL citations of the SAME title group under
        one slug and append -N suffixes. Before fix this produced
        indistinguishable [[-1]](url1)[[-2]](url2) orphans."""
        content = """# Report

Findings [1] and [2].

## Sources

[1] Same Doc
    URL: /library/document/a

[2] Same Doc
    URL: /library/document/b
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS)
        result = formatter.format_document(content)

        assert "[[same-doc-1]](/library/document/a)" in result
        assert "[[same-doc-2]](/library/document/b)" in result
        assert "[[same-doc]/" not in result  # not the no-suffix form
        assert "[[]]" not in result

    def test_domain_id_hyperlinks_empty_title_falls_back_to_citation_number(
        self,
    ):
        """Same last-resort guarantee for DOMAIN_ID_HYPERLINKS. Tested
        via the helper directly (parser requires non-empty title line)."""
        formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS)
        result = formatter._format_domain_id_hyperlinks(
            "see [1]", {"1": ("", "/library/document/abc")}
        )
        assert "[[1]](/library/document/abc)" in result
        assert "[[]]" not in result

    def test_domain_id_hyperlinks_comma_separated_relative_urls(self):
        content = """# Report

Sources [1, 2, 3] confirm this.

## Sources

[1] Doc A
    URL: /library/document/a

[2] Doc B
    URL: /library/document/b

[3] Doc C
    URL: /library/document/c
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS)
        result = formatter.format_document(content)

        # "Doc A" / "Doc B" / "Doc C" slugify to "doc-a" / "doc-b" /
        # "doc-c" -- single docs in DOMAIN_ID mode keep no -N suffix.
        assert (
            "[[doc-a]](/library/document/a)"
            "[[doc-b]](/library/document/b)"
            "[[doc-c]](/library/document/c)"
        ) in result
        assert "[[]]" not in result

    def test_domain_id_hyperlinks_mixed_real_and_relative(self):
        """Real URLs keep their domain labels; relative URLs use title
        slugs. No regression on the real-URL side."""
        content = """# Report

Web source [1] and library source [2].

## Sources

[1] Web Doc
    URL: https://example.com/x

[2] Local Doc
    URL: /library/document/a
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS)
        result = formatter.format_document(content)

        assert "[[example.com]](https://example.com/x)" in result
        assert "[[local-doc]](/library/document/a)" in result
        assert "[[]]" not in result

    # ---- DOMAIN_ID_ALWAYS_HYPERLINKS -------------------------------------

    def test_domain_id_always_hyperlinks_single_relative_url_uses_title_slug(
        self,
    ):
        """Before fix: [[-1]](url). After: [[slug-1]](url)."""
        content = """# Report

See [1].

## Sources

[1] Sumerian King List
    URL: /library/document/abc-123
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS)
        result = formatter.format_document(content)

        assert "[[sumerian-king-list-1]](/library/document/abc-123)" in result
        assert "[[-1]]" not in result
        assert "[[]]" not in result

    def test_domain_id_always_hyperlinks_multi_relative_urls_use_distinct_slugs(
        self,
    ):
        content = """# Report

Findings [1] and [2].

## Sources

[1] Sumerian King List
    URL: /library/document/a

[2] Ancient Mesopotamia
    URL: /library/document/b
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS)
        result = formatter.format_document(content)

        assert "[[sumerian-king-list-1]](/library/document/a)" in result
        assert "[[ancient-mesopotamia-1]](/library/document/b)" in result
        assert "[[-1]]" not in result
        assert "[[-2]]" not in result

    def test_domain_id_always_hyperlinks_same_title_relative_urls_group(
        self,
    ):
        content = """# Report

Findings [1] and [2].

## Sources

[1] Same Doc
    URL: /library/document/a

[2] Same Doc
    URL: /library/document/b
"""
        formatter = CitationFormatter(CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS)
        result = formatter.format_document(content)

        assert "[[same-doc-1]](/library/document/a)" in result
        assert "[[same-doc-2]](/library/document/b)" in result
        assert "[[-1]]" not in result
        assert "[[]]" not in result

    # ---- Real-URL regression (no behavior change) ------------------------

    def test_domain_hyperlinks_real_url_unchanged(self):
        """Guard against accidentally breaking the real-URL path."""
        formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        result = formatter._format_domain_hyperlinks(
            "see [1]", {"1": ("Paper", "https://arxiv.org/abs/1234")}
        )
        assert "[[arxiv.org]](https://arxiv.org/abs/1234)" in result

    def test_domain_id_hyperlinks_real_url_unchanged(self):
        formatter = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS)
        result = formatter._format_domain_id_hyperlinks(
            "see [1] and [2]",
            {
                "1": ("A", "https://arxiv.org/abs/1"),
                "2": ("B", "https://arxiv.org/abs/2"),
            },
        )
        assert "[[arxiv.org-1]](https://arxiv.org/abs/1)" in result
        assert "[[arxiv.org-2]](https://arxiv.org/abs/2)" in result

    def test_domain_id_always_hyperlinks_real_url_unchanged(self):
        formatter = CitationFormatter(CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS)
        result = formatter._format_domain_id_always_hyperlinks(
            "see [1]",
            {"1": ("Paper", "https://example.com/x")},
        )
        assert "[[example.com-1]](https://example.com/x)" in result


class TestApplyInlineHyperlinks:
    """Streaming-path coverage. apply_inline_hyperlinks is the only
    formatter that takes structured dicts of sources (not markdown
    sources blocks); it shares _format_domain_id_hyperlinks so it picks
    up the relative-URL fix transitively."""

    def test_real_url(self):
        fmt = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS)
        result = fmt.apply_inline_hyperlinks(
            "see [1]",
            [{"index": "1", "title": "T", "url": "https://example.com/x"}],
        )
        assert "[[example.com]](https://example.com/x)" in result

    def test_single_relative_url_uses_title_slug(self):
        """Before fix: [[]](url). After: [[slug]](url)."""
        fmt = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS)
        result = fmt.apply_inline_hyperlinks(
            "see [1]",
            [
                {
                    "index": "1",
                    "title": "Sumerian King List",
                    "url": "/library/document/abc",
                }
            ],
        )
        assert "[[sumerian-king-list]](/library/document/abc)" in result
        assert "[[]]" not in result
        assert "[[-1]]" not in result

    def test_multi_relative_urls_use_distinct_slugs(self):
        fmt = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS)
        result = fmt.apply_inline_hyperlinks(
            "see [1] and [2]",
            [
                {
                    "index": "1",
                    "title": "Sumerian King List",
                    "url": "/library/document/a",
                },
                {
                    "index": "2",
                    "title": "Ancient Mesopotamia",
                    "url": "/library/document/b",
                },
            ],
        )
        assert "[[sumerian-king-list]]" in result
        assert "[[ancient-mesopotamia]]" in result
        assert "[[]]" not in result
        assert "[[-1]]" not in result
        assert "[[-2]]" not in result

    def test_no_url_emits_no_link(self):
        """Sources with no URL must not produce empty [[]] links."""
        fmt = CitationFormatter(CitationMode.DOMAIN_ID_HYPERLINKS)
        result = fmt.apply_inline_hyperlinks(
            "see [1]", [{"index": "1", "title": "T", "url": ""}]
        )
        assert "[1]" in result
        assert "[[]]" not in result
        assert "[[1]]" not in result


class TestCitationLabel:
    """Direct unit tests for the _citation_label helper."""

    def test_prefers_domain_when_present(self):
        fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        assert (
            fmt._citation_label("1", "Some Title", "https://example.com/x")
            == "example.com"
        )

    def test_prefers_domain_for_arxiv(self):
        fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        assert (
            fmt._citation_label("1", "Some Title", "https://arxiv.org/abs/1")
            == "arxiv.org"
        )

    def test_falls_back_to_title_slug_for_relative_url(self):
        fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        assert (
            fmt._citation_label(
                "1", "Sumerian King List", "/library/document/abc"
            )
            == "sumerian-king-list"
        )

    def test_falls_back_to_citation_number_when_title_empty(self):
        fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        assert fmt._citation_label("1", "", "/library/document/abc") == "1"

    def test_falls_back_to_citation_number_when_title_whitespace(self):
        fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        assert fmt._citation_label("1", "   ", "/library/document/abc") == "1"

    def test_falls_back_to_title_slug_when_url_empty(self):
        """URL empty but title present -> use slug."""
        fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        assert fmt._citation_label("1", "Title", "") == "title"

    def test_falls_back_to_citation_number_when_both_empty(self):
        fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        assert fmt._citation_label("1", "", "") == "1"

    def test_sanitizes_special_chars_in_title(self):
        fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        assert (
            fmt._citation_label(
                "1", "Title with: weird! chars", "/library/document/a"
            )
            == "title-with-weird-chars"
        )

    def test_legitimate_title_doc_keeps_doc_label(self):
        """A document literally titled ``"Doc"`` slugifies to ``"doc"``;
        the label must keep that slug rather than fall through to the
        citation-number fallback. Regression guard for the case where
        ``_slugify_title`` returned ``"doc"`` as a sentinel."""
        fmt = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        assert fmt._citation_label("1", "Doc", "/library/document/a") == "doc"
        assert fmt._citation_label("1", "DOC", "/library/document/a") == "doc"


class TestSlugifyTitle:
    """Direct unit tests for the _slugify_title helper."""

    def test_empty_returns_none(self):
        assert CitationFormatter._slugify_title("") is None

    def test_whitespace_only_returns_none(self):
        assert CitationFormatter._slugify_title("   ") is None

    def test_punctuation_only_returns_none(self):
        assert CitationFormatter._slugify_title("!!!") is None

    def test_basic_slugification(self):
        assert CitationFormatter._slugify_title("Hello World") == "hello-world"

    def test_special_chars_replaced_with_single_hyphen(self):
        assert (
            CitationFormatter._slugify_title("Some/Weird: Name (with stuff)")
            == "some-weird-name-with-stuff"
        )

    def test_truncates_long_titles(self):
        long = "x" * 100
        slug = CitationFormatter._slugify_title(long)
        assert len(slug) == 32
        assert slug == "x" * 32

    def test_truncation_strips_trailing_hyphen(self):
        """If max_len lands on a hyphen, strip it so callers can append
        -N without producing a malformed --N label."""
        slug = CitationFormatter._slugify_title("x" * 32 + "-tail")
        assert len(slug) == 32
        assert not slug.endswith("-")
        assert slug == "x" * 32

    def test_strips_leading_and_trailing_whitespace(self):
        assert (
            CitationFormatter._slugify_title("  Hello World  ") == "hello-world"
        )

    def test_legitimate_title_doc_is_not_treated_as_missing(self):
        """A document literally titled ``"Doc"`` slugifies to ``"doc"``;
        that is a real label, not a sentinel for "no meaningful slug"."""
        assert CitationFormatter._slugify_title("Doc") == "doc"
        assert CitationFormatter._slugify_title("DOC") == "doc"


class TestSourceTagCacheLookup:
    """_format_source_tagged_hyperlinks must resolve labels from the
    pre-computed tag_cache without re-running _extract_source_label on
    cache hits — evaluating it per citation match is pure dead-weight
    work."""

    def test_label_resolution_runs_once_per_source(self, monkeypatch):
        """_extract_source_label runs once per URL-having source (while
        building tag_cache), not once per citation occurrence."""
        formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS)
        calls = []
        original = CitationFormatter._extract_source_label

        def counting(self, url, collection=None):
            calls.append(url)
            return original(self, url, collection=collection)

        monkeypatch.setattr(
            CitationFormatter, "_extract_source_label", counting
        )

        content = "First [1], again [1], list [1, 2], and [2]."
        sources = {
            "1": ("Paper A", "https://arxiv.org/abs/2024.1111"),
            "2": ("Paper B", "https://example.com/x"),
        }
        result = formatter._format_source_tagged_hyperlinks(
            content, sources, {}
        )

        assert "[[arxiv-1]](https://arxiv.org/abs/2024.1111)" in result
        assert "[[example.com-2]](https://example.com/x)" in result
        assert len(calls) == len(sources)

    def test_url_less_source_uses_cache_miss_fallback(self):
        """Sources without a URL are absent from tag_cache and must
        still resolve via the cache-miss fallback path."""
        formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS)
        result = formatter._format_source_tagged_hyperlinks(
            "see [1]", {"1": ("Title", "")}, {}
        )
        assert "[local-1]" in result
        assert "[[local-1]]" not in result  # no hyperlink for empty URL


class TestUrlClassifierIsNormalImport:
    """#4992 follow-up: the disk-load workaround (PR #4880) is gone.

    ``citation_formatter`` now imports ``URLClassifier``/``URLType``
    through the package like any other module; the classes it uses must
    be the canonical ones, not a shadow copy exec'd from disk (the old
    loader produced a distinct module object whose enum members compared
    unequal to the real ones)."""

    def test_uses_canonical_url_classifier_classes(self):
        from local_deep_research.content_fetcher.url_classifier import (
            URLClassifier,
            URLType,
        )
        from local_deep_research.text_optimization import (
            citation_formatter as cf,
        )

        assert cf.URLClassifier is URLClassifier
        assert cf.URLType is URLType