learningcircuit--local-deep-research/tests/utilities/test_search_utilities_coverage.py

"""
Comprehensive coverage tests for search_utilities.py.

Focuses on gaps not covered by existing test files:
- Nested think tags (multiple levels, interleaved)
- Orphaned tags in various positions
- Empty/whitespace-only strings through every function
- URL deduplication with index merging semantics
- Citation numbering: exact format of [indices] and (source nr: ...) output
- format_findings end-to-end: multiple findings with sources, all-sources footer
"""

from local_deep_research.utilities.search_utilities import (
    extract_links_from_search_results,
    format_findings,
    format_links_to_markdown,
    remove_think_tags,
)


# ═══════════════════════════════════════════════════════════════════════════
# remove_think_tags — nested / interleaved / edge patterns
# ═══════════════════════════════════════════════════════════════════════════


class TestRemoveThinkTagsNested:
    """Deeply-nested and interleaved think tag patterns."""

    def test_double_nested_tags_fully_removed(self):
        """<think>outer<think>inner</think>middle</think> leaves no tags."""
        text = "A <think>outer <think>inner</think> middle</think> B"
        result = remove_think_tags(text)
        assert "<think>" not in result
        assert "</think>" not in result
        assert "A" in result
        assert "B" in result

    def test_triple_nested_tags(self):
        """Three levels of nesting: all tags removed, but non-greedy regex
        leaves residual text between matched pairs.  The key invariant is
        that no <think> or </think> tags survive."""
        text = "<think>L1 <think>L2 <think>L3</think> L2</think> L1</think>keep"
        result = remove_think_tags(text)
        assert "<think>" not in result
        assert "</think>" not in result
        # The non-greedy .*? pairs the first <think> with the innermost
        # </think>, so intermediate text leaks out.  Document that behavior:
        assert "keep" in result

    def test_adjacent_think_blocks(self):
        """Two adjacent think blocks with no gap."""
        text = "<think>first</think><think>second</think>visible"
        result = remove_think_tags(text)
        assert result == "visible"

    def test_interleaved_content_between_think_blocks(self):
        """Content between multiple think blocks is preserved."""
        text = "<think>a</think>ONE<think>b</think>TWO<think>c</think>"
        result = remove_think_tags(text)
        assert result == "ONETWO"

    def test_only_orphaned_opening_tags(self):
        """Multiple orphaned opening tags removed."""
        text = "keep <think> this <think> too"
        result = remove_think_tags(text)
        assert result == "keep  this  too"

    def test_only_orphaned_closing_tags(self):
        """Multiple orphaned closing tags removed."""
        text = "keep </think> this </think> too"
        result = remove_think_tags(text)
        assert result == "keep  this  too"

    def test_orphan_close_before_paired_block(self):
        """Orphan close tag before a valid paired block."""
        text = "</think>prefix <think>hidden</think> suffix"
        result = remove_think_tags(text)
        assert result == "prefix  suffix"

    def test_orphan_open_after_paired_block(self):
        """Orphan open tag after a valid paired block."""
        text = "prefix <think>hidden</think> suffix <think>trailing"
        result = remove_think_tags(text)
        assert result == "prefix  suffix trailing"


class TestRemoveThinkTagsEmptyAndWhitespace:
    """Empty, whitespace-only, and tag-only inputs."""

    def test_empty_string(self):
        assert remove_think_tags("") == ""

    def test_whitespace_only(self):
        assert remove_think_tags("   \n\t  ") == ""

    def test_only_paired_tags_with_whitespace_content(self):
        assert remove_think_tags("<think>  \n  </think>") == ""

    def test_only_orphan_open(self):
        assert remove_think_tags("<think>") == ""

    def test_only_orphan_close(self):
        assert remove_think_tags("</think>") == ""

    def test_newlines_around_tags(self):
        text = "\n\n<think>hidden\n</think>\n\n"
        result = remove_think_tags(text)
        assert result == ""


# ═══════════════════════════════════════════════════════════════════════════
# extract_links_from_search_results — edge cases
# ═══════════════════════════════════════════════════════════════════════════


class TestExtractLinksEdges:
    """Additional edge cases for link extraction."""

    def test_whitespace_only_title_skipped(self):
        """A title of only spaces should be treated as empty after strip."""
        results = [
            {"title": "   ", "link": "https://example.com", "index": "1"}
        ]
        links = extract_links_from_search_results(results)
        assert len(links) == 0

    def test_whitespace_only_link_skipped(self):
        """A link of only spaces should be treated as empty after strip."""
        results = [{"title": "Title", "link": "   ", "index": "1"}]
        links = extract_links_from_search_results(results)
        assert len(links) == 0

    def test_none_index_defaults_to_empty_string(self):
        """None index becomes empty string, not None."""
        results = [{"title": "T", "link": "https://x.com", "index": None}]
        links = extract_links_from_search_results(results)
        assert links[0]["index"] == ""

    def test_missing_index_key_defaults_to_empty_string(self):
        """Missing index key defaults to empty string."""
        results = [{"title": "T", "link": "https://x.com"}]
        links = extract_links_from_search_results(results)
        assert links[0]["index"] == ""

    def test_preserves_insertion_order(self):
        """Output order matches input order."""
        results = [
            {"title": "C", "link": "https://c.com", "index": "3"},
            {"title": "A", "link": "https://a.com", "index": "1"},
            {"title": "B", "link": "https://b.com", "index": "2"},
        ]
        links = extract_links_from_search_results(results)
        assert [item["title"] for item in links] == ["C", "A", "B"]

    def test_large_batch(self):
        """Handles a moderately large list without error."""
        results = [
            {
                "title": f"T{i}",
                "link": f"https://example.com/{i}",
                "index": str(i),
            }
            for i in range(200)
        ]
        links = extract_links_from_search_results(results)
        assert len(links) == 200

    def test_extra_keys_ignored(self):
        """Extra keys in the dict are silently ignored."""
        results = [
            {
                "title": "T",
                "link": "https://x.com",
                "index": "1",
                "snippet": "desc",
                "score": 0.9,
            }
        ]
        links = extract_links_from_search_results(results)
        assert len(links) == 1
        assert "snippet" not in links[0]


# ═══════════════════════════════════════════════════════════════════════════
# format_links_to_markdown — citation numbering & deduplication
# ═══════════════════════════════════════════════════════════════════════════


class TestCitationNumbering:
    """Verify exact citation format: [indices] Title (source nr: indices) URL."""

    def test_single_index_format(self):
        """Single-index link produces [1] Title (source nr: 1)."""
        links = [{"title": "Page", "url": "https://p.com", "index": "1"}]
        result = format_links_to_markdown(links)
        assert "[1] Page (source nr: 1)" in result
        assert "URL: https://p.com" in result

    def test_multiple_indices_sorted(self):
        """Duplicate URL indices are sorted: [1, 3] not [3, 1]."""
        links = [
            {"title": "Page", "url": "https://p.com", "index": "3"},
            {"title": "Page", "url": "https://p.com", "index": "1"},
        ]
        result = format_links_to_markdown(links)
        assert "[1, 3]" in result
        assert "(source nr: 1, 3)" in result

    def test_three_indices_deduplicated_and_sorted(self):
        """[5, 2, 2, 9] becomes [2, 5, 9] after dedup+sort."""
        links = [
            {"title": "X", "url": "https://x.com", "index": "5"},
            {"title": "X", "url": "https://x.com", "index": "2"},
            {"title": "X", "url": "https://x.com", "index": "2"},
            {"title": "X", "url": "https://x.com", "index": "9"},
        ]
        result = format_links_to_markdown(links)
        assert result.count("https://x.com") == 1
        assert "[2, 5, 9]" in result

    def test_empty_index_still_produces_bracket(self):
        """A link with empty string index still produces brackets."""
        links = [{"title": "T", "url": "https://t.com", "index": ""}]
        result = format_links_to_markdown(links)
        assert "[] T" in result

    def test_url_dedup_uses_first_title(self):
        """When URLs collide, the first title seen is used."""
        links = [
            {"title": "First Title", "url": "https://same.com", "index": "1"},
            {"title": "Second Title", "url": "https://same.com", "index": "2"},
        ]
        result = format_links_to_markdown(links)
        assert "First Title" in result
        assert "Second Title" not in result

    def test_different_urls_kept_separate(self):
        """Different URLs are separate entries."""
        links = [
            {"title": "A", "url": "https://a.com", "index": "1"},
            {"title": "B", "url": "https://b.com", "index": "2"},
        ]
        result = format_links_to_markdown(links)
        assert "https://a.com" in result
        assert "https://b.com" in result
        lines_with_url = [
            line for line in result.splitlines() if "URL:" in line
        ]
        assert len(lines_with_url) == 2

    def test_link_key_fallback_produces_same_format(self):
        """Using 'link' key instead of 'url' produces the same output format."""
        links = [{"title": "T", "link": "https://t.com", "index": "7"}]
        result = format_links_to_markdown(links)
        assert "[7] T (source nr: 7)" in result
        assert "URL: https://t.com" in result

    def test_no_url_and_no_link_key_skipped(self):
        """Entry with neither 'url' nor 'link' key is skipped."""
        links = [
            {"title": "Ghost", "index": "1"},
            {"title": "Real", "url": "https://r.com", "index": "2"},
        ]
        result = format_links_to_markdown(links)
        assert "Ghost" not in result
        assert "Real" in result

    def test_mixed_int_and_str_indices_for_same_url_do_not_crash(self):
        """Different strategies can emit int vs str indices for the same URL
        (recursive_decomposition_strategy.py yields int; the fallback in
        report_assembly_service._build_sources_markdown emits str). Before the
        fix, sorted() crashed with TypeError on Python 3 when both reached
        format_links_to_markdown together."""
        links = [
            {"title": "Shared", "url": "https://shared.com", "index": 1},
            {"title": "Shared", "url": "https://shared.com", "index": "2"},
        ]
        result = format_links_to_markdown(links)
        assert "https://shared.com" in result
        assert "[1, 2]" in result

    def test_numeric_indices_sort_numerically_not_lexically(self):
        """Indices like 2, 10 should sort 2 then 10 (numeric), not 10 then 2
        (lexicographic)."""
        links = [
            {"title": "X", "url": "https://x.com", "index": 10},
            {"title": "X", "url": "https://x.com", "index": 2},
        ]
        result = format_links_to_markdown(links)
        assert "[2, 10]" in result

    def test_int_and_str_form_of_same_index_dedup(self):
        """1 (int) and "1" (str) refer to the same citation — collapse to one."""
        links = [
            {"title": "X", "url": "https://x.com", "index": 1},
            {"title": "X", "url": "https://x.com", "index": "1"},
        ]
        result = format_links_to_markdown(links)
        assert "[1]" in result


# ═══════════════════════════════════════════════════════════════════════════
# format_findings — end-to-end with sources
# ═══════════════════════════════════════════════════════════════════════════


class TestFormatFindingsEndToEnd:
    """End-to-end format_findings scenarios combining all sub-functions."""

    def test_all_sources_footer_aggregates_across_findings(self):
        """ALL SOURCES section at the end contains links from every finding."""
        findings = [
            {
                "phase": "Phase 1",
                "content": "C1",
                "search_results": [
                    {"title": "S1", "link": "https://s1.com", "index": "1"}
                ],
            },
            {
                "phase": "Phase 2",
                "content": "C2",
                "search_results": [
                    {"title": "S2", "link": "https://s2.com", "index": "2"}
                ],
            },
        ]
        result = format_findings(findings, "Synth", {})
        # The ALL SOURCES section appears after the last separator
        all_sources_section = result.split("ALL SOURCES")[-1]
        assert "s1.com" in all_sources_section
        assert "s2.com" in all_sources_section

    def test_no_all_sources_when_no_search_results(self):
        """When no finding has search_results, ALL SOURCES section is absent."""
        findings = [
            {"phase": "P", "content": "C", "search_results": []},
        ]
        result = format_findings(findings, "Synth", {})
        assert "ALL SOURCES" not in result

    def test_sources_used_in_section_per_finding(self):
        """Each finding with search results gets its own SOURCES USED section."""
        findings = [
            {
                "phase": "P1",
                "content": "C1",
                "search_results": [
                    {"title": "A", "link": "https://a.com", "index": "1"}
                ],
            },
            {
                "phase": "P2",
                "content": "C2",
                "search_results": [
                    {"title": "B", "link": "https://b.com", "index": "2"}
                ],
            },
        ]
        result = format_findings(findings, "Synth", {})
        assert result.count("SOURCES USED IN THIS SECTION") == 2

    def test_finding_without_search_results_key(self):
        """Finding dict missing 'search_results' key entirely does not crash."""
        findings = [{"phase": "P", "content": "C"}]
        result = format_findings(findings, "Synth", {})
        assert "C" in result
        assert "SOURCES USED IN THIS SECTION" not in result

    def test_followup_question_not_displayed_when_already_shown(self):
        """When phase parsing succeeds, the finding['question'] is NOT also shown."""
        findings = [
            {
                "phase": "Follow-up Iteration 1.1",
                "content": "Answer",
                "question": "Fallback Q",
                "search_results": [],
            }
        ]
        questions = {1: ["Matched question"]}
        result = format_findings(findings, "Synth", questions)
        # The matched question should appear
        assert "Matched question" in result
        # The fallback question should NOT appear because phase parsing succeeded
        assert "Fallback Q" not in result

    def test_subquery_out_of_range_falls_back_to_question_field(self):
        """Sub-query 99 with only 2 sub-queries falls back to finding['question']."""
        findings = [
            {
                "phase": "Sub-query 99",
                "content": "C",
                "question": "My fallback",
                "search_results": [],
            }
        ]
        questions = {0: ["SQ1", "SQ2"]}
        result = format_findings(findings, "Synth", questions)
        assert "My fallback" in result

    def test_followup_single_dot_part_not_two(self):
        """Phase 'Follow-up Iteration 1' (no dot) has len(parts)==1, not 2."""
        findings = [
            {
                "phase": "Follow-up Iteration 1",
                "content": "C",
                "question": "Fallback",
                "search_results": [],
            }
        ]
        questions = {1: ["Q1"]}
        result = format_findings(findings, "Synth", questions)
        # Since split('.') gives ['1'] with len==1, parsing fails, fallback shown
        assert "Fallback" in result

    def test_separator_line_between_findings(self):
        """80-char underscore separator appears between findings."""
        findings = [
            {"phase": "A", "content": "CA", "search_results": []},
            {"phase": "B", "content": "CB", "search_results": []},
        ]
        result = format_findings(findings, "S", {})
        assert "_" * 80 in result

    def test_question_numbering_in_iteration_section(self):
        """Questions are numbered 1., 2., 3. within each iteration."""
        questions = {1: ["Alpha", "Beta", "Gamma"]}
        result = format_findings([], "S", questions)
        assert "1. Alpha" in result
        assert "2. Beta" in result
        assert "3. Gamma" in result

    def test_multiple_iterations_each_numbered_from_one(self):
        """Each iteration re-starts numbering at 1."""
        questions = {1: ["Q1a", "Q1b"], 2: ["Q2a"]}
        result = format_findings([], "S", questions)
        # Iteration 1
        assert "1. Q1a" in result
        assert "2. Q1b" in result
        # Iteration 2 starts from 1 again
        iter2_section = result.split("Iteration 2")[1]
        assert "1. Q2a" in iter2_section