learningcircuit--local-deep-research/tests/utilities/test_search_utilities_behavior.py

"""
Behavioral tests for search_utilities module.

Tests utility functions for processing search results.
"""


class TestRemoveThinkTags:
    """Tests for removing <think> tags from text."""

    def test_removes_paired_think_tags(self):
        """Paired <think>...</think> tags are removed."""
        from local_deep_research.utilities.search_utilities import (
            remove_think_tags,
        )

        text = "Hello <think>internal thought</think> World"
        result = remove_think_tags(text)
        assert result == "Hello  World"

    def test_removes_multiline_think_content(self):
        """Multiline content in think tags is removed."""
        from local_deep_research.utilities.search_utilities import (
            remove_think_tags,
        )

        text = "Before <think>\nline1\nline2\n</think> After"
        result = remove_think_tags(text)
        assert result == "Before  After"

    def test_removes_multiple_think_blocks(self):
        """Multiple think blocks are removed."""
        from local_deep_research.utilities.search_utilities import (
            remove_think_tags,
        )

        text = "<think>first</think> middle <think>second</think>"
        result = remove_think_tags(text)
        assert result == "middle"

    def test_removes_orphaned_opening_tag(self):
        """Orphaned opening <think> tag is removed."""
        from local_deep_research.utilities.search_utilities import (
            remove_think_tags,
        )

        text = "Hello <think> World"
        result = remove_think_tags(text)
        assert result == "Hello  World"

    def test_removes_orphaned_closing_tag(self):
        """Orphaned closing </think> tag is removed."""
        from local_deep_research.utilities.search_utilities import (
            remove_think_tags,
        )

        text = "Hello </think> World"
        result = remove_think_tags(text)
        assert result == "Hello  World"

    def test_preserves_text_without_tags(self):
        """Text without think tags is preserved."""
        from local_deep_research.utilities.search_utilities import (
            remove_think_tags,
        )

        text = "Hello World"
        result = remove_think_tags(text)
        assert result == "Hello World"

    def test_strips_result(self):
        """Result is stripped of whitespace."""
        from local_deep_research.utilities.search_utilities import (
            remove_think_tags,
        )

        text = "  <think>thought</think>  Hello  "
        result = remove_think_tags(text)
        assert result == "Hello"

    def test_handles_empty_string(self):
        """Empty string returns empty."""
        from local_deep_research.utilities.search_utilities import (
            remove_think_tags,
        )

        result = remove_think_tags("")
        assert result == ""

    def test_handles_only_think_tags(self):
        """Only think tags returns empty."""
        from local_deep_research.utilities.search_utilities import (
            remove_think_tags,
        )

        result = remove_think_tags("<think>only this</think>")
        assert result == ""

    def test_nested_tags_handled(self):
        """Nested-looking patterns are handled."""
        from local_deep_research.utilities.search_utilities import (
            remove_think_tags,
        )

        # The regex is greedy, so this becomes empty
        text = "<think>outer <think>inner</think> still outer</think>"
        result = remove_think_tags(text)
        # The first <think> matches with first </think>
        # Then orphaned tags are removed
        assert "<think>" not in result
        assert "</think>" not in result


class TestExtractLinksFromSearchResults:
    """Tests for extracting links from search results."""

    def test_extracts_title_and_url(self):
        """Extracts title and URL from results."""
        from local_deep_research.utilities.search_utilities import (
            extract_links_from_search_results,
        )

        results = [{"title": "Test Page", "link": "https://example.com"}]
        links = extract_links_from_search_results(results)
        assert len(links) == 1
        assert links[0]["title"] == "Test Page"
        assert links[0]["url"] == "https://example.com"

    def test_extracts_index(self):
        """Extracts index from results."""
        from local_deep_research.utilities.search_utilities import (
            extract_links_from_search_results,
        )

        results = [
            {"title": "Test", "link": "https://example.com", "index": "1"}
        ]
        links = extract_links_from_search_results(results)
        assert links[0]["index"] == "1"

    def test_strips_whitespace(self):
        """Strips whitespace from title and URL."""
        from local_deep_research.utilities.search_utilities import (
            extract_links_from_search_results,
        )

        results = [{"title": "  Test  ", "link": "  https://example.com  "}]
        links = extract_links_from_search_results(results)
        assert links[0]["title"] == "Test"
        assert links[0]["url"] == "https://example.com"

    def test_skips_empty_title(self):
        """Skips results with empty title."""
        from local_deep_research.utilities.search_utilities import (
            extract_links_from_search_results,
        )

        results = [{"title": "", "link": "https://example.com"}]
        links = extract_links_from_search_results(results)
        assert len(links) == 0

    def test_skips_empty_url(self):
        """Skips results with empty URL."""
        from local_deep_research.utilities.search_utilities import (
            extract_links_from_search_results,
        )

        results = [{"title": "Test", "link": ""}]
        links = extract_links_from_search_results(results)
        assert len(links) == 0

    def test_skips_none_title(self):
        """Skips results with None title."""
        from local_deep_research.utilities.search_utilities import (
            extract_links_from_search_results,
        )

        results = [{"title": None, "link": "https://example.com"}]
        links = extract_links_from_search_results(results)
        assert len(links) == 0

    def test_skips_none_url(self):
        """Skips results with None URL."""
        from local_deep_research.utilities.search_utilities import (
            extract_links_from_search_results,
        )

        results = [{"title": "Test", "link": None}]
        links = extract_links_from_search_results(results)
        assert len(links) == 0

    def test_handles_empty_list(self):
        """Empty list returns empty list."""
        from local_deep_research.utilities.search_utilities import (
            extract_links_from_search_results,
        )

        links = extract_links_from_search_results([])
        assert links == []

    def test_handles_none_input(self):
        """None input returns empty list."""
        from local_deep_research.utilities.search_utilities import (
            extract_links_from_search_results,
        )

        links = extract_links_from_search_results(None)
        assert links == []

    def test_extracts_multiple_results(self):
        """Extracts multiple results."""
        from local_deep_research.utilities.search_utilities import (
            extract_links_from_search_results,
        )

        results = [
            {"title": "Page 1", "link": "https://example.com/1"},
            {"title": "Page 2", "link": "https://example.com/2"},
        ]
        links = extract_links_from_search_results(results)
        assert len(links) == 2

    def test_handles_missing_index(self):
        """Handles missing index field."""
        from local_deep_research.utilities.search_utilities import (
            extract_links_from_search_results,
        )

        results = [{"title": "Test", "link": "https://example.com"}]
        links = extract_links_from_search_results(results)
        assert links[0]["index"] == ""

    def test_continues_after_error(self):
        """Continues processing after error in one result."""
        from local_deep_research.utilities.search_utilities import (
            extract_links_from_search_results,
        )

        # Include an invalid result that might cause an error
        results = [
            {"title": "Valid", "link": "https://example.com"},
            {"not_a_title": "bad"},  # Missing expected keys
        ]
        links = extract_links_from_search_results(results)
        # Should still get the valid result
        assert len(links) >= 1


class TestFormatLinksToMarkdown:
    """Tests for formatting links to markdown."""

    def test_formats_single_link(self):
        """Formats single link to markdown."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [{"title": "Test", "url": "https://example.com", "index": "1"}]
        result = format_links_to_markdown(links)
        assert "Test" in result
        assert "https://example.com" in result

    def test_includes_index(self):
        """Includes index in formatted output."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [{"title": "Test", "url": "https://example.com", "index": "1"}]
        result = format_links_to_markdown(links)
        assert "[1]" in result

    def test_deduplicates_urls(self):
        """Deduplicates URLs in output."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {"title": "Test", "url": "https://example.com", "index": "1"},
            {"title": "Test", "url": "https://example.com", "index": "2"},
        ]
        result = format_links_to_markdown(links)
        # URL should appear only once
        assert result.count("https://example.com") == 1

    def test_combines_indices_for_duplicate_url(self):
        """Combines indices for duplicate URLs."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {"title": "Test", "url": "https://example.com", "index": "1"},
            {"title": "Test", "url": "https://example.com", "index": "2"},
        ]
        result = format_links_to_markdown(links)
        # Both indices should be mentioned
        assert "1" in result
        assert "2" in result

    def test_handles_empty_list(self):
        """Empty list returns empty string."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        result = format_links_to_markdown([])
        assert result == ""

    def test_handles_link_key_instead_of_url(self):
        """Handles 'link' key instead of 'url'."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [{"title": "Test", "link": "https://example.com", "index": "1"}]
        result = format_links_to_markdown(links)
        assert "https://example.com" in result

    def test_uses_untitled_for_missing_title(self):
        """Uses 'Untitled' for missing title."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [{"url": "https://example.com", "index": "1"}]
        result = format_links_to_markdown(links)
        assert "Untitled" in result

    def test_formats_multiple_links(self):
        """Formats multiple links."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {"title": "Page 1", "url": "https://example.com/1", "index": "1"},
            {"title": "Page 2", "url": "https://example.com/2", "index": "2"},
        ]
        result = format_links_to_markdown(links)
        assert "Page 1" in result
        assert "Page 2" in result


class TestLanguageCodeMap:
    """Tests for the language code mapping."""

    def test_english_maps_to_en(self):
        """English maps to 'en'."""
        from local_deep_research.utilities.search_utilities import (
            LANGUAGE_CODE_MAP,
        )

        assert LANGUAGE_CODE_MAP["english"] == "en"

    def test_french_maps_to_fr(self):
        """French maps to 'fr'."""
        from local_deep_research.utilities.search_utilities import (
            LANGUAGE_CODE_MAP,
        )

        assert LANGUAGE_CODE_MAP["french"] == "fr"

    def test_german_maps_to_de(self):
        """German maps to 'de'."""
        from local_deep_research.utilities.search_utilities import (
            LANGUAGE_CODE_MAP,
        )

        assert LANGUAGE_CODE_MAP["german"] == "de"

    def test_spanish_maps_to_es(self):
        """Spanish maps to 'es'."""
        from local_deep_research.utilities.search_utilities import (
            LANGUAGE_CODE_MAP,
        )

        assert LANGUAGE_CODE_MAP["spanish"] == "es"

    def test_italian_maps_to_it(self):
        """Italian maps to 'it'."""
        from local_deep_research.utilities.search_utilities import (
            LANGUAGE_CODE_MAP,
        )

        assert LANGUAGE_CODE_MAP["italian"] == "it"

    def test_japanese_maps_to_ja(self):
        """Japanese maps to 'ja'."""
        from local_deep_research.utilities.search_utilities import (
            LANGUAGE_CODE_MAP,
        )

        assert LANGUAGE_CODE_MAP["japanese"] == "ja"

    def test_chinese_maps_to_zh(self):
        """Chinese maps to 'zh'."""
        from local_deep_research.utilities.search_utilities import (
            LANGUAGE_CODE_MAP,
        )

        assert LANGUAGE_CODE_MAP["chinese"] == "zh"


class TestFormatFindings:
    """Tests for formatting findings."""

    def test_includes_synthesized_content(self):
        """Includes synthesized content in output."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        result = format_findings([], "This is synthesized content.", {})
        assert "This is synthesized content." in result

    def test_formats_questions_by_iteration(self):
        """Formats questions by iteration."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        questions = {1: ["Question 1", "Question 2"]}
        result = format_findings([], "Content", questions)
        assert "SEARCH QUESTIONS BY ITERATION" in result
        assert "Iteration 1" in result
        assert "Question 1" in result
        assert "Question 2" in result

    def test_formats_detailed_findings(self):
        """Formats detailed findings."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        findings = [
            {"phase": "Initial Research", "content": "Finding content here"}
        ]
        result = format_findings(findings, "Summary", {})
        assert "DETAILED FINDINGS" in result
        assert "Initial Research" in result
        assert "Finding content here" in result

    def test_extracts_sources_from_findings(self):
        """Extracts sources from findings."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        findings = [
            {
                "phase": "Research",
                "content": "Content",
                "search_results": [
                    {
                        "title": "Source",
                        "link": "https://source.com",
                        "index": "1",
                    }
                ],
            }
        ]
        result = format_findings(findings, "Summary", {})
        assert "Source" in result or "source.com" in result

    def test_handles_empty_findings(self):
        """Handles empty findings list."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        result = format_findings([], "Summary content", {})
        assert "Summary content" in result
        # Should not have detailed findings section
        assert (
            "DETAILED FINDINGS" not in result
            or result.count("DETAILED FINDINGS") == 0
        )

    def test_handles_empty_questions(self):
        """Handles empty questions dictionary."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        result = format_findings([], "Content", {})
        # Should not have questions section
        assert result.count("SEARCH QUESTIONS BY ITERATION") == 0

    def test_handles_followup_phase(self):
        """Handles follow-up phase with question matching."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        findings = [{"phase": "Follow-up Iteration 1.1", "content": "Answer"}]
        questions = {1: ["First question"]}
        result = format_findings(findings, "Summary", questions)
        assert "Follow-up" in result or "First question" in result

    def test_handles_subquery_phase(self):
        """Handles sub-query phase from IterDRAG."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        findings = [{"phase": "Sub-query 1", "content": "Answer"}]
        questions = {0: ["Sub question 1"]}
        result = format_findings(findings, "Summary", questions)
        assert "Sub-query" in result or "Sub question" in result

    def test_includes_question_from_finding(self):
        """Includes question field from finding itself."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        findings = [
            {"phase": "Research", "content": "Answer", "question": "What is X?"}
        ]
        result = format_findings(findings, "Summary", {})
        assert "What is X?" in result

    def test_adds_separator_between_findings(self):
        """Adds separator between findings."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        findings = [
            {"phase": "Phase 1", "content": "Content 1"},
            {"phase": "Phase 2", "content": "Content 2"},
        ]
        result = format_findings(findings, "Summary", {})
        # Should have underscores as separator
        assert "_" * 10 in result  # At least some underscores

    def test_handles_missing_phase(self):
        """Handles finding with missing phase."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        findings = [{"content": "Content without phase"}]
        result = format_findings(findings, "Summary", {})
        assert "Unknown Phase" in result

    def test_handles_missing_content(self):
        """Handles finding with missing content."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        findings = [{"phase": "Research"}]
        result = format_findings(findings, "Summary", {})
        assert "No content available" in result