learningcircuit--local-deep-research/tests/utilities/test_search_utilities_formatting.py

"""
Tests for search_utilities.py — format_links_to_markdown and format_findings
edge cases not covered by the existing test_search_utilities.py and
test_search_utilities_extended.py files.

Tests cover:
- format_links_to_markdown: None/empty URL handling, special characters, index
  aggregation for 'link' key fallback
- format_findings: iteration numbering, question_by_iteration edge cases,
  findings with search results that have errors, malformed finding structures
"""


# ---------------------------------------------------------------------------
# format_links_to_markdown — additional edge cases
# ---------------------------------------------------------------------------


class TestFormatLinksToMarkdownNoneUrls:
    """Tests for format_links_to_markdown with None/empty URLs."""

    def test_none_url_skipped(self):
        """Links with None url are skipped."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {"title": "Valid", "url": "https://valid.com", "index": "1"},
            {"title": "None URL", "url": None, "index": "2"},
        ]

        result = format_links_to_markdown(links)

        assert "https://valid.com" in result
        assert "None URL" not in result

    def test_empty_string_url_skipped(self):
        """Links with empty string url are skipped."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {"title": "Empty URL", "url": "", "index": "1"},
            {"title": "Good", "url": "https://good.com", "index": "2"},
        ]

        result = format_links_to_markdown(links)

        assert "https://good.com" in result
        assert "Empty URL" not in result

    def test_no_url_or_link_key_skipped(self):
        """Links with neither 'url' nor 'link' key are skipped."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {"title": "No URL key", "index": "1"},
        ]

        result = format_links_to_markdown(links)

        assert "No URL key" not in result


class TestFormatLinksToMarkdownSpecialChars:
    """Tests for format_links_to_markdown with special characters."""

    def test_url_with_query_params(self):
        """URLs with query parameters are preserved."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {
                "title": "Search",
                "url": "https://example.com/search?q=hello+world&lang=en",
                "index": "1",
            }
        ]

        result = format_links_to_markdown(links)
        assert "https://example.com/search?q=hello+world&lang=en" in result

    def test_url_with_fragment(self):
        """URLs with fragments are stripped in the Sources display.

        Fragments are anchor-only and have no effect on click-through
        landing pages; the canonical form drops them for a cleaner UI.
        """
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {
                "title": "Section",
                "url": "https://example.com/page#section-2",
                "index": "1",
            }
        ]

        result = format_links_to_markdown(links)
        assert "URL: https://example.com/page" in result
        assert "#section-2" not in result

    def test_title_with_special_markdown_chars(self):
        """Titles with markdown-special chars are included."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {
                "title": "Results [2024] — *Important*",
                "url": "https://example.com",
                "index": "1",
            }
        ]

        result = format_links_to_markdown(links)
        assert "Results [2024]" in result


class TestFormatLinksToMarkdownIndexAggregation:
    """Tests for index aggregation in format_links_to_markdown."""

    def test_link_key_fallback_aggregates_indices(self):
        """'link' key fallback still aggregates indices for same URL."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {"title": "Same", "link": "https://same.com", "index": "1"},
            {"title": "Same", "link": "https://same.com", "index": "4"},
        ]

        result = format_links_to_markdown(links)
        assert result.count("https://same.com") == 1
        assert "1" in result
        assert "4" in result

    def test_mixed_url_and_link_keys(self):
        """Handles mix of 'url' and 'link' keys for same actual URL."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {"title": "First", "url": "https://a.com", "index": "1"},
            {"title": "Second", "link": "https://b.com", "index": "2"},
        ]

        result = format_links_to_markdown(links)
        assert "https://a.com" in result
        assert "https://b.com" in result


# ---------------------------------------------------------------------------
# format_findings — additional edge cases
# ---------------------------------------------------------------------------


class TestFormatFindingsIterationNumbering:
    """Tests for iteration numbering in format_findings."""

    def test_single_iteration_with_questions(self):
        """Single iteration with questions formatted correctly."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        questions = {1: ["Q1", "Q2", "Q3"]}

        result = format_findings([], "Summary content", questions)

        assert "Iteration 1" in result
        assert "1. Q1" in result
        assert "2. Q2" in result
        assert "3. Q3" in result

    def test_non_sequential_iteration_numbers(self):
        """Non-sequential iteration numbers (e.g., 1, 3, 5) handled correctly."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        questions = {
            1: ["First iter Q"],
            3: ["Third iter Q"],
            5: ["Fifth iter Q"],
        }

        result = format_findings([], "Summary", questions)

        assert "Iteration 1" in result
        assert "Iteration 3" in result
        assert "Iteration 5" in result
        assert "Iteration 2" not in result

    def test_zero_iteration_key(self):
        """Iteration 0 (used by IterDRAG) is formatted."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        questions = {0: ["Sub-query A", "Sub-query B"]}

        result = format_findings([], "Summary", questions)

        assert "Iteration 0" in result
        assert "Sub-query A" in result


class TestFormatFindingsMalformedStructures:
    """Tests for malformed finding structures."""

    def test_finding_missing_content_key(self):
        """Finding without 'content' key uses default."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        findings = [{"phase": "Phase 1", "search_results": []}]

        result = format_findings(findings, "Summary", {})

        assert "No content available" in result

    def test_finding_missing_phase_key(self):
        """Finding without 'phase' key uses default."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        findings = [{"content": "Some content", "search_results": []}]

        result = format_findings(findings, "Summary", {})

        assert "Unknown Phase" in result
        assert "Some content" in result

    def test_finding_with_only_question_field(self):
        """Finding with only 'question' field displays it."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        findings = [
            {
                "phase": "Custom",
                "content": "Answer text",
                "question": "How does X work?",
                "search_results": [],
            }
        ]

        result = format_findings(findings, "Summary", {})

        assert "How does X work?" in result
        assert "SEARCH QUESTION" in result

    def test_empty_questions_dict_shows_warning_section(self):
        """Empty questions dict (no iterations) omits section."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        result = format_findings([], "Summary", {})

        assert "SEARCH QUESTIONS BY ITERATION" not in result

    def test_findings_with_search_results_error(self):
        """Findings with non-list search_results handled gracefully."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        findings = [
            {
                "phase": "Search",
                "content": "Content",
                "search_results": None,  # None instead of list
            }
        ]

        # Should not crash
        result = format_findings(findings, "Summary", {})
        assert "Content" in result


# ---------------------------------------------------------------------------
# _format_quality_tag — journal quality tags in source lists
# ---------------------------------------------------------------------------


class TestFormatQualityTag:
    """Test the _format_quality_tag helper for source list quality indicators."""

    def test_none_returns_empty(self):
        from local_deep_research.utilities.search_utilities import (
            _format_quality_tag,
        )

        assert _format_quality_tag(None) == ""

    def test_elite_tier(self):
        from local_deep_research.utilities.search_utilities import (
            _format_quality_tag,
        )

        assert "Q1" in _format_quality_tag(10)
        assert "Q1" in _format_quality_tag(9)

    def test_strong_tier(self):
        from local_deep_research.utilities.search_utilities import (
            _format_quality_tag,
        )

        assert "Q1" in _format_quality_tag(7)
        assert "Q1" in _format_quality_tag(8)

    def test_moderate_tier(self):
        from local_deep_research.utilities.search_utilities import (
            _format_quality_tag,
        )

        assert "Q2" in _format_quality_tag(5)
        assert "Q2" in _format_quality_tag(6)

    def test_default_unknown_tier(self):
        """Score 4 (JOURNAL_QUALITY_DEFAULT) is rendered as [Unranked ★]."""
        from local_deep_research.utilities.search_utilities import (
            _format_quality_tag,
        )

        assert _format_quality_tag(4) == " [Unranked ★]"

    def test_predatory_tier(self):
        """Score 1 (predatory — normally auto-removed) falls back to Q4."""
        from local_deep_research.utilities.search_utilities import (
            _format_quality_tag,
        )

        assert "Q4" in _format_quality_tag(1)

    def test_score_boundary_5_is_q2_not_unranked(self):
        """Score 5 crosses the Q2 threshold; must not fall through to Unranked."""
        from local_deep_research.utilities.search_utilities import (
            _format_quality_tag,
        )

        assert _format_quality_tag(5) == " [Q2 ★★★]"

    def test_score_3_renders_unranked_not_q4(self):
        """Score 3 is the filter's 'no scoring data' fallback — rendered as
        Unranked rather than Q4 so it reads as "we don't know this venue"
        rather than "we know it's low-quality"."""
        from local_deep_research.utilities.search_utilities import (
            _format_quality_tag,
        )

        assert _format_quality_tag(3) == " [Unranked ★]"

    def test_preprint_sentinel(self):
        """QUALITY_PREPRINT marks results with no journal_ref at all."""
        from local_deep_research.utilities.search_utilities import (
            QUALITY_PREPRINT,
            _format_quality_tag,
        )

        tag = _format_quality_tag(QUALITY_PREPRINT)
        assert "preprint" in tag
        assert "not in journal catalog" in tag

    def test_pending_sentinel(self):
        from local_deep_research.utilities.search_utilities import (
            QUALITY_PENDING,
            _format_quality_tag,
        )

        assert "downloading" in _format_quality_tag(QUALITY_PENDING)

    def test_out_of_range_value_surfaces_raw(self):
        """Out-of-set inputs must not silently bucket into Q4 — the
        catch-all renders ``[quality=<value>]`` so bad scoring logic is
        visible in the output.
        """
        from local_deep_research.utilities.search_utilities import (
            _format_quality_tag,
        )

        tag = _format_quality_tag(99)
        assert "quality=" in tag
        assert "99" in tag

    def test_every_valid_score_has_explicit_branch(self):
        """Every value in VALID_QUALITY_SCORES must map to a real tier
        tag — none should fall through to the debug catch-all.
        """
        from local_deep_research.constants import VALID_QUALITY_SCORES
        from local_deep_research.utilities.search_utilities import (
            _format_quality_tag,
        )

        for score in VALID_QUALITY_SCORES:
            tag = _format_quality_tag(score)
            assert "quality=" not in tag, (
                f"score {score} fell through to the debug catch-all: {tag!r}"
            )


# ---------------------------------------------------------------------------
# format_links_to_markdown — dedup via canonical URL key
# ---------------------------------------------------------------------------


class TestFormatLinksDedupCanonical:
    """Canonicalized dedup: slight URL variants collapse to one Sources entry."""

    def setup_method(self):
        # canonical_url_key is cached across tests; clear so each test starts
        # from a clean slate (otherwise earlier test inputs can pollute the
        # expected-output behavior if we ever depend on cache-miss timing).
        from local_deep_research.utilities.url_utils import canonical_url_key

        canonical_url_key.cache_clear()

    def test_dedup_trailing_slash(self):
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {"title": "Page", "url": "https://x.com/p", "index": "1"},
            {"title": "Page", "url": "https://x.com/p/", "index": "2"},
        ]
        result = format_links_to_markdown(links)
        # One entry, two indices.
        assert result.count("URL: https://x.com/p") == 1
        assert "[1, 2]" in result

    def test_dedup_tracking_params(self):
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {
                "title": "Vox",
                "url": "https://www.vox.com/a?utm_source=g&utm_medium=s",
                "index": "1",
            },
            {"title": "Vox", "url": "https://www.vox.com/a", "index": "2"},
        ]
        result = format_links_to_markdown(links)
        assert "[1, 2]" in result
        # Canonical (clean) URL is displayed — tracking params stripped.
        assert "utm_source" not in result
        assert "utm_medium" not in result
        assert "URL: https://www.vox.com/a" in result

    def test_dedup_fragment_variants(self):
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {"title": "Wiki", "url": "https://x.com/p", "index": "1"},
            {
                "title": "Wiki",
                "url": "https://x.com/p#section",
                "index": "2",
            },
        ]
        result = format_links_to_markdown(links)
        assert "[1, 2]" in result

    def test_dedup_default_port(self):
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {"title": "Site", "url": "https://x.com/p", "index": "1"},
            {"title": "Site", "url": "https://x.com:443/p", "index": "2"},
        ]
        result = format_links_to_markdown(links)
        assert "[1, 2]" in result

    def test_distinct_urls_stay_distinct(self):
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {"title": "A", "url": "https://x.com/a", "index": "1"},
            {"title": "B", "url": "https://x.com/b", "index": "2"},
        ]
        result = format_links_to_markdown(links)
        assert "https://x.com/a" in result
        assert "https://x.com/b" in result
        assert "[1, 2]" not in result

    def test_display_shows_canonical_url_not_raw(self):
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        # Even when the utm-bearing variant comes first, the Sources
        # section shows the canonical (tracking-stripped) URL — cleaner
        # UI, identical click-through behavior.
        links = [
            {
                "title": "Article",
                "url": "https://x.com/a?utm_source=g",
                "index": "1",
            },
            {"title": "Article", "url": "https://x.com/a", "index": "2"},
        ]
        result = format_links_to_markdown(links)
        assert "URL: https://x.com/a\n" in result
        assert "utm_source" not in result

    def test_display_strips_userinfo(self):
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        # Userinfo in a URL would leak credentials if displayed. Canonical
        # form drops it, so the Sources section is always safe.
        links = [
            {
                "title": "Internal",
                "url": "https://user:secret@internal.example.com/p",
                "index": "1",
            }
        ]
        result = format_links_to_markdown(links)
        assert "secret" not in result
        assert "user:" not in result
        assert "URL: https://internal.example.com/p" in result

    def test_ref_param_not_stripped(self):
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        # ?ref= on GitHub is a branch selector, not a tracker.
        links = [
            {
                "title": "Repo on main",
                "url": "https://github.com/o/r?ref=main",
                "index": "1",
            },
            {
                "title": "Repo on dev",
                "url": "https://github.com/o/r?ref=dev",
                "index": "2",
            },
        ]
        result = format_links_to_markdown(links)
        # Must remain two separate entries.
        assert "ref=main" in result
        assert "ref=dev" in result
        assert "[1, 2]" not in result


# ---------------------------------------------------------------------------
# format_findings — exact character-by-character output
# ---------------------------------------------------------------------------


class TestFormatFindingsExactOutput:
    """Character-by-character tests of the markdown produced by
    ``format_findings`` for predetermined inputs. Locks in the exact
    layout (indentation, blank lines, ``[1]`` citation markers, the
    80-char ``_`` separator) so a renderer regression — even a single
    extra newline or a moved citation — fails loudly.
    """

    def test_exact_output_minimal_findings(self):
        """One finding, one source, one iteration, one question."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        findings_list = [
            {
                "phase": "Initial",
                "content": "Qubits enable quantum computing [1].",
                "search_results": [
                    {
                        "title": "Qubit Basics",
                        "link": "https://example.com/q",
                        "index": "1",
                    },
                ],
            },
        ]
        synthesized_content = "Quantum computing uses qubits [1]."
        questions_by_iteration = {1: ["What is a qubit?"]}

        # Expected output is the verbatim string format_findings produces
        # for the inputs above. Anchored to today's exact layout — any
        # change (added blank line, moved citation, renamed section
        # header) will fail this test, which is the point.
        expected = (
            "Quantum computing uses qubits [1].\n"
            "\n"
            "[1] Qubit Basics (source nr: 1)\n"
            "   URL: https://example.com/q\n"
            "\n"
            "\n"
            "\n"
            "\n"
            "## SEARCH QUESTIONS BY ITERATION\n"
            "\n"
            "\n"
            " #### Iteration 1:\n"
            "1. What is a qubit?\n"
            "\n"
            "\n"
            "\n"
            "## DETAILED FINDINGS\n"
            "\n"
            "\n"
            "### Initial\n"
            "\n"
            "\n"
            "\n"
            "\n"
            "Qubits enable quantum computing [1].\n"
            "\n"
            "### SOURCES USED IN THIS SECTION:\n"
            "[1] Qubit Basics (source nr: 1)\n"
            "   URL: https://example.com/q\n"
            "\n"
            "\n"
            "\n"
            "\n" + ("_" * 80) + "\n"
            "\n"
            "## ALL SOURCES:\n"
            "[1] Qubit Basics (source nr: 1)\n"
            "   URL: https://example.com/q\n"
            "\n"
            "\n"
        )

        actual = format_findings(
            findings_list, synthesized_content, questions_by_iteration
        )
        assert actual == expected, (
            f"format_findings output drifted from the locked-in layout.\n"
            f"--- expected ---\n{expected!r}\n"
            f"--- actual ---\n{actual!r}"
        )

    def test_citations_in_synthesized_content_preserved(self):
        """``[1]`` and ``[2]`` markers in the synthesized content reach
        the final output unchanged. Independent narrower assertion to
        catch citation-handling regressions even when the broader exact-
        match test is in flux for unrelated layout reasons."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        synthesized = "First fact [1]. Second fact [2]. Third fact [1]."
        result = format_findings(
            findings_list=[],
            synthesized_content=synthesized,
            questions_by_iteration={},
        )
        # The synthesized content is the very first thing written.
        assert result.startswith(synthesized + "\n\n"), (
            f"Synthesized content (with citations) was rewritten or "
            f"prefixed by format_findings: result starts with "
            f"{result[: len(synthesized) + 20]!r}"
        )
        assert result.count("[1]") == 2
        assert result.count("[2]") == 1

    def test_source_indices_match_citation_markers(self):
        """When a finding's search_results have indices ``"1"`` and
        ``"2"``, format_links_to_markdown emits ``[1]`` and ``[2]`` —
        these should align with the citation markers in the synthesized
        content. Tests the citation→source alignment that makes the
        markdown navigable."""
        from local_deep_research.utilities.search_utilities import (
            format_findings,
        )

        findings_list = [
            {
                "phase": "Initial",
                "content": "Body referencing [1] and [2].",
                "search_results": [
                    {
                        "title": "Source One",
                        "link": "https://example.com/one",
                        "index": "1",
                    },
                    {
                        "title": "Source Two",
                        "link": "https://example.com/two",
                        "index": "2",
                    },
                ],
            },
        ]
        result = format_findings(
            findings_list,
            synthesized_content="See [1] and [2].",
            questions_by_iteration={},
        )
        # Both [1] and [2] appear in source listings, attached to the
        # right titles and URLs.
        assert "[1] Source One" in result
        assert "https://example.com/one" in result
        assert "[2] Source Two" in result
        assert "https://example.com/two" in result
        # And the synthesized [1]/[2] citations at the top survive.
        assert result.startswith("See [1] and [2].\n\n")


class TestFormatLinksToMarkdownCollections:
    """Tests for the optional ``Collection:`` line surfaced for
    RAG / library results so the source-tagged citation mode can read
    the collection name back from the rendered sources block."""

    def test_emits_collection_line_when_metadata_present(self):
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {
                "title": "Local Paper",
                "url": "/library/document/abc123",
                "index": "1",
                "metadata": {"collection_name": "My Papers"},
            },
        ]
        result = format_links_to_markdown(links)
        assert "URL: /library/document/abc123" in result
        assert "Collection: My Papers" in result

    def test_no_collection_line_when_metadata_absent(self):
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        # Plain web hit, no metadata.
        links = [
            {
                "title": "Web Page",
                "url": "https://example.com/page",
                "index": "1",
            },
        ]
        result = format_links_to_markdown(links)
        assert "URL: https://example.com/page" in result
        assert "Collection:" not in result

    def test_no_collection_line_when_metadata_lacks_collection_name(self):
        """metadata may exist for other reasons (engine_name, score, etc.)
        without carrying a collection name. Don't emit the line then."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {
                "title": "Web Page",
                "url": "https://example.com/page",
                "index": "1",
                "metadata": {"engine_name": "web", "score": 0.91},
            },
        ]
        result = format_links_to_markdown(links)
        assert "Collection:" not in result

    def test_first_non_empty_collection_wins_per_url(self):
        """Two hits for the same canonical URL — the first source with a
        non-empty collection name sets it; later hits don't overwrite.
        Mirrors how title and journal_quality work."""
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )

        links = [
            {
                "title": "Doc",
                "url": "/library/document/abc",
                "index": "1",
                "metadata": {"collection_name": "first"},
            },
            {
                "title": "Doc",
                "url": "/library/document/abc",
                "index": "2",
                "metadata": {"collection_name": "second"},
            },
        ]
        result = format_links_to_markdown(links)
        # Single entry (URLs deduped), carries the first collection.
        assert "Collection: first" in result
        assert "Collection: second" not in result