learningcircuit--local-deep-research/tests/web_search_engines/test_engine_user_agents.py

# allow: no-sut-import — guardian; sweeps engine source files to enforce the centralized USER_AGENT constant
"""Regression tests for the User-Agent invariant across search engines.

Follow-up to issue #4130: nine search-engine modules used to hand-roll
their own User-Agent strings (mostly ``"LocalDeepResearch/1.0 (...)"``
with the version pinned to 1.0, or ad-hoc ``"Local-Deep-Research-Agent"``)
instead of referencing the centralized ``USER_AGENT`` constant. The
strings drifted over time and never carried the actual project version.

This test sweeps the engine source files and asserts that any
``"User-Agent": ...`` header set against the canonical ``USER_AGENT``
constant rather than a literal string, so future copy-pastes can't
silently re-introduce stale version numbers.

Engines that are deliberately NOT included here:
  * ``search_engine_searxng.py`` — sends a full browser UA + Referer +
    Upgrade-Insecure-Requests on purpose, because some public SearXNG
    instances rate-limit non-browser scrapers. Touching this risks
    breaking real users.
  * ``search_engine_wayback.py`` — uses a hybrid bot-identifying UA
    (``"Mozilla/5.0 (Local Deep Research Bot; research project)"``)
    that is honest, just not the canonical constant.
"""

import re
from pathlib import Path

import pytest


SRC_ROOT = (
    Path(__file__).resolve().parents[2]
    / "src"
    / "local_deep_research"
    / "web_search_engines"
    / "engines"
)

# Engines that are required to use ``USER_AGENT`` (with or without a
# polite-pool suffix). See module docstring for the rationale behind
# the SearXNG/Wayback exclusions.
CANONICAL_UA_ENGINES = [
    "search_engine_github.py",
    "search_engine_gutenberg.py",
    "search_engine_stackexchange.py",
    "search_engine_wikinews.py",
    "search_engine_pubchem.py",
    "search_engine_openlibrary.py",
    "search_engine_nasa_ads.py",
    "search_engine_zenodo.py",
    "search_engine_openalex.py",
]


def _source(filename: str) -> str:
    path = SRC_ROOT / filename
    assert path.is_file(), f"engine source not found: {path}"
    return path.read_text(encoding="utf-8")


# The pre-fix strings that must never reappear. These are the exact
# literals that lived in the source before the #4130 follow-up. A more
# general pattern is enforced separately by
# ``test_no_string_literal_user_agent``.
_LEGACY_LITERALS = (
    "LocalDeepResearch/1.0 (https://github.com/LearningCircuit/local-deep-research)",
    "Local-Deep-Research-Agent",
    "local-deep-research-wikinews-search-engine",
)


@pytest.mark.parametrize("filename", CANONICAL_UA_ENGINES)
def test_no_legacy_user_agent_string(filename):
    """No engine should hand-roll one of the pre-fix UA literals."""
    src = _source(filename)
    for legacy in _LEGACY_LITERALS:
        assert legacy not in src, (
            f"{filename} still contains the legacy UA literal {legacy!r}. "
            "Use USER_AGENT from local_deep_research.constants instead."
        )


@pytest.mark.parametrize("filename", CANONICAL_UA_ENGINES)
def test_user_agent_constant_is_imported(filename):
    """USER_AGENT must be imported from the central constants module."""
    src = _source(filename)
    assert re.search(
        r"from\s+\.{2,3}constants\s+import\s+[^\n]*\bUSER_AGENT\b", src
    ), (
        f"{filename} does not import USER_AGENT from ...constants — the "
        "canonical source of truth for the project User-Agent."
    )


@pytest.mark.parametrize("filename", CANONICAL_UA_ENGINES)
def test_no_string_literal_user_agent(filename):
    """Any ``"User-Agent": ...`` header must reference USER_AGENT, not a
    bare string literal.

    Catches new hand-rolled UAs even if they don't match one of the
    historical legacy literals — e.g. someone bumping the version by
    hand to ``"LocalDeepResearch/2.0 (...)"``.
    """
    src = _source(filename)
    # Find every ``"User-Agent": <value>`` site. The regex stops at the
    # first comma or closing brace so multi-key dicts work.
    matches = re.findall(r'"User-Agent"\s*:\s*([^,\n}]+?)(?=[,\n}])', src)
    assert matches, (
        f"Sanity check: expected at least one User-Agent header in "
        f"{filename}, found none — the test is checking the wrong thing."
    )
    for value in matches:
        stripped = value.strip()
        # Accept either the bare constant, or an f-string composition
        # involving the constant (used for polite-pool email suffixing
        # in OpenAlex).
        assert "USER_AGENT" in stripped, (
            f"{filename} sets User-Agent to {stripped!r}, which does not "
            "reference the USER_AGENT constant. Use USER_AGENT (or a "
            "composition like f'{USER_AGENT} ({email})') instead."
        )