learningcircuit--local-deep-research/tests/performance/content_fetcher/test_extraction_benchmark.py

"""Benchmark tests for HTML content extraction pipeline.

Tests extraction quality across 200+ real-world pages from diverse domains.
Skipped in CI (requires network). Run manually with:

    pytest tests/research_library/downloaders/test_extraction_benchmark.py -v -s

Results are printed as a comparison table with content length, boilerplate
count, and timing for each downloader mode.
"""

import time
from dataclasses import dataclass
from typing import Dict, List, Tuple

import pytest

from local_deep_research.content_fetcher import ContentFetcher
from local_deep_research.research_library.downloaders.html import HTMLDownloader
from local_deep_research.research_library.downloaders.playwright_html import (
    AutoHTMLDownloader,
)


# Skip entire module in CI — these hit the network and need a browser.
# `integration` is the marker CI excludes via `-m 'not integration'`; `slow` is
# kept so the tests can be selected/deselected independently when run locally.
pytestmark = [pytest.mark.slow, pytest.mark.integration]


BOILERPLATE_KEYWORDS = [
    "cookie",
    "sign up",
    "newsletter",
    "skip to content",
    "subscribe",
    "accept all",
    "privacy policy",
    "terms of service",
    "log in",
    "sign in",
    "add to cart",
    "checkout",
    "wishlist",
]

# ---------------------------------------------------------------------------
# URL corpus — 200+ pages across 17 categories
# Each entry: (label, url)
# Grouped by expected fetch behaviour: static-friendly vs JS-heavy
# ---------------------------------------------------------------------------

NEWS = [
    ("BBC front", "https://www.bbc.com/news"),
    ("Reuters", "https://www.reuters.com/"),
    ("NYTimes", "https://www.nytimes.com/"),
    ("Guardian", "https://www.theguardian.com/international"),
    ("CNN", "https://www.cnn.com/"),
    ("AP News", "https://apnews.com/"),
    ("Al Jazeera", "https://www.aljazeera.com/"),
    ("DW", "https://www.dw.com/en/"),
    ("France24", "https://www.france24.com/en/"),
    ("NBC News", "https://www.nbcnews.com/"),
    ("USA Today", "https://www.usatoday.com/"),
    ("Forbes", "https://www.forbes.com/"),
    ("Bloomberg", "https://www.bloomberg.com/"),
    ("Politico", "https://www.politico.com/"),
    ("HuffPost", "https://www.huffpost.com/"),
    ("ABC News", "https://abcnews.go.com/"),
    ("Time", "https://time.com/"),
    ("NPR", "https://www.npr.org/"),
    ("BBC Tech", "https://www.bbc.com/news/technology"),
    ("Ars Technica", "https://arstechnica.com/"),
]

TECH = [
    ("GitHub README", "https://github.com/AmiGandhi/WordPredict"),
    ("GH LDR", "https://github.com/LearningCircuit/local-deep-research"),
    (
        "GH Issue",
        "https://github.com/LearningCircuit/local-deep-research/issues/1",
    ),
    (
        "StackOverflow",
        "https://stackoverflow.com/questions/231767/what-does-the-yield-keyword-do-in-python",
    ),
    ("SO tagged", "https://stackoverflow.com/questions/tagged/python"),
    ("HackerNews", "https://news.ycombinator.com/item?id=40956541"),
    ("HN front", "https://news.ycombinator.com/"),
    ("TechCrunch", "https://techcrunch.com/"),
    ("Wired", "https://www.wired.com/"),
    ("The Verge", "https://www.theverge.com/"),
    ("ZDNet", "https://www.zdnet.com/"),
    ("Slashdot", "https://slashdot.org/"),
    ("Dev.to", "https://dev.to/"),
    ("Lobsters", "https://lobste.rs/"),
    ("InfoQ", "https://www.infoq.com/"),
]

REFERENCE = [
    ("Wiki EN", "https://en.wikipedia.org/wiki/Python_(programming_language)"),
    ("Wiki JA", "https://ja.wikipedia.org/wiki/Python"),
    ("Wiki DE", "https://de.wikipedia.org/wiki/Python_(Programmiersprache)"),
    ("Wiki FR", "https://fr.wikipedia.org/wiki/Intelligence_artificielle"),
    ("Wiki ZH", "https://zh.wikipedia.org/wiki/Python"),
    (
        "Wiki AR",
        "https://ar.wikipedia.org/wiki/%D8%A8%D8%A7%D9%8A%D8%AB%D9%88%D9%86",
    ),
    ("Wiki ES", "https://es.wikipedia.org/wiki/Python"),
    (
        "Britannica",
        "https://www.britannica.com/technology/artificial-intelligence",
    ),
    ("M-W Dict", "https://www.merriam-webster.com/dictionary/algorithm"),
    ("W3Schools", "https://www.w3schools.com/python/python_lists.asp"),
]

DOCS = [
    ("Python docs", "https://docs.python.org/3/library/asyncio.html"),
    (
        "MDN Docs",
        "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Functions",
    ),
    (
        "ReadTheDocs",
        "https://requests.readthedocs.io/en/latest/user/quickstart/",
    ),
    (
        "Rust Book",
        "https://doc.rust-lang.org/book/ch04-01-what-is-ownership.html",
    ),
    ("Go Docs", "https://go.dev/doc/effective_go"),
    ("Django Docs", "https://docs.djangoproject.com/en/5.0/topics/http/views/"),
    ("React Docs", "https://react.dev/learn"),
    ("Vue Docs", "https://vuejs.org/guide/introduction.html"),
    (
        "MS Learn",
        "https://learn.microsoft.com/en-us/dotnet/csharp/tour-of-csharp/",
    ),
    ("Kubernetes", "https://kubernetes.io/docs/concepts/overview/"),
]

ACADEMIC = [
    ("ArXiv", "https://arxiv.org/abs/2301.07507"),
    ("ArXiv ICL", "https://arxiv.org/abs/2301.00234"),
    ("ArXiv LLM", "https://arxiv.org/abs/2303.08774"),
    ("ArXiv HTML", "https://arxiv.org/html/2301.07507"),
    ("PubMed", "https://pubmed.ncbi.nlm.nih.gov/37828879/"),
    ("PubMed CRISPR", "https://pubmed.ncbi.nlm.nih.gov/26553966/"),
    ("PubMed COVID", "https://pubmed.ncbi.nlm.nih.gov/32015507/"),
    ("PMC", "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7095418/"),
    (
        "SemScholar",
        "https://www.semanticscholar.org/paper/Attention-Is-All-You-Need-Vaswani-Shazeer/204e3073870fae3d05bcbc2f6a8e263d9b72e776",
    ),
    (
        "SemScholar2",
        "https://www.semanticscholar.org/paper/BERT%3A-Pre-training-of-Deep-Bidirectional-Devlin-Chang/df2b0e26d0599ce3e70df8a9da02e51594e0e992",
    ),
    ("OpenAlex", "https://openalex.org/works/W2741809807"),
    ("OpenAlex2", "https://openalex.org/works/W2963403868"),
    ("Nature", "https://www.nature.com/articles/s41586-024-07487-w"),
    ("Science", "https://www.science.org/"),
    ("ResearchGate", "https://www.researchgate.net/"),
    ("Springer", "https://link.springer.com/"),
    ("IEEE", "https://www.ieee.org/"),
    ("JSTOR", "https://www.jstor.org/"),
    ("ACM DL", "https://dl.acm.org/"),
    ("bioRxiv", "https://www.biorxiv.org/content/10.1101/2024.01.01.573841v1"),
]

GOVERNMENT = [
    ("WhiteHouse", "https://www.whitehouse.gov/"),
    ("NASA", "https://www.nasa.gov/"),
    ("NIH", "https://www.nih.gov/"),
    ("CDC", "https://www.cdc.gov/"),
    ("Europa EU", "https://europa.eu/"),
    ("WHO", "https://www.who.int/"),
    ("UN", "https://www.un.org/en/"),
    ("GOV UK", "https://www.gov.uk/"),
    ("USA.gov", "https://www.usa.gov/"),
    ("Data.gov", "https://data.gov/"),
]

EDUCATION = [
    ("MIT", "https://www.mit.edu/"),
    ("Stanford", "https://www.stanford.edu/"),
    ("Harvard", "https://www.harvard.edu/"),
    ("Berkeley", "https://www.berkeley.edu/"),
    ("Cornell", "https://www.cornell.edu/"),
    ("Oxford", "https://www.ox.ac.uk/"),
    ("Cambridge", "https://www.cam.ac.uk/"),
    ("Coursera", "https://www.coursera.org/"),
    ("Khan Academy", "https://www.khanacademy.org/"),
    ("edX", "https://www.edx.org/"),
]

SHOPPING = [
    ("Amazon", "https://www.amazon.com/"),
    ("eBay", "https://www.ebay.com/"),
    ("Walmart", "https://www.walmart.com/"),
    ("Target", "https://www.target.com/"),
    ("Etsy", "https://www.etsy.com/"),
    ("BestBuy", "https://www.bestbuy.com/"),
    ("Newegg", "https://www.newegg.com/"),
    ("IKEA", "https://www.ikea.com/"),
    ("HomeDepot", "https://www.homedepot.com/"),
    ("AliExpress", "https://www.aliexpress.com/"),
    ("Temu", "https://www.temu.com/"),
    ("Idealo", "https://www.idealo.de/"),
]

SOCIAL = [
    ("Reddit", "https://www.reddit.com/r/Python/"),
    ("Reddit old", "https://old.reddit.com/r/Python/"),
    ("LinkedIn", "https://www.linkedin.com/"),
    ("Pinterest", "https://www.pinterest.com/"),
    ("Quora", "https://www.quora.com/"),
    ("Tumblr", "https://www.tumblr.com/"),
]

ENTERTAINMENT = [
    ("YouTube", "https://www.youtube.com/"),
    ("IMDb", "https://www.imdb.com/title/tt0111161/"),
    ("Rotten Tom", "https://www.rottentomatoes.com/"),
    ("Spotify", "https://open.spotify.com/"),
    ("Goodreads", "https://www.goodreads.com/"),
    ("Letterboxd", "https://letterboxd.com/"),
]

FINANCE = [
    ("Yahoo Fin", "https://finance.yahoo.com/"),
    ("Investing", "https://www.investing.com/"),
    ("MarketWatch", "https://www.marketwatch.com/"),
    ("SeekingAlpha", "https://seekingalpha.com/"),
    ("CoinGecko", "https://www.coingecko.com/"),
    ("Investopedia", "https://www.investopedia.com/"),
]

PACKAGES = [
    ("PyPI justext", "https://pypi.org/project/justext/"),
    ("PyPI trafila", "https://pypi.org/project/trafilatura/"),
    ("npm readab", "https://www.npmjs.com/package/readability"),
    ("npm express", "https://www.npmjs.com/package/express"),
    ("crates serde", "https://crates.io/crates/serde"),
    ("RubyGems rails", "https://rubygems.org/gems/rails"),
    ("Docker Hub", "https://hub.docker.com/"),
]

INTERNATIONAL = [
    ("Baidu", "https://www.baidu.com/"),
    ("Yandex", "https://yandex.ru/"),
    ("Naver", "https://www.naver.com/"),
    ("Rakuten", "https://www.rakuten.co.jp/"),
    ("MercadoLibre", "https://www.mercadolibre.com/"),
    ("Allegro PL", "https://allegro.pl/"),
    ("Bol NL", "https://www.bol.com/"),
    ("Lemonde FR", "https://www.lemonde.fr/"),
    ("Spiegel DE", "https://www.spiegel.de/"),
    ("Corriere IT", "https://www.corriere.it/"),
]

BLOGS = [
    (
        "Medium",
        "https://medium.com/@natassha6789/if-i-had-to-start-learning-data-science-again-how-would-i-do-it-78a02b1b56d2",
    ),
    ("Substack", "https://substack.com/"),
    ("S. Willison", "https://simonwillison.net/2024/Mar/8/gpt-4-barrier/"),
    ("WordPress", "https://wordpress.com/"),
    ("Ghost", "https://ghost.org/"),
    ("Hashnode", "https://hashnode.com/"),
]

HEALTH = [
    ("WebMD", "https://www.webmd.com/"),
    ("Mayo Clinic", "https://www.mayoclinic.org/"),
    ("Healthline", "https://www.healthline.com/"),
    ("MedNews", "https://www.medicalnewstoday.com/"),
    ("Cleveland", "https://my.clevelandclinic.org/"),
]

TRAVEL = [
    ("Booking", "https://www.booking.com/"),
    ("TripAdvisor", "https://www.tripadvisor.com/"),
    ("Airbnb", "https://www.airbnb.com/"),
    ("Expedia", "https://www.expedia.com/"),
    ("Lonely Planet", "https://www.lonelyplanet.com/"),
]

FOOD = [
    ("AllRecipes", "https://www.allrecipes.com/"),
    ("Epicurious", "https://www.epicurious.com/"),
    ("SeriousEats", "https://www.seriouseats.com/"),
    ("Bon Appetit", "https://www.bonappetit.com/"),
    ("Food Network", "https://www.foodnetwork.com/"),
]

MISC = [
    ("Craigslist", "https://www.craigslist.org/"),
    ("Archive.org", "https://archive.org/"),
    ("Wayback", "https://web.archive.org/"),
    ("Pastebin", "https://pastebin.com/"),
    ("Regex101", "https://regex101.com/"),
]


# Combine all categories
ALL_CATEGORIES: Dict[str, List[Tuple[str, str]]] = {
    "news": NEWS,
    "tech": TECH,
    "reference": REFERENCE,
    "docs": DOCS,
    "academic": ACADEMIC,
    "government": GOVERNMENT,
    "education": EDUCATION,
    "shopping": SHOPPING,
    "social": SOCIAL,
    "entertainment": ENTERTAINMENT,
    "finance": FINANCE,
    "packages": PACKAGES,
    "international": INTERNATIONAL,
    "blogs": BLOGS,
    "health": HEALTH,
    "travel": TRAVEL,
    "food": FOOD,
    "misc": MISC,
}

# Flat list of all pages
ALL_PAGES = []
for cat, pages in ALL_CATEGORIES.items():
    for label, url in pages:
        ALL_PAGES.append((cat, label, url))

# Pages expected to work with static fetch (no JS rendering needed)
STATIC_PAGES = []
for cat, label, url in ALL_PAGES:
    if cat not in ("shopping", "social", "entertainment", "travel"):
        STATIC_PAGES.append((label, url))


@dataclass
class FetchResult:
    """Result of a single fetch attempt."""

    category: str = ""
    page: str = ""
    mode: str = ""
    length: int = 0
    boilerplate: int = 0
    time_s: float = 0.0
    sample: str = ""


def _count_boilerplate(data: bytes | None) -> int:
    if not data:
        return 0
    text = data.decode("utf-8", errors="replace").lower()
    return sum(1 for kw in BOILERPLATE_KEYWORDS if kw in text)


def _get_sample(data: bytes | None, n: int = 80) -> str:
    if not data:
        return "(no content)"
    text = data.decode("utf-8", errors="replace")
    idx = text.find("\n\n")
    start = text[idx + 2 :] if idx > 0 else text
    return start[:n].replace("\n", " ").strip()


def _run_downloader(
    dl, category: str, name: str, url: str, mode: str
) -> FetchResult:
    t0 = time.time()
    try:
        data = dl.download(url)
    except Exception:
        data = None
    elapsed = time.time() - t0

    return FetchResult(
        category=category,
        page=name,
        mode=mode,
        length=len(data) if data else 0,
        boilerplate=_count_boilerplate(data),
        time_s=round(elapsed, 2),
        sample=_get_sample(data),
    )


def _print_category_summary(results: list[FetchResult], mode: str):
    """Print per-category success rates."""
    from collections import defaultdict

    by_cat: Dict[str, list[FetchResult]] = defaultdict(list)
    for r in results:
        if r.mode == mode:
            by_cat[r.category].append(r)

    print(f"\n  Per-category breakdown ({mode}):")
    for cat in ALL_CATEGORIES:
        cat_results = by_cat.get(cat, [])
        if not cat_results:
            continue
        success = sum(1 for r in cat_results if r.length > 100)
        total = len(cat_results)
        avg_len = sum(r.length for r in cat_results if r.length > 100) // max(
            success, 1
        )
        avg_bp = sum(r.boilerplate for r in cat_results) / max(total, 1)
        bar = "█" * success + "░" * (total - success)
        print(
            f"    {cat:<15} {bar} {success}/{total}  "
            f"avg_len={avg_len:>6}  avg_bp={avg_bp:.1f}"
        )


class TestExtractionBenchmark:
    """Benchmark extraction quality across downloader modes.

    Tests 200+ real-world pages from 17 categories.
    Not purely assertions-based — prints a comparison table for human review.
    """

    @pytest.fixture(autouse=True)
    def _setup(self):
        self.static_dl = HTMLDownloader(timeout=20)
        # Benchmark exercises the JS-rendering fallback path explicitly,
        # so opt in regardless of the disable-by-default ctor.
        self.auto_dl = AutoHTMLDownloader(timeout=20, enable_js_rendering=True)
        yield
        self.static_dl.close()
        self.auto_dl.close()

    def test_static_pages_return_content(self):
        """Static downloader should extract content from most static pages."""
        failures = []
        for name, url in STATIC_PAGES[:30]:  # Test first 30 for speed
            result = _run_downloader(self.static_dl, "", name, url, "static")
            if result.length < 100:
                failures.append(f"{name}: {result.length} chars")

        # Allow up to 20% failure rate (bot protection, geo-blocks, etc.)
        max_failures = len(STATIC_PAGES[:30]) * 0.2
        assert len(failures) <= max_failures, (
            f"Static downloader failed on too many pages "
            f"({len(failures)}/{len(STATIC_PAGES[:30])}): "
            f"{', '.join(failures[:10])}"
        )

    def test_full_benchmark(self):
        """Full benchmark across all pages with Auto downloader.

        Prints detailed results table grouped by category.
        """
        results: list[FetchResult] = []

        total = len(ALL_PAGES)
        print(f"\n{'=' * 90}")
        print(
            f"  EXTRACTION BENCHMARK: {total} pages across "
            f"{len(ALL_CATEGORIES)} categories"
        )
        print(f"{'=' * 90}")

        for i, (cat, name, url) in enumerate(ALL_PAGES):
            r = _run_downloader(self.auto_dl, cat, name, url, "Auto")
            results.append(r)

            # Progress indicator
            status = "✓" if r.length > 100 else "✗"
            print(
                f"  [{i + 1:>3}/{total}] {status} {cat:<14} "
                f"{name:<16} {r.length:>7} chars  "
                f"{r.time_s:>5.1f}s  bp={r.boilerplate}"
            )

        # Summary table by category
        print(f"\n{'=' * 90}")
        print("  SUMMARY")
        print(f"{'=' * 90}")

        total_success = sum(1 for r in results if r.length > 100)
        total_chars = sum(r.length for r in results)
        total_bp = sum(r.boilerplate for r in results)
        avg_time = sum(r.time_s for r in results) / len(results)

        print(
            f"\n  Overall: {total_success}/{total} pages extracted "
            f"({total_success / total * 100:.0f}%)"
        )
        print(f"  Total chars: {total_chars:,}")
        print(f"  Total boilerplate hits: {total_bp}")
        print(f"  Avg time per page: {avg_time:.1f}s")

        _print_category_summary(results, "Auto")

        # Show failures
        failures = [r for r in results if r.length <= 100]
        if failures:
            print(f"\n  Failed pages ({len(failures)}):")
            for r in failures:
                print(
                    f"    {r.category:<14} {r.page:<16} "
                    f"{r.length} chars  {r.sample[:60]}"
                )

        # Show high-boilerplate pages (potential quality issues)
        high_bp = [r for r in results if r.boilerplate >= 3 and r.length > 100]
        if high_bp:
            print(f"\n  High boilerplate pages ({len(high_bp)}):")
            for r in sorted(high_bp, key=lambda x: -x.boilerplate):
                print(
                    f"    {r.category:<14} {r.page:<16} "
                    f"bp={r.boilerplate}  {r.length} chars"
                )

        # Soft assertion: at least 60% of pages should extract content
        # (many sites have bot protection, geo-blocks, paywalls)
        assert total_success >= total * 0.6, (
            f"Too many failures: {total_success}/{total} "
            f"({total_success / total * 100:.0f}%) — expected at least 60%"
        )


# URLs that ContentFetcher should route to specialized downloaders
CONTENT_FETCHER_URLS = [
    ("ArXiv abs", "https://arxiv.org/abs/2301.07507", "arXiv"),
    ("ArXiv GPT-4", "https://arxiv.org/abs/2303.08774", "arXiv"),
    ("PubMed", "https://pubmed.ncbi.nlm.nih.gov/37828879/", "PubMed"),
    ("PubMed CRISPR", "https://pubmed.ncbi.nlm.nih.gov/26553966/", "PubMed"),
    ("PubMed COVID", "https://pubmed.ncbi.nlm.nih.gov/32015507/", "PubMed"),
    ("PMC", "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7095418/", "PMC"),
    (
        "SemScholar",
        "https://www.semanticscholar.org/paper/Attention-Is-All-You-Need-Vaswani-Shazeer/204e3073870fae3d05bcbc2f6a8e263d9b72e776",
        "Semantic Scholar",
    ),
    ("OpenAlex", "https://openalex.org/works/W2741809807", "OpenAlex"),
    (
        "bioRxiv",
        "https://www.biorxiv.org/content/10.1101/2024.01.01.573841v1",
        "bioRxiv",
    ),
    (
        "HTML page",
        "https://en.wikipedia.org/wiki/Python_(programming_language)",
        "Web Page",
    ),
]


class TestContentFetcherRouting:
    """Test that ContentFetcher routes academic URLs to specialized downloaders.

    Verifies the new fetch_batch path used by FullSearchResults actually
    works end-to-end with real URLs.
    """

    def test_academic_urls_via_content_fetcher(self):
        """ContentFetcher.fetch_batch returns content for academic URLs."""
        urls = [url for _, url, _ in CONTENT_FETCHER_URLS]

        with ContentFetcher(timeout=30) as fetcher:
            results = fetcher.fetch_batch(urls)

        print(f"\n{'=' * 90}")
        print("  CONTENT FETCHER ROUTING BENCHMARK")
        print(f"{'=' * 90}")

        failures = []
        for label, url, expected_source in CONTENT_FETCHER_URLS:
            content = results.get(url)
            length = len(content) if content else 0
            sample = (
                content[:80].replace("\n", " ") if content else "(no content)"
            )

            # Check URL classification
            info = fetcher.get_url_info(url)
            detected = info["source_name"]

            status = "✓" if length > 50 else "✗"
            match = "✓" if detected == expected_source else "✗"
            print(
                f"  {status} {label:<16} "
                f"route={match} {detected:<18} "
                f"{length:>7} chars  {sample[:50]}"
            )

            if length <= 50:
                failures.append(f"{label} ({detected}): {length} chars")

        print(
            f"\n  Results: {len(CONTENT_FETCHER_URLS) - len(failures)}"
            f"/{len(CONTENT_FETCHER_URLS)} URLs returned content"
        )

        if failures:
            print(f"  Failures: {', '.join(failures)}")

        # At least 60% should work (some may be rate-limited or paywalled)
        max_failures = len(CONTENT_FETCHER_URLS) * 0.4
        assert len(failures) <= max_failures, (
            f"ContentFetcher failed on too many URLs: "
            f"{len(failures)}/{len(CONTENT_FETCHER_URLS)}: "
            f"{', '.join(failures)}"
        )