learningcircuit--local-deep-research/scripts/dev/analyze_passthroughs.py

#!/usr/bin/env python3
"""
Passthrough analysis: catalogue every paper that fell through every
local tier of the journal filter and ended up at the passthrough score
(== threshold). Helps decide whether the LLM tier is still worth keeping.

Reuses the same arxiv/openalex/pubmed/s2 dataset as
test_journal_filter_arxiv.py but routes results through __score_journal
directly so we can also see *which* tier actually scored each paper.
"""

import argparse
import re
import sys
from collections import Counter, defaultdict
from unittest.mock import Mock

from local_deep_research.advanced_search_system.filters import (
    journal_reputation_filter as _jrf,
)

# Stub SearXNG so the filter can be instantiated standalone
_jrf.create_search_engine = lambda *a, **kw: None

from local_deep_research.utilities.thread_context import search_context  # noqa: E402
from local_deep_research.web_search_engines.engines.search_engine_arxiv import (  # noqa: E402
    ArXivSearchEngine,
)
from local_deep_research.web_search_engines.engines.search_engine_openalex import (  # noqa: E402
    OpenAlexSearchEngine,
)
from local_deep_research.web_search_engines.engines.search_engine_pubmed import (  # noqa: E402
    PubMedSearchEngine,
)
from local_deep_research.web_search_engines.engines.search_engine_semantic_scholar import (  # noqa: E402
    SemanticScholarSearchEngine,
)

DOMAIN_QUERIES = {
    "fusion": "tokamak plasma confinement",
    "llm": "large language model alignment",
    "graph_nn": "graph neural networks",
    "astro": "exoplanet atmosphere spectroscopy",
    "biomed": "CRISPR gene editing therapy",
    "condmat": "high temperature superconductor cuprate",
    "climate": "climate model ocean heat content",
    "quantum": "quantum error correction surface code",
    "math": "riemann hypothesis zeta function",
    "robotics": "reinforcement learning robotic manipulation",
}

ENGINES = {
    "arxiv": ArXivSearchEngine,
    "openalex": OpenAlexSearchEngine,
    "pubmed": PubMedSearchEngine,
    "s2": SemanticScholarSearchEngine,
}


def categorize(journal_ref: str, cleaned: str) -> str:
    """Bucket a passthrough journal_ref into a category."""
    j = journal_ref.strip()
    c = cleaned.strip()
    # Citation-like strings (author initials, "et al.", quotes around title)
    if re.search(r"^[A-Z]\.\s*[A-Z]", j):
        return "citation_author"
    if '"' in j or "“" in j:
        return "citation_quoted"
    if re.search(r"\bet al\b", j, re.I):
        return "citation_et_al"
    # Cleaning debris (bare year/page leftover)
    if re.search(r"\b(19|20)\d{2}\b", c):
        return "cleaning_debris"
    if re.search(r"\d", c[-10:]):
        return "cleaning_trailing_num"
    # Conference (the regex tier already handles these via score=5)
    if re.search(
        r"(proc(eedings|\.)?|conference|symp|workshop|colloq)", c, re.I
    ):
        return "conference_uncaught"
    # Truncated long names
    if len(j) > 50 and j.endswith(("…", "...")):
        return "truncated"
    # Looks like a real journal name we just don't have
    return "real_journal_unknown"


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("-n", type=int, default=15)
    ap.add_argument("-t", "--threshold", type=int, default=4)
    args = ap.parse_args()

    snapshot = {
        "search.journal_reputation.threshold": args.threshold,
        "search.journal_reputation.exclude_non_published": False,
        "search.journal_reputation.max_context": 3000,
        "search.journal_reputation.reanalysis_period": 365,
        "search.engine.web.arxiv.journal_reputation.enabled": True,
        "search.engine.web.openalex.journal_reputation.enabled": True,
        "search.engine.web.pubmed.journal_reputation.enabled": True,
        "search.engine.web.semantic_scholar.journal_reputation.enabled": True,
    }

    passthroughs: list[tuple[str, str, str, str, str]] = []
    # (engine, domain, raw_journal_ref, cleaned, category)
    totals: Counter = Counter()

    for engine_name, cls in ENGINES.items():
        for domain, q in DOMAIN_QUERIES.items():
            print(f"  • {engine_name:<9} {domain:<10} {q!r} ...", flush=True)
            kwargs = {
                "max_results": args.n,
                "llm": Mock(),
                "settings_snapshot": snapshot,
            }
            if engine_name in ("pubmed", "s2"):
                kwargs["optimize_queries"] = False
            try:
                with search_context({"username": None, "user_password": None}):
                    engine = cls(**kwargs)
                    out = engine.run(q)
            except Exception as e:
                print(f"      ! {e}")
                continue

            for r in out:
                jref = r.get("journal_ref")
                if not jref:
                    totals["no_journal_ref"] += 1
                    continue
                qual = r.get("journal_quality")
                if qual == args.threshold:
                    # Passthrough — local tiers couldn't score
                    cleaned = jref  # Filter caches private; approximate
                    cat = categorize(jref, cleaned)
                    passthroughs.append(
                        (engine_name, domain, jref, cleaned, cat)
                    )
                    totals[f"passthrough_{cat}"] += 1
                elif qual is not None:
                    totals[f"scored_{qual}"] += 1

    print()
    print("=" * 90)
    print("OVERALL")
    print("=" * 90)
    for k in sorted(totals):
        print(f"  {k:<35} {totals[k]:>5}")

    print()
    print("=" * 90)
    print(f"PASSTHROUGH BY CATEGORY ({len(passthroughs)} total)")
    print("=" * 90)
    by_cat: dict[str, list] = defaultdict(list)
    for row in passthroughs:
        by_cat[row[4]].append(row)
    for cat in sorted(by_cat, key=lambda c: -len(by_cat[c])):
        rows = by_cat[cat]
        print(f"\n[{cat}]  ({len(rows)} entries)")
        for engine_name, domain, jref, cleaned, _ in rows[:8]:
            print(f"  {engine_name:<9} {domain:<10} {jref[:65]}")
        if len(rows) > 8:
            print(f"  ... and {len(rows) - 8} more")

    return 0


if __name__ == "__main__":
    sys.exit(main())