learningcircuit--local-deep-research/pyproject.toml

[build-system]
requires = ["pdm-backend>=2.4.3"]
build-backend = "pdm.backend"

[project]
name = "local-deep-research"
dynamic = ["version"]
description = "AI-powered research assistant with deep, iterative analysis using LLMs and web searches"
readme = "README.md"
requires-python = ">=3.12,<3.15"
license = {file = "LICENSE"}
authors = [
    {name = "LearningCircuit", email = "185559241+LearningCircuit@users.noreply.github.com"},
    {name = "HashedViking", email = "6432677+HashedViking@users.noreply.github.com"},
    {name = "djpetti", email = "djpetti@gmail.com"},
]
classifiers = [
    "Programming Language :: Python :: 3",
    "License :: OSI Approved :: MIT License",
    "Operating System :: OS Independent",
]
dependencies = [
    "langchain~=1.2",  # lock pulls 1.3.9: GHSA-gr75-jv2w-4656 fix
    "langchain-community~=0.4",
    "langchain-core~=1.2",
    "langchain-ollama~=1.0",
    "langchain-openai~=1.1",
    "langchain-anthropic~=1.3",  # lock pulls 1.4.6: GHSA-gr75-jv2w-4656 fix
    "openai>=2",  # direct dep: error_handling/openai_compat_errors.py imports exception classes
    "httpx>=0.27",  # direct dep: error_handling/openai_compat_errors.py imports ConnectError/ReadTimeout
    "duckduckgo-search~=8.1",
    "python-dateutil~=2.9",
    "typing-extensions~=4.15",
    "justext~=3.0",
    "readabilipy~=0.3",
    "trafilatura~=2.0",
    "extruct~=0.18",
    "playwright~=1.58",
    "beautifulsoup4~=4.14",
    "flask~=3.1.3",  # CVE-2026-27205 fix: session Vary: Cookie header
    "werkzeug~=3.1.6",  # CVE-2026-27199 fix: safe_join() Windows device names
    "flask-cors~=6.0",
    "flask-socketio~=5.6.1",  # Fix for Flask 3.1.3 session property compatibility
    "alembic~=1.17",
    "sqlalchemy~=2.0",
    "sqlalchemy-utc~=0.14",
    "wikipedia~=1.4",
    "arxiv~=2.4",
    "pypdf~=6.12",  # CVE-2026-33123, CVE-2026-31826, GHSA-9m86-7pmv-2852, CVE-2026-27888, CVE-2026-27024, CVE-2026-0777, CVE-2025-55197, GHSA-3crg-w4f6-42mx, GHSA-248m-82v9-q6g6, GHSA-cj93-chg6-vgv8, GHSA-jm82-fx9c-mx94 fixes
    "sentence-transformers~=5.2",
    "faiss-cpu~=1.13.0",  # 1.14.x wheels SIGILL on CPUs without AVX2 (#4480); unpin criteria tracked in #4499
    "pydantic~=2.12",
    "pydantic-settings~=2.12",  # lock pulls 2.14.2: GHSA-4xgf-cpjx-pc3j fix
    "toml~=0.10",
    "platformdirs~=4.5",
    "dynaconf~=3.2",
    "requests>=2.32",  # CVE-2026-25645 patched in 2.33; PDM override ensures 2.33+ in dev
    "urllib3~=2.7",  # CVE-2026-44431 CVE-2026-44432 fix
    "tiktoken~=0.12",
    "xmltodict~=1.0",
    "lxml>=5.3",  # >=5.3 to satisfy crawl4ai; PDM override ensures >=6.0 in dev
    "newspaper4k~=0.9",
    "crawl4ai~=0.9",  # 0.9.0: GHSA-r253-r9jw-qg44 (RCE via Chromium launch-arg injection), GHSA-2jq4-q6vv-4cp3 (path-traversal file write), GHSA-wm69-2pc3-rmmf fixes; supersedes 0.8.9 fixes
    "defusedxml~=0.7",  # XXE-safe XML parsing
    "nh3~=0.3",  # Backend HTML sanitization
    "pdfplumber~=0.11",
    "unstructured~=0.18",
    # Runtime parsers for the unstructured office-document loaders. These are
    # NOT pulled in by bare `unstructured` (only by its `[docx]`/`[odt]`/`[pptx]`/
    # `[xlsx]` extras), yet the loader registry advertises .doc/.docx/.odt/.ppt/
    # .pptx/.xls/.xlsx as supported. Without them those uploads fail at runtime
    # with `ModuleNotFoundError: No module named 'docx'` (see issue #4414).
    "python-docx~=1.2",  # .doc, .docx, .odt (odt parser imports python-docx)
    "python-pptx~=1.0",  # .ppt, .pptx
    "python-slugify~=8.0",  # citation_formatter._slugify_title
    "openpyxl~=3.1",  # .xlsx (unstructured Excel partition)
    "msoffcrypto-tool~=6.0",  # imported at module load by unstructured.partition.xlsx
    "xlrd~=2.0",  # legacy .xls reading (pandas engine)
    "google-search-results~=2.4",
    "jaraco-context~=6.1",  # CVE GHSA-58pv-8j8x-9vj2 fix for path traversal (also pinned as transitive dep)
    "flask-wtf~=1.2",
    "optuna~=4.7",
    "elasticsearch~=9.3",
    "methodtools~=0.4",
    "loguru~=0.7",
    "cachetools~=7.0",
    "matplotlib~=3.10",
    "pandas~=3.0",
    "plotly~=6.5",
    "kaleido~=1.2",
    "aiohttp~=3.13",  # lock pulls 3.14.1: CVE-2026-54273..54280 fixes
    "tenacity~=9.1",
    "apscheduler~=3.11",
    "rich~=14.3",
    "click~=8.3",
    "flask-login~=0.6",
    "flask-limiter~=4.1",
    "sqlcipher3-binary~=0.6; sys_platform == \"linux\" and platform_machine == \"x86_64\"",
    "sqlcipher3~=0.6; (platform_machine == \"aarch64\" or platform_machine == \"arm64\") and sys_platform == \"linux\"",
    "sqlcipher3~=0.6; sys_platform != \"linux\"",
    "lxml-html-clean>=0.4.4",  # CVE-2026-28350: base tag injection bypass
    "weasyprint~=68.1",
    "Pillow>=12.1.1",  # CVE-2026-25990: out-of-bounds write in PSD
    "cryptography>=48.0.1,<49",  # CVE-2026-26007 (SECT curves), CVE-2026-39892 (buffer overflow), GHSA-537c-gmf6-5ccf (bundled OpenSSL, fixed by 48.0.1's OpenSSL 4.0.1 recompile); <49 avoids cryptography 49.0.0 breaking changes (removed PUBLIC/PRIVATE_KEY_TYPES aliases, stricter X.509 loading)
    "apprise~=1.9",
    "markdown~=3.10",
    "pypandoc-binary~=1.16",
    "datasets~=4.5",
    "pyarrow~=23.0",
    "langchain-experimental~=0.4",
]

[project.optional-dependencies]
mcp = [
    "mcp[cli]~=1.2",
]

[project.urls]
"Homepage" = "https://github.com/LearningCircuit/local-deep-research"
"Bug Tracker" = "https://github.com/LearningCircuit/local-deep-research/issues"

[project.scripts]
# `ldr` was removed in commit c1f80e7a8 ("Remove deprecated CLI", #319) but the
# entry-point line was left behind, so the installed binary has crashed on
# import for every release since. Caught by the entry-point smoke in
# release-gate.yml. Restore only when a replacement CLI exists.
ldr-web = "local_deep_research.web.app:main"
ldr-mcp = "local_deep_research.mcp:run_server"

# Note: build backend is pdm.backend (see [build-system]). These
# [tool.setuptools] sections are kept for compatibility with editable
# installs via `pip install -e .` which may fall back to setuptools.
[tool.setuptools]
include-package-data = true

[tool.setuptools.package-data]
"local_deep_research.web" = ["templates/*", "static/*", "static/**/*"]
"local_deep_research.defaults" = ["*.toml", "*.py", "*.template", ".env.template", "*.json"]
"local_deep_research.notifications" = ["templates/*.jinja2"]
# Note: there is no `local_deep_research.data` package any more. Every
# academic dataset is downloaded at runtime into the user data directory
# by the `local_deep_research.journal_quality.data_sources` package.
















[tool.pdm]
distribution = true
version = { source = "file", path = "src/local_deep_research/__version__.py" }
[[tool.pdm.source]]
url = "https://download.pytorch.org/whl/cpu"
name = "torch"
include_packages = ["torch", "torch*"]

[tool.pdm.resolution.overrides]
# arxiv pins requests~=2.32.0, but 2.32.x has CVE-2026-25645
requests = ">=2.33"
# newspaper4k and crawl4ai pin lxml<6.0 but work fine with 6.x
# (tested). Override to get Python 3.14 pre-built wheels.
lxml = ">=6.0"

[dependency-groups]
dev = [
    "virtualenv~=20.36",
    "pre-commit~=4.5",
    "cookiecutter~=2.6",
    "pytest-mock~=3.15",
    "pytest~=9.0",
    "pytest-cov~=7.0",
    "pytest-timeout~=2.4",
    "pytest-asyncio~=1.3",
    "pytest-xdist~=3.8",
    "ruff~=0.15",
    "freezegun~=1.5",
    "mypy~=1.19",
    "types-requests~=2.32",
    "types-toml~=0.10",
    "types-flask~=1.1",
    "hypothesis~=6.151",
    "vulture~=2.14",
    "towncrier~=24.8",
    "xlwt~=1.3",  # test-only: writes legacy .xls fixtures for XLSLoader tests
]

[tool.pytest.ini_options]
markers = [
    "requires_llm: marks tests that require a real LLM (not fallback)",
    "integration: marks integration tests",
    "slow: marks slow tests",
    "asyncio: marks async tests that use asyncio",
    "fd_canary: heavy file-descriptor-leak regression tests (#3816). CPU-bound real-event-loop spins that exceeded the legacy 60s timeout under coverage + '-n auto' contention, and whose FD-count assertions are noisy under parallel execution; run in a dedicated serial, no-coverage CI step instead of the parallel suite.",
]
# Global timeout for all tests - fail fast instead of hanging.
# "thread" (not "signal") because SIGALRM was firing inside weakref/GC on
# 3.14 in CI, corrupting xdist workers and losing their coverage data.
timeout = 180
timeout_method = "thread"

[tool.coverage.run]
source = ["src"]
branch = true
omit = [
    "*/tests/*",
    "*/__pycache__/*",
    "*/migrations/*",
    # Experimental strategies (will be cleaned up)
    "*/strategies/*",
    # CLI entry points (hard to unit test)
    "*/cli.py",
    "*/cli/*",
    # Visualization code
    "*/visualization.py",
    # Experimental browsecomp decoding
    "*/answer_decoding/*",
]

[tool.coverage.report]
exclude_lines = [
    "pragma: no cover",
    "if TYPE_CHECKING:",
    "raise NotImplementedError",
]
# Conservative baseline — ratchet up as coverage improves
fail_under = 50
show_missing = true

[tool.coverage.html]
directory = "coverage/htmlcov"

[tool.ruff]
# Exclude a variety of commonly ignored directories.
exclude = [
    ".bzr",
    ".direnv",
    ".eggs",
    ".git",
    ".git-rewrite",
    ".hg",
    ".ipynb_checkpoints",
    ".mypy_cache",
    ".nox",
    ".pants.d",
    ".pyenv",
    ".pytest_cache",
    ".pytype",
    ".ruff_cache",
    ".svn",
    ".tox",
    ".venv",
    ".vscode",
    "__pypackages__",
    "_build",
    "buck-out",
    "build",
    "dist",
    "node_modules",
    "site-packages",
    "venv",
]

# Same as Black.
line-length = 80
indent-width = 4

[tool.ruff.lint]
# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default.
select = [
    "E4", "E7", "E9", "F", "S",
    # --- Wave 2: logging, performance, return style, exceptions, print, comprehensions ---
    "G",      # flake8-logging-format: lazy formatting in logger calls
    "PERF",   # perflint: performance anti-patterns
    "RET",    # flake8-return: unnecessary assign-before-return, superfluous else
    "TRY",    # tryceratops: exception handling patterns
    "T20",    # flake8-print: print statement detection in production code
    "C4",     # flake8-comprehensions: unnecessary comprehensions/calls
]
ignore = [
    # --- Bandit (S) rules suppressed globally for existing code patterns ---
    # try-except-pass/continue: pervasive cleanup pattern throughout codebase
    "S110",   # try-except-pass
    "S112",   # try-except-continue
    # subprocess calls with known-safe partial paths (e.g. "git", "pip")
    "S603",   # subprocess-without-shell-equals-true
    "S606",   # start-process-with-no-shell
    "S607",   # start-process-with-partial-path
    # --- Wave 2: rules ignored initially due to high violation counts ---
    # TODO: fix violations and remove ignore
    "TRY003",   # raise-vanilla-args — overly pedantic for built-in exceptions
    "TRY004",   # type-check-without-type-error — breaks API contracts; ruff autofix was removed
    "TRY300",   # try-consider-else — unintuitive syntax; conflicts with RET505
    "PERF401",  # manual-list-comprehension — no reliable autofix; negligible real-world impact
]

# Allow fix for all enabled rules (when `--fix`) is provided.
fixable = ["ALL"]
unfixable = []

# Allow unused variables when underscore-prefixed.
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"

[tool.ruff.lint.per-file-ignores]
# Tests: assert is the standard pattern; hardcoded values are test fixtures; print is fine
"tests/**" = ["S101", "S104", "S105", "S106", "S107", "S108", "S113", "S311", "S324", "S602", "S608", "T20", "TRY201", "G004", "C420", "TRY002", "C408", "C401", "C416", "C414", "PERF102", "PERF402", "PERF403", "RET503", "RET504", "TRY301"]
# Examples: hardcoded values are demo fixtures; print is fine; bare exceptions acceptable
"examples/**" = ["S105", "S311", "T201", "TRY002", "C408", "C401", "C416", "PERF102", "PERF402", "PERF403", "RET503", "RET502", "RET504", "TRY301"]
# Cookiecutter hooks: shell=True is needed for cross-platform compat
"cookiecutter-docker/**" = ["S602", "T201"]
# CI scripts: test password is intentional
"scripts/ci/**" = ["S105"]
# Pre-commit hooks: print is the interface
".pre-commit-hooks/**" = ["T201", "RET503", "RET504"]
# Scripts: print is the interface
"scripts/**" = ["T201"]
".github/scripts/**" = ["T201"]
# CLI modules: print is the interface
"src/local_deep_research/cli/**" = ["T201"]
# Benchmarks: print is used for output/reporting
"src/local_deep_research/benchmarks/**" = ["T201"]
# Web/settings files with intentional print for startup/debug
"src/local_deep_research/settings/logger.py" = ["T201"]
"src/local_deep_research/web/routes/route_registry.py" = ["T201"]
"src/local_deep_research/web_search_engines/rate_limiting/cli.py" = ["T201"]
# Server config: binding to 0.0.0.0 is intentional for containerized deployment
"src/local_deep_research/web/server_config.py" = ["S104"]
"src/local_deep_research/web/routes/settings_routes.py" = ["S104"]
"src/local_deep_research/security/network_utils.py" = ["S104"]
"src/local_deep_research/security/notification_validator.py" = ["S104"]
"src/local_deep_research/security/url_builder.py" = ["S104"]
"src/local_deep_research/utilities/llm_utils.py" = ["S104"]
"src/local_deep_research/web_search_engines/rate_limiting/llm/wrapper.py" = ["S104"]
# Non-cryptographic random used for jitter/sampling, not security
"src/local_deep_research/scheduler/background.py" = ["S311"]
"src/local_deep_research/web/auth/middleware_optimizer.py" = ["S311"]
"src/local_deep_research/web_search_engines/engines/search_engine_google_pse.py" = ["S311"]
"src/local_deep_research/web_search_engines/rate_limiting/tracker.py" = ["S311"]
# Hardcoded password strings are default/placeholder values, not real secrets
"src/local_deep_research/database/models/library.py" = ["S105"]
"src/local_deep_research/web/auth/database_middleware.py" = ["S105"]
# Requests without timeout: safe_requests module handles timeouts at a higher level
"src/local_deep_research/security/safe_requests.py" = ["S113"]
# Markup usage is intentional and inputs are pre-sanitized
"src/local_deep_research/web/themes/__init__.py" = ["S704"]
"src/local_deep_research/web/utils/vite_helper.py" = ["S704"]
# SQL expressions use parameterized queries; ruff flags f-string construction
"src/local_deep_research/advanced_search_system/filters/followup_relevance_filter.py" = ["S608"]
"src/local_deep_research/web_search_engines/search_engine_base.py" = ["S608"]
# XML parsing uses defusedxml elsewhere; this specific usage is on trusted data
"src/local_deep_research/research_library/downloaders/arxiv.py" = ["S314"]

[tool.ruff.format]
# Like Black, use double quotes for strings.
quote-style = "double"
# Like Black, indent with spaces, rather than tabs.
indent-style = "space"
# Like Black, respect magic trailing commas.
skip-magic-trailing-comma = false
# Like Black, automatically detect the appropriate line ending.
line-ending = "auto"

[tool.mypy]
python_version = "3.12"
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = false
check_untyped_defs = true
warn_redundant_casts = true
warn_unused_ignores = true
no_implicit_optional = true
strict_equality = true
show_error_codes = true

# Start permissive - allow missing imports for third-party packages without stubs
ignore_missing_imports = true

# Ignore test files
[[tool.mypy.overrides]]
module = "tests.*"
ignore_errors = true

# Third-party packages without type stubs — suppress import-untyped errors
[[tool.mypy.overrides]]
module = ["cachetools", "cachetools.*", "dateutil", "dateutil.*"]
ignore_missing_imports = true

# Existing modules with type errors — ignore until cleaned up.
# New modules are checked by default. Remove entries as modules are fixed.
[[tool.mypy.overrides]]
module = [
    "local_deep_research.advanced_search_system.*",
    "local_deep_research.api.*",
    "local_deep_research.benchmarks.*",
    "local_deep_research.config.*",
    "local_deep_research.database.*",
    "local_deep_research.domain_classifier.*",
    "local_deep_research.embeddings.splitters.*",
    "local_deep_research.library.*",
    "local_deep_research.metrics.*",
    "local_deep_research.research_library.*",
    "local_deep_research.security.*",
    "local_deep_research.settings.*",
    "local_deep_research.storage.*",
    "local_deep_research.utilities.*",
    "local_deep_research.web.*",
]
ignore_errors = true

# ---------------------------------------------------------------------------
# towncrier — release-notes news fragments
# ---------------------------------------------------------------------------
# Each PR drops a tiny markdown file under changelog.d/ named
# <id>.<category>.md (or +slug.<category>.md for orphan fragments
# without a PR/issue number). At release prep time the maintainer runs:
#
#     pdm run towncrier build --version <X.Y.Z> --yes
#
# which renders the fragments into docs/release_notes/<X.Y.Z>.md (per
# the {version}-templated filename below) and `git rm`s the consumed
# fragments. The release workflow (.github/workflows/release.yml) then
# reads that per-version file as the human-narrative input to the
# release body (alongside the AI TL;DR and the auto-generated PR list).
#
# Categories are intentionally lean — the auto-generated PR list already
# catches dep bumps, CI churn, etc., so changelog.d/ fragments are just
# for user-visible changes worth highlighting.
[tool.towncrier]
# `name` is set explicitly so towncrier never tries to import the
# `local_deep_research` package. The release workflow runs towncrier
# in a sparse checkout that omits src/, so an import-based lookup
# would fail with ModuleNotFoundError. `package`/`package_dir` are
# intentionally not set — with `name` provided and `--version` always
# passed on the CLI, towncrier has no remaining reason to import.
name = "local-deep-research"
# single_file=false + a {version}-templated filename makes towncrier
# write a fresh per-release file each build instead of appending to one
# growing CHANGELOG. Matches the pre-existing
# docs/release_notes/<version>.md convention used by the release
# workflow.
single_file = false
filename = "docs/release_notes/{version}.md"
directory = "changelog.d"
# title_format=false suppresses the `## <version> (<date>)` header in
# the rendered output — the GitHub release page already shows the
# version as its title, and per-version files don't need an inline
# version header either.
title_format = false
issue_format = "[#{issue}](https://github.com/LearningCircuit/local-deep-research/pull/{issue})"
underlines = ["", "", ""]

[[tool.towncrier.type]]
directory = "breaking"
name = "💥 Breaking Changes"
showcontent = true

[[tool.towncrier.type]]
directory = "security"
name = "🔒 Security"
showcontent = true

[[tool.towncrier.type]]
directory = "feature"
name = "✨ New Features"
showcontent = true

[[tool.towncrier.type]]
directory = "bugfix"
name = "🐛 Bug Fixes"
showcontent = true

[[tool.towncrier.type]]
directory = "removal"
name = "🗑️ Removed"
showcontent = true

[[tool.towncrier.type]]
directory = "misc"
name = "📝 Other Changes"
showcontent = true