open-metadata--openmetadata/ingestion/pyproject.toml

[build-system]
requires = ["setuptools~=70.3.0"]
build-backend = "setuptools.build_meta"

# We will keep handling dependencies in setup.py
# since it helps us organize and isolate version management
[project]
name = "openmetadata-ingestion"
version = "2.0.0.0.dev0"
dynamic = ["readme", "dependencies", "optional-dependencies"]
authors = [
    { name = "OpenMetadata Committers" }
]
license = { file = "LICENSE" }
description = "Ingestion Framework for OpenMetadata"
requires-python = ">=3.10"

[project.urls]
Homepage = "https://open-metadata.org/"
Documentation = "https://docs.open-metadata.org/"
Source = "https://github.com/open-metadata/OpenMetadata"

[tool.setuptools.dynamic]
readme = { file = ["README.md"] }

[tool.setuptools.packages.find]
where = ["./src"]
exclude = ["tests*"]
namespaces = true

[tool.setuptools.package-data]
"metadata.examples" = ["workflows/*.yaml"]
"_openmetadata_testutils" = ["data/**/*"]
"metadata.data_quality" = ["data/**/*"]

[project.scripts]
metadata = "metadata.cmd:metadata"

[project.entry-points.apache_airflow_provider]
provider_info = "airflow_provider_openmetadata:get_provider_config"

[tool.coverage.run]
source = ["metadata"]
relative_files = true
branch = true
parallel = true

# Remap installed-package paths back to the source tree so that
# ``coverage combine`` (in a lightweight CI job without the package
# installed) produces paths that resolve to the checkout.
[tool.coverage.paths]
metadata = [
    "src/metadata",
    "*/site-packages/metadata",
]

[tool.coverage.report]
omit = [
    "*__init__*",
    "*/generated/*",
    "tests/*",
    "ingestion/src/*",
    "*/src/metadata/ingestion/source/database/sample_*"
]

[tool.coverage.xml]
output = "ci-coverage.xml"

[tool.coverage.html]
show_contexts = true

[tool.pytest.ini_options]
markers = [
    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
    "xdist_group(name): group tests to run on the same xdist worker"
]

# Skip tests that are known to fail or are not ready to run until issue #21736 is solved
addopts = "--ignore=ingestion/tests/unit/topology/database/test_deltalake.py --ignore=ingestion/tests/integration/sources/mlmodels/mlflow/test_mlflow.py"
[project.entry-points.pytest11]
pytest_openmetadata = "_openmetadata_testutils.pytest_openmetadata.plugin"

[tool.ruff]
line-length = 120
target-version = "py310"
# Pin source roots so first-party isort detection is stable regardless of
# whether ruff is invoked from the repo root (pre-commit, CI) or from
# `ingestion/` (`make py_format_check`). Each path is listed twice — once
# relative to `ingestion/` (cwd for `make py_format_check`) and once with
# the `ingestion/` prefix (cwd is repo root for pre-commit hooks). Same
# pattern as `extend-exclude` below.
src = [
    "src",
    "ingestion/src",
    "../openmetadata-airflow-apis",
    "openmetadata-airflow-apis",
]
# Each excluded path is listed twice: once relative to the pyproject root
# (cwd = ingestion/) and once with the `ingestion/` prefix (for invocations
# from the repo root, e.g. `make py_format_check`). Without the prefixed form
# ruff's multi-root crawler races on extend-exclude matching and intermittently
# scans the generated/ tree.
extend-exclude = [
    "src/metadata/generated",
    "ingestion/src/metadata/generated",
    "build",
    "ingestion/build",
    "env",
    "ingestion/env",
    "../openmetadata-airflow-apis/build",
    "openmetadata-airflow-apis/build",
]

[tool.ruff.format]
# Black-compatible formatter; defaults match black output.

[tool.ruff.lint]
# ─── Selected rule families ──────────────────────────────────────────────
select = [
    # Style + correctness baseline
    "E",        # pycodestyle errors  — PEP 8 errors (whitespace, indent)
    "W",        # pycodestyle warnings — PEP 8 warnings (deprecated forms)
    "F",        # pyflakes             — unused vars/imports, undefined names
    "I",        # isort                — import grouping/ordering (auto-fix)
    "N",        # pep8-naming          — snake_case / PascalCase (replaces pylint C0103)
    "UP",       # pyupgrade            — modernize syntax for >=3.10

    # Likely-bug catchers
    "B",        # flake8-bugbear       — high-signal bug heuristics
    "C4",       # flake8-comprehensions — rewrite list/dict/set comps cleanly
    "C90",      # mccabe                — cyclomatic complexity cap per function
    "RET",      # flake8-return         — cleanup return-statement logic
    "SIM",      # flake8-simplify       — collapse trivial patterns
    "TRY",      # tryceratops           — better exception handling

    # Hygiene
    "PIE",      # flake8-pie            — small misc cleanups
    "ICN",      # flake8-import-conventions — enforce `import pandas as pd` etc.
    "T20",      # flake8-print          — no print/pprint (replaces print_checker plugin)
    "TC",       # flake8-type-checking  — move type-only imports under TYPE_CHECKING
    "TID",      # flake8-tidy-imports   — relative-import + banned-api (replaces import_checker plugin)
    "PTH",      # flake8-use-pathlib    — prefer pathlib over os.path
    "PERF",     # perflint              — common perf antipatterns

    # Pylint port (split intentionally)
    "PLE",      # pylint Errors         — real error rules from pylint
    "PLC",      # pylint Conventions    — style rules from pylint
    "PLW",      # pylint Warnings       — warning rules from pylint
    "PLR",      # pylint Refactor       — see ignore list for the noisy ones we drop

    # Ruff-specific
    "RUF",      # ruff-native rules     — Astral's own catches (incl. RUF100 unused-noqa)
]

# ─── Ignores: justify EVERY one ──────────────────────────────────────────
ignore = [
    # ── Formatter conflicts: enabling these creates oscillating fixes ──
    "E501",     # line-too-long           — `ruff format` handles wrapping
    "W191",     # tab-indentation         — `ruff format` handles whitespace
    "COM812",   # missing-trailing-comma  — fights with the formatter
    "ISC001",   # implicit-string-concat  — fights with the formatter

    # ── Star imports: real cleanup, separate refactor scope ──
    "F403",     # `from x import *`       — pervasive in connector __init__.py
    "F405",     # may-be-undefined        — cascade of F403; same scope decision

    # ── Bugbear taste calls ──
    "B008",     # function call in default arg — FastAPI/Typer Depends() pattern relies on this

    # ── Pyupgrade taste ──
    "UP015",    # redundant-open-mode     — explicit `open(p, "r")` is a readability call

    # ── RUF nits unsuited to our patterns ──
    "RUF005",   # iterable concat → unpack — Pydantic / SA models read better as concat
    "RUF022",   # `__all__` not sorted     — manual ordering communicates groupings

    # ── Simplify rules that are taste, not bugs ──
    "SIM108",   # use ternary              — often hurts readability for guard clauses
    "SIM114",   # combine-if-branches      — readability is the point of separate branches

    # ── Tryceratops nits ──
    "TRY003",   # long exception messages  — community consensus: skip
    "TRY400",   # use logger.exception over logger.error in except — the
                # codebase intentionally logs `logger.debug(traceback.format_exc())`
                # separately from the summary log line, splitting traceback
                # (debug-only) from the user-facing summary so production
                # log levels can filter independently. Forcing
                # `logger.exception` would conflate the two.

    # ── Perflint unavoidable cases ──
    "PERF203",  # try inside for-loop      — sometimes the only correct shape (row-by-row)

    # ── PLR complexity caps: noisy on connector dispatchers ──
    "PLR0904",  # too-many-public-methods
    "PLR0911",  # too-many-return-statements
    "PLR0912",  # too-many-branches
    "PLR0913",  # too-many-arguments
    "PLR0914",  # too-many-locals
    "PLR0915",  # too-many-statements
    "PLR0916",  # too-many-boolean-expressions
    "PLR0917",  # too-many-positional-arguments

    # ── PLR taste calls ──
    "PLR2004",  # magic-value comparison   — fires on every HTTP status, port number, retry count
    "PLR1711",  # useless-return None      — explicit-is-better-than-implicit defense
    "PLR2044",  # empty-comment            — too pedantic for the value
]

# Existing violations were grandfathered via `ruff check --add-noqa` during
# the pylint→ruff migration. Cleanup follow-up PRs should drop the noqas one
# rule at a time and fix the underlying issues.

[tool.ruff.lint.per-file-ignores]
# Re-exports are the whole point of __init__; F401/F403 are intended there.
"**/__init__.py" = ["F401", "F403"]
# Tests legitimately use assert; magic numbers and lazy imports are fine in test code.
# Each path listed twice — once relative to `ingestion/` (cwd for `make
# py_format_check`) and once with the `ingestion/` prefix (cwd is repo root
# for pre-commit hooks). Same pattern as `extend-exclude` above.
# `S101` (assert in test) is forward-looking — flake8-bandit (`S`) is not
# yet selected; the rule is no-op today but the entry stays so when `S`
# lands in a later stage tests don't immediately error out.
"tests/**/*.py" = ["S101", "PLR2004", "PLC0415"]
"ingestion/tests/**/*.py" = ["S101", "PLR2004", "PLC0415"]
# Auto-generated from JSON Schema — never edit, never lint.
"src/metadata/generated/**" = ["ALL"]
"ingestion/src/metadata/generated/**" = ["ALL"]

[tool.ruff.lint.isort]
known-first-party = [
    "metadata",
    "ingestion",
    "_openmetadata_testutils",
    "airflow_provider_openmetadata",
]

[tool.ruff.lint.mccabe]
# Default 10 flags every connector yield_table; 15 is the common compromise.
max-complexity = 15

[tool.ruff.lint.pylint]
# Match the pylint config we replaced (max-args=7, max-public-methods=25)
# even though we ignored PLR0913/PLR0904 — devs running ruff-check locally
# still see the limits in any future per-file unignores.
max-args = 7
max-public-methods = 25

[tool.ruff.lint.flake8-tidy-imports]
# Banning relative imports keeps absolute paths everywhere (matches existing
# isort + import-checker plugin behaviour).
ban-relative-imports = "all"

[tool.ruff.lint.flake8-tidy-imports.banned-api]
# Direct port of ingestion/plugins/import_checker.py — both prefixes are
# "build artifacts, don't depend on the layout of the source tree". Use
# the `metadata.` package directly.
"ingestion.src.metadata".msg     = "Use `metadata.` imports instead of `ingestion.src.metadata`."
"ingestion.build.lib".msg        = "Don't import build artifacts; use `metadata.` directly."

[tool.basedpyright]
include = ["src"]
stubPath = "stubs"
exclude = [
  "**/__pycache__",
  "src/metadata/generated/*",
  "src/metadata/__version__.py",
]

# Pin the analysis target to the lowest supported Python version so
# basedpyright catches forward-incompatible code (e.g. tomllib usage,
# PEP 695 generics) at type-check time. Runtime support across 3.10/3.11/
# 3.12 is verified separately by the unit-test matrix in py-tests.yml.
# Keep this in sync with `requires-python` above.
pythonVersion = "3.10"

# `standard` is the surveyed-2025-2026 production-default. `recommended`
# (basedpyright's default) enables `reportUnknown*` which is catastrophic
# on a 75-connector codebase with partially-typed third-party deps.
typeCheckingMode = "standard"
# Don't fail CI on warnings; only errors gate. Warnings are signals to
# tighten later via the ratchet plan below.
failOnWarnings = false

# Existing violations are grandfathered via .basedpyright/baseline.json
# (regenerate with `basedpyright -p ingestion/pyproject.toml --writebaseline`
# from the ingestion/ directory). New violations in any file fail CI.

# ── Real-bug catchers held to error ──────────────────────────────────────
# All of these are at `error` in `standard` defaults; restating to document
# intent and prevent a future `typeCheckingMode = "basic"` regression from
# silently dropping them.
reportPossiblyUnboundVariable = "error"
reportOptionalMemberAccess = "error"
reportAttributeAccessIssue = "error"
reportCallIssue = "error"
reportArgumentType = "error"
reportReturnType = "error"
reportAssignmentType = "error"
reportIncompatibleMethodOverride = "error"
reportInvalidTypeArguments = "error"

# ── Cheap promotions: real-bug rules that fire rarely ────────────────────
reportMatchNotExhaustive = "warning"
reportUnreachable = "warning"
reportInvalidCast = "warning"

# ── Stub-gap noise: explicitly off (matches `standard` defaults) ─────────
# These rules are "none" in standard mode by default. Restating them is
# documentation, not behavior change — it just makes the team's stance
# explicit so a future config refactor doesn't accidentally turn them on.
reportUnknownMemberType = "none"
reportUnknownArgumentType = "none"
reportUnknownVariableType = "none"
reportUnknownParameterType = "none"
reportMissingParameterType = "warning"
reportUnusedCallResult = "none"
reportImplicitStringConcatenation = "none"    # ruff already handles ISC001 conflict
reportUnannotatedClassAttribute = "none"      # fights with Pydantic patterns

# ── Project-specific waivers ─────────────────────────────────────────────
reportDeprecated = false                      # sqlalchemy 2.x deprecations;
reportMissingTypeStubs = false                # connector deps lack stubs; covered via allowedUntypedLibraries
reportAny = false                             # pandas/SA patterns produce intentional Any
reportExplicitAny = false                     # same
# @override was only added in python 3.12: https://docs.python.org/3/library/typing.html#typing.override
reportImplicitOverride = false                # we're at Python 3.10
# Cross-platform stub drift (macOS arm64 vs Linux x86_64) means a `# pyright: ignore`
# that is necessary on one platform may look unused on the other. Disable the
# unused-ignore warning so contributors can land platform-specific residuals
# without a back-and-forth flap between local and CI.
reportUnnecessaryTypeIgnoreComment = false
reportUntypedFunctionDecorator = false        # Pydantic, Click, Typer, pytest fixture all trip it
reportPrivateUsage = false
reportImportCycles = false

# ── Allowed untyped libraries: scope Unknown noise to known boundaries ──
# When we ratchet `reportUnknown*` to `warning` later (see executionEnvironments
# scaffold below), these libraries stay silent — keeping focus on our own
# type debt rather than third-party stub gaps.
# Populate when promoting `reportUnknown*` rules to `warning` (see ratchet plan).
allowedUntypedLibraries = []

# ── Ratchet scaffold: enable subtree-by-subtree promotions later ────────
# Empty for now; uncomment + populate as well-typed subtrees are ready.
# Each block holds files under `root` to a stricter rule subset without
# affecting the rest of the codebase.
#
# Example progression:
#
# [[tool.basedpyright.executionEnvironments]]
# root = "src/metadata/data_quality"
# strict = true
#
# [[tool.basedpyright.executionEnvironments]]
# root = "src/metadata/utils"
# reportMissingParameterType = "error"