wehub-resource-sync
212 lines
6.0 KiB
Python
212 lines
6.0 KiB
Python
try:
|
||
import sys
|
||
import pysqlite3 as sqlite3 # type: ignore
|
||
|
||
sys.modules["sqlite3"] = sqlite3
|
||
sys.modules["sqlite3.dbapi2"] = sqlite3.dbapi2
|
||
except Exception:
|
||
pass
|
||
|
||
import os
|
||
import pytest
|
||
import tenacity
|
||
|
||
from typing import TYPE_CHECKING
|
||
from pathlib import Path
|
||
|
||
from deepeval.tracing.tracing import trace_manager
|
||
from deepeval.config.settings import get_settings, reset_settings, Settings
|
||
|
||
if TYPE_CHECKING:
|
||
from _pytest.fixtures import FixtureRequest
|
||
|
||
|
||
PRESERVE_FROM_BASELINE = {
|
||
name
|
||
for name in Settings.model_fields.keys()
|
||
if name.endswith("_API_KEY") and not name.startswith("CONFIDENT")
|
||
}
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _ensure_hidden_store_dir(tmp_path: Path):
|
||
d = tmp_path / ".deepeval"
|
||
d.mkdir(exist_ok=True)
|
||
# some code expects the file to be there after a run,
|
||
# but at minimum the directory must exist to avoid FileNotFoundError
|
||
yield
|
||
|
||
|
||
@pytest.fixture
|
||
def hidden_store_dir(tmp_path: Path) -> Path:
|
||
d = tmp_path / ".deepeval"
|
||
d.mkdir(parents=True, exist_ok=True)
|
||
return d
|
||
|
||
|
||
# Silence telemetry for all tests so we don't have to deal with the noise
|
||
@pytest.fixture(autouse=True)
|
||
def _telemetry_opt_out(monkeypatch):
|
||
monkeypatch.setenv("DEEPEVAL_TELEMETRY_OPT_OUT", "1")
|
||
yield
|
||
|
||
|
||
# Run every test in its own temp CWD so .deepeval/.deepeval is sandboxed
|
||
@pytest.fixture(autouse=True)
|
||
def _isolate_cwd(tmp_path: Path, monkeypatch):
|
||
monkeypatch.chdir(tmp_path)
|
||
yield
|
||
|
||
|
||
# Default dotenv path most tests can reuse; override in tests as needed
|
||
@pytest.fixture
|
||
def env_path(monkeypatch, tmp_path: Path) -> Path:
|
||
monkeypatch.setenv("ENV_DIR_PATH", str(tmp_path))
|
||
return tmp_path / ".env.local"
|
||
|
||
|
||
@pytest.fixture
|
||
def env_dir(monkeypatch, tmp_path: Path) -> Path:
|
||
monkeypatch.setenv("ENV_DIR_PATH", str(tmp_path))
|
||
return tmp_path
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def no_sleep(monkeypatch):
|
||
monkeypatch.setattr(tenacity.nap, "sleep", lambda _: None, raising=True)
|
||
|
||
|
||
@pytest.fixture()
|
||
def settings():
|
||
settings = get_settings()
|
||
yield settings
|
||
|
||
|
||
@pytest.fixture(scope="session")
|
||
def _session_env_baseline():
|
||
# capture the environment as it existed when pytest started
|
||
return os.environ.copy()
|
||
|
||
|
||
def _restore_env_to(baseline: dict[str, str]) -> None:
|
||
# remove any keys not in the baseline
|
||
for k in list(os.environ.keys()):
|
||
if k not in baseline:
|
||
os.environ.pop(k, None)
|
||
# update differing values to match the baseline
|
||
for k, v in baseline.items():
|
||
if os.environ.get(k) != v:
|
||
os.environ[k] = v
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _env_sandbox(_session_env_baseline, request, monkeypatch):
|
||
# Start from the session baseline (CI secrets included)
|
||
_restore_env_to(_session_env_baseline)
|
||
|
||
# Save whitelisted secrets from the session baseline
|
||
preserved = {
|
||
k: v
|
||
for k, v in _session_env_baseline.items()
|
||
if k in PRESERVE_FROM_BASELINE and isinstance(v, str) and v.strip()
|
||
}
|
||
|
||
# Clear ALL Settings keys to avoid leaking config (file system mode, default save, etc.)
|
||
for setting_key in list(Settings.model_fields.keys()):
|
||
monkeypatch.delenv(setting_key, raising=False)
|
||
|
||
# Re-inject only the secrets we explicitly want to preserve
|
||
for k, v in preserved.items():
|
||
monkeypatch.setenv(k, v)
|
||
|
||
# Never open the Confident AI browser UI during tests
|
||
monkeypatch.setenv("CONFIDENT_OPEN_BROWSER", "0")
|
||
|
||
# Disable dotenv by default unless the test opts in via @pytest.mark.enable_dotenv
|
||
if not request.node.get_closest_marker("enable_dotenv"):
|
||
monkeypatch.setenv("DEEPEVAL_DISABLE_DOTENV", "1")
|
||
else:
|
||
monkeypatch.delenv("DEEPEVAL_DISABLE_DOTENV", raising=False)
|
||
|
||
# Fresh Settings for this test
|
||
reset_settings(reload_dotenv=False)
|
||
|
||
yield
|
||
|
||
# Restore to the session baseline after the test
|
||
_restore_env_to(_session_env_baseline)
|
||
reset_settings(reload_dotenv=False)
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _core_mode_no_confident(
|
||
_env_sandbox, monkeypatch, request: "FixtureRequest"
|
||
):
|
||
|
||
# Ensure no Confident keys come from the process env in this test
|
||
for key in ("CONFIDENT_API_KEY", "CONFIDENTAI_API_KEY"):
|
||
monkeypatch.delenv(key, raising=False)
|
||
|
||
# Prevent dotenv from re-injecting keys from files during the test
|
||
# core tests shouldn’t depend on local .env anyway
|
||
if not request.node.get_closest_marker("enable_dotenv"):
|
||
monkeypatch.setenv("DEEPEVAL_DISABLE_DOTENV", "1")
|
||
|
||
# Rebuild the Settings singleton from the now-clean process env
|
||
reset_settings(reload_dotenv=False)
|
||
|
||
# Clear the in-memory Settings fields (no persistence)
|
||
s = get_settings()
|
||
with s.edit(persist=False) as ctx:
|
||
ctx.s.CONFIDENT_API_KEY = None
|
||
|
||
# Yield control to the test
|
||
yield
|
||
|
||
|
||
@pytest.fixture()
|
||
def enable_dotenv(monkeypatch):
|
||
monkeypatch.setenv("DEEPEVAL_DISABLE_DOTENV", "0")
|
||
# rebuild Settings after changing the env
|
||
reset_settings(reload_dotenv=False)
|
||
|
||
|
||
@pytest.fixture(autouse=False)
|
||
def unpatch_openai_after():
|
||
from deepeval.openai.patch import unpatch_openai_classes
|
||
|
||
yield
|
||
unpatch_openai_classes()
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _reset_tracing_state():
|
||
from deepeval.tracing.types import EvalSession
|
||
|
||
trace_manager.clear_traces()
|
||
# Atomic reset: dropping the session clears mode + every per-run
|
||
# collection (pending_traces, traces_to_evaluate, trace_uuid_to_golden,
|
||
# test_case_metrics) in one go, so a test cannot leak eval state to its
|
||
# neighbors via a forgotten field.
|
||
trace_manager.eval_session = EvalSession()
|
||
try:
|
||
trace_manager.task_bindings.clear()
|
||
except Exception:
|
||
pass
|
||
yield
|
||
|
||
|
||
@pytest.fixture
|
||
def completed_traces(monkeypatch):
|
||
"""Capture completed traces before they are evicted from trace_manager.traces."""
|
||
captured = []
|
||
_original = trace_manager.end_trace
|
||
|
||
def _capturing(trace_uuid):
|
||
if trace_uuid in trace_manager.active_traces:
|
||
captured.append(trace_manager.active_traces[trace_uuid])
|
||
_original(trace_uuid)
|
||
|
||
monkeypatch.setattr(trace_manager, "end_trace", _capturing)
|
||
return captured
|