from __future__ import annotations import json import os import shutil import pyfiglet import typer import webbrowser from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from pydantic import ValidationError from pydantic.fields import FieldInfo from enum import Enum from pathlib import Path from rich import print from typing import ( Any, Dict, Iterable, Tuple, Optional, get_args, get_origin, Union, ) from opentelemetry.trace import Span from deepeval.config.settings import Settings, get_settings from deepeval.key_handler import ( KEY_FILE_HANDLER, ModelKeyValues, EmbeddingKeyValues, ) from deepeval.test_run.test_run import ( global_test_run_manager, ) from deepeval.confident.api import get_confident_api_key, set_confident_api_key from deepeval.cli.dotenv_handler import DotenvHandler StrOrEnum = Union[str, "Enum"] PROD = "https://app.confident-ai.com" WWW = "https://www.confident-ai.com" # Hosts considered "browser-clickable" Confident AI properties. Programmatic # hosts (api.*, deepeval.*, otel.*) are intentionally excluded. _CONFIDENT_UTM_HOSTS = frozenset( {"confident-ai.com", "www.confident-ai.com", "app.confident-ai.com"} ) _UTM_SOURCE = "deepeval" def with_utm( url: str, *, medium: str, content: str, ) -> str: """Append standardized UTM params to a Confident AI URL. Schema: - utm_source = "deepeval" (constant; identifies all deepeval-driven traffic) - utm_medium = surface type ("cli" / "python_sdk") - utm_content = location on the source surface (e.g. "login_pair_browser_open") `utm_campaign` is intentionally omitted: this is evergreen referral, not a time-bound marketing push. `ref_page` is intentionally NOT supported here: CLI invocations and Python SDK call sites are not pages. `utm_medium` already identifies the surface type and `utm_content` pinpoints the call site. `ref_page` is exclusively a docs-site concept (set by the remark plugin / runtime client module). No-ops if the URL is not a tracked Confident AI host or already carries a `utm_source` (don't clobber upstream tagging). """ if not url: return url parts = urlsplit(url) if parts.hostname not in _CONFIDENT_UTM_HOSTS: return url query = dict(parse_qsl(parts.query, keep_blank_values=True)) if "utm_source" in query: return url query["utm_source"] = _UTM_SOURCE query["utm_medium"] = medium query["utm_content"] = content return urlunsplit(parts._replace(query=urlencode(query))) # List all mutually exclusive USE_* keys USE_LLM_KEYS = [ key for key in Settings.model_fields if key.startswith("USE_") and key in ModelKeyValues.__members__ ] USE_EMBED_KEYS = [ key for key in Settings.model_fields if key.startswith("USE_") and key in EmbeddingKeyValues.__members__ ] def handle_save_result( *, handled: bool, path: Optional[str], updates: dict, save: Optional[str], quiet: bool, success_msg: Optional[str] = None, updated_msg: str = "Saved environment variables to {path} (ensure it's git-ignored).", no_changes_msg: str = "No changes to save in {path}.", tip_msg: Optional[str] = None, ) -> bool: if not handled and save is not None: raise typer.BadParameter( "Unsupported --save option. Use --save=dotenv[:path].", param_hint="--save", ) if quiet: return False if path and updates: print(updated_msg.format(path=path)) elif path: print(no_changes_msg.format(path=path)) elif tip_msg: print(tip_msg) if success_msg: print(success_msg) return True def render_confident_banner(): # pyfiglet defaults to width=80, which wraps the banner mid-word; render # at the real terminal width so it stays on one line whenever it fits. width = shutil.get_terminal_size(fallback=(120, 24)).columns print( pyfiglet.Figlet(font="big_money-ne", width=width).renderText( "Confident AI" ) ) def render_login_message(): print( "๐Ÿฅณ Welcome to [rgb(106,0,255)]Confident AI[/rgb(106,0,255)], the evals cloud platform ๐Ÿกโค๏ธ" ) print("") render_confident_banner() def upload_and_open_link(_span: Optional[Span] = None): last_test_run_data = global_test_run_manager.get_latest_test_run_data() if last_test_run_data: confident_api_key = get_confident_api_key() if confident_api_key == "" or confident_api_key is None: render_login_message() login_url = with_utm( PROD, medium="cli", content="upload_and_open_link" ) print( f"๐Ÿ”‘ You'll need to get an API key at [link={login_url}]{login_url}[/link] to view your results (free)" ) webbrowser.open(login_url) while True: confident_api_key = input("๐Ÿ” Enter your API Key: ").strip() if confident_api_key: set_confident_api_key(confident_api_key) print( "\n๐ŸŽ‰๐Ÿฅณ Congratulations! You've successfully logged in! :raising_hands: " ) if _span is not None: _span.set_attribute("completed", True) break else: print("โŒ API Key cannot be empty. Please try again.\n") print("๐Ÿ“ค Uploading test run to Confident AI...") global_test_run_manager.post_test_run(last_test_run_data) else: print( "โŒ No test run found in cache. Run 'deepeval login' + an evaluation to get started ๐Ÿš€." ) def clear_evaluation_model_keys(): for key in ModelKeyValues: KEY_FILE_HANDLER.remove_key(key) def clear_embedding_model_keys(): for key in EmbeddingKeyValues: KEY_FILE_HANDLER.remove_key(key) def _to_str_key(k: StrOrEnum) -> str: return k.name if hasattr(k, "name") else str(k) def _normalize_kv(updates: Dict[StrOrEnum, str]) -> Dict[str, str]: return {_to_str_key(k): v for k, v in updates.items()} def _normalize_keys(keys: Iterable[StrOrEnum]) -> list[str]: return [_to_str_key(k) for k in keys] def _normalize_setting_key(raw_key: str) -> str: """Normalize CLI keys like 'log-level' / 'LOG_LEVEL' to model field names.""" return raw_key.strip().lower().replace("-", "_") def _parse_save_option( save_opt: Optional[str] = None, default_path: str = ".env.local" ) -> Tuple[bool, Optional[str]]: if not save_opt: return False, None kind, *rest = save_opt.split(":", 1) if kind != "dotenv": return False, None path = rest[0] if rest else default_path return True, path def resolve_save_target(save_opt: Optional[str]) -> Optional[str]: """ Returns a normalized save target string like 'dotenv:.env.local' or None. Precedence: 1) --save=... 2) DEEPEVAL_DEFAULT_SAVE (opt-in project default) 3) None (no save) """ if save_opt: return save_opt env_default = os.getenv("DEEPEVAL_DEFAULT_SAVE") if env_default and env_default.strip(): return env_default.strip() return None def save_environ_to_store( updates: Dict[StrOrEnum, str], save_opt: Optional[str] = None ) -> Tuple[bool, Optional[str]]: """ Save 'updates' into the selected store (currently only dotenv). Idempotent upsert. Returns (handled, path). """ ok, path = _parse_save_option(save_opt) if not ok: return False, None if updates: DotenvHandler(path).upsert(_normalize_kv(updates)) return True, path def unset_environ_in_store( keys: Iterable[StrOrEnum], save_opt: Optional[str] = None ) -> Tuple[bool, Optional[str]]: """ Remove keys from the selected store (currently only dotenv). Returns (handled, path). """ ok, path = _parse_save_option(save_opt) if not ok: return False, None norm = _normalize_keys(keys) if norm: DotenvHandler(path).unset(norm) return True, path def _as_legacy_use_key( k: str, ) -> Union[ModelKeyValues, EmbeddingKeyValues, None]: if k in ModelKeyValues.__members__: return ModelKeyValues[k] if k in EmbeddingKeyValues.__members__: return EmbeddingKeyValues[k] return None def switch_model_provider( target: Union[ModelKeyValues, EmbeddingKeyValues], save: Optional[str] = None, ) -> Tuple[bool, Optional[str]]: """ Ensure exactly one USE_* flag is enabled. We *unset* all other USE_* keys (instead of writing explicit "NO") to: - keep dotenv clean - preserve Optional[bool] semantics (unset vs explicit false) """ keys_to_clear = ( USE_LLM_KEYS if isinstance(target, ModelKeyValues) else USE_EMBED_KEYS ) target_key = target.name # or _to_str_key(target) if target_key not in keys_to_clear: raise ValueError(f"{target} is not a recognized USE_* model key") # Clear legacy JSON store entries for k in keys_to_clear: legacy = _as_legacy_use_key(k) if legacy is not None: KEY_FILE_HANDLER.remove_key(legacy) KEY_FILE_HANDLER.write_key(target, "YES") if not save: return True, None handled, path = unset_environ_in_store(keys_to_clear, save) if not handled: return False, None return save_environ_to_store({target: "true"}, save) def coerce_blank_to_none(value: Optional[str]) -> Optional[str]: """Return None if value is None/blank/whitespace; otherwise return stripped string.""" if value is None: return None value = value.strip() return value or None def load_service_account_key_file(path: Path) -> str: try: raw = path.read_text(encoding="utf-8").strip() except OSError as e: raise typer.BadParameter( f"Could not read service account file: {path}", param_hint="--service-account-file", ) from e if not raw: raise typer.BadParameter( f"Service account file is empty: {path}", param_hint="--service-account-file", ) # Validate it's JSON and normalize to a single-line string for dotenv. try: obj = json.loads(raw) except json.JSONDecodeError as e: raise typer.BadParameter( f"Service account file does not contain valid JSON: {path}", param_hint="--service-account-file", ) from e return json.dumps(obj, separators=(",", ":")) def unwrap_optional(annotation: Any) -> Any: """ If `annotation` is Optional[T] (i.e. Union[T, None]), return T. Otherwise return `annotation` unchanged. Note: If it's a Union with multiple non-None members, we leave it unchanged. """ origin = get_origin(annotation) if origin is Union: non_none = [a for a in get_args(annotation) if a is not type(None)] if len(non_none) == 1: return non_none[0] return annotation def looks_like_json_container_literal(raw_value: str) -> bool: setting = raw_value.strip() return (setting.startswith("{") and setting.endswith("}")) or ( setting.startswith("[") and setting.endswith("]") ) def should_parse_json_for_field(field_info: FieldInfo) -> bool: annotation = unwrap_optional(field_info.annotation) origin = get_origin(annotation) or annotation return origin in (list, dict, tuple, set) def maybe_parse_json_literal(raw_value: str, field_info) -> object: if not isinstance(raw_value, str): return raw_value if not looks_like_json_container_literal(raw_value): return raw_value if not should_parse_json_for_field(field_info): return raw_value try: return json.loads(raw_value) except Exception as e: raise typer.BadParameter(f"Invalid JSON for {field_info}: {e}") from e def resolve_field_names(settings, query: str) -> list[str]: """Return matching Settings fields for a case-insensitive partial query.""" fields = type(settings).model_fields query = _normalize_setting_key(query) # exact match (case-insensitive) first exact = [ name for name in fields.keys() if _normalize_setting_key(name) == query ] if exact: return exact # substring matches return [ name for name in fields.keys() if query in _normalize_setting_key(name) ] def is_optional(annotation) -> bool: origin = get_origin(annotation) if origin is Union: return type(None) in get_args(annotation) return False def parse_and_validate(field_name: str, field_info, raw: str): """ Validate and coerce a CLI value by delegating to the Settings model. Field validators like LOG_LEVEL coercion (e.g. 'error' -> numeric log level) are applied. """ settings = get_settings() value: object = maybe_parse_json_literal(raw, field_info) payload = settings.model_dump(mode="python") payload[field_name] = value try: validated = type(settings).model_validate(payload) except ValidationError as e: # Surface field-specific error(s) if possible field_errors: list[str] = [] for err in e.errors(): loc = err.get("loc") or () if loc and loc[0] == field_name: field_errors.append(err.get("msg") or str(err)) detail = "; ".join(field_errors) if field_errors else str(e) raise typer.BadParameter( f"Invalid value for {field_name}: {raw!r}. {detail}" ) from e return getattr(validated, field_name)