confident-ai--deepeval/deepeval/cli/generate/utils.py

import json
from enum import Enum
from pathlib import Path
from typing import List, Optional, Union

import typer

from deepeval.dataset import EvaluationDataset
from deepeval.dataset.golden import Golden, ConversationalGolden
from deepeval.synthesizer.config import (
    ConversationalStylingConfig,
    StylingConfig,
)


class GenerationMethod(str, Enum):
    DOCS = "docs"
    CONTEXTS = "contexts"
    SCRATCH = "scratch"
    GOLDENS = "goldens"


class GoldenVariation(str, Enum):
    SINGLE_TURN = "single-turn"
    MULTI_TURN = "multi-turn"


class FileType(str, Enum):
    JSON = "json"
    CSV = "csv"
    JSONL = "jsonl"


def require_method_option(value, option_name: str, method: GenerationMethod):
    if value is None or value == []:
        raise typer.BadParameter(
            f"`{option_name}` is required when --method is `{method.value}`.",
            param_hint=option_name,
        )
    return value


def load_contexts_file(contexts_file: Path) -> List[List[str]]:
    try:
        raw_contexts = json.loads(contexts_file.read_text(encoding="utf-8"))
    except FileNotFoundError:
        raise typer.BadParameter(
            f"Contexts file not found: {contexts_file}",
            param_hint="--contexts-file",
        )
    except json.JSONDecodeError as exc:
        raise typer.BadParameter(
            f"Contexts file must be valid JSON: {exc}",
            param_hint="--contexts-file",
        )

    if not isinstance(raw_contexts, list):
        raise typer.BadParameter(
            "Contexts file must contain a JSON list of context lists.",
            param_hint="--contexts-file",
        )

    for context in raw_contexts:
        if not isinstance(context, list) or not all(
            isinstance(chunk, str) for chunk in context
        ):
            raise typer.BadParameter(
                'Contexts file must be shaped like [["chunk 1", "chunk 2"], ...].',
                param_hint="--contexts-file",
            )

    return raw_contexts


def load_goldens_file(
    goldens_file: Path,
) -> Union[List[Golden], List[ConversationalGolden]]:
    if not goldens_file.exists():
        raise typer.BadParameter(
            f"Goldens file not found: {goldens_file}",
            param_hint="--goldens-file",
        )

    dataset = EvaluationDataset()
    suffix = goldens_file.suffix.lower()
    if suffix == ".json":
        dataset.add_goldens_from_json_file(str(goldens_file))
        return dataset.goldens

    if suffix == ".csv":
        dataset.add_goldens_from_csv_file(str(goldens_file))
        return dataset.goldens

    if suffix == ".jsonl":
        dataset.add_goldens_from_jsonl_file(str(goldens_file))
        return dataset.goldens

    raise typer.BadParameter(
        "Goldens file must be a .json, .csv, or .jsonl file.",
        param_hint="--goldens-file",
    )


def validate_golden_variation(
    goldens: Union[List[Golden], List[ConversationalGolden]],
    variation: GoldenVariation,
) -> None:
    if not goldens:
        raise typer.BadParameter(
            "Goldens file does not contain any goldens.",
            param_hint="--goldens-file",
        )

    first_golden = goldens[0]
    is_multi_turn = isinstance(first_golden, ConversationalGolden)
    if variation == GoldenVariation.MULTI_TURN and not is_multi_turn:
        raise typer.BadParameter(
            "`--variation multi-turn` requires conversational goldens.",
            param_hint="--variation",
        )
    if variation == GoldenVariation.SINGLE_TURN and is_multi_turn:
        raise typer.BadParameter(
            "`--variation single-turn` requires single-turn goldens.",
            param_hint="--variation",
        )


def single_turn_styling_config(
    scenario: Optional[str],
    task: Optional[str],
    input_format: Optional[str],
    expected_output_format: Optional[str],
) -> Optional[StylingConfig]:
    if not any([scenario, task, input_format, expected_output_format]):
        return None
    return StylingConfig(
        scenario=scenario,
        task=task,
        input_format=input_format,
        expected_output_format=expected_output_format,
    )


def multi_turn_styling_config(
    scenario_context: Optional[str],
    conversational_task: Optional[str],
    participant_roles: Optional[str],
    scenario_format: Optional[str],
    expected_outcome_format: Optional[str],
) -> Optional[ConversationalStylingConfig]:
    if not any(
        [
            scenario_context,
            conversational_task,
            participant_roles,
            scenario_format,
            expected_outcome_format,
        ]
    ):
        return None
    return ConversationalStylingConfig(
        scenario_context=scenario_context,
        conversational_task=conversational_task,
        participant_roles=participant_roles,
        scenario_format=scenario_format,
        expected_outcome_format=expected_outcome_format,
    )


def validate_scratch_styling(
    variation: GoldenVariation,
    scenario: Optional[str],
    task: Optional[str],
    input_format: Optional[str],
    scenario_context: Optional[str],
    conversational_task: Optional[str],
    participant_roles: Optional[str],
) -> None:
    if variation == GoldenVariation.SINGLE_TURN:
        missing = [
            option
            for option, value in [
                ("--scenario", scenario),
                ("--task", task),
                ("--input-format", input_format),
            ]
            if value is None
        ]
    else:
        missing = [
            option
            for option, value in [
                ("--scenario-context", scenario_context),
                ("--conversational-task", conversational_task),
                ("--participant-roles", participant_roles),
            ]
            if value is None
        ]

    if missing:
        raise typer.BadParameter(
            "Scratch generation requires: " + ", ".join(missing),
            param_hint=missing[0],
        )