confident-ai--deepeval/deepeval/cli/generate/command.py

import sys
from pathlib import Path
from typing import Any, List, Optional

import typer
from rich import print

from deepeval.cli.generate.utils import (
    FileType,
    GenerationMethod,
    GoldenVariation,
    load_contexts_file,
    load_goldens_file,
    multi_turn_styling_config,
    require_method_option,
    single_turn_styling_config,
    validate_golden_variation,
    validate_scratch_styling,
)


# Lazy module-level attrs: ``Synthesizer`` and ``ContextConstructionConfig``
# materialize on first access (PEP 562) so unrelated CLI commands like
# ``deepeval test run`` don't pay for the synthesizer chain at startup.
# Tests still see them as module attributes so ``monkeypatch.setattr(
# generate_cli, "Synthesizer", _Fake)`` works.
def __getattr__(name: str) -> Any:
    if name == "Synthesizer":
        from deepeval.synthesizer import Synthesizer

        globals()["Synthesizer"] = Synthesizer
        return Synthesizer
    if name == "ContextConstructionConfig":
        from deepeval.synthesizer.config import ContextConstructionConfig

        globals()["ContextConstructionConfig"] = ContextConstructionConfig
        return ContextConstructionConfig
    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")


def generate_command(
    method: GenerationMethod = typer.Option(
        ...,
        "--method",
        help="Golden generation method to use.",
        case_sensitive=False,
    ),
    variation: GoldenVariation = typer.Option(
        ...,
        "--variation",
        help="Golden variation to generate.",
        case_sensitive=False,
    ),
    output_dir: str = typer.Option(
        "./synthetic_data",
        "--output-dir",
        help="Directory where generated goldens will be saved.",
    ),
    file_type: FileType = typer.Option(
        FileType.JSON,
        "--file-type",
        help="File type to save generated goldens as.",
        case_sensitive=False,
    ),
    file_name: Optional[str] = typer.Option(
        None,
        "--file-name",
        help="Optional output filename without extension.",
    ),
    model: Optional[str] = typer.Option(
        None,
        "--model",
        help="Model to use for generation.",
    ),
    async_mode: bool = typer.Option(
        True,
        "--async-mode/--sync-mode",
        help="Whether to generate goldens concurrently.",
    ),
    max_concurrent: int = typer.Option(
        100,
        "--max-concurrent",
        help="Maximum number of concurrent generation tasks.",
    ),
    include_expected: bool = typer.Option(
        True,
        "--include-expected/--no-include-expected",
        help="Whether to generate expected output or expected outcome.",
    ),
    cost_tracking: bool = typer.Option(
        False,
        "--cost-tracking",
        help="Print generation cost when supported by the model.",
    ),
    documents: Optional[List[str]] = typer.Option(
        None,
        "--documents",
        help="Document path to use with --method docs. Can be passed multiple times.",
    ),
    contexts_file: Optional[Path] = typer.Option(
        None,
        "--contexts-file",
        help='JSON file shaped like [["chunk 1", "chunk 2"], ...].',
    ),
    goldens_file: Optional[Path] = typer.Option(
        None,
        "--goldens-file",
        help="Existing goldens file to augment (.json, .csv, or .jsonl).",
    ),
    num_goldens: Optional[int] = typer.Option(
        None,
        "--num-goldens",
        help="Number of goldens to generate with --method scratch.",
    ),
    max_goldens_per_context: int = typer.Option(
        2,
        "--max-goldens-per-context",
        help="Maximum goldens to generate per context.",
    ),
    max_goldens_per_golden: int = typer.Option(
        2,
        "--max-goldens-per-golden",
        help="Maximum goldens to generate per existing golden.",
    ),
    max_contexts_per_document: int = typer.Option(
        3,
        "--max-contexts-per-document",
        help="Maximum contexts to construct per document.",
    ),
    min_contexts_per_document: int = typer.Option(
        1,
        "--min-contexts-per-document",
        help="Minimum contexts to construct per document.",
    ),
    chunk_size: int = typer.Option(
        1024,
        "--chunk-size",
        help="Token chunk size for document parsing.",
    ),
    chunk_overlap: int = typer.Option(
        0,
        "--chunk-overlap",
        help="Token overlap between document chunks.",
    ),
    context_quality_threshold: float = typer.Option(
        0.5,
        "--context-quality-threshold",
        help="Minimum context quality threshold.",
    ),
    context_similarity_threshold: float = typer.Option(
        0.0,
        "--context-similarity-threshold",
        help="Minimum context grouping similarity threshold.",
    ),
    max_retries: int = typer.Option(
        3,
        "--max-retries",
        help="Maximum retries for context construction quality checks.",
    ),
    scenario: Optional[str] = typer.Option(
        None,
        "--scenario",
        help="Single-turn generation scenario.",
    ),
    task: Optional[str] = typer.Option(
        None,
        "--task",
        help="Single-turn generation task.",
    ),
    input_format: Optional[str] = typer.Option(
        None,
        "--input-format",
        help="Single-turn input format.",
    ),
    expected_output_format: Optional[str] = typer.Option(
        None,
        "--expected-output-format",
        help="Single-turn expected output format.",
    ),
    scenario_context: Optional[str] = typer.Option(
        None,
        "--scenario-context",
        help="Multi-turn scenario context.",
    ),
    conversational_task: Optional[str] = typer.Option(
        None,
        "--conversational-task",
        help="Multi-turn conversational task.",
    ),
    participant_roles: Optional[str] = typer.Option(
        None,
        "--participant-roles",
        help="Multi-turn participant roles.",
    ),
    scenario_format: Optional[str] = typer.Option(
        None,
        "--scenario-format",
        help="Multi-turn scenario format.",
    ),
    expected_outcome_format: Optional[str] = typer.Option(
        None,
        "--expected-outcome-format",
        help="Multi-turn expected outcome format.",
    ),
):
    """Generate synthetic goldens with the golden synthesizer."""
    # Go through the module so test monkeypatches stick. Direct
    # ``from deepeval.synthesizer import Synthesizer`` would always
    # fetch the real class and ignore patched module attrs.
    _self = sys.modules[__name__]
    Synthesizer = _self.Synthesizer
    ContextConstructionConfig = _self.ContextConstructionConfig

    document_paths = None
    contexts = None
    goldens = None

    if method == GenerationMethod.DOCS:
        document_paths = require_method_option(documents, "--documents", method)
    elif method == GenerationMethod.CONTEXTS:
        contexts_path = require_method_option(
            contexts_file, "--contexts-file", method
        )
        contexts = load_contexts_file(contexts_path)
    elif method == GenerationMethod.SCRATCH:
        require_method_option(num_goldens, "--num-goldens", method)
        validate_scratch_styling(
            variation=variation,
            scenario=scenario,
            task=task,
            input_format=input_format,
            scenario_context=scenario_context,
            conversational_task=conversational_task,
            participant_roles=participant_roles,
        )
    elif method == GenerationMethod.GOLDENS:
        goldens_path = require_method_option(
            goldens_file, "--goldens-file", method
        )
        goldens = load_goldens_file(goldens_path)
        validate_golden_variation(goldens, variation)

    styling_config = single_turn_styling_config(
        scenario=scenario,
        task=task,
        input_format=input_format,
        expected_output_format=expected_output_format,
    )
    conversational_styling_config = multi_turn_styling_config(
        scenario_context=scenario_context,
        conversational_task=conversational_task,
        participant_roles=participant_roles,
        scenario_format=scenario_format,
        expected_outcome_format=expected_outcome_format,
    )
    synthesizer = Synthesizer(
        model=model,
        async_mode=async_mode,
        max_concurrent=max_concurrent,
        styling_config=styling_config,
        conversational_styling_config=conversational_styling_config,
        cost_tracking=cost_tracking,
    )

    if method == GenerationMethod.DOCS:
        context_construction_config = ContextConstructionConfig(
            max_contexts_per_document=max_contexts_per_document,
            min_contexts_per_document=min_contexts_per_document,
            chunk_size=chunk_size,
            chunk_overlap=chunk_overlap,
            context_quality_threshold=context_quality_threshold,
            context_similarity_threshold=context_similarity_threshold,
            max_retries=max_retries,
        )
        if variation == GoldenVariation.SINGLE_TURN:
            synthesizer.generate_goldens_from_docs(
                document_paths=document_paths,
                include_expected_output=include_expected,
                max_goldens_per_context=max_goldens_per_context,
                context_construction_config=context_construction_config,
            )
        else:
            synthesizer.generate_conversational_goldens_from_docs(
                document_paths=document_paths,
                include_expected_outcome=include_expected,
                max_goldens_per_context=max_goldens_per_context,
                context_construction_config=context_construction_config,
            )

    elif method == GenerationMethod.CONTEXTS:
        if variation == GoldenVariation.SINGLE_TURN:
            synthesizer.generate_goldens_from_contexts(
                contexts=contexts,
                include_expected_output=include_expected,
                max_goldens_per_context=max_goldens_per_context,
            )
        else:
            synthesizer.generate_conversational_goldens_from_contexts(
                contexts=contexts,
                include_expected_outcome=include_expected,
                max_goldens_per_context=max_goldens_per_context,
            )

    elif method == GenerationMethod.SCRATCH:
        if variation == GoldenVariation.SINGLE_TURN:
            synthesizer.generate_goldens_from_scratch(num_goldens=num_goldens)
        else:
            synthesizer.generate_conversational_goldens_from_scratch(
                num_goldens=num_goldens
            )

    elif method == GenerationMethod.GOLDENS:
        if variation == GoldenVariation.SINGLE_TURN:
            synthesizer.generate_goldens_from_goldens(
                goldens=goldens,
                max_goldens_per_golden=max_goldens_per_golden,
                include_expected_output=include_expected,
            )
        else:
            synthesizer.generate_conversational_goldens_from_goldens(
                goldens=goldens,
                max_goldens_per_golden=max_goldens_per_golden,
                include_expected_outcome=include_expected,
            )

    output_path = synthesizer.save_as(
        file_type=file_type.value,
        directory=output_dir,
        file_name=file_name,
        quiet=True,
    )
    print(f"Synthetic goldens saved at {output_path}!")