zzet--gortex/eval/tests/test_config_properties.py

"""Property-based tests for eval.config module.

Feature: eval-framework
Uses hypothesis to verify config merge and validation properties.
"""

from __future__ import annotations

import re
from typing import Any

import pytest
from hypothesis import given, settings
from hypothesis import strategies as st

from config import merge_configs, validate_config

# ---------------------------------------------------------------------------
# Strategies
# ---------------------------------------------------------------------------

# Keys: non-empty strings without dots (dots are used as path separators in
# validate_config, so keeping keys simple avoids confusion).
_key_st = st.text(
    alphabet=st.characters(whitelist_categories=("L", "N"), whitelist_characters=("_", "-")),
    min_size=1,
    max_size=12,
)

# Leaf values: scalars that YAML configs typically hold.
_leaf_st = st.one_of(
    st.text(min_size=0, max_size=30),
    st.integers(min_value=-1000, max_value=1000),
    st.floats(allow_nan=False, allow_infinity=False, min_value=-1e6, max_value=1e6),
    st.booleans(),
    st.none(),
)

# Shallow config dict (1 level deep) — sufficient for merge precedence tests.
_flat_dict_st = st.dictionaries(keys=_key_st, values=_leaf_st, max_size=8)

# Nested config dict (up to 2 levels) — mirrors real YAML configs.
_nested_value_st = st.one_of(
    _leaf_st,
    st.dictionaries(keys=_key_st, values=_leaf_st, max_size=5),
)
_config_dict_st = st.dictionaries(keys=_key_st, values=_nested_value_st, max_size=8)


# Required field paths used by validate_config.
_REQUIRED_FIELDS = [
    "model.model_name",
    "agent.agent_class",
    "environment.environment_class",
]


def _build_full_config() -> dict[str, Any]:
    """Return a config dict that passes validation (all required fields present)."""
    return {
        "model": {"model_name": "test-model"},
        "agent": {"agent_class": "eval.agents.TestAgent"},
        "environment": {"environment_class": "eval.environments.TestEnv"},
    }


def _set_nested(d: dict, dotted_key: str, value: Any) -> None:
    """Set a value in a nested dict using dot notation."""
    parts = dotted_key.split(".")
    current = d
    for part in parts[:-1]:
        current = current.setdefault(part, {})
    current[parts[-1]] = value


# ---------------------------------------------------------------------------
# Property 10: Config merge precedence
# Feature: eval-framework, Property 10: Config merge precedence
# **Validates: Requirements 10.1, 10.2**
# ---------------------------------------------------------------------------


class TestConfigMergePrecedence:
    """For any two config dicts, mode values override model values on shared
    keys; unique keys preserved."""

    @given(model=_config_dict_st, mode=_config_dict_st)
    @settings(max_examples=100)
    def test_mode_overrides_model_on_shared_keys(
        self, model: dict[str, Any], mode: dict[str, Any]
    ) -> None:
        """Shared top-level keys take the mode value (or deep-merged sub-dict)."""
        merged = merge_configs(model, mode)

        for key in mode:
            if key in model:
                model_val = model[key]
                mode_val = mode[key]
                merged_val = merged[key]

                if isinstance(model_val, dict) and isinstance(mode_val, dict):
                    # When both are dicts, mode sub-keys override model sub-keys.
                    for sub_key in mode_val:
                        assert merged_val[sub_key] == mode_val[sub_key]
                else:
                    # Scalar or type mismatch: mode wins entirely.
                    assert merged_val == mode_val

    @given(model=_config_dict_st, mode=_config_dict_st)
    @settings(max_examples=100)
    def test_unique_keys_preserved(
        self, model: dict[str, Any], mode: dict[str, Any]
    ) -> None:
        """Keys present in only one config appear unchanged in the merged result."""
        merged = merge_configs(model, mode)

        # Model-only keys preserved.
        for key in model:
            if key not in mode:
                assert merged[key] == model[key]

        # Mode-only keys preserved.
        for key in mode:
            if key not in model:
                assert merged[key] == mode[key]

    @given(model=_config_dict_st, mode=_config_dict_st)
    @settings(max_examples=100)
    def test_merged_contains_all_keys(
        self, model: dict[str, Any], mode: dict[str, Any]
    ) -> None:
        """The merged dict contains every key from both inputs."""
        merged = merge_configs(model, mode)
        assert set(merged.keys()) == set(model.keys()) | set(mode.keys())


# ---------------------------------------------------------------------------
# Property 11: Config validation catches missing required fields
# Feature: eval-framework, Property 11: Config validation catches missing required fields
# **Validates: Requirements 10.3**
# ---------------------------------------------------------------------------

# Strategy: pick a non-empty subset of required fields to remove.
_required_subsets_st = st.lists(
    st.sampled_from(_REQUIRED_FIELDS),
    min_size=1,
    max_size=len(_REQUIRED_FIELDS),
    unique=True,
)


class TestConfigValidationMissingFields:
    """For any merged config missing one or more of (model_name, agent_class,
    environment_class), validation fails naming the missing field(s)."""

    @given(missing_fields=_required_subsets_st)
    @settings(max_examples=100)
    def test_validation_fails_naming_missing_fields(
        self, missing_fields: list[str]
    ) -> None:
        """Removing any subset of required fields causes ValueError listing them."""
        config = _build_full_config()

        # Remove the selected required fields.
        for field in missing_fields:
            parts = field.split(".")
            section = parts[0]
            key = parts[1]
            if section in config and isinstance(config[section], dict):
                config[section].pop(key, None)

        with pytest.raises(ValueError, match="Missing required config fields") as exc_info:
            validate_config(config)

        error_msg = str(exc_info.value)
        for field in missing_fields:
            assert field in error_msg, (
                f"Expected '{field}' to be named in error but got: {error_msg}"
            )

    @given(missing_fields=_required_subsets_st)
    @settings(max_examples=100)
    def test_validation_only_reports_actually_missing_fields(
        self, missing_fields: list[str]
    ) -> None:
        """Fields that ARE present should NOT appear in the error message."""
        config = _build_full_config()
        present_fields = [f for f in _REQUIRED_FIELDS if f not in missing_fields]

        for field in missing_fields:
            parts = field.split(".")
            section = parts[0]
            key = parts[1]
            if section in config and isinstance(config[section], dict):
                config[section].pop(key, None)

        with pytest.raises(ValueError) as exc_info:
            validate_config(config)

        error_msg = str(exc_info.value)
        for field in present_fields:
            assert field not in error_msg, (
                f"Field '{field}' is present but was reported as missing: {error_msg}"
            )

    @given(st.just(_build_full_config()))
    @settings(max_examples=10)
    def test_complete_config_passes_validation(self, config: dict[str, Any]) -> None:
        """A config with all required fields should pass validation."""
        validate_config(config)  # should not raise