# Copyright (c) Microsoft. All rights reserved. # type: ignore """ This file is not carefully reviewed. It is to ensure the *somewhat* correctness of the code in agentlightning/config.py. It can contain logically erroroneous expected values. Please do not use the file as a reference for the expected behavior of config. """ import argparse import inspect import sys from typing import TypeVar # Added for completeness if testing TypeVars directly from typing import _GenericAlias # type: ignore from typing import ( Any, Callable, Dict, List, Optional, Tuple, Type, Union, get_args, get_origin, ) from unittest import mock # For mock.patch.object, mock.call, MagicMock etc. import pytest from agentlightning import config # TypeVar as used in the original code CliConfigurable = Any _C = TypeVar("_C", bound=CliConfigurable) # --- Helper classes for testing --- class SimpleConfig: def __init__(self, name: str, value: int = 10): self.name = name self.value = value class ComplexConfig: def __init__( self, text: str, is_active: bool, count: Optional[int] = None, items: List[str] = [], # Default mutable is usually discouraged, but for testing CLI it's okay. maybe_text: Optional[str] = "default_text", optional_list: Optional[List[int]] = None, list_of_optionals: List[Optional[str]] = [], any_param: Any = None, untyped_param="untyped_default", # Parameter without type hint but with a default ): self.text = text self.is_active = is_active self.count = count self.items = items self.maybe_text = maybe_text self.optional_list = optional_list self.list_of_optionals = list_of_optionals self.any_param = any_param self.untyped_param = untyped_param class NoInitParamsConfig: def __init__(self): pass class OnlySelfConfig: # For completeness, though __init__ without params is similar def __init__(self): pass class RequiredOnlyConfig: def __init__(self, req_str: str, req_int: int): self.req_str = req_str self.req_int = req_int class OptionalNoDefaultConfig: def __init__(self, opt_val: Optional[str]): # No __init__ default self.opt_val = opt_val # --- Tests for nullable_str --- @pytest.mark.parametrize( "input_val, expected_output", [ ("none", None), ("None", None), ("NONE", None), ("null", None), ("Null", None), ("NULL", None), ("~", None), ("nil", None), ("Nil", None), ("NIL", None), ("actual_string", "actual_string"), ("", ""), (" ", " "), ("NoneValue", "NoneValue"), # Should not convert if not exact keyword ], ) def test_nullable_str(input_val: str, expected_output: Optional[str]) -> None: """Tests the nullable_str function for various inputs.""" assert config.nullable_str(input_val) == expected_output # --- Tests for _str_to_bool --- @pytest.mark.parametrize( "input_val, expected_output", [ ("yes", True), ("Yes", True), ("YES", True), ("true", True), ("True", True), ("TRUE", True), ("t", True), ("T", True), ("y", True), ("Y", True), ("1", True), (True, True), # Direct bool pass-through ("no", False), ("No", False), ("NO", False), ("false", False), ("False", False), ("FALSE", False), ("f", False), ("F", False), ("n", False), ("N", False), ("0", False), (False, False), # Direct bool pass-through ], ) def test_str_to_bool_valid(input_val: Union[str, bool], expected_output: bool) -> None: """Tests _str_to_bool with valid boolean string representations.""" assert config._str_to_bool(input_val) == expected_output @pytest.mark.parametrize("invalid_input", ["maybe", "2", "", " ", "trueish", "falsey"]) def test_str_to_bool_invalid(invalid_input: str) -> None: """Tests _str_to_bool with invalid inputs, expecting ArgumentTypeError.""" with pytest.raises(argparse.ArgumentTypeError): config._str_to_bool(invalid_input) # --- Tests for _get_param_type_details --- @pytest.mark.parametrize( "annotation, expected_core_type, expected_is_optional, expected_is_list", [ (str, str, False, False), (int, int, False, False), (bool, bool, False, False), (Any, Any, False, False), (Optional[str], str, True, False), (Union[str, None], str, True, False), # Equivalent to Optional[str] (Optional[int], int, True, False), (List[str], List[str], False, True), # core_type is List[str] (List[int], List[int], False, True), (Optional[List[str]], List[str], True, True), # core_type is List[str], outer is Optional (Union[List[str], None], List[str], True, True), (List[Optional[str]], List[Optional[str]], False, True), # core_type is List[Optional[str]] (Optional[List[Optional[int]]], List[Optional[int]], True, True), (inspect.Parameter.empty, inspect.Parameter.empty, False, False), # No annotation (Union[int, str], Union[int, str], False, False), # Non-optional Union (Tuple[str, int], Tuple[str, int], False, False), # Tuple is not a List (Callable[[int], str], Callable[[int], str], False, False), (Dict[str, int], Dict[str, int], False, False), (List[Any], List[Any], False, True), (Optional[List[Any]], List[Any], True, True), ], ) def test_get_param_type_details( annotation: Any, expected_core_type: Any, expected_is_optional: bool, expected_is_list: bool ) -> None: """Tests _get_param_type_details for various type annotations.""" core_type, is_optional, is_list = config._get_param_type_details(annotation) # Handle comparison for complex types like Union by comparing origin and args if get_origin(expected_core_type) is Union and get_origin(core_type) is Union: assert get_origin(core_type) == get_origin(expected_core_type) # Sort args for comparison as order doesn't matter in Union and get_args might not preserve input order assert sorted(map(str, get_args(core_type))) == sorted(map(str, get_args(expected_core_type))) else: assert core_type == expected_core_type assert is_optional == expected_is_optional assert is_list == expected_is_list # --- Tests for _determine_argparse_type_and_nargs --- @pytest.mark.parametrize( "core_param_type, is_param_list, expected_kwargs", [ # List cases (List[str], True, {"nargs": "*", "type": str}), (List[int], True, {"nargs": "*", "type": int}), (List[float], True, {"nargs": "*", "type": float}), (List[bool], True, {"nargs": "*", "type": config._str_to_bool}), (List[Optional[str]], True, {"nargs": "*", "type": config.nullable_str}), (List[Any], True, {"nargs": "*", "type": str}), # Defaults to str for List[Any] items # Non-list cases (str, False, {"type": str}), (int, False, {"type": int}), (float, False, {"type": float}), (bool, False, {"type": config._str_to_bool}), (Optional[str], False, {"type": config.nullable_str}), # Handled by re-checking details (Union[str, None], False, {"type": config.nullable_str}), # Equivalent to Optional[str] (Any, False, {"type": str}), # No specific argparse type for Any (inspect.Parameter.empty, False, {"type": str}), # No specific type for unannotated # List of complex types (e.g. List[Dict]), type of item is treated as string (List[Dict[str, int]], True, {"nargs": "*", "type": str}), ], ) def test_determine_argparse_type_and_nargs( core_param_type: Any, is_param_list: bool, expected_kwargs: Dict[str, Any] ) -> None: """Tests _determine_argparse_type_and_nargs for type and nargs mapping.""" assert config._determine_argparse_type_and_nargs(core_param_type, is_param_list) == expected_kwargs # --- Tests for _build_help_string --- @pytest.mark.parametrize( "cls_name, param_name, core_type, is_optional, is_list, expected_help", [ ("C", "p", str, False, False, "For C: 'p'. Inferred type: str."), ("C", "p", int, True, False, "For C: 'p'. Inferred type: Optional[int]."), ("C", "p_list", List[str], False, True, "For C: 'p_list'. Inferred type: List[str]."), ("C", "p_opt_list", List[int], True, True, "For C: 'p_opt_list'. Inferred type: Optional[List[int]]."), ( "C", "p_list_opt", List[Optional[str]], False, True, "For C: 'p_list_opt'. Inferred type: List[Optional[str]].", ), ("C", "p_any", Any, False, False, "For C: 'p_any'. Inferred type: Any."), ("C", "p_opt_any", Any, True, False, "For C: 'p_opt_any'. Inferred type: Optional[Any]."), ("C", "p_list_any", List[Any], False, True, "For C: 'p_list_any'. Inferred type: List[Any]."), ("C", "p_empty", inspect.Parameter.empty, False, False, "For C: 'p_empty'. Inferred type: Any."), ], ) def test_build_help_string( cls_name: str, param_name: str, core_type: Any, is_optional: bool, is_list: bool, expected_help: str ) -> None: """Tests _build_help_string for generating correct help messages.""" assert config._build_help_string(cls_name, param_name, core_type, is_optional, is_list) == expected_help # --- Tests for _add_argument_for_parameter --- @pytest.fixture def mock_parser(): """Fixture to create a mock ArgumentParser.""" parser = mock.MagicMock(spec=argparse.ArgumentParser) parser.add_argument = mock.MagicMock() return parser def get_param_obj(cls, param_name): """Helper to get an inspect.Parameter object from a class's __init__.""" sig = inspect.signature(cls.__init__) return sig.parameters[param_name] @pytest.mark.parametrize( "param_name, cls, expected_cli_name_part, expected_argparse_kwargs_subset", [ ("name", SimpleConfig, "simpleconfig.name", {"type": str, "required": True}), ("value", SimpleConfig, "simpleconfig.value", {"type": int, "default": 10}), ("text", ComplexConfig, "complexconfig.text", {"type": str, "required": True}), ("is_active", ComplexConfig, "complexconfig.is-active", {"type": config._str_to_bool, "required": True}), ( "count", ComplexConfig, "complexconfig.count", {"type": config.nullable_int, "default": None}, ), # Optional[int]=None ("items", ComplexConfig, "complexconfig.items", {"type": str, "nargs": "*", "default": []}), ( "maybe_text", ComplexConfig, "complexconfig.maybe-text", {"type": config.nullable_str, "default": "default_text"}, ), ("optional_list", ComplexConfig, "complexconfig.optional-list", {"type": int, "nargs": "*", "default": None}), ( "list_of_optionals", ComplexConfig, "complexconfig.list-of-optionals", {"type": config.nullable_str, "nargs": "*", "default": []}, ), ( "any_param", ComplexConfig, "complexconfig.any-param", {"default": None}, ), # Type Any implies no specific argparse type ( "untyped_param", ComplexConfig, "complexconfig.untyped-param", {"type": str, "default": "untyped_default"}, ), # No type hint ( "opt_val", OptionalNoDefaultConfig, "optionalnodefaultconfig.opt-val", {"type": config.nullable_str, "default": None}, ), ], ) def test_add_argument_for_parameter( mock_parser, param_name, cls, expected_cli_name_part, expected_argparse_kwargs_subset ): """Tests _add_argument_for_parameter for correct argument configuration.""" param_obj = get_param_obj(cls, param_name) dest_name = f"{cls.__name__.lower()}_{param_name}" config._add_argument_for_parameter(mock_parser, cls, param_name, param_obj, dest_name) # Build expected CLI argument name (e.g., --simpleconfig.name) expected_cli_arg_name = f"--{expected_cli_name_part.replace('_', '-')}" # Check that add_argument was called assert mock_parser.add_argument.called # Get the actual call arguments actual_call_args, actual_call_kwargs = mock_parser.add_argument.call_args assert actual_call_args[0] == expected_cli_arg_name assert actual_call_kwargs["dest"] == dest_name # Check for the presence and correctness of specified kwargs for key, expected_value in expected_argparse_kwargs_subset.items(): assert key in actual_call_kwargs, f"Key '{key}' not in argparse kwargs for {param_name}" assert actual_call_kwargs[key] == expected_value, f"Value for key '{key}' incorrect for {param_name}" # Check 'required' status carefully if expected_argparse_kwargs_subset.get("required", False): assert actual_call_kwargs.get("required") is True else: # Not required or default makes it not required assert not actual_call_kwargs.get("required", False) # Either 'required' is False or not present # Check help string was generated assert "help" in actual_call_kwargs assert isinstance(actual_call_kwargs["help"], str) # --- Tests for _add_arguments_for_class --- def test_add_arguments_for_class(mock_parser: Any) -> None: """Tests _add_arguments_for_class by checking calls to _add_argument_for_parameter.""" class_arg_configs_maps = {} with mock.patch("agentlightning.config._add_argument_for_parameter") as mock_add_param_func: config._add_arguments_for_class(mock_parser, ComplexConfig, class_arg_configs_maps) assert ComplexConfig in class_arg_configs_maps init_params = inspect.signature(ComplexConfig.__init__).parameters expected_calls = 0 for p_name, p_obj in init_params.items(): if p_name == "self": continue expected_calls += 1 dest = f"complexconfig_{p_name}" assert class_arg_configs_maps[ComplexConfig][p_name] == dest # Check if mock_add_param_func was called with these specific arguments found_call = any( call_args[0] == mock_parser and call_args[1] == ComplexConfig and call_args[2] == p_name and call_args[3] == p_obj and call_args[4] == dest for call_args, _ in mock_add_param_func.call_args_list ) assert found_call, f"Expected call for {p_name} not found or with wrong arguments." assert mock_add_param_func.call_count == expected_calls def test_add_arguments_for_class_no_init_params(mock_parser: Any) -> None: """Tests _add_arguments_for_class with a class having no __init__ parameters.""" class_arg_configs_maps = {} with mock.patch("agentlightning.config._add_argument_for_parameter") as mock_add_param_func: config._add_arguments_for_class(mock_parser, NoInitParamsConfig, class_arg_configs_maps) mock_add_param_func.assert_not_called() assert class_arg_configs_maps[NoInitParamsConfig] == {} def test_create_argument_parser() -> None: """Tests _create_argument_parser for basic parser properties.""" parser = config._create_argument_parser() assert isinstance(parser, argparse.ArgumentParser) assert parser.description == "CLI configurator for application components." # The formatter_class is a type, so compare types assert type(parser.formatter_class) == type(argparse.ArgumentDefaultsHelpFormatter) def test_instantiate_classes() -> None: """Tests _instantiate_classes with various argument types and defaults.""" parsed_args = argparse.Namespace( simpleconfig_name="TestName", simpleconfig_value=99, complexconfig_text="Hello", complexconfig_is_active=True, complexconfig_count=None, # Explicitly None complexconfig_items=["item1", "item2"], # maybe_text not provided, should use its default "default_text" from __init__ # Argparse default for complexconfig_maybe_text would be "default_text" complexconfig_maybe_text="default_text", # Simulate argparse providing the default complexconfig_optional_list=[1, 2, 3], complexconfig_list_of_optionals=["a", None, "b"], complexconfig_any_param=object(), complexconfig_untyped_param="custom_untyped", ) # This simulates that 'complexconfig_maybe_text' was given a default by add_argument # and thus present in parsed_args. # If not provided on CLI and having __init__ default, argparse gives it default. class_arg_configs_maps = { SimpleConfig: {"name": "simpleconfig_name", "value": "simpleconfig_value"}, ComplexConfig: { p: f"complexconfig_{p}" for p in inspect.signature(ComplexConfig.__init__).parameters if p != "self" }, } instances = config._instantiate_classes(parsed_args, (SimpleConfig, ComplexConfig), class_arg_configs_maps) assert len(instances) == 2 sc, cc = instances assert isinstance(sc, SimpleConfig) assert sc.name == "TestName" assert sc.value == 99 assert isinstance(cc, ComplexConfig) assert cc.text == "Hello" assert cc.is_active is True assert cc.count is None assert cc.items == ["item1", "item2"] assert cc.maybe_text == "default_text" # From __init__ via argparse default assert cc.optional_list == [1, 2, 3] assert cc.list_of_optionals == ["a", None, "b"] assert cc.any_param is parsed_args.complexconfig_any_param assert cc.untyped_param == "custom_untyped" def test_instantiate_classes_error_handling(caplog): """Tests error logging during class instantiation failure.""" class FailingConfig: def __init__(self, param): raise ValueError("Instantiaion failed") parsed_args = argparse.Namespace(failingconfig_param="value") class_arg_configs_maps = {FailingConfig: {"param": "failingconfig_param"}} with pytest.raises(ValueError, match="Instantiaion failed"): config._instantiate_classes(parsed_args, (FailingConfig,), class_arg_configs_maps) assert "Error instantiating FailingConfig" in caplog.text assert "{'param': 'value'}" in caplog.text # constructor_args assert "Error: Instantiaion failed" in caplog.text # --- Integration Tests for lightning_cli --- def run_lightning_cli(classes_to_configure, cli_args_list): """Helper to run lightning_cli with mocked sys.argv.""" # Prepend a dummy program name to cli_args_list for sys.argv with mock.patch.object(sys, "argv", ["test_program.py"] + cli_args_list): result = config.lightning_cli(*classes_to_configure) if not isinstance(result, tuple): return (result,) return result def test_lightning_cli_no_classes(): """Tests lightning_cli with no classes provided.""" assert run_lightning_cli([], []) == tuple() def test_lightning_cli_simple_config(): """Tests lightning_cli with a simple class and various argument scenarios.""" # Only required arg (sc1,) = run_lightning_cli([SimpleConfig], ["--simpleconfig.name", "MyName"]) assert sc1.name == "MyName" assert sc1.value == 10 # Default from __init__ # All args (sc2,) = run_lightning_cli([SimpleConfig], ["--simpleconfig.name", "Another", "--simpleconfig.value", "77"]) assert sc2.name == "Another" assert sc2.value == 77 def test_lightning_cli_complex_config_types(): """Tests lightning_cli with ComplexConfig, checking various type conversions.""" cli_args = [ "--complexconfig.text", "CLI Text", "--complexconfig.is-active", "yes", "--complexconfig.count", "123", "--complexconfig.items", "apple", "banana", "--complexconfig.maybe-text", "None", # Test nullable_str "--complexconfig.optional-list", "10", "20", "30", "--complexconfig.list-of-optionals", "first", "nil", "third", "null", "--complexconfig.any-param", "AnythingGoes", "--complexconfig.untyped-param", "FromCLI", ] (cc,) = run_lightning_cli([ComplexConfig], cli_args) assert cc.text == "CLI Text" assert cc.is_active is True assert cc.count == 123 assert cc.items == ["apple", "banana"] assert cc.maybe_text is None assert cc.optional_list == [10, 20, 30] assert cc.list_of_optionals == ["first", None, "third", None] assert cc.any_param == "AnythingGoes" assert cc.untyped_param == "FromCLI" def test_lightning_cli_complex_config_defaults(): """Tests that __init__ defaults are used if CLI args are not provided.""" # Only provide required arguments for ComplexConfig cli_args = ["--complexconfig.text", "Minimal", "--complexconfig.is-active", "f"] (cc,) = run_lightning_cli([ComplexConfig], cli_args) assert cc.text == "Minimal" assert cc.is_active is False assert cc.count is None # Default from __init__ assert cc.items == [] # Default from __init__ assert cc.maybe_text == "default_text" # Default from __init__ assert cc.optional_list is None # Default from __init__ assert cc.list_of_optionals == [] # Default from __init__ assert cc.any_param is None # Default from __init__ assert cc.untyped_param == "untyped_default" # Default from __init__ def test_lightning_cli_multiple_classes(): """Tests configuring multiple classes simultaneously.""" cli_args = [ "--simpleconfig.name", "SC_Multi", "--simpleconfig.value", "5", "--complexconfig.text", "CC_Multi", "--complexconfig.is-active", "true", "--complexconfig.maybe-text", "NotNone", ] sc, cc = run_lightning_cli([SimpleConfig, ComplexConfig], cli_args) assert isinstance(sc, SimpleConfig) assert sc.name == "SC_Multi" assert sc.value == 5 assert isinstance(cc, ComplexConfig) assert cc.text == "CC_Multi" assert cc.is_active is True assert cc.maybe_text == "NotNone" assert cc.items == [] # Default def test_lightning_cli_missing_required_arg_exits(capsys): """Tests that argparse exits if a required argument is missing.""" with pytest.raises(SystemExit) as e: run_lightning_cli([RequiredOnlyConfig], ["--requiredonlyconfig.req-int", "123"]) assert e.value.code != 0 # Argparse exits with non-zero for error captured = capsys.readouterr() # Capture stderr assert "the following arguments are required: --requiredonlyconfig.req-str" in captured.err def test_lightning_cli_optional_no_default_behavior(): """Tests Optional parameter without __init__ default.""" # Not provided: should become None (cfg1,) = run_lightning_cli([OptionalNoDefaultConfig], []) assert cfg1.opt_val is None # Provided as "None" via nullable_str (cfg2,) = run_lightning_cli([OptionalNoDefaultConfig], ["--optionalnodefaultconfig.opt-val", "None"]) assert cfg2.opt_val is None # Provided with a value (cfg3,) = run_lightning_cli([OptionalNoDefaultConfig], ["--optionalnodefaultconfig.opt-val", "ActualValue"]) assert cfg3.opt_val == "ActualValue"