"""Tests for omnigent.spec.validator.""" from __future__ import annotations import pytest from omnigent.inner.datamodel import OSEnvSandboxSpec, OSEnvSpec from omnigent.spec.types import ( AgentSpec, CompactionConfig, ExecutorSpec, InteractionConfig, LLMConfig, LocalToolInfo, MCPServerConfig, ModalityConfig, SkillSpec, ToolsConfig, ) from omnigent.spec.validator import validate def _minimal_spec(**overrides: object) -> AgentSpec: """Build a minimal valid AgentSpec with optional overrides. Mirrors the parser's consolidation: when ``llm`` is supplied, ``executor.model`` and ``executor.connection`` are synced from the LLM config so the validator sees the canonical fields. """ defaults: dict[str, object] = { "spec_version": 1, "executor": ExecutorSpec(config={"harness": "claude-sdk"}), } defaults.update(overrides) # Consolidate llm → executor (same as parser does at load time). llm = defaults.get("llm") if isinstance(llm, LLMConfig): executor = defaults.get("executor") if not isinstance(executor, ExecutorSpec): executor = ExecutorSpec(config={"harness": "claude-sdk"}) if executor.model is None and llm.model: executor = ExecutorSpec( type=executor.type, timeout=executor.timeout, max_iterations=executor.max_iterations, profile=executor.profile, config=executor.config, model=llm.model, connection=llm.connection if executor.connection is None else executor.connection, context_window=executor.context_window, ) defaults["executor"] = executor return AgentSpec(**defaults) # type: ignore[arg-type] def test_minimal_spec_valid() -> None: result = validate(_minimal_spec()) assert result.valid def test_invalid_spec_version() -> None: result = validate(_minimal_spec(spec_version=2)) assert not result.valid assert any("spec_version" in e.path for e in result.errors) def test_llm_valid() -> None: spec = _minimal_spec(llm=LLMConfig(model="openai/gpt-5.4", connection={"api_key": "sk-test"})) result = validate(spec) assert result.valid def test_llm_empty_model() -> None: spec = _minimal_spec(llm=LLMConfig(model="")) result = validate(spec) assert not result.valid assert any("llm.model" in e.path for e in result.errors) def test_llm_arbitrary_extra_passes_validation() -> None: """Extra keys are passed through — validator does not reject them.""" spec = _minimal_spec( llm=LLMConfig( model="openai/gpt-5.4", connection={"api_key": "sk-test"}, extra={"temperature": 0.7, "reasoning_effort": "extreme"}, ) ) result = validate(spec) assert result.valid def test_valid_input_modalities() -> None: spec = _minimal_spec( interaction=InteractionConfig( modalities=ModalityConfig(input=["text", "image", "audio", "video", "file"]) ) ) result = validate(spec) assert result.valid def test_invalid_input_modality() -> None: spec = _minimal_spec( interaction=InteractionConfig(modalities=ModalityConfig(input=["text", "smell"])) ) result = validate(spec) assert not result.valid assert any("smell" in e.message for e in result.errors) def test_invalid_output_modality() -> None: spec = _minimal_spec( interaction=InteractionConfig(modalities=ModalityConfig(output=["text", "file"])) ) result = validate(spec) assert not result.valid assert any("file" in e.message for e in result.errors) def test_valid_output_modalities() -> None: spec = _minimal_spec( interaction=InteractionConfig(modalities=ModalityConfig(output=["text", "image", "audio"])) ) result = validate(spec) assert result.valid def test_skill_valid() -> None: spec = _minimal_spec( skills=[ SkillSpec( name="deep-search", description="Search the web.", content="Use search.web.", ) ] ) result = validate(spec) assert result.valid def test_skill_name_invalid_pattern() -> None: spec = _minimal_spec(skills=[SkillSpec(name="Bad_Name", description="Bad.", content=".")]) result = validate(spec) assert not result.valid assert any("must match" in e.message for e in result.errors) def test_skill_name_too_long() -> None: spec = _minimal_spec(skills=[SkillSpec(name="a" * 65, description="Long name.", content=".")]) result = validate(spec) assert not result.valid assert any("at most 64" in e.message for e in result.errors) def test_skill_description_too_long() -> None: spec = _minimal_spec(skills=[SkillSpec(name="ok", description="x" * 1025, content=".")]) result = validate(spec) assert not result.valid assert any("at most 1024" in e.message for e in result.errors) def test_duplicate_skill_names() -> None: spec = _minimal_spec( skills=[ SkillSpec(name="dupe", description="First.", content="."), SkillSpec(name="dupe", description="Second.", content="."), ] ) result = validate(spec) assert not result.valid assert any("duplicate skill name" in e.message for e in result.errors) def test_mcp_http_valid() -> None: spec = _minimal_spec(mcp_servers=[MCPServerConfig(name="svc", url="http://localhost:9000")]) result = validate(spec) assert result.valid def test_duplicate_mcp_names() -> None: spec = _minimal_spec( mcp_servers=[ MCPServerConfig(name="dupe", url="http://a"), MCPServerConfig(name="dupe", url="http://b"), ] ) result = validate(spec) assert not result.valid assert any("duplicate MCP server name" in e.message for e in result.errors) def test_duplicate_tool_names_across_mcp_and_local() -> None: spec = _minimal_spec( mcp_servers=[MCPServerConfig(name="search", url="http://localhost:9000")], local_tools=[ LocalToolInfo(name="search", path="tools/python/search.py", language="python") ], ) result = validate(spec) assert not result.valid assert any("duplicate tool name" in e.message for e in result.errors) def test_sub_agent_reference_valid() -> None: sub = _minimal_spec( name="helper", llm=LLMConfig(model="openai/gpt-4o", connection={"api_key": "sk-test"}), ) spec = _minimal_spec( tools=ToolsConfig(agents=["helper"]), sub_agents=[sub], ) result = validate(spec) assert result.valid def test_sub_agent_reference_missing() -> None: spec = _minimal_spec( tools=ToolsConfig(agents=["ghost"]), ) result = validate(spec) assert not result.valid assert any("ghost" in e.message for e in result.errors) @pytest.mark.parametrize( "invalid_name", [ "has.dot", # dot is the tunneled model field delimiter "has/slash", # slash is the litellm provider/model separator "has space", # whitespace confuses API clients and log pipelines "has\ttab", # tab is also whitespace "", # empty string has no meaningful identity ], ) def test_agent_name_invalid_characters(invalid_name: str) -> None: """ Agent names with dots, slashes, whitespace, or empty string are rejected. Each of these characters would break either the tunneled model field (dots), litellm routing (slashes), or client parsing (whitespace/empty). """ spec = _minimal_spec(name=invalid_name) result = validate(spec) assert not result.valid assert any("name" in e.path for e in result.errors) @pytest.mark.parametrize( "valid_name", [ "researcher", "my-agent", "agent_v2", "Agent123", "CamelCase", "a", ], ) def test_agent_name_valid(valid_name: str) -> None: """Agent names using alphanumeric, hyphens, and underscores are accepted.""" spec = _minimal_spec(name=valid_name) result = validate(spec) assert result.valid def test_agent_name_invalid_in_sub_agent() -> None: """Invalid name on a sub-agent (not just the root) is caught.""" sub = _minimal_spec( name="bad.name", llm=LLMConfig(model="openai/gpt-4o", connection={"api_key": "sk-test"}), ) spec = _minimal_spec( tools=ToolsConfig(agents=["bad.name"]), sub_agents=[sub], ) result = validate(spec) assert not result.valid assert any("name" in e.path for e in result.errors) def test_agent_name_reserved_ui_rejected() -> None: """ The reserved name ``"ui"`` is rejected even though it matches the name pattern. ``"ui"`` is the Web UI "Add agent" title-prefix sentinel (``"ui::