chore: import upstream snapshot with attribution

tests/unittests/evaluation/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/unittests/evaluation/mock_gcs_utils.py

@@ -0,0 +1,131 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+from typing import Union
+
+
+class MockBlob:
+  """Mocks a GCS Blob object.
+
+  This class provides mock implementations for a few common GCS Blob methods,
+  allowing the user to test code that interacts with GCS without actually
+  connecting to a real bucket.
+  """
+
+  def __init__(self, name: str) -> None:
+    """Initializes a MockBlob.
+
+    Args:
+        name: The name of the blob.
+    """
+    self.name = name
+    self.content: Optional[bytes] = None
+    self.content_type: Optional[str] = None
+    self._exists: bool = False
+
+  def upload_from_string(
+      self, data: Union[str, bytes], content_type: Optional[str] = None
+  ) -> None:
+    """Mocks uploading data to the blob (from a string or bytes).
+
+    Args:
+        data: The data to upload (string or bytes).
+        content_type:  The content type of the data (optional).
+    """
+    if isinstance(data, str):
+      self.content = data.encode("utf-8")
+    elif isinstance(data, bytes):
+      self.content = data
+    else:
+      raise TypeError("data must be str or bytes")
+
+    if content_type:
+      self.content_type = content_type
+    self._exists = True
+
+  def download_as_text(self) -> str:
+    """Mocks downloading the blob's content as text.
+
+    Returns:
+        str: The content of the blob as text.
+
+    Raises:
+        Exception: If the blob doesn't exist (hasn't been uploaded to).
+    """
+    if self.content is None:
+      return b""
+    return self.content
+
+  def delete(self) -> None:
+    """Mocks deleting a blob."""
+    self.content = None
+    self.content_type = None
+    self._exists = False
+
+  def exists(self) -> bool:
+    """Mocks checking if the blob exists."""
+    return self._exists
+
+
+class MockBucket:
+  """Mocks a GCS Bucket object."""
+
+  def __init__(self, name: str) -> None:
+    """Initializes a MockBucket.
+
+    Args:
+        name: The name of the bucket.
+    """
+    self.name = name
+    self.blobs: dict[str, MockBlob] = {}
+
+  def blob(self, blob_name: str) -> MockBlob:
+    """Mocks getting a Blob object (doesn't create it in storage).
+
+    Args:
+        blob_name: The name of the blob.
+
+    Returns:
+        A MockBlob instance.
+    """
+    if blob_name not in self.blobs:
+      self.blobs[blob_name] = MockBlob(blob_name)
+    return self.blobs[blob_name]
+
+  def list_blobs(self, prefix: Optional[str] = None) -> list[MockBlob]:
+    """Mocks listing blobs in a bucket, optionally with a prefix."""
+    if prefix:
+      return [
+          blob for name, blob in self.blobs.items() if name.startswith(prefix)
+      ]
+    return list(self.blobs.values())
+
+  def exists(self) -> bool:
+    """Mocks checking if the bucket exists."""
+    return True
+
+
+class MockClient:
+  """Mocks the GCS Client."""
+
+  def __init__(self) -> None:
+    """Initializes MockClient."""
+    self.buckets: dict[str, MockBucket] = {}
+
+  def bucket(self, bucket_name: str) -> MockBucket:
+    """Mocks getting a Bucket object."""
+    if bucket_name not in self.buckets:
+      self.buckets[bucket_name] = MockBucket(bucket_name)
+    return self.buckets[bucket_name]

tests/unittests/evaluation/simulation/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/unittests/evaluation/simulation/test_llm_backed_user_simulator.py

@@ -0,0 +1,420 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation import conversation_scenarios
+from google.adk.evaluation.simulation.llm_backed_user_simulator import LlmBackedUserSimulator
+from google.adk.evaluation.simulation.llm_backed_user_simulator import LlmBackedUserSimulatorConfig
+from google.adk.evaluation.simulation.user_simulator import Status
+from google.adk.evaluation.simulation.user_simulator_personas import UserBehavior
+from google.adk.evaluation.simulation.user_simulator_personas import UserPersona
+from google.adk.events.event import Event
+from google.genai import types
+from pydantic import ValidationError
+import pytest
+
+_INPUT_EVENTS = [
+    Event(
+        author="user",
+        content=types.Content(
+            parts=[types.Part(text="Can you help me?")], role="user"
+        ),
+        invocation_id="inv1",
+    ),
+    Event(
+        author="helpful_assistant",
+        content=types.Content(
+            parts=[
+                types.Part(
+                    text="I'll get the user's name and greet them first.",
+                    thought=True,
+                ),
+                types.Part(
+                    function_call=types.FunctionCall(name="get_user_name")
+                ),
+                types.Part(
+                    function_response=types.FunctionResponse(
+                        name="get_user_name",
+                        response={"name": "John Doe"},
+                    )
+                ),
+                types.Part(text="Hi John, what can I do for you?"),
+            ],
+            role="model",
+        ),
+        invocation_id="inv1",
+    ),
+]
+
+_INPUT_EVENTS_LONG = _INPUT_EVENTS + [
+    Event(
+        author="user",
+        content=types.Content(
+            parts=[types.Part(text="I need to book a flight.")], role="user"
+        ),
+        invocation_id="inv2",
+    ),
+    Event(
+        author="helpful_assistant",
+        content=types.Content(
+            parts=[
+                types.Part(
+                    text="Sure, what is your departure date and destination?",
+                ),
+            ],
+            role="model",
+        ),
+        invocation_id="inv2",
+    ),
+]
+
+_EXPECTED_REWRITTEN_DIALOGUE = """user: Can you help me?
+
+helpful_assistant: Hi John, what can I do for you?"""
+
+_EXPECTED_REWRITTEN_DIALOGUE_LONG = _EXPECTED_REWRITTEN_DIALOGUE + """
+
+user: I need to book a flight.
+
+helpful_assistant: Sure, what is your departure date and destination?"""
+
+
+def test_llm_backed_user_simulator_config_validation():
+  """Tests for LlmBackedUserSimulatorConfig."""
+  config = LlmBackedUserSimulatorConfig(custom_instructions=None)
+  assert config.custom_instructions is None
+  valid_instructions = (
+      "{{ stop_signal }} {{ conversation_plan }} {{ conversation_history }}"
+  )
+  config = LlmBackedUserSimulatorConfig(custom_instructions=valid_instructions)
+  assert config.custom_instructions == valid_instructions
+  invalid_instructions = "Instructions with missing formatting placeholders"
+  with pytest.raises(ValidationError):
+    LlmBackedUserSimulatorConfig(custom_instructions=invalid_instructions)
+
+
+class TestHelperMethods:
+  """Test cases for LlmBackedUserSimulator helper methods."""
+
+  def test_convert_conversation_to_user_sim_pov(self):
+    """Tests _convert_conversation_to_user_sim_pov method."""
+    rewritten_dialogue = LlmBackedUserSimulator._summarize_conversation(
+        _INPUT_EVENTS
+    )
+    assert rewritten_dialogue == _EXPECTED_REWRITTEN_DIALOGUE
+    rewritten_dialogue = LlmBackedUserSimulator._summarize_conversation(
+        _INPUT_EVENTS_LONG
+    )
+    assert rewritten_dialogue == _EXPECTED_REWRITTEN_DIALOGUE_LONG
+
+  def test_summarize_conversation_with_function_calls(self):
+    """Tests _summarize_conversation with include_function_calls=True."""
+    rewritten_dialogue = LlmBackedUserSimulator._summarize_conversation(
+        _INPUT_EVENTS, include_function_calls=True
+    )
+    expected = (
+        "user: Can you help me?\n\n"
+        "helpful_assistant called tool 'get_user_name' with args: None\n\n"
+        "Tool 'get_user_name' returned: {'name': 'John Doe'}\n\n"
+        "helpful_assistant: Hi John, what can I do for you?"
+    )
+    assert rewritten_dialogue == expected
+
+
+async def to_async_iter(items):
+  for item in items:
+    yield item
+
+
+@pytest.fixture
+def mock_llm_agent(mocker):
+  """Provides a mock LLM agent."""
+  mock_llm_registry_cls = mocker.patch(
+      "google.adk.evaluation.simulation.llm_backed_user_simulator.LLMRegistry",
+      autospec=True,
+  )
+  mock_llm_registry = mocker.MagicMock()
+  mock_llm_registry_cls.return_value = mock_llm_registry
+  mock_agent = mocker.MagicMock()
+  mock_llm_registry.resolve.return_value.return_value = mock_agent
+  return mock_agent
+
+
+@pytest.fixture
+def conversation_scenario():
+  """Provides a test conversation scenario."""
+  return conversation_scenarios.ConversationScenario(
+      starting_prompt="Hello", conversation_plan="test plan"
+  )
+
+
+@pytest.fixture
+def user_persona():
+  """Provides a test user persona."""
+  return UserPersona(
+      id="test_persona",
+      description="A test persona",
+      behaviors=[
+          UserBehavior(
+              name="polite",
+              description="is polite",
+              behavior_instructions=["Always say please and thank you."],
+              violation_rubrics=["is rude"],
+          )
+      ],
+  )
+
+
+@pytest.fixture
+def conversation_scenario_with_persona(user_persona):
+  """Provides a test conversation scenario with a user persona."""
+  return conversation_scenarios.ConversationScenario(
+      starting_prompt="Hello",
+      conversation_plan="test plan with persona",
+      user_persona=user_persona,
+  )
+
+
+@pytest.fixture
+def simulator(mock_llm_agent, conversation_scenario):
+  """Provides an LlmBackedUserSimulator instance for testing."""
+  config = LlmBackedUserSimulatorConfig(
+      model="test-model",
+  )
+  sim = LlmBackedUserSimulator(
+      config=config, conversation_scenario=conversation_scenario
+  )
+  sim._invocation_count = 1  # Bypass starting prompt by default for tests
+  return sim
+
+
+@pytest.fixture
+def simulator_with_persona(mock_llm_agent, conversation_scenario_with_persona):
+  """Provides an LlmBackedUserSimulator instance for testing."""
+  config = LlmBackedUserSimulatorConfig(
+      model="test-model",
+  )
+  sim = LlmBackedUserSimulator(
+      config=config, conversation_scenario=conversation_scenario_with_persona
+  )
+  sim._invocation_count = 1  # Bypass starting prompt by default for tests
+  return sim
+
+
+class TestLlmBackedUserSimulator:
+  """Test cases for LlmBackedUserSimulator main methods."""
+
+  @pytest.mark.asyncio
+  async def test_get_llm_response_return_value(
+      self, simulator, mock_llm_agent, mocker
+  ):
+    """Tests that _get_llm_response returns the full response correctly."""
+    mock_llm_response = mocker.create_autospec(
+        types.GenerateContentResponse, instance=True
+    )
+    mock_llm_response.error_code = None
+    mock_llm_response.content = types.Content(
+        parts=[
+            types.Part(text="some thought", thought=True),
+            types.Part(text="Hello world!"),
+        ]
+    )
+    mock_llm_response.parts = mock_llm_response.content.parts
+    mock_llm_agent.generate_content_async.return_value = to_async_iter(
+        [mock_llm_response]
+    )
+    response, error_reason = await simulator._get_llm_response(
+        rewritten_dialogue=""
+    )
+    assert response == "Hello world!"
+    assert error_reason is None
+
+  @pytest.mark.asyncio
+  async def test_get_next_user_message_first_invocation(
+      self, simulator, mock_llm_agent, conversation_scenario
+  ):
+    """Tests that the first invocation returns the starting prompt."""
+    simulator._invocation_count = 0  # override testing default
+    next_user_message = await simulator.get_next_user_message(events=[])
+
+    expected_user_message = types.Content(
+        parts=[types.Part(text=conversation_scenario.starting_prompt)],
+        role="user",
+    )
+    assert next_user_message.status == Status.SUCCESS
+    assert next_user_message.user_message == expected_user_message
+    mock_llm_agent.generate_content_async.assert_not_called()
+
+  @pytest.mark.asyncio
+  async def test_turn_limit_reached(self, conversation_scenario):
+    """Tests get_next_user_message when the turn limit is reached."""
+    config = LlmBackedUserSimulatorConfig(
+        max_allowed_invocations=1,
+    )
+    simulator = LlmBackedUserSimulator(
+        config=config, conversation_scenario=conversation_scenario
+    )
+    simulator._invocation_count = 1
+
+    next_user_message = await simulator.get_next_user_message(
+        events=_INPUT_EVENTS
+    )
+
+    assert next_user_message.status == Status.TURN_LIMIT_REACHED
+    assert next_user_message.user_message is None
+
+  @pytest.mark.asyncio
+  async def test_stop_signal_detected(self, simulator, mock_llm_agent, mocker):
+    """Tests get_next_user_message when the stop signal is detected."""
+    mock_llm_response = mocker.create_autospec(
+        types.GenerateContentResponse, instance=True
+    )
+    mock_llm_response.error_code = None
+    mock_llm_response.content = types.Content(
+        parts=[types.Part(text="Thanks! Bye!</finished>")]
+    )
+    mock_llm_response.parts = mock_llm_response.content.parts
+    mock_llm_agent.generate_content_async.return_value = to_async_iter(
+        [mock_llm_response]
+    )
+
+    next_user_message = await simulator.get_next_user_message(
+        events=_INPUT_EVENTS
+    )
+
+    assert next_user_message.status == Status.STOP_SIGNAL_DETECTED
+    assert next_user_message.user_message is None
+
+  @pytest.mark.asyncio
+  async def test_no_message_generated_empty_response(
+      self, simulator, mock_llm_agent
+  ):
+    """Tests get_next_user_message when no message is generated (empty stream)."""
+    mock_llm_agent.generate_content_async.return_value = to_async_iter([])
+
+    with pytest.raises(
+        RuntimeError,
+        match="Failed to generate a user message: LLM returned empty response",
+    ):
+      await simulator.get_next_user_message(events=_INPUT_EVENTS)
+
+  @pytest.mark.asyncio
+  async def test_get_next_user_message_safety_blocked(
+      self, simulator, mock_llm_agent, mocker
+  ):
+    """Tests get_next_user_message when response is safety blocked."""
+    mock_llm_response = mocker.create_autospec(
+        types.GenerateContentResponse, instance=True
+    )
+    mock_llm_response.content = None
+    mock_llm_response.error_code = "SAFETY"
+    mock_llm_response.error_message = "Blocked by safety"
+    mock_llm_response.parts = []
+    mock_llm_agent.generate_content_async.return_value = to_async_iter(
+        [mock_llm_response]
+    )
+
+    with pytest.raises(
+        RuntimeError,
+        match=(
+            "Failed to generate a user message: safety filters or other error"
+            " \\(code=SAFETY\\)"
+        ),
+    ):
+      await simulator.get_next_user_message(events=_INPUT_EVENTS)
+
+  @pytest.mark.asyncio
+  async def test_get_next_user_message_thinking_only(
+      self, simulator, mock_llm_agent, mocker
+  ):
+    """Tests get_next_user_message when response contains only thinking tokens."""
+    mock_llm_response = mocker.create_autospec(
+        types.GenerateContentResponse, instance=True
+    )
+    mock_llm_response.content = types.Content(
+        parts=[
+            types.Part(text="thinking...", thought=True),
+        ]
+    )
+    mock_llm_response.error_code = None
+    mock_llm_response.parts = mock_llm_response.content.parts
+    mock_llm_agent.generate_content_async.return_value = to_async_iter(
+        [mock_llm_response]
+    )
+
+    with pytest.raises(
+        RuntimeError,
+        match=(
+            "Failed to generate a user message: LLM returned only thinking"
+            " tokens"
+        ),
+    ):
+      await simulator.get_next_user_message(events=_INPUT_EVENTS)
+
+  @pytest.mark.asyncio
+  async def test_get_next_user_message_success(
+      self, simulator, mock_llm_agent, mocker
+  ):
+    """Tests get_next_user_message when the user message is generated successfully."""
+    mock_llm_response = mocker.create_autospec(
+        types.GenerateContentResponse, instance=True
+    )
+    mock_llm_response.error_code = None
+    mock_llm_response.content = types.Content(
+        parts=[types.Part(text="I need to book a flight.")]
+    )
+    mock_llm_response.parts = mock_llm_response.content.parts
+    mock_llm_agent.generate_content_async.return_value = to_async_iter(
+        [mock_llm_response]
+    )
+
+    next_user_message = await simulator.get_next_user_message(
+        events=_INPUT_EVENTS
+    )
+
+    expected_user_message = types.Content(
+        parts=[types.Part(text="I need to book a flight.")], role="user"
+    )
+
+    assert next_user_message.status == Status.SUCCESS
+    assert next_user_message.user_message == expected_user_message
+
+  @pytest.mark.asyncio
+  async def test_get_next_user_message_with_persona_success(
+      self, simulator_with_persona, mock_llm_agent, mocker
+  ):
+    """Tests get_next_user_message when the user message is generated successfully."""
+    mock_llm_response = mocker.create_autospec(
+        types.GenerateContentResponse, instance=True
+    )
+    mock_llm_response.error_code = None
+    mock_llm_response.content = types.Content(
+        parts=[types.Part(text="I need to book a flight.")]
+    )
+    mock_llm_response.parts = mock_llm_response.content.parts
+    mock_llm_agent.generate_content_async.return_value = to_async_iter(
+        [mock_llm_response]
+    )
+
+    next_user_message = await simulator_with_persona.get_next_user_message(
+        events=_INPUT_EVENTS
+    )
+
+    expected_user_message = types.Content(
+        parts=[types.Part(text="I need to book a flight.")], role="user"
+    )
+
+    assert next_user_message.status == Status.SUCCESS
+    assert next_user_message.user_message == expected_user_message

tests/unittests/evaluation/simulation/test_llm_backed_user_simulator_prompts.py

@@ -0,0 +1,280 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import textwrap
+
+from google.adk.evaluation.simulation.llm_backed_user_simulator_prompts import _DEFAULT_USER_SIMULATOR_INSTRUCTIONS_TEMPLATE
+from google.adk.evaluation.simulation.llm_backed_user_simulator_prompts import _get_user_simulator_instructions_template
+from google.adk.evaluation.simulation.llm_backed_user_simulator_prompts import _USER_SIMULATOR_INSTRUCTIONS_WITH_PERSONA_TEMPLATE
+from google.adk.evaluation.simulation.llm_backed_user_simulator_prompts import get_llm_backed_user_simulator_prompt
+from google.adk.evaluation.simulation.llm_backed_user_simulator_prompts import is_valid_user_simulator_template
+from google.adk.evaluation.simulation.user_simulator_personas import UserBehavior
+from google.adk.evaluation.simulation.user_simulator_personas import UserPersona
+from jinja2.exceptions import SecurityError
+import pytest
+
+_MOCK_DEFAULT_TEMPLATE = textwrap.dedent("""\
+  Default template
+
+  # Conversation Plan
+  {{conversation_plan}}
+
+  # Conversation History
+  {{conversation_history}}
+
+  # Stop signal
+  {{stop_signal}}
+""").strip()
+
+_MOCK_PERSONA_TEMPLATE = textwrap.dedent("""\
+  Persona template
+
+  # Persona Description
+  {{persona.description}}
+  {% for b in persona.behaviors %}
+  ## {{ b.name }}
+  {{ b.description }}
+
+  Instructions:
+  {{ b.get_behavior_instructions_str() }}
+  {% endfor %}
+  # Conversation Plan
+  {{conversation_plan}}
+
+  # Conversation History
+  {{conversation_history}}
+
+  # Stop signal
+  {{stop_signal}}
+""").strip()
+
+
+class TestGetUserSimulatorInstructionsTemplate:
+  """Test cases for _get_user_simulator_instructions_template."""
+
+  def test_get_user_simulator_instructions_template_default(self):
+    assert (
+        _get_user_simulator_instructions_template()
+        == _DEFAULT_USER_SIMULATOR_INSTRUCTIONS_TEMPLATE
+    )
+
+  def test_get_user_simulator_instructions_template_with_custom_instructions(
+      self,
+  ):
+    custom_instructions = "custom instructions"
+    assert (
+        _get_user_simulator_instructions_template(
+            custom_instructions=custom_instructions
+        )
+        == custom_instructions
+    )
+
+  def test_get_user_simulator_instructions_template_with_persona(self):
+    user_persona = UserPersona(
+        id="test_persona", description="Test persona", behaviors=[]
+    )
+    assert (
+        _get_user_simulator_instructions_template(user_persona=user_persona)
+        == _USER_SIMULATOR_INSTRUCTIONS_WITH_PERSONA_TEMPLATE
+    )
+
+  def test_get_user_simulator_instructions_template_with_bad_custom_instructions_raises_error(
+      self,
+  ):
+    custom_instructions = "custom instructions"
+    user_persona = UserPersona(
+        id="test_persona", description="Test persona", behaviors=[]
+    )
+    with pytest.raises(ValueError):
+      _get_user_simulator_instructions_template(
+          custom_instructions=custom_instructions, user_persona=user_persona
+      )
+
+
+sample_persona = UserPersona(
+    id="test_persona",
+    description="Test persona description",
+    behaviors=[
+        UserBehavior(
+            name="Test behavior",
+            description="Test behavior description",
+            behavior_instructions=["instruction 1", "instruction 2"],
+            violation_rubrics=["rubric 1"],
+        )
+    ],
+)
+
+
+class TestGetLlmBackedUserSimulatorPrompt:
+  """Test cases for get_llm_backed_user_simulator_prompt."""
+
+  def test_get_llm_backed_user_simulator_prompt_default(self, mocker):
+    mocker.patch(
+        "google.adk.evaluation.simulation.llm_backed_user_simulator_prompts._DEFAULT_USER_SIMULATOR_INSTRUCTIONS_TEMPLATE",
+        _MOCK_DEFAULT_TEMPLATE,
+    )
+    prompt = get_llm_backed_user_simulator_prompt(
+        conversation_plan="test plan",
+        conversation_history="test history",
+        stop_signal="test stop",
+    )
+    expected_prompt = textwrap.dedent("""\
+      Default template
+
+      # Conversation Plan
+      test plan
+
+      # Conversation History
+      test history
+
+      # Stop signal
+      test stop""").strip()
+
+    assert prompt == expected_prompt
+
+  def test_get_llm_backed_user_simulator_prompt_with_custom_instructions(self):
+    custom_instructions = textwrap.dedent("""\
+      Custom instructions:
+
+      # Past history
+      {{conversation_plan}}
+
+      # Plan
+      {{conversation_plan}}
+
+      # Finished!
+      {{stop_signal}}""").strip()
+    prompt = get_llm_backed_user_simulator_prompt(
+        conversation_plan="test plan",
+        conversation_history="test history",
+        stop_signal="test stop",
+        custom_instructions=custom_instructions,
+    )
+
+    expected_prompt = textwrap.dedent("""\
+      Custom instructions:
+
+      # Past history
+      test plan
+
+      # Plan
+      test plan
+
+      # Finished!
+      test stop""").strip()
+    assert prompt == expected_prompt
+
+  def test_get_llm_backed_user_simulator_prompt_with_persona(self, mocker):
+    mocker.patch(
+        "google.adk.evaluation.simulation.llm_backed_user_simulator_prompts._USER_SIMULATOR_INSTRUCTIONS_WITH_PERSONA_TEMPLATE",
+        _MOCK_PERSONA_TEMPLATE,
+    )
+    prompt = get_llm_backed_user_simulator_prompt(
+        conversation_plan="test plan",
+        conversation_history="test history",
+        stop_signal="test stop",
+        user_persona=sample_persona,
+    )
+    expected_prompt = textwrap.dedent("""\
+      Persona template
+
+      # Persona Description
+      Test persona description
+
+      ## Test behavior
+      Test behavior description
+
+      Instructions:
+        * instruction 1
+        * instruction 2
+
+      # Conversation Plan
+      test plan
+
+      # Conversation History
+      test history
+
+      # Stop signal
+      test stop""").strip()
+    assert prompt == expected_prompt
+
+  def test_get_llm_backed_user_simulator_prompt_renders_persona_templates_in_sandbox(
+      self,
+  ):
+    user_persona = UserPersona(
+        id="test_persona",
+        description="Test persona description",
+        behaviors=[
+            UserBehavior(
+                name="Behavior {{ stop_signal }}",
+                description="Description {{ stop_signal }}",
+                behavior_instructions=["instruction {{ stop_signal }}"],
+                violation_rubrics=["rubric 1"],
+            )
+        ],
+    )
+
+    prompt = get_llm_backed_user_simulator_prompt(
+        conversation_plan="test plan",
+        conversation_history="test history",
+        stop_signal="test stop",
+        user_persona=user_persona,
+    )
+
+    assert "## Behavior test stop" in prompt
+    assert "Description test stop" in prompt
+    assert "  * instruction test stop" in prompt
+
+  def test_get_llm_backed_user_simulator_prompt_blocks_unsafe_persona_templates(
+      self,
+  ):
+    user_persona = UserPersona(
+        id="test_persona",
+        description="Test persona description",
+        behaviors=[
+            UserBehavior(
+                name="{{ ''.__class__.__mro__ }}",
+                description="Test behavior description",
+                behavior_instructions=["instruction 1"],
+                violation_rubrics=["rubric 1"],
+            )
+        ],
+    )
+
+    with pytest.raises(SecurityError):
+      get_llm_backed_user_simulator_prompt(
+          conversation_plan="test plan",
+          conversation_history="test history",
+          stop_signal="test stop",
+          user_persona=user_persona,
+      )
+
+
+class TestIsValidUserSimulatorTemplate:
+  """Test cases for is_valid_user_simulator_template."""
+
+  def test_valid_template(self):
+    template = "Hello {{ name }}"
+    params = ["name"]
+    assert is_valid_user_simulator_template(template, params) is True
+
+  def test_invalid_syntax(self):
+    template = "Hello {{ name"
+    params = ["name"]
+    assert is_valid_user_simulator_template(template, params) is False
+
+  def test_missing_parameter(self):
+    template = "Hello"
+    params = ["name"]
+    assert is_valid_user_simulator_template(template, params) is False

tests/unittests/evaluation/simulation/test_per_turn_user_simulation_quality_prompts.py

@@ -0,0 +1,239 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import textwrap
+
+from google.adk.evaluation.simulation.per_turn_user_simulator_quality_prompts import _get_latest_turn_user_simulator_quality_prompt_template
+from google.adk.evaluation.simulation.per_turn_user_simulator_quality_prompts import _LATEST_TURN_USER_SIMULATOR_EVALUATOR_PROMPT_TEMPLATE
+from google.adk.evaluation.simulation.per_turn_user_simulator_quality_prompts import _LATEST_TURN_USER_SIMULATOR_WITH_PERSONA_EVALUATOR_PROMPT_TEMPLATE
+from google.adk.evaluation.simulation.per_turn_user_simulator_quality_prompts import get_per_turn_user_simulator_quality_prompt
+from google.adk.evaluation.simulation.user_simulator_personas import UserBehavior
+from google.adk.evaluation.simulation.user_simulator_personas import UserPersona
+from jinja2.exceptions import SecurityError
+import pytest
+
+_MOCK_DEFAULT_TEMPLATE = textwrap.dedent("""\
+  Default template
+
+  # Conversation Plan
+  {{conversation_plan}}
+
+  # Conversation History
+  {{conversation_history}}
+
+  # Generated User Response
+  {{generated_user_response}}
+
+  # Stop signal
+  {{stop_signal}}
+""").strip()
+
+_MOCK_PERSONA_TEMPLATE = textwrap.dedent("""\
+  Persona template
+
+  # Persona Description
+  {{persona.description}}
+  {% for b in persona.behaviors %}
+  ## Criteria: {{ b.name | render_string_filter}}
+  {{ b.description | render_string_filter}}
+
+  Mark as FAIL if any of the following Violations occur:
+  {{ b.get_violation_rubrics_str() | render_string_filter}}
+  {% endfor %}
+  # Conversation Plan
+  {{conversation_plan}}
+
+  # Conversation History
+  {{conversation_history}}
+
+  # Generated User Response
+  {{generated_user_response}}
+
+  # Stop signal
+  {{stop_signal}}
+""").strip()
+
+
+class TestGetLatestTurnUserSimulatorQualityPrompt:
+  """Test cases for get_latest_turn_user_simulator_quality_prompt."""
+
+  def test_get_get_latest_turn_user_simulator_quality_prompt_template_default(
+      self,
+  ):
+    prompt = _get_latest_turn_user_simulator_quality_prompt_template(
+        user_persona=None
+    )
+    assert prompt == _LATEST_TURN_USER_SIMULATOR_EVALUATOR_PROMPT_TEMPLATE
+
+  def test_get_latest_turn_user_simulator_quality_prompt_template_with_persona(
+      self,
+  ):
+    """Tests that the correct prompt is returned when a persona is provided."""
+    persona = UserPersona(
+        id="test_persona",
+        description="Test persona description.",
+        behaviors=[
+            UserBehavior(
+                name="test_behavior",
+                description="Test behavior description.",
+                behavior_instructions=["instruction1"],
+                violation_rubrics=["violation1"],
+            )
+        ],
+    )
+    prompt = _get_latest_turn_user_simulator_quality_prompt_template(
+        user_persona=persona
+    )
+    assert (
+        prompt
+        == _LATEST_TURN_USER_SIMULATOR_WITH_PERSONA_EVALUATOR_PROMPT_TEMPLATE
+    )
+
+
+class TestGetPerTurnUserSimulatorQualityPrompt:
+  """Test cases for get_per_turn_user_simulator_quality_prompt."""
+
+  def test_get_per_turn_user_simulator_quality_prompt_default(self, mocker):
+    """Tests that the correct prompt is returned when no persona is provided."""
+    mocker.patch(
+        "google.adk.evaluation.simulation.per_turn_user_simulator_quality_prompts._LATEST_TURN_USER_SIMULATOR_EVALUATOR_PROMPT_TEMPLATE",
+        _MOCK_DEFAULT_TEMPLATE,
+    )
+    prompt = get_per_turn_user_simulator_quality_prompt(
+        conversation_plan="plan",
+        conversation_history="history",
+        generated_user_response="response",
+        stop_signal="stop",
+        user_persona=None,
+    )
+    expected_prompt = textwrap.dedent("""\
+      Default template
+
+      # Conversation Plan
+      plan
+
+      # Conversation History
+      history
+
+      # Generated User Response
+      response
+
+      # Stop signal
+      stop""").strip()
+    assert prompt == expected_prompt
+
+  def test_get_per_turn_user_simulator_quality_prompt_with_persona(
+      self, mocker
+  ):
+    """Tests that the correct prompt is returned when a persona is provided."""
+    mocker.patch(
+        "google.adk.evaluation.simulation.per_turn_user_simulator_quality_prompts._LATEST_TURN_USER_SIMULATOR_WITH_PERSONA_EVALUATOR_PROMPT_TEMPLATE",
+        _MOCK_PERSONA_TEMPLATE,
+    )
+    persona = UserPersona(
+        id="test_persona",
+        description="Test persona description.",
+        behaviors=[
+            UserBehavior(
+                name="test_behavior",
+                description="Test behavior description.",
+                behavior_instructions=["instruction1"],
+                violation_rubrics=["violation1"],
+            )
+        ],
+    )
+    prompt = get_per_turn_user_simulator_quality_prompt(
+        conversation_plan="plan",
+        conversation_history="history",
+        generated_user_response="response",
+        stop_signal="stop",
+        user_persona=persona,
+    )
+    expected_prompt = textwrap.dedent("""\
+      Persona template
+
+      # Persona Description
+      Test persona description.
+
+      ## Criteria: test_behavior
+      Test behavior description.
+
+      Mark as FAIL if any of the following Violations occur:
+        * violation1
+
+      # Conversation Plan
+      plan
+
+      # Conversation History
+      history
+
+      # Generated User Response
+      response
+
+      # Stop signal
+      stop""").strip()
+    assert prompt == expected_prompt
+
+  def test_get_per_turn_user_simulator_quality_prompt_renders_persona_templates_in_sandbox(
+      self,
+  ):
+    persona = UserPersona(
+        id="test_persona",
+        description="Test persona description.",
+        behaviors=[
+            UserBehavior(
+                name="criteria {{ stop_signal }}",
+                description="Test behavior {{ stop_signal }}.",
+                behavior_instructions=["instruction1"],
+                violation_rubrics=["violation {{ stop_signal }}"],
+            )
+        ],
+    )
+
+    prompt = get_per_turn_user_simulator_quality_prompt(
+        conversation_plan="plan",
+        conversation_history="history",
+        generated_user_response="response",
+        stop_signal="stop",
+        user_persona=persona,
+    )
+
+    assert "## Criteria: criteria stop" in prompt
+    assert "Test behavior stop." in prompt
+    assert "  * violation stop" in prompt
+
+  def test_get_per_turn_user_simulator_quality_prompt_blocks_unsafe_persona_templates(
+      self,
+  ):
+    persona = UserPersona(
+        id="test_persona",
+        description="Test persona description.",
+        behaviors=[
+            UserBehavior(
+                name="{{ ''.__class__.__mro__ }}",
+                description="Test behavior description.",
+                behavior_instructions=["instruction1"],
+                violation_rubrics=["violation1"],
+            )
+        ],
+    )
+
+    with pytest.raises(SecurityError):
+      get_per_turn_user_simulator_quality_prompt(
+          conversation_plan="plan",
+          conversation_history="history",
+          generated_user_response="response",
+          stop_signal="stop",
+          user_persona=persona,
+      )

tests/unittests/evaluation/simulation/test_per_turn_user_simulation_quality_v1.py

@@ -0,0 +1,698 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation.eval_case import ConversationScenario
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.eval_metrics import EvalStatus
+from google.adk.evaluation.eval_metrics import JudgeModelOptions
+from google.adk.evaluation.eval_metrics import LlmBackedUserSimulatorCriterion
+from google.adk.evaluation.evaluator import PerInvocationResult
+from google.adk.evaluation.llm_as_judge import AutoRaterScore
+from google.adk.evaluation.llm_as_judge_utils import Label
+from google.adk.evaluation.simulation.per_turn_user_simulator_quality_v1 import _format_conversation_history
+from google.adk.evaluation.simulation.per_turn_user_simulator_quality_v1 import _parse_llm_response
+from google.adk.evaluation.simulation.per_turn_user_simulator_quality_v1 import PerTurnUserSimulatorQualityV1
+from google.adk.evaluation.simulation.user_simulator_personas import UserBehavior
+from google.adk.evaluation.simulation.user_simulator_personas import UserPersona
+from google.adk.models.llm_response import LlmResponse
+from google.genai import types
+from google.genai import types as genai_types
+import pytest
+
+
+@pytest.mark.parametrize(
+    "response_text",
+    [
+        """```json
+  {
+    "criteria": [
+      {
+        "name": "TEST_NAME",
+        "reasoning": "test_resonining",
+        "passes": True
+      }
+    ],
+    "is_valid_undefined_key": True
+  }
+  ```""",
+        """```json
+  {
+    "criteria": [
+      {
+        "name": "TEST_NAME",
+        "reasoning": "test_resonining",
+        "passes": True
+      }
+    ],
+    "is_valid": "undefined label",
+  }
+  ```""",
+    ],
+)
+def test_parse_llm_response_label_not_found(response_text):
+  label = _parse_llm_response(response_text)
+  assert label == Label.NOT_FOUND
+
+
+@pytest.mark.parametrize(
+    "response_text",
+    [
+        """```json
+  {
+    "criteria": [
+      {
+        "name": "TEST_NAME",
+        "reasoning": "test_resonining",
+        "passes": True
+      }
+    ],
+    "is_valid": True
+  }
+  ```""",
+        """```json
+  {
+    "criteria": [
+      {
+        "name": "TEST_NAME",
+        "reasoning": "test_resonining",
+        "passes": True
+      }
+    ],
+    "is_valid": "true"
+  }
+  ```""",
+        """```json
+  {
+    "criteria": [
+      {
+        "name": "TEST_NAME",
+        "reasoning": "test_resonining",
+        "passes": True
+      }
+    ],
+    "is_valid": "valid"
+  }
+  ```""",
+    ],
+)
+def test_parse_llm_response_label_valid(response_text):
+  label = _parse_llm_response(response_text)
+  assert label == Label.VALID
+
+
+@pytest.mark.parametrize(
+    "response_text",
+    [
+        """```json
+  {
+    "criteria": [
+      {
+        "name": "TEST_NAME",
+        "reasoning": "test_resonining",
+        "passes": False
+      }
+    ],
+    "is_valid": False
+  }
+  ```""",
+        """```json
+  {
+    "criteria": [
+      {
+        "name": "TEST_NAME",
+        "reasoning": "test_resonining",
+        "passes": False
+      }
+    ],
+    "is_valid": "false",
+  }
+  ```""",
+        """```json
+  {
+    "criteria": [
+      {
+        "name": "TEST_NAME",
+        "reasoning": "test_resonining",
+        "passes": False
+      }
+    ],
+    "is_valid": "invalid",
+  }
+  ```""",
+        """```json
+  {
+    "criteria": [
+      {
+        "name": "TEST_NAME",
+        "reasoning": "test_resonining",
+        "passes": False
+      }
+    ],
+    "is_valid": "almost",
+  }
+  ```""",
+        """```json
+  {
+    "criteria": [
+      {
+        "name": "TEST_NAME",
+        "reasoning": "test_resonining",
+        "passes": False
+      }
+    ],
+    "is_valid": "partially_valid",
+  }
+  ```""",
+        """```json
+  {
+    "criteria": [
+      {
+        "name": "TEST_NAME",
+        "reasoning": "test_resonining",
+        "passes": False
+      }
+    ],
+    "is_valid": "partially valid",
+  }
+  ```""",
+        """```json
+  {
+    "criteria": [
+      {
+        "name": "TEST_NAME",
+        "reasoning": "test_resonining",
+        "passes": False
+      }
+    ],
+    "is_valid": "partially",
+  }
+  ```""",
+    ],
+)
+def test_parse_llm_response_label_invalid(response_text):
+  label = _parse_llm_response(response_text)
+  assert label == Label.INVALID
+
+
+def create_test_template() -> str:
+  return """This is a test template with stop signal: `{{stop_signal}}`.
+
+# Conversation Plan
+{{conversation_plan}}
+
+# Conversation History
+{{conversation_history}}
+
+# Generated User Response
+{{generated_user_response}}
+""".strip()
+
+
+def _create_test_evaluator(
+    threshold: float = 1.0, stop_signal: str = "test stop signal"
+) -> PerTurnUserSimulatorQualityV1:
+  evaluator = PerTurnUserSimulatorQualityV1(
+      EvalMetric(
+          metric_name="test_per_turn_user_simulator_quality_v1",
+          threshold=threshold,
+          criterion=LlmBackedUserSimulatorCriterion(
+              threshold=threshold,
+              stop_signal=stop_signal,
+              judge_model_options=JudgeModelOptions(
+                  judge_model="gemini-2.5-flash",
+                  judge_model_config=genai_types.GenerateContentConfig(),
+                  num_samples=3,
+              ),
+          ),
+      ),
+  )
+  return evaluator
+
+
+def _create_test_conversation_scenario(
+    conversation_plan: str = "test conversation plan",
+    starting_prompt: str = "test starting prompt",
+    user_persona: UserPersona = None,
+) -> ConversationScenario:
+  """Returns a ConversationScenario."""
+  return ConversationScenario(
+      starting_prompt=starting_prompt,
+      conversation_plan=conversation_plan,
+      user_persona=user_persona,
+  )
+
+
+def _create_test_invocation(
+    invocation_id: str,
+    user_content: str = "user content",
+    model_content: str = "model content",
+) -> Invocation:
+  return Invocation(
+      invocation_id=invocation_id,
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text=user_content)],
+          role="user",
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text=model_content)],
+          role="model",
+      ),
+  )
+
+
+def _create_test_invocations(
+    conversation_history: list[str],
+) -> list[Invocation]:
+  conversation_length = len(conversation_history)
+
+  assert conversation_length % 2 == 0
+
+  invocations = []
+  for i in range(conversation_length // 2):
+    user_message = conversation_history[2 * i]
+    model_message = conversation_history[2 * i + 1]
+
+    invocations.append(
+        _create_test_invocation(
+            "turn {i}", user_content=user_message, model_content=model_message
+        )
+    )
+
+  return invocations
+
+
+def test_format_llm_prompt_raises_error_if_previous_invocations_is_none():
+  evaluator = _create_test_evaluator()
+  with pytest.raises(
+      ValueError, match="Previous invocations should have a set value"
+  ):
+    evaluator._format_llm_prompt(
+        invocation=_create_test_invocation("1"),
+        conversation_scenario=_create_test_conversation_scenario(),
+        previous_invocations=None,
+    )
+
+
+def test_format_llm_prompt_raises_error_if_conversation_scenario_is_none():
+  evaluator = _create_test_evaluator()
+  with pytest.raises(
+      ValueError, match="Conversation scenario should have a set value"
+  ):
+    evaluator._format_llm_prompt(
+        invocation=_create_test_invocation("1"),
+        conversation_scenario=None,
+        previous_invocations=[],
+    )
+
+
+def test_convert_llm_response_to_score_pass():
+  evaluator = _create_test_evaluator()
+  auto_rater_response = """```json
+{
+  "is_valid": True,
+}
+```"""
+  llm_response = LlmResponse(
+      content=genai_types.Content(
+          parts=[genai_types.Part(text=auto_rater_response)],
+          role="model",
+      )
+  )
+  auto_rater_score = evaluator._convert_llm_response_to_score(llm_response)
+  assert auto_rater_score == AutoRaterScore(score=1.0)
+
+
+def test_convert_llm_response_to_score_failure():
+  evaluator = _create_test_evaluator()
+  auto_rater_response = """```json
+{
+  "is_valid": False,
+}
+```"""
+  llm_response = LlmResponse(
+      content=genai_types.Content(
+          parts=[genai_types.Part(text=auto_rater_response)],
+          role="model",
+      )
+  )
+  auto_rater_score = evaluator._convert_llm_response_to_score(llm_response)
+  assert auto_rater_score == AutoRaterScore(score=0.0)
+
+
+def test_convert_llm_response_to_score_invalid_json():
+  evaluator = _create_test_evaluator()
+  llm_response = LlmResponse(
+      content=genai_types.Content(
+          parts=[genai_types.Part(text="invalid json")],
+          role="model",
+      )
+  )
+  auto_rater_score = evaluator._convert_llm_response_to_score(llm_response)
+  assert auto_rater_score == AutoRaterScore()
+
+
+def test_convert_llm_response_to_score_missing_key():
+  evaluator = _create_test_evaluator()
+  llm_response = LlmResponse(
+      content=genai_types.Content(
+          parts=[genai_types.Part(text="{}")],
+          role="model",
+      )
+  )
+  auto_rater_score = evaluator._convert_llm_response_to_score(llm_response)
+  assert auto_rater_score == AutoRaterScore()
+
+
+def test_aggregate_samples_not_evaluated():
+  evaluator = _create_test_evaluator()
+  samples = [
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("1"),
+          score=None,
+          eval_status=EvalStatus.NOT_EVALUATED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("2"),
+          score=None,
+          eval_status=EvalStatus.NOT_EVALUATED,
+      ),
+  ]
+
+  aggregation = evaluator._aggregate_samples(samples)
+  assert aggregation == samples[0]
+
+
+def test_aggregate_samples_pass():
+  evaluator = _create_test_evaluator()
+  # The majority of results should be positive.
+  samples = [
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("1"),
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("2"),
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("3"),
+          score=0.0,
+          eval_status=EvalStatus.FAILED,
+      ),
+  ]
+
+  aggregation_result = evaluator._aggregate_samples(samples)
+
+  assert aggregation_result.score == 1.0
+  assert aggregation_result.eval_status == EvalStatus.PASSED
+
+
+def test_aggregate_samples_failure():
+  evaluator = _create_test_evaluator()
+
+  # The majority of results should be negative.
+  samples = [
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("1"),
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("2"),
+          score=0.0,
+          eval_status=EvalStatus.FAILED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("3"),
+          score=0.0,
+          eval_status=EvalStatus.FAILED,
+      ),
+  ]
+
+  aggregation_result = evaluator._aggregate_samples(samples)
+
+  assert aggregation_result.score == 0.0
+  assert aggregation_result.eval_status == EvalStatus.FAILED
+
+
+def test_format_conversation_history_with_none_values():
+  """Tests that _format_conversation_history handles None values."""
+  invocations = [
+      Invocation(
+          invocation_id="1",
+          user_content=types.Content(),
+          final_response=None,
+      )
+  ]
+  formatted_history = _format_conversation_history(invocations)
+  assert formatted_history == ""
+
+
+def test_format_conversation_history():
+  conversation_history = [
+      "first user prompt.",
+      "first agent response.",
+      "second user prompt.",
+      "second agent response.",
+  ]
+  invocation_history = _create_test_invocations(conversation_history)
+  formatted_history = _format_conversation_history(invocation_history)
+  assert formatted_history == """user: first user prompt.
+
+model: first agent response.
+
+user: second user prompt.
+
+model: second agent response."""
+
+
+def test_evaluate_first_turn_pass():
+  evaluator = _create_test_evaluator(
+      threshold=0.8, stop_signal="test stop signal"
+  )
+  conversation_scenario = _create_test_conversation_scenario(
+      conversation_plan="plan",
+      starting_prompt="test starting prompt",
+  )
+  invocation = _create_test_invocation("1", user_content="test starting prompt")
+
+  result = evaluator._evaluate_first_turn(invocation, conversation_scenario)
+
+  assert result.score == 1.0
+  assert result.eval_status == EvalStatus.PASSED
+
+
+def test_evaluate_first_turn_failure():
+  evaluator = _create_test_evaluator(
+      threshold=1.0, stop_signal="test stop signal"
+  )
+  conversation_scenario = _create_test_conversation_scenario(
+      conversation_plan="plan",
+      starting_prompt="test starting prompt",
+  )
+  invocation = _create_test_invocation("1", "wrong starting prompt")
+
+  result = evaluator._evaluate_first_turn(invocation, conversation_scenario)
+
+  assert result.score == 0.0
+  assert result.eval_status == EvalStatus.FAILED
+
+
+def test_aggregate_conversation_results_all_pass_produces_pass():
+  evaluator = _create_test_evaluator()
+  results = [
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("1"),
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("2"),
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("3"),
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("4"),
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+  ]
+  aggregation = evaluator._aggregate_conversation_results(results)
+  assert aggregation.overall_score == 1.0
+  assert aggregation.overall_eval_status == EvalStatus.PASSED
+
+
+def test_aggregate_conversation_results_percentage_above_threshold_produces_pass():
+  evaluator = _create_test_evaluator(threshold=0.7)
+  results = [
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("1"),
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("2"),
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("3"),
+          score=0.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("4"),
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+  ]
+  aggregation = evaluator._aggregate_conversation_results(results)
+  assert aggregation.overall_score == 0.75
+  assert aggregation.overall_eval_status == EvalStatus.PASSED
+
+
+def test_aggregate_conversation_results_all_failures_produces_failure():
+  evaluator = _create_test_evaluator()
+  results = [
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("1"),
+          score=0.0,
+          eval_status=EvalStatus.FAILED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("2"),
+          score=0.0,
+          eval_status=EvalStatus.FAILED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("3"),
+          score=0.0,
+          eval_status=EvalStatus.FAILED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("4"),
+          score=0.0,
+          eval_status=EvalStatus.FAILED,
+      ),
+  ]
+  aggregation = evaluator._aggregate_conversation_results(results)
+  assert aggregation.overall_score == 0.0
+  assert aggregation.overall_eval_status == EvalStatus.FAILED
+
+
+def test_aggregate_conversation_percentage_below_threshold_produces_failure():
+  evaluator = _create_test_evaluator(threshold=1.0)
+  results = [
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("1"),
+          score=0.0,
+          eval_status=EvalStatus.FAILED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("2"),
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("3"),
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=_create_test_invocation("4"),
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+  ]
+  aggregation = evaluator._aggregate_conversation_results(results)
+  assert aggregation.overall_score == 0.75
+  assert aggregation.overall_eval_status == EvalStatus.FAILED
+
+
+@pytest.mark.asyncio
+async def test_evaluate_invocations_all_pass():
+  evaluator = _create_test_evaluator()
+
+  async def sample_llm_valid(*args, **kwargs):  # pylint: disable=unused-argument
+    return AutoRaterScore(score=1.0)
+
+  evaluator._sample_llm = sample_llm_valid  # pylint: disable=protected-access
+  starting_prompt = "first user prompt."
+  conversation_scenario = _create_test_conversation_scenario(
+      starting_prompt=starting_prompt
+  )
+  invocations = _create_test_invocations(
+      [starting_prompt, "model 1.", "user 2.", "model 2."]
+  )
+  result = await evaluator.evaluate_invocations(
+      actual_invocations=invocations,
+      expected_invocations=None,
+      conversation_scenario=conversation_scenario,
+  )
+
+  assert result.overall_score == 1.0
+  assert result.overall_eval_status == EvalStatus.PASSED
+  assert len(result.per_invocation_results) == 2
+  assert result.per_invocation_results[0].score == 1.0
+  assert result.per_invocation_results[1].score == 1.0
+
+
+@pytest.mark.asyncio
+async def test_evaluate_invocations_none_judge_model_config():
+  """Tests evaluation when judge_model_config is None."""
+  evaluator = PerTurnUserSimulatorQualityV1(
+      EvalMetric(
+          metric_name="test_per_turn_user_simulator_quality_v1",
+          threshold=1.0,
+          criterion=LlmBackedUserSimulatorCriterion(
+              threshold=1.0,
+              stop_signal="test stop signal",
+              judge_model_options=JudgeModelOptions(
+                  judge_model="gemini-2.5-flash",
+                  judge_model_config=None,
+                  num_samples=1,
+              ),
+          ),
+      ),
+  )
+
+  async def sample_llm_valid(*args, **kwargs):  # pylint: disable=unused-argument
+    return AutoRaterScore(score=1.0)
+
+  evaluator._sample_llm = sample_llm_valid  # pylint: disable=protected-access
+  starting_prompt = "first user prompt."
+  conversation_scenario = _create_test_conversation_scenario(
+      starting_prompt=starting_prompt
+  )
+  invocations = _create_test_invocations(
+      [starting_prompt, "model 1.", "user 2.", "model 2."]
+  )
+  result = await evaluator.evaluate_invocations(
+      actual_invocations=invocations,
+      expected_invocations=None,
+      conversation_scenario=conversation_scenario,
+  )
+
+  assert result.overall_score == 1.0
+  assert result.overall_eval_status == EvalStatus.PASSED

tests/unittests/evaluation/simulation/test_pre_built_personas.py

@@ -0,0 +1,20 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.adk.evaluation.simulation.pre_built_personas import get_default_persona_registry
+
+
+def test_get_default_persona_registry():
+  """Tests that the default persona registry can be loaded."""
+  assert get_default_persona_registry() is not None

tests/unittests/evaluation/simulation/test_static_user_simulator.py

@@ -0,0 +1,54 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.simulation import static_user_simulator
+from google.adk.evaluation.simulation import user_simulator
+from google.genai import types
+import pytest
+
+
+class TestStaticUserSimulator:
+  """Test cases for StaticUserSimulator."""
+
+  @pytest.mark.asyncio
+  async def test_get_next_user_message(self):
+    """Tests that the provided messages are returned in order followed by the stop signal."""
+    conversation = [
+        Invocation(
+            invocation_id="inv1",
+            user_content=types.Content(parts=[types.Part(text="message 1")]),
+        ),
+        Invocation(
+            invocation_id="inv2",
+            user_content=types.Content(parts=[types.Part(text="message 2")]),
+        ),
+    ]
+    simulator = static_user_simulator.StaticUserSimulator(
+        static_conversation=conversation
+    )
+
+    next_message_1 = await simulator.get_next_user_message(events=[])
+    assert user_simulator.Status.SUCCESS == next_message_1.status
+    assert "message 1" == next_message_1.user_message.parts[0].text
+
+    next_message_2 = await simulator.get_next_user_message(events=[])
+    assert user_simulator.Status.SUCCESS == next_message_2.status
+    assert "message 2" == next_message_2.user_message.parts[0].text
+
+    next_message_3 = await simulator.get_next_user_message(events=[])
+    assert user_simulator.Status.STOP_SIGNAL_DETECTED == next_message_3.status
+    assert next_message_3.user_message is None

tests/unittests/evaluation/simulation/test_user_simulator.py

@@ -0,0 +1,106 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation.simulation import user_simulator as user_simulator_module
+from google.adk.evaluation.simulation.user_simulator import BaseUserSimulatorConfig
+from google.adk.evaluation.simulation.user_simulator import NextUserMessage
+from google.adk.evaluation.simulation.user_simulator import register_user_simulator
+from google.adk.evaluation.simulation.user_simulator import Status
+from google.adk.evaluation.simulation.user_simulator import UserSimulator
+from google.genai.types import Content
+from pydantic import Field
+import pytest
+from typing_extensions import Literal
+
+
+def test_next_user_message_validation():
+  """Tests post-init validation of NextUserMessage."""
+  with pytest.raises(
+      ValueError,
+      match=(
+          "A user_message should be provided if and only if the status is"
+          " SUCCESS"
+      ),
+  ):
+    NextUserMessage(status=Status.SUCCESS)
+
+  with pytest.raises(
+      ValueError,
+      match=(
+          "A user_message should be provided if and only if the status is"
+          " SUCCESS"
+      ),
+  ):
+    NextUserMessage(status=Status.TURN_LIMIT_REACHED, user_message=Content())
+
+  # these two should not cause exceptions
+  NextUserMessage(status=Status.SUCCESS, user_message=Content())
+  NextUserMessage(status=Status.TURN_LIMIT_REACHED)
+
+
+# -----------------------------------------------------------------------------
+# `register_user_simulator` -- the extension-point API in `user_simulator.py`.
+# End-to-end dispatch through `UserSimulatorProvider` is covered separately in
+# `test_user_simulator_provider.py`.
+# -----------------------------------------------------------------------------
+
+
+class _FakeConfig(BaseUserSimulatorConfig):
+  """Test-only config subclass with a unique Literal discriminator."""
+
+  type: Literal["fake_sim"] = Field(default="fake_sim")
+
+
+class _FakeSimulator(UserSimulator):
+  """Test-only simulator; internals do not matter, only its type identity."""
+
+
+def test_register_user_simulator_writes_to_shared_registry():
+  """`register_user_simulator(config_type, simulator_type)` must write the
+
+  mapping into `_SIMULATOR_BY_CONFIG_TYPE` so that any consumer -- including
+  `UserSimulatorProvider` in another module -- can look it up.
+  """
+  try:
+    register_user_simulator(_FakeConfig, _FakeSimulator)
+    assert (
+        user_simulator_module._SIMULATOR_BY_CONFIG_TYPE.get(_FakeConfig)
+        is _FakeSimulator
+    )
+  finally:
+    # Clean up so we don't leak state into other tests.
+    user_simulator_module._SIMULATOR_BY_CONFIG_TYPE.pop(_FakeConfig, None)
+
+
+def test_register_user_simulator_overwrites_existing_entry():
+  """Re-registering the same config type must overwrite the previous entry.
+
+  This lets a test or an out-of-tree extension swap in an alternative
+  implementation for the same config type without having to unregister first.
+  """
+
+  class _AlternativeFakeSimulator(UserSimulator):
+    pass
+
+  try:
+    register_user_simulator(_FakeConfig, _FakeSimulator)
+    register_user_simulator(_FakeConfig, _AlternativeFakeSimulator)
+    assert (
+        user_simulator_module._SIMULATOR_BY_CONFIG_TYPE.get(_FakeConfig)
+        is _AlternativeFakeSimulator
+    )
+  finally:
+    user_simulator_module._SIMULATOR_BY_CONFIG_TYPE.pop(_FakeConfig, None)

tests/unittests/evaluation/simulation/test_user_simulator_personas.py

@@ -0,0 +1,133 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.errors.not_found_error import NotFoundError
+from google.adk.evaluation.simulation.user_simulator_personas import UserBehavior
+from google.adk.evaluation.simulation.user_simulator_personas import UserPersona
+from google.adk.evaluation.simulation.user_simulator_personas import UserPersonaRegistry
+import pytest
+
+
+class TestUserBehavior:
+  """Test cases for UserBehavior."""
+
+  def test_create_user_behavior(self):
+    """Tests UserBehavior creation."""
+    behavior = UserBehavior(
+        name="test_behavior",
+        description="Test behavior description.",
+        behavior_instructions=["instruction1", "instruction2"],
+        violation_rubrics=["violation1", "violation2"],
+    )
+    assert behavior.name == "test_behavior"
+    assert behavior.description == "Test behavior description."
+    assert behavior.behavior_instructions == ["instruction1", "instruction2"]
+    assert behavior.violation_rubrics == ["violation1", "violation2"]
+
+  def test_get_behavior_instructions_str(self):
+    """Tests get_behavior_instructions_str method."""
+    behavior = UserBehavior(
+        name="test_behavior",
+        description="Test behavior description.",
+        behavior_instructions=["instruction1", "instruction2"],
+        violation_rubrics=[],
+    )
+    assert (
+        behavior.get_behavior_instructions_str()
+        == "  * instruction1\n  * instruction2"
+    )
+
+  def test_get_violation_rubrics_str(self):
+    """Tests get_violation_rubrics_str method."""
+    behavior = UserBehavior(
+        name="test_behavior",
+        description="Test behavior description.",
+        behavior_instructions=[],
+        violation_rubrics=["violation1", "violation2"],
+    )
+    assert (
+        behavior.get_violation_rubrics_str() == "  * violation1\n  * violation2"
+    )
+
+
+class TestUserPersona:
+  """Test cases for UserPersona."""
+
+  def test_create_user_persona(self):
+    """Tests UserPersona creation."""
+    behavior = UserBehavior(
+        name="test_behavior",
+        description="Test behavior description.",
+        behavior_instructions=["instruction1"],
+        violation_rubrics=["violation1"],
+    )
+    persona = UserPersona(
+        id="test_persona",
+        description="Test persona description.",
+        behaviors=[behavior],
+    )
+    assert persona.id == "test_persona"
+    assert persona.description == "Test persona description."
+    assert persona.behaviors == [behavior]
+
+
+class TestUserPersonaRegistry:
+  """Test cases for UserPersonaRegistry."""
+
+  def test_register_and_get_persona(self):
+    """Tests register_persona and get_persona methods."""
+    registry = UserPersonaRegistry()
+    persona = UserPersona(
+        id="test_persona", description="Test persona", behaviors=[]
+    )
+    registry.register_persona("persona1", persona)
+    assert registry.get_persona("persona1") == persona
+
+  def test_get_persona_not_found(self):
+    """Tests get_persona for a non-existent persona."""
+    registry = UserPersonaRegistry()
+    with pytest.raises(NotFoundError, match="persona2 not found in registry."):
+      registry.get_persona("persona2")
+
+  def test_update_persona(self):
+    """Tests updating an existing persona in the registry."""
+    registry = UserPersonaRegistry()
+    persona1 = UserPersona(
+        id="test_persona1", description="Test persona 1", behaviors=[]
+    )
+    persona2 = UserPersona(
+        id="test_persona2", description="Test persona 2", behaviors=[]
+    )
+    registry.register_persona("persona1", persona1)
+    assert registry.get_persona("persona1") == persona1
+    registry.register_persona("persona1", persona2)
+    assert registry.get_persona("persona1") == persona2
+
+  def test_get_registered_personas(self):
+    """Tests get_registered_personas method."""
+    registry = UserPersonaRegistry()
+    persona1 = UserPersona(
+        id="test_persona1", description="Test persona 1", behaviors=[]
+    )
+    persona2 = UserPersona(
+        id="test_persona2", description="Test persona 2", behaviors=[]
+    )
+    registry.register_persona("persona1", persona1)
+    registry.register_persona("persona2", persona2)
+    registered_personas = registry.get_registered_personas()
+    assert len(registered_personas) == 2
+    assert persona1 in registered_personas
+    assert persona2 in registered_personas

tests/unittests/evaluation/simulation/test_user_simulator_provider.py

@@ -0,0 +1,200 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation import conversation_scenarios
+from google.adk.evaluation import eval_case
+from google.adk.evaluation.simulation import user_simulator as user_simulator_module
+from google.adk.evaluation.simulation import user_simulator_provider
+from google.adk.evaluation.simulation.llm_backed_user_simulator import LlmBackedUserSimulator
+from google.adk.evaluation.simulation.llm_backed_user_simulator import LlmBackedUserSimulatorConfig
+from google.adk.evaluation.simulation.static_user_simulator import StaticUserSimulator
+from google.adk.evaluation.simulation.user_simulator import BaseUserSimulatorConfig
+from google.genai import types
+from pydantic import Field
+import pytest
+from typing_extensions import Literal
+
+_TEST_CONVERSATION = [
+    eval_case.Invocation(
+        invocation_id='inv1',
+        user_content=types.Content(parts=[types.Part(text='Hello!')]),
+    ),
+]
+
+_TEST_CONVERSATION_SCENARIO = conversation_scenarios.ConversationScenario(
+    starting_prompt='Hello!', conversation_plan='test plan'
+)
+
+
+class TestUserSimulatorProvider:
+  """Test cases for the UserSimulatorProvider."""
+
+  def test_provide_static_user_simulator(self):
+    """Tests the case when a StaticUserSimulator should be provided."""
+    provider = user_simulator_provider.UserSimulatorProvider()
+    test_eval_case = eval_case.EvalCase(
+        eval_id='test_eval_id',
+        conversation=_TEST_CONVERSATION,
+    )
+    simulator = provider.provide(test_eval_case)
+    assert isinstance(simulator, StaticUserSimulator)
+    assert simulator.static_conversation == _TEST_CONVERSATION
+
+  def test_provide_llm_backed_user_simulator(self, mocker):
+    """Tests the case when a LlmBackedUserSimulator should be provided."""
+    mock_llm_registry = mocker.patch(
+        'google.adk.evaluation.simulation.llm_backed_user_simulator.LLMRegistry',
+        autospec=True,
+    )
+    mock_llm_registry.return_value.resolve.return_value = mocker.Mock()
+    # Test case 1: No config in provider.
+    provider = user_simulator_provider.UserSimulatorProvider()
+    test_eval_case = eval_case.EvalCase(
+        eval_id='test_eval_id',
+        conversation_scenario=_TEST_CONVERSATION_SCENARIO,
+    )
+    simulator = provider.provide(test_eval_case)
+    assert isinstance(simulator, LlmBackedUserSimulator)
+    assert simulator._conversation_scenario == _TEST_CONVERSATION_SCENARIO
+
+    # Test case 2: Config in provider.
+    llm_config = LlmBackedUserSimulatorConfig(
+        model='test_model',
+    )
+    provider = user_simulator_provider.UserSimulatorProvider(
+        user_simulator_config=llm_config
+    )
+    simulator = provider.provide(test_eval_case)
+    assert isinstance(simulator, LlmBackedUserSimulator)
+    assert simulator._conversation_scenario == _TEST_CONVERSATION_SCENARIO
+    assert simulator._config.model == 'test_model'
+
+  # ---------------------------------------------------------------------------
+  # Backward-compat + discriminator + registry
+  # ---------------------------------------------------------------------------
+
+  def test_init_accepts_bare_base_config_but_provide_raises(self):
+    """A bare `BaseUserSimulatorConfig` is a valid *instance* of the base,
+
+    so `__init__` accepts it -- but the base has no registered simulator,
+    so `provide()` should raise a clear error pointing the caller at the
+    fix. Callers wanting the default should pass `None` instead.
+    """
+    provider = user_simulator_provider.UserSimulatorProvider(
+        user_simulator_config=BaseUserSimulatorConfig()
+    )
+    # __init__ stores the bare base as-is; no silent up-conversion.
+    assert type(provider._user_simulator_config) is BaseUserSimulatorConfig
+
+    test_eval_case = eval_case.EvalCase(
+        eval_id='test_eval_id',
+        conversation_scenario=_TEST_CONVERSATION_SCENARIO,
+    )
+    with pytest.raises(
+        ValueError,
+        match=(
+            r'No UserSimulator registered for config type'
+            r' `BaseUserSimulatorConfig`'
+        ),
+    ):
+      provider.provide(test_eval_case)
+
+  def test_init_rejects_non_config_argument(self):
+    """Passing something that isn't a `BaseUserSimulatorConfig` should raise
+
+    a clear ValueError.
+    """
+    with pytest.raises(
+        ValueError,
+        match=r'Expect config of type `.*BaseUserSimulatorConfig.*`\.',
+    ):
+      user_simulator_provider.UserSimulatorProvider(
+          user_simulator_config='not a config'  # type: ignore[arg-type]
+      )
+
+  # NOTE: The "both / neither of conversation, conversation_scenario"
+  # checks in `provide()` are defensive; `EvalCase` itself enforces the same
+  # invariant at construction time via a model_validator, so those branches
+  # in the provider are effectively unreachable and don't warrant a unit
+  # test at this layer.
+
+  def test_base_config_type_defaults_to_none(self):
+    """The base `BaseUserSimulatorConfig.type` must default to `None` -- the
+
+    base class must not hard-code a specific subclass's discriminator value.
+    Concrete subclasses supply their own `Literal[...]` default.
+    """
+    base = BaseUserSimulatorConfig()
+    assert base.type is None
+
+  def test_llm_backed_config_has_locked_type_literal(self):
+    """The `type` discriminator on `LlmBackedUserSimulatorConfig` must be a
+
+    Literal locked to `"llm_backed"`, so future subclasses can dispatch
+    correctly via pydantic's discriminated union.
+    """
+    config = LlmBackedUserSimulatorConfig()
+    assert config.type == 'llm_backed'
+    # Attempting to construct with a different `type` value must fail
+    # validation (Literal constraint).
+    with pytest.raises(Exception):
+      LlmBackedUserSimulatorConfig(type='something_else')
+
+  def test_llm_backed_user_simulator_registered_by_provider_module(self):
+    """Importing `user_simulator_provider` must wire the built-in
+
+    `LlmBackedUserSimulator` into the shared registry. This is the "batteries
+    included" contract callers rely on: they can `UserSimulatorProvider()`
+    without ever touching `register_user_simulator(...)`. If the registration
+    line at the top of the provider module is removed, this test catches it
+    immediately -- otherwise dispatch would silently fall through to the
+    "unregistered config type" error path.
+    """
+    assert (
+        user_simulator_module._SIMULATOR_BY_CONFIG_TYPE.get(
+            LlmBackedUserSimulatorConfig
+        )
+        is LlmBackedUserSimulator
+    )
+
+  def test_provide_raises_for_unregistered_config_type(self, mocker):
+    """If the caller supplies a config subclass that no one has registered,
+
+    provide() must raise a clear error naming the offending type.
+    """
+    mocker.patch(
+        'google.adk.evaluation.simulation.llm_backed_user_simulator.LLMRegistry',
+        autospec=True,
+    )
+
+    class _UnregisteredConfig(BaseUserSimulatorConfig):
+      type: Literal['unregistered'] = Field(default='unregistered')
+
+    provider = user_simulator_provider.UserSimulatorProvider(
+        user_simulator_config=_UnregisteredConfig()
+    )
+    test_eval_case = eval_case.EvalCase(
+        eval_id='test_eval_id',
+        conversation_scenario=_TEST_CONVERSATION_SCENARIO,
+    )
+    with pytest.raises(
+        ValueError,
+        match=(
+            r'No UserSimulator registered for config type'
+            r' `_UnregisteredConfig`'
+        ),
+    ):
+      provider.provide(test_eval_case)

tests/unittests/evaluation/test__path_validation.py

@@ -0,0 +1,52 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation._path_validation import validate_path_segment
+import pytest
+
+
+@pytest.mark.parametrize(
+    "value", ["eval_set_1", "my-app", "App Name 1", "résumé", "a.b.c"]
+)
+def test_validate_path_segment_accepts_valid_value(value):
+  validate_path_segment(value, "field")
+
+
+def test_validate_path_segment_rejects_empty():
+  with pytest.raises(ValueError, match="must not be empty"):
+    validate_path_segment("", "field")
+
+
+def test_validate_path_segment_rejects_null_byte():
+  with pytest.raises(ValueError, match="must not contain null bytes"):
+    validate_path_segment("foo\x00bar", "field")
+
+
+@pytest.mark.parametrize("value", ["foo/bar", "foo\\bar", "/", "\\"])
+def test_validate_path_segment_rejects_path_separators(value):
+  with pytest.raises(ValueError, match="must not contain path separators"):
+    validate_path_segment(value, "field")
+
+
+@pytest.mark.parametrize("value", [".", ".."])
+def test_validate_path_segment_rejects_traversal_segments(value):
+  with pytest.raises(ValueError, match="must not contain traversal segments"):
+    validate_path_segment(value, "field")
+
+
+def test_validate_path_segment_includes_field_name_in_error():
+  with pytest.raises(ValueError, match="eval_set_id"):
+    validate_path_segment("", "eval_set_id")

tests/unittests/evaluation/test_app_details.py

@@ -0,0 +1,73 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation.app_details import AgentDetails
+from google.adk.evaluation.app_details import AppDetails
+from google.genai import types as genai_types
+from pytest import raises
+
+
+def test_get_developer_instructions_existing_agent():
+  agent_details = {
+      'agent1': AgentDetails(
+          name='agent1', instructions='instruction for agent1'
+      ),
+      'agent2': AgentDetails(
+          name='agent2', instructions='instruction for agent2'
+      ),
+  }
+  app_details = AppDetails(
+      agent_details=agent_details,
+  )
+
+  # Test for existing agent
+  instructions = app_details.get_developer_instructions('agent1')
+  assert instructions == 'instruction for agent1'
+
+
+def test_get_developer_instructions_non_existing_Agent():
+  agent_details = {
+      'agent1': AgentDetails(
+          name='agent1', instructions='instruction for agent1'
+      ),
+      'agent2': AgentDetails(
+          name='agent2', instructions='instruction for agent2'
+      ),
+  }
+  app_details = AppDetails(
+      agent_details=agent_details,
+  )
+
+  # Test for existing agent
+  with raises(ValueError, match='`agent3` not found in the agentic system.'):
+    app_details.get_developer_instructions('agent3')
+
+
+def test_get_tools_by_agent_name():
+  tool1 = genai_types.Tool(
+      function_declarations=[genai_types.FunctionDeclaration(name='tool1_func')]
+  )
+  agent_details = {
+      'agent1': AgentDetails(name='agent1', tool_declarations=[tool1]),
+      'agent2': AgentDetails(name='agent2', tool_declarations=[]),
+  }
+  app_details = AppDetails(
+      agent_details=agent_details,
+  )
+
+  tools = app_details.get_tools_by_agent_name()
+  expected_tools = {'agent1': [tool1], 'agent2': []}
+  assert tools == expected_tools

tests/unittests/evaluation/test_custom_metric_evaluator.py

@@ -0,0 +1,110 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from unittest import mock
+
+from google.adk.evaluation.custom_metric_evaluator import _CustomMetricEvaluator
+from google.adk.evaluation.custom_metric_evaluator import _get_metric_function
+from google.adk.evaluation.eval_case import ConversationScenario
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.evaluator import EvaluationResult
+import pytest
+
+
+def my_sync_metric_function(
+    eval_metric: EvalMetric,
+    actual_invocations: list[Invocation],
+    expected_invocations: list[Invocation] | None,
+    conversation_scenario: ConversationScenario | None,
+) -> EvaluationResult:
+  """Sync metric function for testing."""
+  return EvaluationResult(overall_score=1.0)
+
+
+async def my_async_metric_function(
+    eval_metric: EvalMetric,
+    actual_invocations: list[Invocation],
+    expected_invocations: list[Invocation] | None,
+    conversation_scenario: ConversationScenario | None,
+) -> EvaluationResult:
+  """Async metric function for testing."""
+  return EvaluationResult(overall_score=0.5)
+
+
+@mock.patch("importlib.import_module")
+def test_get_metric_function_success(mock_import_module):
+  """Tests that _get_metric_function successfully returns a function."""
+  mock_module = mock.MagicMock()
+  mock_module.my_sync_metric_function = my_sync_metric_function
+  mock_import_module.return_value = mock_module
+  func = _get_metric_function(
+      "test_custom_metric_evaluator.my_sync_metric_function"
+  )
+  assert func == my_sync_metric_function
+
+
+@mock.patch("importlib.import_module", side_effect=ImportError)
+def test_get_metric_function_module_not_found(mock_import_module):
+  """Tests that _get_metric_function raises ImportError for non-existent module."""
+  with pytest.raises(ImportError):
+    _get_metric_function("non_existent_module.my_sync_metric_function")
+
+
+@mock.patch("importlib.import_module")
+def test_get_metric_function_function_not_found(mock_import_module):
+  """Tests that _get_metric_function raises ImportError for non-existent function."""
+  mock_import_module.return_value = object()
+  with pytest.raises(ImportError):
+    _get_metric_function(
+        "google.adk.tests.unittests.evaluation.test_custom_metric_evaluator.non_existent_function"
+    )
+
+
+def test_get_metric_function_malformed_path():
+  """Tests that _get_metric_function raises ImportError for malformed path."""
+  with pytest.raises(ImportError):
+    _get_metric_function("malformed_path")
+
+
+@mock.patch(
+    "google.adk.evaluation.custom_metric_evaluator._get_metric_function",
+    return_value=my_sync_metric_function,
+)
+@pytest.mark.asyncio
+async def test_custom_metric_evaluator_sync_function(mock_get_metric_function):
+  """Tests that _CustomMetricEvaluator works with a sync metric function."""
+  eval_metric = EvalMetric(metric_name="sync_metric")
+  evaluator = _CustomMetricEvaluator(
+      eval_metric=eval_metric,
+      custom_function_path="google.adk.tests.unittests.evaluation.test_custom_metric_evaluator.my_sync_metric_function",
+  )
+  result = await evaluator.evaluate_invocations([], None)
+  assert result.overall_score == 1.0
+
+
+@mock.patch(
+    "google.adk.evaluation.custom_metric_evaluator._get_metric_function",
+    return_value=my_async_metric_function,
+)
+@pytest.mark.asyncio
+async def test_custom_metric_evaluator_async_function(mock_get_metric_function):
+  """Tests that _CustomMetricEvaluator works with an async metric function."""
+  eval_metric = EvalMetric(metric_name="async_metric")
+  evaluator = _CustomMetricEvaluator(
+      eval_metric=eval_metric,
+      custom_function_path="google.adk.tests.unittests.evaluation.test_custom_metric_evaluator.my_async_metric_function",
+  )
+  result = await evaluator.evaluate_invocations([], None)
+  assert result.overall_score == 0.5

tests/unittests/evaluation/test_eval_case.py

@@ -0,0 +1,311 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation.conversation_scenarios import ConversationScenario
+from google.adk.evaluation.eval_case import EvalCase
+from google.adk.evaluation.eval_case import get_all_tool_calls
+from google.adk.evaluation.eval_case import get_all_tool_calls_with_responses
+from google.adk.evaluation.eval_case import get_all_tool_responses
+from google.adk.evaluation.eval_case import IntermediateData
+from google.adk.evaluation.eval_case import InvocationEvent
+from google.adk.evaluation.eval_case import InvocationEvents
+from google.adk.evaluation.eval_case import SessionInput
+from google.genai import types as genai_types
+import pytest
+
+
+def test_eval_models_preserve_extra_metadata():
+  session_input = SessionInput(
+      app_name='app',
+      user_id='user',
+      eval_group='retrieval',
+      source='nightly',
+  )
+
+  assert session_input.model_extra == {
+      'eval_group': 'retrieval',
+      'source': 'nightly',
+  }
+  assert session_input.model_dump()['eval_group'] == 'retrieval'
+
+  eval_case = EvalCase(
+      eval_id='case_1',
+      conversation=[],
+      session_input=session_input,
+      owner='platform',
+  )
+
+  assert eval_case.model_extra == {'owner': 'platform'}
+  dumped = eval_case.model_dump()
+  assert dumped['owner'] == 'platform'
+  assert dumped['session_input']['source'] == 'nightly'
+
+
+def test_get_all_tool_calls_with_none_input():
+  """Tests that an empty list is returned when intermediate_data is None."""
+  assert get_all_tool_calls(None) == []
+
+
+def test_get_all_tool_calls_with_intermediate_data_no_tools():
+  """Tests IntermediateData with no tool calls."""
+  intermediate_data = IntermediateData(tool_uses=[])
+  assert get_all_tool_calls(intermediate_data) == []
+
+
+def test_get_all_tool_calls_with_intermediate_data():
+  """Tests that tool calls are correctly extracted from IntermediateData."""
+  tool_call1 = genai_types.FunctionCall(
+      name='search', args={'query': 'weather'}
+  )
+  tool_call2 = genai_types.FunctionCall(name='lookup', args={'id': '123'})
+  intermediate_data = IntermediateData(tool_uses=[tool_call1, tool_call2])
+  assert get_all_tool_calls(intermediate_data) == [tool_call1, tool_call2]
+
+
+def test_get_all_tool_calls_with_empty_invocation_events():
+  """Tests InvocationEvents with an empty list of invocation events."""
+  intermediate_data = InvocationEvents(invocation_events=[])
+  assert get_all_tool_calls(intermediate_data) == []
+
+
+def test_get_all_tool_calls_with_invocation_events_no_tools():
+  """Tests InvocationEvents containing events without any tool calls."""
+  invocation_event = InvocationEvent(
+      author='agent',
+      content=genai_types.Content(
+          parts=[genai_types.Part(text='Thinking...')], role='model'
+      ),
+  )
+  intermediate_data = InvocationEvents(invocation_events=[invocation_event])
+  assert get_all_tool_calls(intermediate_data) == []
+
+
+def test_get_all_tool_calls_with_invocation_events():
+  """Tests that tool calls are correctly extracted from a InvocationSteps object."""
+  tool_call1 = genai_types.FunctionCall(
+      name='search', args={'query': 'weather'}
+  )
+  tool_call2 = genai_types.FunctionCall(name='lookup', args={'id': '123'})
+
+  invocation_event1 = InvocationEvent(
+      author='agent1',
+      content=genai_types.Content(
+          parts=[genai_types.Part(function_call=tool_call1)],
+          role='model',
+      ),
+  )
+  invocation_event2 = InvocationEvent(
+      author='agent2',
+      content=genai_types.Content(
+          parts=[
+              genai_types.Part(text='Found something.'),
+              genai_types.Part(function_call=tool_call2),
+          ],
+          role='model',
+      ),
+  )
+  intermediate_data = InvocationEvents(
+      invocation_events=[invocation_event1, invocation_event2]
+  )
+  assert get_all_tool_calls(intermediate_data) == [tool_call1, tool_call2]
+
+
+def test_get_all_tool_calls_with_unsupported_type():
+  """Tests that a ValueError is raised for unsupported intermediate_data types."""
+  with pytest.raises(
+      ValueError, match='Unsupported type for intermediate_data'
+  ):
+    get_all_tool_calls('this is not a valid type')
+
+
+def test_get_all_tool_responses_with_none_input():
+  """Tests that an empty list is returned when intermediate_data is None."""
+  assert get_all_tool_responses(None) == []
+
+
+def test_get_all_tool_responses_with_empty_invocation_events():
+  """Tests InvocationEvents with an empty list of events."""
+  intermediate_data = InvocationEvents(invocation_events=[])
+  assert get_all_tool_responses(intermediate_data) == []
+
+
+def test_get_all_tool_responses_with_invocation_events_no_tools():
+  """Tests InvocationEvents containing events without any tool responses."""
+  invocation_event = InvocationEvent(
+      author='agent',
+      content=genai_types.Content(
+          parts=[genai_types.Part(text='Thinking...')], role='model'
+      ),
+  )
+  intermediate_data = InvocationEvents(invocation_events=[invocation_event])
+  assert get_all_tool_responses(intermediate_data) == []
+
+
+def test_get_all_tool_responses_with_invocation_events():
+  """Tests that tool responses are correctly extracted from a InvocationEvents object."""
+  tool_response1 = genai_types.FunctionResponse(
+      name='search', response={'result': 'weather is good'}
+  )
+  tool_response2 = genai_types.FunctionResponse(
+      name='lookup', response={'id': '123'}
+  )
+  invocation_event1 = InvocationEvent(
+      author='agent1',
+      content=genai_types.Content(
+          parts=[genai_types.Part(function_response=tool_response1)],
+          role='model',
+      ),
+  )
+  invocation_event2 = InvocationEvent(
+      author='agent2',
+      content=genai_types.Content(
+          parts=[
+              genai_types.Part(text='Found something.'),
+              genai_types.Part(function_response=tool_response2),
+          ],
+          role='model',
+      ),
+  )
+  intermediate_data = InvocationEvents(
+      invocation_events=[invocation_event1, invocation_event2]
+  )
+  assert get_all_tool_responses(intermediate_data) == [
+      tool_response1,
+      tool_response2,
+  ]
+
+
+def test_get_all_tool_responses_with_unsupported_type():
+  """Tests that a ValueError is raised for unsupported intermediate_data types."""
+  with pytest.raises(
+      ValueError, match='Unsupported type for intermediate_data'
+  ):
+    get_all_tool_responses('this is not a valid type')
+
+
+def test_get_all_tool_calls_with_responses_with_none_input():
+  """Tests that an empty list is returned when intermediate_data is None."""
+  assert get_all_tool_calls_with_responses(None) == []
+
+
+def test_get_all_tool_calls_with_responses_with_intermediate_data_no_tool_calls():
+  """Tests get_all_tool_calls_with_responses with IntermediateData with no tool calls."""
+  # No tool calls
+  intermediate_data = IntermediateData(tool_uses=[], tool_responses=[])
+  assert get_all_tool_calls_with_responses(intermediate_data) == []
+
+
+def test_get_all_tool_calls_with_responses_with_intermediate_data_with_tool_calls():
+  """Tests get_all_tool_calls_with_responses with IntermediateData with tools."""
+  # With matching and non-matching tool calls
+  tool_call1 = genai_types.FunctionCall(
+      name='search', args={'query': 'weather'}, id='call1'
+  )
+  tool_response1 = genai_types.FunctionResponse(
+      name='search', response={'result': 'sunny'}, id='call1'
+  )
+  tool_call2 = genai_types.FunctionCall(
+      name='lookup', args={'id': '123'}, id='call2'
+  )
+  intermediate_data = IntermediateData(
+      tool_uses=[tool_call1, tool_call2], tool_responses=[tool_response1]
+  )
+  assert get_all_tool_calls_with_responses(intermediate_data) == [
+      (tool_call1, tool_response1),
+      (tool_call2, None),
+  ]
+
+
+def test_get_all_tool_calls_with_responses_with_steps_no_tool_calls():
+  """Tests get_all_tool_calls_with_responses with Steps that don't have tool calls."""
+  # No tool calls
+  intermediate_data = InvocationEvents(invocation_events=[])
+  assert get_all_tool_calls_with_responses(intermediate_data) == []
+
+
+def test_get_all_tool_calls_with_responses_with_invocation_events():
+  """Tests get_all_tool_calls_with_responses with InvocationEvents."""
+  # No tools
+  intermediate_data = InvocationEvents(invocation_events=[])
+  assert get_all_tool_calls_with_responses(intermediate_data) == []
+
+  # With matching and non-matching tool calls
+  tool_call1 = genai_types.FunctionCall(
+      name='search', args={'query': 'weather'}, id='call1'
+  )
+  tool_response1 = genai_types.FunctionResponse(
+      name='search', response={'result': 'sunny'}, id='call1'
+  )
+  tool_call2 = genai_types.FunctionCall(
+      name='lookup', args={'id': '123'}, id='call2'
+  )
+  invocation_event1 = InvocationEvent(
+      author='agent',
+      content=genai_types.Content(
+          parts=[
+              genai_types.Part(function_call=tool_call1),
+              genai_types.Part(function_call=tool_call2),
+          ],
+          role='model',
+      ),
+  )
+  invocation_event2 = InvocationEvent(
+      author='tool',
+      content=genai_types.Content(
+          parts=[genai_types.Part(function_response=tool_response1)],
+          role='tool',
+      ),
+  )
+  intermediate_data = InvocationEvents(
+      invocation_events=[invocation_event1, invocation_event2]
+  )
+  assert get_all_tool_calls_with_responses(intermediate_data) == [
+      (tool_call1, tool_response1),
+      (tool_call2, None),
+  ]
+
+
+def test_conversation_and_conversation_scenario_mutual_exclusion():
+  """Tests the ensure_conversation_xor_conversation_scenario validator."""
+  test_conversation_scenario = ConversationScenario(
+      starting_prompt='', conversation_plan=''
+  )
+
+  with pytest.raises(
+      ValueError,
+      match=(
+          'Exactly one of conversation and conversation_scenario must be'
+          ' provided in an EvalCase.'
+      ),
+  ):
+    EvalCase(eval_id='test_id')
+
+  with pytest.raises(
+      ValueError,
+      match=(
+          'Exactly one of conversation and conversation_scenario must be'
+          ' provided in an EvalCase.'
+      ),
+  ):
+    EvalCase(
+        eval_id='test_id',
+        conversation=[],
+        conversation_scenario=test_conversation_scenario,
+    )
+
+  # these two should not cause exceptions
+  EvalCase(eval_id='test_id', conversation=[])
+  EvalCase(eval_id='test_id', conversation_scenario=test_conversation_scenario)

tests/unittests/evaluation/test_eval_config.py

@@ -0,0 +1,266 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation.eval_config import _DEFAULT_EVAL_CONFIG
+from google.adk.evaluation.eval_config import EvalConfig
+from google.adk.evaluation.eval_config import get_eval_metrics_from_config
+from google.adk.evaluation.eval_config import get_evaluation_criteria_or_default
+from google.adk.evaluation.eval_rubrics import Rubric
+from google.adk.evaluation.eval_rubrics import RubricContent
+from google.adk.evaluation.simulation.llm_backed_user_simulator import LlmBackedUserSimulatorConfig
+from pydantic import ValidationError
+import pytest
+
+
+def test_get_evaluation_criteria_or_default_returns_default():
+  assert get_evaluation_criteria_or_default("") == _DEFAULT_EVAL_CONFIG
+
+
+def test_get_evaluation_criteria_or_default_reads_from_file(mocker):
+  mocker.patch("os.path.exists", return_value=True)
+  eval_config = EvalConfig(
+      criteria={"tool_trajectory_avg_score": 0.5, "response_match_score": 0.5}
+  )
+  mocker.patch(
+      "builtins.open", mocker.mock_open(read_data=eval_config.model_dump_json())
+  )
+  assert get_evaluation_criteria_or_default("dummy_path") == eval_config
+
+
+def test_get_evaluation_criteria_or_default_returns_default_if_file_not_found(
+    mocker,
+):
+  mocker.patch("os.path.exists", return_value=False)
+  assert (
+      get_evaluation_criteria_or_default("dummy_path") == _DEFAULT_EVAL_CONFIG
+  )
+
+
+def test_get_eval_metrics_from_config():
+  rubric_1 = Rubric(
+      rubric_id="test-rubric",
+      rubric_content=RubricContent(text_property="test"),
+  )
+  eval_config = EvalConfig(
+      criteria={
+          "tool_trajectory_avg_score": 1.0,
+          "response_match_score": 0.8,
+          "final_response_match_v2": {
+              "threshold": 0.5,
+              "judge_model_options": {
+                  "judge_model": "gemini-pro",
+                  "num_samples": 1,
+              },
+          },
+          "rubric_based_final_response_quality_v1": {
+              "threshold": 0.9,
+              "judge_model_options": {
+                  "judge_model": "gemini-ultra",
+                  "num_samples": 1,
+              },
+              "rubrics": [rubric_1],
+          },
+      }
+  )
+  eval_metrics = get_eval_metrics_from_config(eval_config)
+
+  assert len(eval_metrics) == 4
+  assert eval_metrics[0].metric_name == "tool_trajectory_avg_score"
+  assert eval_metrics[0].threshold == 1.0
+  assert eval_metrics[0].criterion.threshold == 1.0
+  assert eval_metrics[1].metric_name == "response_match_score"
+  assert eval_metrics[1].threshold == 0.8
+  assert eval_metrics[1].criterion.threshold == 0.8
+  assert eval_metrics[2].metric_name == "final_response_match_v2"
+  assert eval_metrics[2].threshold == 0.5
+  assert eval_metrics[2].criterion.threshold == 0.5
+  assert (
+      eval_metrics[2].criterion.judge_model_options["judge_model"]
+      == "gemini-pro"
+  )
+  assert eval_metrics[3].metric_name == "rubric_based_final_response_quality_v1"
+  assert eval_metrics[3].threshold == 0.9
+  assert eval_metrics[3].criterion.threshold == 0.9
+  assert (
+      eval_metrics[3].criterion.judge_model_options["judge_model"]
+      == "gemini-ultra"
+  )
+  assert len(eval_metrics[3].criterion.rubrics) == 1
+  assert eval_metrics[3].criterion.rubrics[0] == rubric_1
+
+
+def test_get_eval_metrics_from_config_with_custom_metrics():
+  eval_config = EvalConfig(
+      criteria={
+          "custom_metric_1": 1.0,
+          "custom_metric_2": {
+              "threshold": 0.5,
+          },
+      },
+      custom_metrics={
+          "custom_metric_1": {
+              "code_config": {"name": "path/to/custom/metric_1"},
+          },
+          "custom_metric_2": {
+              "code_config": {"name": "path/to/custom/metric_2"},
+          },
+      },
+  )
+  eval_metrics = get_eval_metrics_from_config(eval_config)
+
+  assert len(eval_metrics) == 2
+  assert eval_metrics[0].metric_name == "custom_metric_1"
+  assert eval_metrics[0].threshold == 1.0
+  assert eval_metrics[0].criterion.threshold == 1.0
+  assert eval_metrics[0].custom_function_path == "path/to/custom/metric_1"
+  assert eval_metrics[1].metric_name == "custom_metric_2"
+  assert eval_metrics[1].threshold == 0.5
+  assert eval_metrics[1].criterion.threshold == 0.5
+  assert eval_metrics[1].custom_function_path == "path/to/custom/metric_2"
+
+
+def test_get_eval_metrics_from_config_empty_criteria():
+  eval_config = EvalConfig(criteria={})
+  eval_metrics = get_eval_metrics_from_config(eval_config)
+  assert not eval_metrics
+
+
+# -----------------------------------------------------------------------------
+# `user_simulator_config` discriminator + backward-compat coverage
+# -----------------------------------------------------------------------------
+
+
+def test_user_simulator_config_default_is_none():
+  """A brand-new EvalConfig has no user simulator config by default."""
+  eval_config = EvalConfig()
+  assert eval_config.user_simulator_config is None
+
+
+def test_user_simulator_config_json_with_explicit_type():
+  """A JSON config that carries `type=llm_backed` should deserialize to the
+
+  concrete subclass, not just the base.
+  """
+  payload = (
+      '{"criteria": {"tool_trajectory_avg_score": 1.0},'
+      ' "userSimulatorConfig": {"type": "llm_backed",'
+      ' "model": "my-model", "maxAllowedInvocations": 5}}'
+  )
+  eval_config = EvalConfig.model_validate_json(payload)
+
+  assert isinstance(
+      eval_config.user_simulator_config, LlmBackedUserSimulatorConfig
+  )
+  assert eval_config.user_simulator_config.type == "llm_backed"
+  assert eval_config.user_simulator_config.model == "my-model"
+  assert eval_config.user_simulator_config.max_allowed_invocations == 5
+
+
+def test_user_simulator_config_json_without_type_backward_compat():
+  """Pre-discriminator JSON (no `type` field) must still deserialize into
+
+  `LlmBackedUserSimulatorConfig` -- this is the backward-compat contract.
+  """
+  # Note the ABSENCE of `type`: this shape is what existing configs on disk
+  # look like today.
+  payload = (
+      '{"criteria": {"tool_trajectory_avg_score": 1.0},'
+      ' "userSimulatorConfig": {"model": "legacy-model"}}'
+  )
+  eval_config = EvalConfig.model_validate_json(payload)
+
+  assert isinstance(
+      eval_config.user_simulator_config, LlmBackedUserSimulatorConfig
+  )
+  assert eval_config.user_simulator_config.type == "llm_backed"
+  assert eval_config.user_simulator_config.model == "legacy-model"
+
+
+def test_user_simulator_config_json_without_type_snake_case():
+  """The default-type injector must handle snake_case JSON keys too, since
+
+  users may serialize with `by_alias=False`.
+  """
+  payload = (
+      '{"criteria": {"tool_trajectory_avg_score": 1.0},'
+      ' "user_simulator_config": {"model": "legacy-model-snake"}}'
+  )
+  eval_config = EvalConfig.model_validate_json(payload)
+
+  assert isinstance(
+      eval_config.user_simulator_config, LlmBackedUserSimulatorConfig
+  )
+  assert eval_config.user_simulator_config.model == "legacy-model-snake"
+
+
+def test_user_simulator_config_json_with_explicit_null_type():
+  """`type: null` in JSON (the shape produced by a `BaseUserSimulatorConfig`
+
+  whose default `type=None` gets serialized) must be treated the same as a
+  missing `type` key: default to the legacy subclass.
+  """
+  payload = (
+      '{"criteria": {},'
+      ' "userSimulatorConfig": {"type": null, "model": "explicit-null"}}'
+  )
+  eval_config = EvalConfig.model_validate_json(payload)
+
+  assert isinstance(
+      eval_config.user_simulator_config, LlmBackedUserSimulatorConfig
+  )
+  assert eval_config.user_simulator_config.type == "llm_backed"
+  assert eval_config.user_simulator_config.model == "explicit-null"
+
+
+def test_user_simulator_config_json_with_unknown_type_raises():
+  """An unknown discriminator value must fail validation loudly."""
+  payload = (
+      '{"criteria": {}, "userSimulatorConfig": {"type": "typo_type_name"}}'
+  )
+  with pytest.raises(ValidationError):
+    EvalConfig.model_validate_json(payload)
+
+
+def test_user_simulator_config_round_trip_via_model_dump_json():
+  """Serialize -> deserialize preserves the concrete subclass (and the
+
+  `type` tag survives the round-trip).
+  """
+  original = EvalConfig(
+      user_simulator_config=LlmBackedUserSimulatorConfig(
+          model="round-trip-model"
+      )
+  )
+  restored = EvalConfig.model_validate_json(original.model_dump_json())
+  assert isinstance(
+      restored.user_simulator_config, LlmBackedUserSimulatorConfig
+  )
+  assert restored.user_simulator_config.model == "round-trip-model"
+  assert restored.user_simulator_config.type == "llm_backed"
+
+
+def test_user_simulator_config_python_construction():
+  """Direct Python construction with a concrete subclass instance also
+
+  works -- the discriminator on `Field` doesn't interfere with that path.
+  """
+  eval_config = EvalConfig(
+      user_simulator_config=LlmBackedUserSimulatorConfig(model="py-model"),
+  )
+  assert isinstance(
+      eval_config.user_simulator_config, LlmBackedUserSimulatorConfig
+  )
+  assert eval_config.user_simulator_config.model == "py-model"

tests/unittests/evaluation/test_evaluation_generator.py

@@ -0,0 +1,968 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import asyncio
+
+from google.adk.evaluation.app_details import AgentDetails
+from google.adk.evaluation.app_details import AppDetails
+from google.adk.evaluation.conversation_scenarios import ConversationScenario
+from google.adk.evaluation.eval_case import EvalCase
+from google.adk.evaluation.eval_case import get_all_tool_calls
+from google.adk.evaluation.eval_set import EvalSet
+from google.adk.evaluation.evaluation_generator import _LiveSession
+from google.adk.evaluation.evaluation_generator import EvaluationGenerator
+from google.adk.evaluation.request_intercepter_plugin import _RequestIntercepterPlugin
+from google.adk.evaluation.simulation.llm_backed_user_simulator import LlmBackedUserSimulator
+from google.adk.evaluation.simulation.llm_backed_user_simulator import LlmBackedUserSimulatorConfig
+from google.adk.evaluation.simulation.user_simulator import NextUserMessage
+from google.adk.evaluation.simulation.user_simulator import Status as UserSimulatorStatus
+from google.adk.evaluation.simulation.user_simulator import UserSimulator
+from google.adk.events.event import Event
+from google.adk.events.event_actions import EventActions
+from google.adk.models.llm_request import LlmRequest
+from google.genai import types
+import pytest
+
+
+def _build_event(
+    author: str, parts: list[types.Part], invocation_id: str
+) -> Event:
+  """Builds an Event object with specified parts."""
+
+  return Event(
+      author=author,
+      content=types.Content(parts=parts),
+      invocation_id=invocation_id,
+  )
+
+
+class TestConvertEventsToEvalInvocation:
+  """Test cases for EvaluationGenerator.convert_events_to_eval_invocations method."""
+
+  def test_convert_events_to_eval_invocations_empty(
+      self,
+  ):
+    """Tests conversion with an empty list of events."""
+    invocations = EvaluationGenerator.convert_events_to_eval_invocations([])
+    assert invocations == []
+
+  def test_convert_single_turn_text_only(
+      self,
+  ):
+    """Tests a single turn with a text response."""
+    events = [
+        _build_event("user", [types.Part(text="Hello")], "inv1"),
+        _build_event("agent", [types.Part(text="Hi there!")], "inv1"),
+    ]
+
+    invocations = EvaluationGenerator.convert_events_to_eval_invocations(events)
+
+    assert len(invocations) == 1
+    invocation = invocations[0]
+    assert invocation.invocation_id == "inv1"
+    assert invocation.user_content.parts[0].text == "Hello"
+    assert invocation.final_response.parts[0].text == "Hi there!"
+    assert len(invocation.intermediate_data.invocation_events) == 0
+
+  def test_convert_single_turn_tool_call(
+      self,
+  ):
+    """Tests a single turn with a tool call."""
+    events = [
+        _build_event("user", [types.Part(text="what is the weather?")], "inv1"),
+        _build_event(
+            "agent",
+            [
+                types.Part(
+                    function_call=types.FunctionCall(
+                        name="get_weather", args={}
+                    )
+                )
+            ],
+            "inv1",
+        ),
+    ]
+
+    invocations = EvaluationGenerator.convert_events_to_eval_invocations(events)
+
+    assert len(invocations) == 1
+    invocation = invocations[0]
+    assert invocation.user_content.parts[0].text == "what is the weather?"
+    assert invocation.final_response is None
+    events = invocation.intermediate_data.invocation_events
+    assert len(events) == 1
+    assert events[0].author == "agent"
+    assert events[0].content.parts[0].function_call.name == "get_weather"
+
+  def test_convert_single_turn_tool_and_text_response(
+      self,
+  ):
+    """Tests a single turn with a tool call and a final text response."""
+    events = [
+        _build_event("user", [types.Part(text="what is the weather?")], "inv1"),
+        _build_event(
+            "agent",
+            [
+                types.Part(
+                    function_call=types.FunctionCall(
+                        name="get_weather", args={}
+                    )
+                )
+            ],
+            "inv1",
+        ),
+        _build_event("agent", [types.Part(text="It is sunny in SF.")], "inv1"),
+    ]
+
+    invocations = EvaluationGenerator.convert_events_to_eval_invocations(events)
+
+    assert len(invocations) == 1
+    invocation = invocations[0]
+    assert invocation.final_response.parts[0].text == "It is sunny in SF."
+    events = invocation.intermediate_data.invocation_events
+    assert len(events) == 1
+    assert events[0].content.parts[0].function_call.name == "get_weather"
+
+  def test_multi_turn(
+      self,
+  ):
+    """Tests a conversation with multiple turns."""
+    events = [
+        _build_event("user", [types.Part(text="Hello")], "inv1"),
+        _build_event("agent", [types.Part(text="Hi there!")], "inv1"),
+        _build_event("user", [types.Part(text="How are you?")], "inv2"),
+        _build_event("agent", [types.Part(text="I am fine.")], "inv2"),
+    ]
+
+    invocations = EvaluationGenerator.convert_events_to_eval_invocations(events)
+
+    assert len(invocations) == 2
+    assert invocations[0].user_content.parts[0].text == "Hello"
+    assert invocations[0].final_response.parts[0].text == "Hi there!"
+    assert invocations[1].user_content.parts[0].text == "How are you?"
+    assert invocations[1].final_response.parts[0].text == "I am fine."
+
+  def test_multi_agent(
+      self,
+  ):
+    """Tests a multi-agent scenario creating multiple steps."""
+    events = [
+        _build_event("user", [types.Part(text="Do something")], "inv1"),
+        _build_event(
+            "root_agent",
+            [
+                types.Part(
+                    function_call=types.FunctionCall(name="tool1", args={})
+                )
+            ],
+            "inv1",
+        ),
+        _build_event(
+            "sub_agent_1",
+            [
+                types.Part(
+                    function_call=types.FunctionCall(name="tool2", args={})
+                )
+            ],
+            "inv1",
+        ),
+        _build_event(
+            "sub_agent_1",
+            [
+                types.Part(
+                    function_call=types.FunctionCall(name="tool3", args={})
+                ),
+                types.Part(text="intermediate response"),
+            ],
+            "inv1",
+        ),
+        _build_event(
+            "sub_agent_2",
+            [
+                types.Part(
+                    function_call=types.FunctionCall(name="tool4", args={})
+                )
+            ],
+            "inv1",
+        ),
+        _build_event("root_agent", [types.Part(text="All done.")], "inv1"),
+    ]
+
+    invocations = EvaluationGenerator.convert_events_to_eval_invocations(events)
+
+    assert len(invocations) == 1
+    invocation = invocations[0]
+    assert invocation.final_response.parts[0].text == "All done."
+    events = invocation.intermediate_data.invocation_events
+
+    assert len(events) == 4
+    assert events[0].author == "root_agent"
+    assert events[1].author == "sub_agent_1"
+    assert events[2].author == "sub_agent_1"
+    assert events[3].author == "sub_agent_2"
+
+  def test_convert_multi_agent_final_responses(
+      self,
+  ):
+    """Tests that only the last final response is excluded from intermediate data."""
+    events = [
+        _build_event("user", [types.Part(text="Hello")], "inv1"),
+        _build_event("agent1", [types.Part(text="First response")], "inv1"),
+        _build_event("agent2", [types.Part(text="Second response")], "inv1"),
+    ]
+
+    invocations = EvaluationGenerator.convert_events_to_eval_invocations(events)
+
+    assert len(invocations) == 1
+    invocation = invocations[0]
+    assert invocation.final_response.parts[0].text == "Second response"
+
+    intermediate_events = invocation.intermediate_data.invocation_events
+    # agent1 is included because it is not the final_event (which is agent2)
+    assert len(intermediate_events) == 1
+    assert intermediate_events[0].author == "agent1"
+    assert intermediate_events[0].content.parts[0].text == "First response"
+
+
+class TestGetAppDetailsByInvocationId:
+  """Test cases for EvaluationGenerator._get_app_details_by_invocation_id method."""
+
+  def test_get_app_details_by_invocation_id_empty(self, mocker):
+    """Tests with an empty list of events."""
+    mock_request_intercepter = mocker.MagicMock(spec=_RequestIntercepterPlugin)
+    app_details = EvaluationGenerator._get_app_details_by_invocation_id(
+        [], mock_request_intercepter
+    )
+    assert app_details == {}
+
+  def test_get_app_details_by_invocation_id_no_model_requests(self, mocker):
+    """Tests when request_intercepter returns no model requests."""
+    mock_request_intercepter = mocker.MagicMock(spec=_RequestIntercepterPlugin)
+    mock_request_intercepter.get_model_request.return_value = None
+    events = [
+        _build_event("user", [types.Part(text="Hello")], "inv1"),
+        _build_event("agent", [types.Part(text="Hi there!")], "inv1"),
+    ]
+    app_details = EvaluationGenerator._get_app_details_by_invocation_id(
+        events, mock_request_intercepter
+    )
+    assert app_details == {"inv1": AppDetails(agent_details={})}
+    mock_request_intercepter.get_model_request.assert_called_once_with(
+        events[1]
+    )
+
+  def test_get_app_details_single_invocation_single_agent(self, mocker):
+    """Tests a single invocation with one agent."""
+    mock_request_intercepter = mocker.MagicMock(spec=_RequestIntercepterPlugin)
+    mock_llm_request = LlmRequest(model="test")
+    mock_llm_request.config.system_instruction = "instruction1"
+    mock_llm_request.config.tools = [types.Tool()]
+    mock_request_intercepter.get_model_request.return_value = mock_llm_request
+
+    events = [
+        _build_event("user", [types.Part(text="Hello")], "inv1"),
+        _build_event("agent", [types.Part(text="Hi there!")], "inv1"),
+    ]
+    app_details = EvaluationGenerator._get_app_details_by_invocation_id(
+        events, mock_request_intercepter
+    )
+
+    expected_app_details = {
+        "inv1": AppDetails(
+            agent_details={
+                "agent": AgentDetails(
+                    name="agent",
+                    instructions="instruction1",
+                    tool_declarations=[types.Tool()],
+                )
+            }
+        )
+    }
+    assert app_details == expected_app_details
+    mock_request_intercepter.get_model_request.assert_called_once_with(
+        events[1]
+    )
+
+  def test_get_app_details_multiple_invocations_multiple_agents(self, mocker):
+    """Tests multiple invocations with multiple agents."""
+    mock_request_intercepter = mocker.MagicMock(spec=_RequestIntercepterPlugin)
+
+    def get_model_request_side_effect(event):
+      mock_llm_request = LlmRequest(model="test")
+      if event.invocation_id == "inv1" and event.author == "agent1":
+        mock_llm_request.config.system_instruction = "instruction1"
+        mock_llm_request.config.tools = [
+            types.Tool(
+                function_declarations=[types.FunctionDeclaration(name="tool1")]
+            )
+        ]
+        return mock_llm_request
+      if event.invocation_id == "inv2" and event.author == "agent2":
+        mock_llm_request.config.system_instruction = "instruction2"
+        return mock_llm_request
+      return None
+
+    mock_request_intercepter.get_model_request.side_effect = (
+        get_model_request_side_effect
+    )
+
+    events = [
+        _build_event("user", [types.Part(text="Hello")], "inv1"),
+        _build_event("agent1", [types.Part(text="Hi there!")], "inv1"),
+        _build_event("user", [types.Part(text="Hello again")], "inv2"),
+        _build_event("agent2", [types.Part(text="Hi again!")], "inv2"),
+        _build_event(
+            "agent1", [types.Part(text="Hi again from agent1")], "inv2"
+        ),  # no request
+    ]
+    app_details = EvaluationGenerator._get_app_details_by_invocation_id(
+        events, mock_request_intercepter
+    )
+
+    expected_app_details = {
+        "inv1": AppDetails(
+            agent_details={
+                "agent1": AgentDetails(
+                    name="agent1",
+                    instructions="instruction1",
+                    tool_declarations=[
+                        types.Tool(
+                            function_declarations=[
+                                types.FunctionDeclaration(name="tool1")
+                            ]
+                        )
+                    ],
+                )
+            }
+        ),
+        "inv2": AppDetails(
+            agent_details={
+                "agent2": AgentDetails(
+                    name="agent2",
+                    instructions="instruction2",
+                    tool_declarations=[],
+                )
+            }
+        ),
+    }
+    assert app_details == expected_app_details
+    assert mock_request_intercepter.get_model_request.call_count == 3
+
+
+class TestGenerateInferencesForSingleUserInvocation:
+  """Test cases for EvaluationGenerator._generate_inferences_for_single_user_invocation method."""
+
+  @pytest.mark.asyncio
+  async def test_generate_inferences_with_mock_runner(self, mocker):
+    """Tests inference generation with a mocked runner."""
+    runner = mocker.MagicMock()
+
+    agent_parts = [types.Part(text="Agent response")]
+
+    async def mock_run_async(*args, **kwargs):
+      yield _build_event(
+          author="agent",
+          parts=agent_parts,
+          invocation_id="inv1",
+      )
+
+    runner.run_async.return_value = mock_run_async()
+
+    user_content = types.Content(parts=[types.Part(text="User query")])
+
+    events = [
+        event
+        async for event in (
+            EvaluationGenerator._generate_inferences_for_single_user_invocation(
+                runner, "test_user", "test_session", user_content
+            )
+        )
+    ]
+
+    assert len(events) == 2
+    assert events[0].author == "user"
+    assert events[0].content == user_content
+    assert events[0].invocation_id == "inv1"
+    assert events[1].author == "agent"
+    assert events[1].content.parts == agent_parts
+
+    runner.run_async.assert_called_once_with(
+        user_id="test_user",
+        session_id="test_session",
+        new_message=user_content,
+    )
+
+
+class TestGenerateInferencesForSingleUserInvocationLive:
+  """Test cases for EvaluationGenerator._generate_inferences_for_single_user_invocation_live method."""
+
+  @pytest.mark.asyncio
+  async def test_generate_inferences_live(self, mocker):
+    """Tests live inference generation."""
+    mock_live_request_queue = mocker.MagicMock()
+    event_queue = asyncio.Queue()
+    turn_complete_event = asyncio.Event()
+
+    user_content = types.Content(parts=[types.Part(text="User query")])
+    invocation_id = "inv1"
+
+    agent_event = _build_event(
+        "agent", [types.Part(text="Agent response")], invocation_id
+    )
+    other_event = _build_event(
+        "agent", [types.Part(text="Other response")], "inv2"
+    )
+
+    gen = EvaluationGenerator._generate_inferences_for_single_user_invocation_live(
+        live_request_queue=mock_live_request_queue,
+        event_queue=event_queue,
+        user_message=user_content,
+        current_invocation_id=invocation_id,
+        turn_complete_event=turn_complete_event,
+        live_timeout_seconds=300,
+    )
+
+    # First yield should be the user message
+    first_event = await gen.__anext__()
+    assert first_event.author == "user"
+    assert first_event.content == user_content
+    assert first_event.invocation_id == invocation_id
+
+    # Mock turn_complete_event.wait to avoid blocking
+    turn_complete_event.wait = mocker.AsyncMock()
+
+    # Put events in queue BEFORE advancing
+    await event_queue.put(agent_event)
+    await event_queue.put(other_event)
+
+    # Now advance to get the next event
+    second_event = await gen.__anext__()
+
+    assert mock_live_request_queue.send_content.called
+    mock_live_request_queue.send_content.assert_called_once_with(user_content)
+
+    assert second_event == agent_event
+
+    # The generator should be exhausted now because other_event doesn't match invocation_id
+    with pytest.raises(StopAsyncIteration):
+      await gen.__anext__()
+
+  @pytest.mark.asyncio
+  async def test_generate_inferences_live_with_synthetic_events(self, mocker):
+    """Tests live inference generation with synthetic events."""
+    mock_live_request_queue = mocker.MagicMock()
+    event_queue = asyncio.Queue()
+    turn_complete_event = asyncio.Event()
+
+    user_content = types.Content(parts=[types.Part(text="User query")])
+    invocation_id = "inv1"
+
+    transcription = types.Transcription(text="Partial transcription")
+    partial_event = Event(
+        author="agent",
+        content=types.Content(parts=[]),
+        invocation_id=invocation_id,
+        output_transcription=transcription,
+        partial=True,
+    )
+
+    gen = EvaluationGenerator._generate_inferences_for_single_user_invocation_live(
+        live_request_queue=mock_live_request_queue,
+        event_queue=event_queue,
+        user_message=user_content,
+        current_invocation_id=invocation_id,
+        turn_complete_event=turn_complete_event,
+        live_timeout_seconds=300,
+        agent_name="custom_agent_name",
+    )
+
+    # First yield should be the user message
+    first_event = await gen.__anext__()
+    assert first_event.author == "user"
+    assert first_event.content == user_content
+    assert first_event.invocation_id == invocation_id
+
+    # Mock turn_complete_event.wait to avoid blocking
+    turn_complete_event.wait = mocker.AsyncMock()
+
+    # Put the partial event in the queue
+    await event_queue.put(partial_event)
+
+    # Now advance
+    second_event = await gen.__anext__()
+    assert second_event == partial_event
+
+    # Next should be the synthetic event
+    third_event = await gen.__anext__()
+    assert third_event.author == "custom_agent_name"
+    assert third_event.invocation_id == invocation_id
+    assert third_event.content.role == "model"
+    assert third_event.content.parts[0].text == "Partial transcription"
+
+    # The generator should be exhausted now
+    with pytest.raises(StopAsyncIteration):
+      await gen.__anext__()
+
+
+@pytest.fixture
+def mock_runner(mocker):
+  """Provides a mock Runner for testing."""
+  mock_runner_cls = mocker.patch(
+      "google.adk.evaluation.evaluation_generator.Runner"
+  )
+  mock_runner_instance = mocker.AsyncMock()
+  mock_runner_instance.__aenter__.return_value = mock_runner_instance
+  mock_runner_cls.return_value = mock_runner_instance
+  yield mock_runner_instance
+
+
+@pytest.fixture
+def mock_session_service(mocker):
+  """Provides a mock InMemorySessionService for testing."""
+  mock_session_service_cls = mocker.patch(
+      "google.adk.evaluation.evaluation_generator.InMemorySessionService"
+  )
+  mock_session_service_instance = mocker.MagicMock()
+  mock_session_service_instance.create_session = mocker.AsyncMock()
+  mock_session_service_cls.return_value = mock_session_service_instance
+  yield mock_session_service_instance
+
+
+class TestGenerateInferencesFromRootAgent:
+  """Test cases for EvaluationGenerator._generate_inferences_from_root_agent method."""
+
+  @pytest.mark.asyncio
+  async def test_generates_inferences_with_user_simulator(
+      self, mocker, mock_runner, mock_session_service
+  ):
+    """Tests that inferences are generated by interacting with a user simulator."""
+    mock_agent = mocker.MagicMock()
+    mock_user_sim = mocker.MagicMock(spec=UserSimulator)
+
+    # Mock user simulator will produce one message, then stop.
+    async def get_next_user_message_side_effect(*args, **kwargs):
+      if mock_user_sim.get_next_user_message.call_count == 1:
+        return NextUserMessage(
+            status=UserSimulatorStatus.SUCCESS,
+            user_message=types.Content(parts=[types.Part(text="message 1")]),
+        )
+      return NextUserMessage(status=UserSimulatorStatus.STOP_SIGNAL_DETECTED)
+
+    mock_user_sim.get_next_user_message = mocker.AsyncMock(
+        side_effect=get_next_user_message_side_effect
+    )
+
+    mock_generate_inferences = mocker.patch(
+        "google.adk.evaluation.evaluation_generator.EvaluationGenerator._generate_inferences_for_single_user_invocation"
+    )
+    mocker.patch(
+        "google.adk.evaluation.evaluation_generator.EvaluationGenerator._get_app_details_by_invocation_id"
+    )
+    mocker.patch(
+        "google.adk.evaluation.evaluation_generator.EvaluationGenerator.convert_events_to_eval_invocations"
+    )
+
+    # Each call to _generate_inferences_for_single_user_invocation will
+    # yield one user and one agent event.
+    async def mock_generate_inferences_side_effect(
+        runner, user_id, session_id, user_content
+    ):
+      yield _build_event("user", user_content.parts, "inv1")
+      yield _build_event("agent", [types.Part(text="agent_response")], "inv1")
+
+    mock_generate_inferences.side_effect = mock_generate_inferences_side_effect
+
+    await EvaluationGenerator._generate_inferences_from_root_agent(
+        root_agent=mock_agent,
+        user_simulator=mock_user_sim,
+    )
+
+    # Check that user simulator was called until it stopped.
+    assert mock_user_sim.get_next_user_message.call_count == 2
+
+    # Check that we generated inferences for each user message.
+    assert mock_generate_inferences.call_count == 1
+
+    # Check the content of the user messages passed to inference generation
+    mock_generate_inferences.assert_called_once()
+    called_with_content = mock_generate_inferences.call_args.args[3]
+    assert called_with_content.parts[0].text == "message 1"
+
+  @pytest.mark.asyncio
+  async def test_generates_inferences_with_user_simulator_live(
+      self, mocker, mock_runner, mock_session_service
+  ):
+    """Tests that inferences are generated by interacting with a user simulator in live mode."""
+    mock_agent = mocker.MagicMock()
+    mock_user_sim = mocker.MagicMock(spec=UserSimulator)
+
+    # Mock user simulator will produce one message, then stop.
+    async def get_next_user_message_side_effect(*args, **kwargs):
+      if mock_user_sim.get_next_user_message.call_count == 1:
+        return NextUserMessage(
+            status=UserSimulatorStatus.SUCCESS,
+            user_message=types.Content(parts=[types.Part(text="message 1")]),
+        )
+      return NextUserMessage(status=UserSimulatorStatus.STOP_SIGNAL_DETECTED)
+
+    mock_user_sim.get_next_user_message = mocker.AsyncMock(
+        side_effect=get_next_user_message_side_effect
+    )
+
+    mock_generate_inferences_live = mocker.patch(
+        "google.adk.evaluation.evaluation_generator.EvaluationGenerator._generate_inferences_for_single_user_invocation_live"
+    )
+    mocker.patch(
+        "google.adk.evaluation.evaluation_generator.EvaluationGenerator._get_app_details_by_invocation_id"
+    )
+    mocker.patch(
+        "google.adk.evaluation.evaluation_generator.EvaluationGenerator.convert_events_to_eval_invocations"
+    )
+
+    # Mock _LiveSession context manager
+    mock_live_session = mocker.MagicMock()
+    mock_live_session.__aenter__ = mocker.AsyncMock(
+        return_value=mock_live_session
+    )
+    mock_live_session.__aexit__ = mocker.AsyncMock(return_value=None)
+    mock_live_session.live_request_queue = mocker.MagicMock()
+    mock_live_session.event_queue = asyncio.Queue()
+    mock_live_session.turn_complete_event = asyncio.Event()
+    mock_live_session.live_finished = asyncio.Event()
+
+    mock_live_session_cls = mocker.patch(
+        "google.adk.evaluation.evaluation_generator._LiveSession",
+        return_value=mock_live_session,
+    )
+
+    # Each call to _generate_inferences_for_single_user_invocation_live will
+    # yield one user and one agent event.
+    async def mock_generate_inferences_live_side_effect(*args, **kwargs):
+      yield _build_event("user", [types.Part(text="message 1")], "inv1")
+      yield _build_event("agent", [types.Part(text="agent_response")], "inv1")
+
+    mock_generate_inferences_live.side_effect = (
+        mock_generate_inferences_live_side_effect
+    )
+
+    await EvaluationGenerator._generate_inferences_from_root_agent_live(
+        root_agent=mock_agent,
+        user_simulator=mock_user_sim,
+        live_timeout_seconds=600,
+    )
+
+    # Check that user simulator was called until it stopped.
+    assert mock_user_sim.get_next_user_message.call_count == 2
+
+    # Check that we generated inferences for each user message.
+    mock_generate_inferences_live.assert_called_once()
+    called_with_content = mock_generate_inferences_live.call_args.kwargs[
+        "user_message"
+    ]
+    assert called_with_content.parts[0].text == "message 1"
+    assert (
+        mock_generate_inferences_live.call_args.kwargs["live_timeout_seconds"]
+        == 600
+    )
+
+    # Verify that the agent response was collected
+    mock_convert = EvaluationGenerator.convert_events_to_eval_invocations
+    mock_convert.assert_called_once()
+    events_passed = mock_convert.call_args.args[0]
+
+    agent_events = [e for e in events_passed if e.author == "agent"]
+    assert len(agent_events) == 1
+    assert agent_events[0].content.parts[0].text == "agent_response"
+
+    # Verify that the _LiveSession constructor was called
+    mock_live_session_cls.assert_called_once()
+
+
+class TestGenerateResponses:
+  """Test cases for EvaluationGenerator.generate_responses method."""
+
+  @pytest.mark.asyncio
+  async def test_generate_responses_passes_config_to_simulator_instance(
+      self, mocker
+  ):
+    """Tests that user_simulator_config reaches the actual UserSimulator instance when UserSimulatorProvider is not mocked."""
+    mock_process_query = mocker.patch(
+        "google.adk.evaluation.evaluation_generator.EvaluationGenerator._process_query",
+        new_callable=mocker.AsyncMock,
+        return_value=[],
+    )
+
+    user_simulator_config = LlmBackedUserSimulatorConfig(
+        model="gemini-2.5-flash",
+        max_allowed_invocations=5,
+        custom_instructions=(
+            "custom {{ stop_signal }} {{ conversation_plan }} {{"
+            " conversation_history }}"
+        ),
+    )
+    eval_set = EvalSet(
+        eval_set_id="test_set",
+        eval_cases=[
+            EvalCase(
+                eval_id="case_0",
+                conversation_scenario=ConversationScenario(
+                    starting_prompt="hello",
+                    conversation_plan="test plan",
+                ),
+            )
+        ],
+    )
+
+    await EvaluationGenerator.generate_responses(
+        eval_set=eval_set,
+        agent_module_path="some.agent.module",
+        repeat_num=1,
+        user_simulator_config=user_simulator_config,
+    )
+
+    mock_process_query.assert_called_once()
+    user_simulator = mock_process_query.call_args.args[1]
+    assert isinstance(user_simulator, LlmBackedUserSimulator)
+    assert user_simulator._config.model == "gemini-2.5-flash"
+    assert user_simulator._config.max_allowed_invocations == 5
+    assert (
+        user_simulator._config.custom_instructions
+        == "custom {{ stop_signal }} {{ conversation_plan }} {{"
+        " conversation_history }}"
+    )
+
+
+class TestLiveSessionCallbacks:
+  """Unit tests verifying that _LiveSession manually triggers callbacks."""
+
+  @pytest.mark.asyncio
+  async def test_live_session_manually_triggers_callbacks(self, mocker):
+    from google.adk.agents.callback_context import CallbackContext
+    from google.adk.agents.llm_agent import Agent
+    from google.adk.models.llm_request import LlmRequest
+
+    # 1. Setup mock runner, agent, and session
+    mock_runner = mocker.MagicMock()
+    mock_runner.session_service.append_event = mocker.AsyncMock()
+    mock_session = mocker.MagicMock()
+    mock_agent = mocker.MagicMock(spec=Agent)
+    mock_runner.agent = mock_agent
+    mock_runner._find_agent_to_run.return_value = mock_agent
+
+    mock_agent.name = "test_agent"
+
+    # Mock _llm_flow._preprocess_async to set dummy instruction
+    async def mock_preprocess_async(invocation_context, llm_request):
+      llm_request.config.system_instruction = "mock instruction"
+      return
+      yield  # make it an async generator
+
+    mock_flow = mocker.MagicMock()
+    mock_flow._preprocess_async = mock_preprocess_async
+    mock_agent._llm_flow = mock_flow
+
+    # Mock run_live stream yielding one event
+    mock_event = Event(
+        author="agent",
+        content=types.Content(parts=[types.Part(text="Hello")]),
+        invocation_id="test_invocation_id",
+    )
+
+    async def mock_run_live(*args, **kwargs):
+      yield mock_event
+
+    mock_agent.run_live.return_value = mock_run_live()
+
+    # Mock plugin_manager on invocation context
+    mock_plugin_manager = mocker.MagicMock()
+    mock_plugin_manager.run_before_model_callback = mocker.AsyncMock()
+    mock_plugin_manager.run_after_model_callback = mocker.AsyncMock()
+    mock_runner._new_invocation_context_for_live.return_value.plugin_manager = (
+        mock_plugin_manager
+    )
+    mock_runner._new_invocation_context_for_live.return_value.agent = mock_agent
+
+    # 2. Instantiate and enter _LiveSession
+    live_session = _LiveSession(
+        runner=mock_runner,
+        session=mock_session,
+        user_id="test_user",
+        session_id="test_session",
+    )
+
+    # Directly run _consume_events as a coroutine for synchronous-style testing
+    await live_session._consume_events()
+
+    # 3. Assertions
+    mock_plugin_manager.run_before_model_callback.assert_called_once()
+    called_before_args = mock_plugin_manager.run_before_model_callback.call_args
+    assert isinstance(
+        called_before_args.kwargs["callback_context"], CallbackContext
+    )
+    assert isinstance(called_before_args.kwargs["llm_request"], LlmRequest)
+    assert (
+        called_before_args.kwargs["llm_request"].config.system_instruction
+        == "mock instruction"
+    )
+
+    mock_plugin_manager.run_after_model_callback.assert_called_once()
+    called_after_args = mock_plugin_manager.run_after_model_callback.call_args
+    assert isinstance(
+        called_after_args.kwargs["callback_context"], CallbackContext
+    )
+    assert isinstance(called_after_args.kwargs["llm_response"], Event)
+    assert called_after_args.kwargs["llm_response"] == mock_event
+
+  @pytest.mark.asyncio
+  async def test_live_session_manually_triggers_callbacks_with_tools(
+      self, mocker
+  ):
+    from google.adk.agents.callback_context import CallbackContext
+    from google.adk.agents.llm_agent import Agent
+    from google.adk.models.llm_request import LlmRequest
+
+    # 1. Setup mock runner, agent, and session
+    mock_runner = mocker.MagicMock()
+    mock_runner.session_service.append_event = mocker.AsyncMock()
+    mock_session = mocker.MagicMock()
+    mock_agent = mocker.MagicMock(spec=Agent)
+    mock_runner.agent = mock_agent
+    mock_runner._find_agent_to_run.return_value = mock_agent
+
+    mock_agent.name = "test_agent"
+
+    # Set up a mock tool
+    mock_tool = mocker.MagicMock()
+    mock_tool.name = "get_weather"
+    mock_decl = types.FunctionDeclaration(
+        name="get_weather",
+        description="Get weather details",
+    )
+    mock_tool._get_declaration.return_value = mock_decl
+
+    # Mock _llm_flow._preprocess_async to set instruction and append tool
+    async def mock_preprocess_async(invocation_context, llm_request):
+      llm_request.config.system_instruction = "mock instruction"
+      llm_request.append_tools([mock_tool])
+      return
+      yield  # make it an async generator
+
+    mock_flow = mocker.MagicMock()
+    mock_flow._preprocess_async = mock_preprocess_async
+    mock_agent._llm_flow = mock_flow
+
+    # Mock run_live stream yielding one event
+    mock_event = Event(
+        author="agent",
+        content=types.Content(parts=[types.Part(text="Hello")]),
+        invocation_id="test_invocation_id",
+    )
+
+    async def mock_run_live(*args, **kwargs):
+      yield mock_event
+
+    mock_agent.run_live.return_value = mock_run_live()
+
+    # Mock plugin_manager on invocation context
+    mock_plugin_manager = mocker.MagicMock()
+    mock_plugin_manager.run_before_model_callback = mocker.AsyncMock()
+    mock_plugin_manager.run_after_model_callback = mocker.AsyncMock()
+    mock_runner._new_invocation_context_for_live.return_value.plugin_manager = (
+        mock_plugin_manager
+    )
+    mock_runner._new_invocation_context_for_live.return_value.agent = mock_agent
+
+    # 2. Instantiate and enter _LiveSession
+    live_session = _LiveSession(
+        runner=mock_runner,
+        session=mock_session,
+        user_id="test_user",
+        session_id="test_session",
+    )
+
+    # Directly run _consume_events as a coroutine
+    await live_session._consume_events()
+
+    # 3. Assertions
+    mock_plugin_manager.run_before_model_callback.assert_called_once()
+    called_before_args = mock_plugin_manager.run_before_model_callback.call_args
+    assert isinstance(
+        called_before_args.kwargs["callback_context"], CallbackContext
+    )
+
+    llm_request = called_before_args.kwargs["llm_request"]
+    assert isinstance(llm_request, LlmRequest)
+    assert llm_request.config.system_instruction == "mock instruction"
+
+    # Assert that tool was correctly wrapped under types.Tool format
+    assert len(llm_request.config.tools) == 1
+    wrapped_tool = llm_request.config.tools[0]
+    assert isinstance(wrapped_tool, types.Tool)
+    assert len(wrapped_tool.function_declarations) == 1
+    assert wrapped_tool.function_declarations[0].name == "get_weather"
+
+    mock_plugin_manager.run_after_model_callback.assert_called_once()
+    called_after_args = mock_plugin_manager.run_after_model_callback.call_args
+    assert isinstance(
+        called_after_args.kwargs["callback_context"], CallbackContext
+    )
+    assert isinstance(called_after_args.kwargs["llm_response"], Event)
+    assert called_after_args.kwargs["llm_response"] == mock_event
+
+
+def test_convert_events_preserves_tool_calls_when_skip_summarization():
+  """Regression test for #5410.
+
+  When an event has skip_summarization=True, is_final_response() returns True
+  even if the event contains function calls.  Previously such an event was
+  treated as final_event and excluded from invocation_events, causing
+  get_all_tool_calls() to return an empty list and tool_trajectory_avg_score
+  to always be 0.0 despite matching tool calls.
+  """
+  events = [
+      Event(
+          invocation_id="inv1",
+          author="user",
+          content=types.Content(
+              parts=[types.Part(text="run a query")], role="user"
+          ),
+          timestamp=1000.0,
+      ),
+      Event(
+          invocation_id="inv1",
+          author="agent",
+          content=types.Content(
+              parts=[
+                  types.Part(
+                      function_call=types.FunctionCall(
+                          id="call_01",
+                          name="execute_sql",
+                          args={"project_id": "my-proj", "query": "SELECT 1"},
+                      )
+                  )
+              ]
+          ),
+          actions=EventActions(skip_summarization=True),
+      ),
+  ]
+
+  invocations = EvaluationGenerator.convert_events_to_eval_invocations(events)
+  assert len(invocations) == 1
+
+  tool_calls = get_all_tool_calls(invocations[0].intermediate_data)
+  assert len(tool_calls) == 1
+  assert tool_calls[0].name == "execute_sql"
+  assert tool_calls[0].args == {"project_id": "my-proj", "query": "SELECT 1"}

tests/unittests/evaluation/test_final_response_match_v1.py

@@ -0,0 +1,141 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.eval_metrics import PrebuiltMetrics
+from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.final_response_match_v1 import _calculate_rouge_1_scores
+from google.adk.evaluation.final_response_match_v1 import RougeEvaluator
+from google.genai import types as genai_types
+import pytest
+
+
+def _create_test_rouge_evaluator(threshold: float) -> RougeEvaluator:
+  return RougeEvaluator(
+      EvalMetric(metric_name="response_match_score", threshold=threshold)
+  )
+
+
+def _create_test_invocations(
+    candidate: str, reference: str
+) -> tuple[Invocation, Invocation]:
+  """Returns tuple of (actual_invocation, expected_invocation)."""
+  return Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="This is a test query.")]
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text=candidate)]
+      ),
+  ), Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="This is a test query.")]
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text=reference)]
+      ),
+  )
+
+
+def test_calculate_rouge_1_scores_empty_candidate_and_reference():
+  candidate = ""
+  reference = ""
+  rouge_1_score = _calculate_rouge_1_scores(candidate, reference)
+  assert rouge_1_score.precision == 0
+  assert rouge_1_score.recall == 0
+  assert rouge_1_score.fmeasure == 0
+
+
+def test_calculate_rouge_1_scores_empty_candidate():
+  candidate = ""
+  reference = "This is a test reference."
+  rouge_1_score = _calculate_rouge_1_scores(candidate, reference)
+  assert rouge_1_score.precision == 0
+  assert rouge_1_score.recall == 0
+  assert rouge_1_score.fmeasure == 0
+
+
+def test_calculate_rouge_1_scores_empty_reference():
+  candidate = "This is a test candidate response."
+  reference = ""
+  rouge_1_score = _calculate_rouge_1_scores(candidate, reference)
+  assert rouge_1_score.precision == 0
+  assert rouge_1_score.recall == 0
+  assert rouge_1_score.fmeasure == 0
+
+
+def test_calculate_rouge_1_scores():
+  candidate = "This is a test candidate response."
+  reference = "This is a test reference."
+  rouge_1_score = _calculate_rouge_1_scores(candidate, reference)
+  assert rouge_1_score.precision == pytest.approx(2 / 3)
+  assert rouge_1_score.recall == pytest.approx(4 / 5)
+  assert rouge_1_score.fmeasure == pytest.approx(8 / 11)
+
+
+@pytest.mark.parametrize(
+    "candidates, references, expected_score, expected_status",
+    [
+        (
+            ["The quick brown fox jumps.", "hello world"],
+            ["The quick brown fox jumps over the lazy dog.", "hello"],
+            0.69048,  # (5/7 + 2/3) / 2
+            EvalStatus.FAILED,
+        ),
+        (
+            ["This is a test.", "Another test case."],
+            ["This is a test.", "This is a different test."],
+            0.625,  # (1 + 1/4) / 2
+            EvalStatus.FAILED,
+        ),
+        (
+            ["No matching words here.", "Second candidate."],
+            ["Completely different text.", "Another reference."],
+            0.0,  # (0 + 1/2) / 2
+            EvalStatus.FAILED,
+        ),
+        (
+            ["Same words", "Same words"],
+            ["Same words", "Same words"],
+            1.0,
+            EvalStatus.PASSED,
+        ),
+    ],
+)
+def test_rouge_evaluator_multiple_invocations(
+    candidates: list[str],
+    references: list[str],
+    expected_score: float,
+    expected_status: EvalStatus,
+):
+  rouge_evaluator = _create_test_rouge_evaluator(threshold=0.8)
+  actual_invocations = []
+  expected_invocations = []
+  for candidate, reference in zip(candidates, references):
+    actual_invocation, expected_invocation = _create_test_invocations(
+        candidate, reference
+    )
+    actual_invocations.append(actual_invocation)
+    expected_invocations.append(expected_invocation)
+
+  evaluation_result = rouge_evaluator.evaluate_invocations(
+      actual_invocations, expected_invocations
+  )
+  assert evaluation_result.overall_score == pytest.approx(
+      expected_score, rel=1e-3
+  )
+  assert evaluation_result.overall_eval_status == expected_status

tests/unittests/evaluation/test_final_response_match_v2.py

@@ -0,0 +1,594 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_case import InvocationEvent
+from google.adk.evaluation.eval_case import InvocationEvents
+from google.adk.evaluation.eval_metrics import BaseCriterion
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.eval_metrics import EvalStatus
+from google.adk.evaluation.eval_metrics import JudgeModelOptions
+from google.adk.evaluation.eval_metrics import PrebuiltMetrics
+from google.adk.evaluation.evaluator import PerInvocationResult
+from google.adk.evaluation.final_response_match_v2 import _parse_critique
+from google.adk.evaluation.final_response_match_v2 import FinalResponseMatchV2Evaluator
+from google.adk.evaluation.llm_as_judge import AutoRaterScore
+from google.adk.evaluation.llm_as_judge_utils import Label
+from google.adk.models.llm_response import LlmResponse
+from google.genai import types as genai_types
+import pytest
+
+
+@pytest.mark.parametrize(
+    "response_text",
+    [
+        """```json
+  {
+    "is_the_agent_response_valid_or_invalid": "valid",
+    "reasoning": "The response is valid."
+  }
+  ```""",
+        """```json
+  {
+    "is_the_agent_response_valid": "undefined label",
+  }
+  ```""",
+    ],
+)
+def test_parse_critique_label_not_found(response_text):
+  label = _parse_critique(response_text)
+  assert label == Label.NOT_FOUND
+
+
+@pytest.mark.parametrize(
+    "response_text",
+    [
+        """```json
+  {
+    "is_the_agent_response_valid": "valid",
+    "reasoning": "The response is valid."
+  }
+  ```""",
+        """```json
+  {
+    "is_the_agent_response_valid": ["valid"],
+    "reasoning": "The response is valid."
+  }
+  ```""",
+        """```json
+  {
+    "is_the_agent_response_valid":\n    [ "valid\n"],
+    "reasoning": "The response is valid."
+  }
+  ```""",
+    ],
+)
+def test_parse_critique(response_text):
+  label = _parse_critique(response_text)
+  assert label == Label.VALID
+
+
+@pytest.mark.parametrize(
+    "response_text",
+    [
+        """```json
+  {
+    "is_the_agent_response_invalid": "invalid",
+    "reasoning": "The response is invalid."
+  }
+  ```""",
+        """```json
+  {
+    "is_the_agent_response_invalid": ["invalid"],
+    "reasoning": "The response is invalid."
+  }
+  ```""",
+        """```json
+  {
+    "is_the_agent_response_invalid":\n    [ "invalid\n"],
+    "reasoning": "The response is invalid."
+  }
+  ```""",
+    ],
+)
+def test_parse_critique_invalid(response_text):
+  label = _parse_critique(response_text)
+  assert label == Label.INVALID
+
+
+def create_test_template() -> str:
+  return """
+This is a test template.
+
+{{
+  "User prompt": {prompt},
+  "Agent response": {response},
+  "Reference response": {golden_response},
+}}
+
+The answer should be a json alone which follows the json structure below:
+{{
+  "is_the_agent_response_valid": [valid or invalid],
+  "reasoning":
+}}
+"""
+
+
+def _create_test_evaluator_gemini(
+    threshold: float,
+    *,
+    include_intermediate_responses_in_final: bool = False,
+) -> FinalResponseMatchV2Evaluator:
+  evaluator = FinalResponseMatchV2Evaluator(
+      EvalMetric(
+          metric_name="final_response_match_v2",
+          threshold=threshold,
+          criterion=BaseCriterion(
+              threshold=0.5,
+              include_intermediate_responses_in_final=(
+                  include_intermediate_responses_in_final
+              ),
+          ),
+      ),
+  )
+  evaluator._auto_rater_prompt_template = create_test_template()
+  return evaluator
+
+
+def _create_test_invocations(
+    candidate: str, reference: str
+) -> tuple[Invocation, Invocation]:
+  """Returns tuple of (actual_invocation, expected_invocation)."""
+  actual_invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="This is a test query.")],
+          role="user",
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text=candidate)],
+          role="model",
+      ),
+  )
+  expected_invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="This is a test query.")],
+          role="user",
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text=reference)],
+          role="model",
+      ),
+  )
+  return actual_invocation, expected_invocation
+
+
+def _add_intermediate_text(invocation: Invocation, text: str) -> Invocation:
+  invocation.intermediate_data = InvocationEvents(
+      invocation_events=[
+          InvocationEvent(
+              author="agent",
+              content=genai_types.Content(
+                  parts=[genai_types.Part(text=text)],
+                  role="model",
+              ),
+          ),
+      ]
+  )
+  return invocation
+
+
+def test_format_auto_rater_prompt():
+  evaluator = _create_test_evaluator_gemini(threshold=0.8)
+  actual_invocation, expected_invocation = _create_test_invocations(
+      "candidate text", "reference text"
+  )
+  prompt = evaluator.format_auto_rater_prompt(
+      actual_invocation, expected_invocation
+  )
+  assert prompt == """
+This is a test template.
+
+{
+  "User prompt": This is a test query.,
+  "Agent response": candidate text,
+  "Reference response": reference text,
+}
+
+The answer should be a json alone which follows the json structure below:
+{
+  "is_the_agent_response_valid": [valid or invalid],
+  "reasoning":
+}
+"""
+
+
+def test_format_auto_rater_prompt_uses_empty_text_for_missing_final_response():
+  evaluator = _create_test_evaluator_gemini(threshold=0.8)
+  actual_invocation, expected_invocation = _create_test_invocations(
+      "candidate text", "reference text"
+  )
+  actual_invocation.final_response = None
+  expected_invocation.final_response = None
+
+  prompt = evaluator.format_auto_rater_prompt(
+      actual_invocation, expected_invocation
+  )
+
+  assert "None" not in prompt
+  assert '"Agent response": ,' in prompt
+  assert '"Reference response": ,' in prompt
+
+
+def test_format_auto_rater_prompt_ignores_intermediate_by_default():
+  evaluator = _create_test_evaluator_gemini(threshold=0.8)
+  actual_invocation, expected_invocation = _create_test_invocations(
+      "candidate final", "reference final"
+  )
+  _add_intermediate_text(actual_invocation, "candidate intro")
+  _add_intermediate_text(expected_invocation, "reference intro")
+
+  prompt = evaluator.format_auto_rater_prompt(
+      actual_invocation, expected_invocation
+  )
+
+  assert "candidate final" in prompt
+  assert "reference final" in prompt
+  assert "candidate intro" not in prompt
+  assert "reference intro" not in prompt
+
+
+def test_format_auto_rater_prompt_includes_intermediate_when_enabled():
+  evaluator = _create_test_evaluator_gemini(
+      threshold=0.8, include_intermediate_responses_in_final=True
+  )
+  actual_invocation, expected_invocation = _create_test_invocations(
+      "candidate final", "reference final"
+  )
+  _add_intermediate_text(actual_invocation, "candidate intro")
+  _add_intermediate_text(expected_invocation, "reference intro")
+
+  prompt = evaluator.format_auto_rater_prompt(
+      actual_invocation, expected_invocation
+  )
+
+  assert "candidate intro\ncandidate final" in prompt
+  assert "reference intro\nreference final" in prompt
+
+
+def test_convert_auto_rater_response_to_score_valid():
+  evaluator = _create_test_evaluator_gemini(threshold=0.8)
+  auto_rater_response = """```json
+{
+  "is_the_agent_response_valid": "valid",
+  "reasoning": "The response is valid."
+}
+```"""
+  llm_response = LlmResponse(
+      content=genai_types.Content(
+          parts=[genai_types.Part(text=auto_rater_response)],
+          role="model",
+      )
+  )
+  auto_rater_score = evaluator.convert_auto_rater_response_to_score(
+      llm_response
+  )
+  assert auto_rater_score == AutoRaterScore(score=1.0)
+
+
+def test_convert_auto_rater_response_to_score_invalid():
+  evaluator = _create_test_evaluator_gemini(threshold=0.8)
+  auto_rater_response = """```json
+{
+  "is_the_agent_response_valid": "invalid",
+  "reasoning": "The response is invalid."
+}
+```"""
+  llm_response = LlmResponse(
+      content=genai_types.Content(
+          parts=[genai_types.Part(text=auto_rater_response)],
+          role="model",
+      )
+  )
+  auto_rater_score = evaluator.convert_auto_rater_response_to_score(
+      llm_response
+  )
+  assert auto_rater_score == AutoRaterScore(score=0.0)
+
+
+def test_convert_auto_rater_response_to_score_invalid_json():
+  evaluator = _create_test_evaluator_gemini(threshold=0.8)
+  llm_response = LlmResponse(
+      content=genai_types.Content(
+          parts=[genai_types.Part(text="invalid json")],
+          role="model",
+      )
+  )
+  auto_rater_score = evaluator.convert_auto_rater_response_to_score(
+      llm_response
+  )
+  assert auto_rater_score == AutoRaterScore()
+
+
+def test_convert_auto_rater_response_to_score_missing_key():
+  evaluator = _create_test_evaluator_gemini(threshold=0.8)
+  llm_response = LlmResponse(
+      content=genai_types.Content(
+          parts=[genai_types.Part(text="{}")],
+          role="model",
+      )
+  )
+  auto_rater_score = evaluator.convert_auto_rater_response_to_score(
+      llm_response
+  )
+  assert auto_rater_score == AutoRaterScore()
+
+
+def test_aggregate_per_invocation_samples_none_evaluated():
+  evaluator = _create_test_evaluator_gemini(threshold=0.5)
+
+  actual_invocation, expected_invocation = _create_test_invocations(
+      "candidate text", "reference text"
+  )
+
+  per_invocation_result_samples = [
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=None,
+          eval_status=EvalStatus.NOT_EVALUATED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=None,
+          eval_status=EvalStatus.NOT_EVALUATED,
+      ),
+  ]
+
+  assert (
+      evaluator.aggregate_per_invocation_samples(per_invocation_result_samples)
+      == per_invocation_result_samples[0]
+  )
+
+
+def test_aggregate_per_invocation_samples_valid():
+  evaluator = _create_test_evaluator_gemini(threshold=0.5)
+
+  actual_invocation, expected_invocation = _create_test_invocations(
+      "candidate text", "reference text"
+  )
+
+  per_invocation_result_samples = [
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=0.0,
+          eval_status=EvalStatus.FAILED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=0.0,
+          eval_status=EvalStatus.FAILED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=1.0,
+          eval_status=EvalStatus.NOT_EVALUATED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=None,
+          eval_status=EvalStatus.NOT_EVALUATED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=0.0,
+          eval_status=EvalStatus.NOT_EVALUATED,
+      ),
+  ]
+
+  per_invocation_result = evaluator.aggregate_per_invocation_samples(
+      per_invocation_result_samples
+  )
+
+  assert per_invocation_result.score == 1.0
+  assert per_invocation_result.eval_status == EvalStatus.PASSED
+
+
+def test_aggregate_per_invocation_samples_invalid():
+  evaluator = _create_test_evaluator_gemini(threshold=0.5)
+
+  actual_invocation, expected_invocation = _create_test_invocations(
+      "candidate text", "reference text"
+  )
+
+  per_invocation_result_samples = [
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=0.0,
+          eval_status=EvalStatus.FAILED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=0.0,
+          eval_status=EvalStatus.FAILED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=0.0,
+          eval_status=EvalStatus.FAILED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=1.0,
+          eval_status=EvalStatus.NOT_EVALUATED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=None,
+          eval_status=EvalStatus.NOT_EVALUATED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=0.0,
+          eval_status=EvalStatus.NOT_EVALUATED,
+      ),
+  ]
+
+  per_invocation_result = evaluator.aggregate_per_invocation_samples(
+      per_invocation_result_samples
+  )
+
+  assert per_invocation_result.score == 0.0
+  assert per_invocation_result.eval_status == EvalStatus.FAILED
+
+
+def test_aggregate_invocation_results():
+  evaluator = _create_test_evaluator_gemini(threshold=0.5)
+
+  actual_invocation, expected_invocation = _create_test_invocations(
+      "candidate text", "reference text"
+  )
+
+  per_invocation_results = [
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=1.0,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=0.0,
+          eval_status=EvalStatus.FAILED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=0.0,
+          eval_status=EvalStatus.FAILED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=None,
+          eval_status=EvalStatus.PASSED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=100.0,
+          eval_status=EvalStatus.NOT_EVALUATED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=None,
+          eval_status=EvalStatus.NOT_EVALUATED,
+      ),
+  ]
+
+  aggregated_result = evaluator.aggregate_invocation_results(
+      per_invocation_results
+  )
+
+  # Only 4 / 8 invocations are evaluated, and 2 / 4 are valid.
+  assert aggregated_result.overall_score == 0.5
+  assert aggregated_result.overall_eval_status == EvalStatus.PASSED
+
+
+def test_aggregate_invocation_results_none_evaluated():
+  evaluator = _create_test_evaluator_gemini(threshold=0.5)
+
+  actual_invocation, expected_invocation = _create_test_invocations(
+      "candidate text", "reference text"
+  )
+
+  per_invocation_results = [
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=None,
+          eval_status=EvalStatus.NOT_EVALUATED,
+      ),
+      PerInvocationResult(
+          actual_invocation=actual_invocation,
+          expected_invocation=expected_invocation,
+          score=1.0,
+          eval_status=EvalStatus.NOT_EVALUATED,
+      ),
+  ]
+
+  aggregated_result = evaluator.aggregate_invocation_results(
+      per_invocation_results
+  )
+
+  assert aggregated_result.overall_score is None
+  assert aggregated_result.overall_eval_status == EvalStatus.NOT_EVALUATED
+  assert aggregated_result.per_invocation_results == per_invocation_results

tests/unittests/evaluation/test_gcs_eval_set_results_manager.py

@@ -0,0 +1,220 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+from google.adk.errors.not_found_error import NotFoundError
+from google.adk.evaluation._eval_set_results_manager_utils import _sanitize_eval_set_result_name
+from google.adk.evaluation._eval_set_results_manager_utils import create_eval_set_result
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_metrics import EvalMetricResult
+from google.adk.evaluation.eval_metrics import EvalMetricResultPerInvocation
+from google.adk.evaluation.eval_result import EvalCaseResult
+from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.gcs_eval_set_results_manager import GcsEvalSetResultsManager
+from google.genai import types as genai_types
+import pytest
+
+from .mock_gcs_utils import MockBucket
+from .mock_gcs_utils import MockClient
+
+
+def _get_test_eval_case_results():
+  # Create mock Invocation objects
+  actual_invocation_1 = Invocation(
+      invocation_id="actual_1",
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="input_1")]
+      ),
+  )
+  expected_invocation_1 = Invocation(
+      invocation_id="expected_1",
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="expected_input_1")]
+      ),
+  )
+  actual_invocation_2 = Invocation(
+      invocation_id="actual_2",
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="input_2")]
+      ),
+  )
+  expected_invocation_2 = Invocation(
+      invocation_id="expected_2",
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="expected_input_2")]
+      ),
+  )
+
+  eval_metric_result_1 = EvalMetricResult(
+      metric_name="metric",
+      threshold=0.8,
+      score=1.0,
+      eval_status=EvalStatus.PASSED,
+  )
+  eval_metric_result_2 = EvalMetricResult(
+      metric_name="metric",
+      threshold=0.8,
+      score=0.5,
+      eval_status=EvalStatus.FAILED,
+  )
+  eval_metric_result_per_invocation_1 = EvalMetricResultPerInvocation(
+      actual_invocation=actual_invocation_1,
+      expected_invocation=expected_invocation_1,
+      eval_metric_results=[eval_metric_result_1],
+  )
+  eval_metric_result_per_invocation_2 = EvalMetricResultPerInvocation(
+      actual_invocation=actual_invocation_2,
+      expected_invocation=expected_invocation_2,
+      eval_metric_results=[eval_metric_result_2],
+  )
+  return [
+      EvalCaseResult(
+          eval_set_id="eval_set",
+          eval_id="eval_case_1",
+          final_eval_status=EvalStatus.PASSED,
+          overall_eval_metric_results=[eval_metric_result_1],
+          eval_metric_result_per_invocation=[
+              eval_metric_result_per_invocation_1
+          ],
+          session_id="session_1",
+      ),
+      EvalCaseResult(
+          eval_set_id="eval_set",
+          eval_id="eval_case_2",
+          final_eval_status=EvalStatus.FAILED,
+          overall_eval_metric_results=[eval_metric_result_2],
+          eval_metric_result_per_invocation=[
+              eval_metric_result_per_invocation_2
+          ],
+          session_id="session_2",
+      ),
+  ]
+
+
+class TestGcsEvalSetResultsManager:
+
+  @pytest.fixture
+  def gcs_eval_set_results_manager(self, mocker):
+    mock_storage_client = MockClient()
+    bucket_name = "test_bucket"
+    mock_bucket = MockBucket(bucket_name)
+    mocker.patch.object(mock_storage_client, "bucket", return_value=mock_bucket)
+    mocker.patch(
+        "google.cloud.storage.Client", return_value=mock_storage_client
+    )
+    return GcsEvalSetResultsManager(bucket_name=bucket_name)
+
+  def test_save_eval_set_result(self, gcs_eval_set_results_manager, mocker):
+    mocker.patch("time.time", return_value=12345678)
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_results = _get_test_eval_case_results()
+    eval_set_result = create_eval_set_result(
+        app_name, eval_set_id, eval_case_results
+    )
+    blob_name = gcs_eval_set_results_manager._get_eval_set_result_blob_name(
+        app_name, eval_set_result.eval_set_result_id
+    )
+    mock_write_eval_set_result = mocker.patch.object(
+        gcs_eval_set_results_manager,
+        "_write_eval_set_result",
+    )
+    gcs_eval_set_results_manager.save_eval_set_result(
+        app_name, eval_set_id, eval_case_results
+    )
+    mock_write_eval_set_result.assert_called_once_with(
+        blob_name,
+        eval_set_result,
+    )
+
+  def test_get_eval_set_result_not_found(
+      self, gcs_eval_set_results_manager, mocker
+  ):
+    mocker.patch("time.time", return_value=12345678)
+    app_name = "test_app"
+    with pytest.raises(NotFoundError) as e:
+      gcs_eval_set_results_manager.get_eval_set_result(
+          app_name, "non_existent_id"
+      )
+
+  def test_get_eval_set_result(self, gcs_eval_set_results_manager, mocker):
+    mocker.patch("time.time", return_value=12345678)
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_results = _get_test_eval_case_results()
+    gcs_eval_set_results_manager.save_eval_set_result(
+        app_name, eval_set_id, eval_case_results
+    )
+    eval_set_result = create_eval_set_result(
+        app_name, eval_set_id, eval_case_results
+    )
+    retrieved_eval_set_result = (
+        gcs_eval_set_results_manager.get_eval_set_result(
+            app_name, eval_set_result.eval_set_result_id
+        )
+    )
+    assert retrieved_eval_set_result == eval_set_result
+
+  def test_get_eval_set_result_double_encoded_legacy(
+      self, gcs_eval_set_results_manager, mocker
+  ):
+    mocker.patch("time.time", return_value=12345678)
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_results = _get_test_eval_case_results()
+    eval_set_result = create_eval_set_result(
+        app_name, eval_set_id, eval_case_results
+    )
+
+    blob_name = gcs_eval_set_results_manager._get_eval_set_result_blob_name(
+        app_name, eval_set_result.eval_set_result_id
+    )
+    blob = gcs_eval_set_results_manager.bucket.blob(blob_name)
+    double_encoded_json = json.dumps(eval_set_result.model_dump_json())
+    blob.upload_from_string(
+        double_encoded_json, content_type="application/json"
+    )
+
+    retrieved_eval_set_result = (
+        gcs_eval_set_results_manager.get_eval_set_result(
+            app_name, eval_set_result.eval_set_result_id
+        )
+    )
+    assert retrieved_eval_set_result == eval_set_result
+
+  def test_list_eval_set_results(self, gcs_eval_set_results_manager, mocker):
+    mocker.patch("time.time", return_value=123)
+    app_name = "test_app"
+    eval_set_ids = ["test_eval_set_1", "test_eval_set_2", "test_eval_set_3"]
+    for eval_set_id in eval_set_ids:
+      eval_case_results = _get_test_eval_case_results()
+      gcs_eval_set_results_manager.save_eval_set_result(
+          app_name, eval_set_id, eval_case_results
+      )
+    retrieved_eval_set_result_ids = (
+        gcs_eval_set_results_manager.list_eval_set_results(app_name)
+    )
+    assert retrieved_eval_set_result_ids == [
+        "test_app_test_eval_set_1_123",
+        "test_app_test_eval_set_2_123",
+        "test_app_test_eval_set_3_123",
+    ]
+
+  def test_list_eval_set_results_empty(self, gcs_eval_set_results_manager):
+    app_name = "test_app"
+    retrieved_eval_set_result_ids = (
+        gcs_eval_set_results_manager.list_eval_set_results(app_name)
+    )
+    assert retrieved_eval_set_result_ids == []

tests/unittests/evaluation/test_gcs_eval_sets_manager.py

@@ -0,0 +1,421 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.adk.errors.not_found_error import NotFoundError
+from google.adk.evaluation.eval_case import EvalCase
+from google.adk.evaluation.eval_set import EvalSet
+from google.adk.evaluation.gcs_eval_sets_manager import _EVAL_SET_FILE_EXTENSION
+from google.adk.evaluation.gcs_eval_sets_manager import GcsEvalSetsManager
+from google.cloud import exceptions as cloud_exceptions
+import pytest
+
+from .mock_gcs_utils import MockBlob
+from .mock_gcs_utils import MockBucket
+from .mock_gcs_utils import MockClient
+
+
+class TestGcsEvalSetsManager:
+  """Tests for GcsEvalSetsManager."""
+
+  @pytest.fixture
+  def gcs_eval_sets_manager(self, mocker):
+    mock_storage_client = MockClient()
+    bucket_name = "test_bucket"
+    mock_bucket = MockBucket(bucket_name)
+    mocker.patch.object(mock_storage_client, "bucket", return_value=mock_bucket)
+    mocker.patch(
+        "google.cloud.storage.Client", return_value=mock_storage_client
+    )
+    return GcsEvalSetsManager(bucket_name=bucket_name)
+
+  def test_gcs_eval_sets_manager_get_eval_set_success(
+      self, gcs_eval_sets_manager
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    mock_eval_set = EvalSet(eval_set_id=eval_set_id, eval_cases=[])
+    mock_bucket = gcs_eval_sets_manager.bucket
+    mock_blob = mock_bucket.blob(
+        f"{app_name}/evals/eval_sets/{eval_set_id}{_EVAL_SET_FILE_EXTENSION}"
+    )
+    mock_blob.upload_from_string(mock_eval_set.model_dump_json())
+
+    eval_set = gcs_eval_sets_manager.get_eval_set(app_name, eval_set_id)
+
+    assert eval_set == mock_eval_set
+
+  def test_gcs_eval_sets_manager_get_eval_set_not_found(
+      self, gcs_eval_sets_manager
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set_not_exist"
+    eval_set = gcs_eval_sets_manager.get_eval_set(app_name, eval_set_id)
+
+    assert eval_set is None
+
+  def test_gcs_eval_sets_manager_create_eval_set_success(
+      self, gcs_eval_sets_manager, mocker
+  ):
+    mocked_time = 12345678
+    mocker.patch("time.time", return_value=mocked_time)
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    mock_write_eval_set_to_blob = mocker.patch.object(
+        gcs_eval_sets_manager,
+        "_write_eval_set_to_blob",
+    )
+    eval_set_blob_name = gcs_eval_sets_manager._get_eval_set_blob_name(
+        app_name, eval_set_id
+    )
+
+    created_eval_set = gcs_eval_sets_manager.create_eval_set(
+        app_name, eval_set_id
+    )
+
+    expected_eval_set = EvalSet(
+        eval_set_id=eval_set_id,
+        name=eval_set_id,
+        eval_cases=[],
+        creation_timestamp=mocked_time,
+    )
+    mock_write_eval_set_to_blob.assert_called_once_with(
+        eval_set_blob_name,
+        expected_eval_set,
+    )
+    assert created_eval_set == expected_eval_set
+
+  def test_gcs_eval_sets_manager_create_eval_set_invalid_id(
+      self, gcs_eval_sets_manager
+  ):
+    app_name = "test_app"
+    eval_set_id = "invalid-id"
+
+    with pytest.raises(ValueError, match="Invalid Eval Set ID"):
+      gcs_eval_sets_manager.create_eval_set(app_name, eval_set_id)
+
+  def test_gcs_eval_sets_manager_list_eval_sets_success(
+      self, gcs_eval_sets_manager
+  ):
+    app_name = "test_app"
+    mock_blob_1 = MockBlob(
+        f"test_app/evals/eval_sets/eval_set_1{_EVAL_SET_FILE_EXTENSION}"
+    )
+    mock_blob_2 = MockBlob(
+        f"test_app/evals/eval_sets/eval_set_2{_EVAL_SET_FILE_EXTENSION}"
+    )
+    mock_blob_3 = MockBlob("test_app/evals/eval_sets/not_an_eval_set.txt")
+    mock_bucket = gcs_eval_sets_manager.bucket
+    mock_bucket.blobs = {
+        mock_blob_1.name: mock_blob_1,
+        mock_blob_2.name: mock_blob_2,
+        mock_blob_3.name: mock_blob_3,
+    }
+
+    eval_sets = gcs_eval_sets_manager.list_eval_sets(app_name)
+
+    assert eval_sets == ["eval_set_1", "eval_set_2"]
+
+  def test_gcs_eval_sets_manager_list_eval_sets_fails(
+      self, gcs_eval_sets_manager, mocker
+  ):
+    mocker.patch.object(
+        gcs_eval_sets_manager.bucket,
+        "list_blobs",
+        side_effect=cloud_exceptions.NotFound("not found"),
+    )
+
+    with pytest.raises(NotFoundError):
+      gcs_eval_sets_manager.list_eval_sets("test_app")
+
+  def test_gcs_eval_sets_manager_add_eval_case_success(
+      self, gcs_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+    mock_eval_set = EvalSet(eval_set_id=eval_set_id, eval_cases=[])
+    mocker.patch.object(
+        gcs_eval_sets_manager, "get_eval_set", return_value=mock_eval_set
+    )
+    mock_write_eval_set_to_blob = mocker.patch.object(
+        gcs_eval_sets_manager, "_write_eval_set_to_blob"
+    )
+    eval_set_blob_name = gcs_eval_sets_manager._get_eval_set_blob_name(
+        app_name, eval_set_id
+    )
+
+    gcs_eval_sets_manager.add_eval_case(app_name, eval_set_id, mock_eval_case)
+
+    assert len(mock_eval_set.eval_cases) == 1
+    assert mock_eval_set.eval_cases[0] == mock_eval_case
+    mock_write_eval_set_to_blob.assert_called_once_with(
+        eval_set_blob_name, mock_eval_set
+    )
+
+  def test_gcs_eval_sets_manager_add_eval_case_eval_set_not_found(
+      self, gcs_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+    mocker.patch.object(
+        gcs_eval_sets_manager, "get_eval_set", return_value=None
+    )
+
+    with pytest.raises(
+        NotFoundError, match="Eval set `test_eval_set` not found."
+    ):
+      gcs_eval_sets_manager.add_eval_case(app_name, eval_set_id, mock_eval_case)
+
+  def test_gcs_eval_sets_manager_add_eval_case_eval_case_id_exists(
+      self, gcs_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+    mock_eval_set = EvalSet(
+        eval_set_id=eval_set_id, eval_cases=[mock_eval_case]
+    )
+    mocker.patch.object(
+        gcs_eval_sets_manager, "get_eval_set", return_value=mock_eval_set
+    )
+
+    with pytest.raises(
+        ValueError,
+        match=(
+            f"Eval id `{eval_case_id}` already exists in `{eval_set_id}` eval"
+            " set."
+        ),
+    ):
+      gcs_eval_sets_manager.add_eval_case(app_name, eval_set_id, mock_eval_case)
+
+  def test_gcs_eval_sets_manager_get_eval_case_success(
+      self, gcs_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+    mock_eval_set = EvalSet(
+        eval_set_id=eval_set_id, eval_cases=[mock_eval_case]
+    )
+    mocker.patch.object(
+        gcs_eval_sets_manager, "get_eval_set", return_value=mock_eval_set
+    )
+
+    eval_case = gcs_eval_sets_manager.get_eval_case(
+        app_name, eval_set_id, eval_case_id
+    )
+
+    assert eval_case == mock_eval_case
+
+  def test_gcs_eval_sets_manager_get_eval_case_eval_set_not_found(
+      self, gcs_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mocker.patch.object(
+        gcs_eval_sets_manager, "get_eval_set", return_value=None
+    )
+
+    eval_case = gcs_eval_sets_manager.get_eval_case(
+        app_name, eval_set_id, eval_case_id
+    )
+
+    assert eval_case is None
+
+  def test_gcs_eval_sets_manager_get_eval_case_eval_case_not_found(
+      self, gcs_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_set = EvalSet(eval_set_id=eval_set_id, eval_cases=[])
+    mocker.patch.object(
+        gcs_eval_sets_manager, "get_eval_set", return_value=mock_eval_set
+    )
+
+    eval_case = gcs_eval_sets_manager.get_eval_case(
+        app_name, eval_set_id, eval_case_id
+    )
+
+    assert eval_case is None
+
+  def test_gcs_eval_sets_manager_update_eval_case_success(
+      self, gcs_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_case = EvalCase(
+        eval_id=eval_case_id, conversation=[], creation_timestamp=456
+    )
+    updated_eval_case = EvalCase(
+        eval_id=eval_case_id, conversation=[], creation_timestamp=123
+    )
+    mock_eval_set = EvalSet(
+        eval_set_id=eval_set_id, eval_cases=[mock_eval_case]
+    )
+    mocker.patch.object(
+        gcs_eval_sets_manager, "get_eval_set", return_value=mock_eval_set
+    )
+    mocker.patch.object(
+        gcs_eval_sets_manager, "get_eval_case", return_value=mock_eval_case
+    )
+    mock_write_eval_set_to_blob = mocker.patch.object(
+        gcs_eval_sets_manager, "_write_eval_set_to_blob"
+    )
+    eval_set_blob_name = gcs_eval_sets_manager._get_eval_set_blob_name(
+        app_name, eval_set_id
+    )
+
+    gcs_eval_sets_manager.update_eval_case(
+        app_name, eval_set_id, updated_eval_case
+    )
+
+    assert len(mock_eval_set.eval_cases) == 1
+    assert mock_eval_set.eval_cases[0] == updated_eval_case
+    mock_write_eval_set_to_blob.assert_called_once_with(
+        eval_set_blob_name,
+        EvalSet(eval_set_id=eval_set_id, eval_cases=[updated_eval_case]),
+    )
+
+  def test_gcs_eval_sets_manager_update_eval_case_eval_set_not_found(
+      self, gcs_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    updated_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+    mocker.patch.object(
+        gcs_eval_sets_manager, "get_eval_case", return_value=None
+    )
+
+    with pytest.raises(
+        NotFoundError,
+        match=f"Eval set `{eval_set_id}` not found.",
+    ):
+      gcs_eval_sets_manager.update_eval_case(
+          app_name, eval_set_id, updated_eval_case
+      )
+
+  def test_gcs_eval_sets_manager_update_eval_case_eval_case_not_found(
+      self, gcs_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_set = EvalSet(eval_set_id=eval_set_id, eval_cases=[])
+    mocker.patch.object(
+        gcs_eval_sets_manager, "get_eval_set", return_value=mock_eval_set
+    )
+    updated_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+
+    with pytest.raises(
+        NotFoundError,
+        match=(
+            f"Eval case `{eval_case_id}` not found in eval set `{eval_set_id}`."
+        ),
+    ):
+      gcs_eval_sets_manager.update_eval_case(
+          app_name, eval_set_id, updated_eval_case
+      )
+
+  def test_gcs_eval_sets_manager_delete_eval_case_success(
+      self, gcs_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+    mock_eval_set = EvalSet(
+        eval_set_id=eval_set_id, eval_cases=[mock_eval_case]
+    )
+    mock_bucket = gcs_eval_sets_manager.bucket
+    mock_blob = mock_bucket.blob(
+        f"{app_name}/evals/eval_sets/{eval_set_id}{_EVAL_SET_FILE_EXTENSION}"
+    )
+    mock_blob.upload_from_string(mock_eval_set.model_dump_json())
+    mocker.patch.object(
+        gcs_eval_sets_manager, "get_eval_set", return_value=mock_eval_set
+    )
+    mocker.patch.object(
+        gcs_eval_sets_manager, "get_eval_case", return_value=mock_eval_case
+    )
+    mock_write_eval_set_to_blob = mocker.patch.object(
+        gcs_eval_sets_manager, "_write_eval_set_to_blob"
+    )
+    eval_set_blob_name = gcs_eval_sets_manager._get_eval_set_blob_name(
+        app_name, eval_set_id
+    )
+
+    gcs_eval_sets_manager.delete_eval_case(app_name, eval_set_id, eval_case_id)
+
+    assert len(mock_eval_set.eval_cases) == 0
+    mock_write_eval_set_to_blob.assert_called_once_with(
+        eval_set_blob_name,
+        EvalSet(eval_set_id=eval_set_id, eval_cases=[]),
+    )
+
+  def test_gcs_eval_sets_manager_delete_eval_case_eval_set_not_found(
+      self, gcs_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+
+    mock_write_eval_set_to_blob = mocker.patch.object(
+        gcs_eval_sets_manager, "_write_eval_set_to_blob"
+    )
+
+    with pytest.raises(
+        NotFoundError,
+        match=f"Eval set `{eval_set_id}` not found.",
+    ):
+      gcs_eval_sets_manager.delete_eval_case(
+          app_name, eval_set_id, eval_case_id
+      )
+    mock_write_eval_set_to_blob.assert_not_called()
+
+  def test_gcs_eval_sets_manager_delete_eval_case_eval_case_not_found(
+      self, gcs_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_set = EvalSet(eval_set_id=eval_set_id, eval_cases=[])
+    mocker.patch.object(
+        gcs_eval_sets_manager, "get_eval_set", return_value=mock_eval_set
+    )
+    mocker.patch.object(
+        gcs_eval_sets_manager, "get_eval_case", return_value=None
+    )
+    mock_write_eval_set_to_blob = mocker.patch.object(
+        gcs_eval_sets_manager, "_write_eval_set_to_blob"
+    )
+
+    with pytest.raises(
+        NotFoundError,
+        match=(
+            f"Eval case `{eval_case_id}` not found in eval set `{eval_set_id}`."
+        ),
+    ):
+      gcs_eval_sets_manager.delete_eval_case(
+          app_name, eval_set_id, eval_case_id
+      )
+    mock_write_eval_set_to_blob.assert_not_called()

tests/unittests/evaluation/test_in_memory_eval_sets_manager.py

@@ -0,0 +1,198 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+from google.adk.errors.not_found_error import NotFoundError
+from google.adk.evaluation.eval_case import EvalCase
+from google.adk.evaluation.in_memory_eval_sets_manager import InMemoryEvalSetsManager
+import pytest
+
+
+@pytest.fixture
+def app_name():
+  return "test_app"
+
+
+@pytest.fixture
+def manager():
+  return InMemoryEvalSetsManager()
+
+
+@pytest.fixture
+def eval_set_id():
+  return "test_eval_set"
+
+
+@pytest.fixture
+def eval_case_id():
+  return "test_eval_case"
+
+
+def test_create_eval_set(manager, app_name, eval_set_id):
+  eval_set = manager.create_eval_set(app_name, eval_set_id)
+  assert eval_set is not None
+  assert eval_set.eval_set_id == eval_set_id
+  assert eval_set.eval_cases == []
+
+
+def test_create_eval_set_already_exists(manager, app_name, eval_set_id):
+  manager.create_eval_set(app_name, eval_set_id)
+  with pytest.raises(ValueError):
+    manager.create_eval_set(app_name, eval_set_id)
+
+
+def test_get_eval_set(manager, app_name, eval_set_id):
+  manager.create_eval_set(app_name, eval_set_id)
+  eval_set = manager.get_eval_set(app_name, eval_set_id)
+  assert eval_set is not None
+  assert eval_set.eval_set_id == eval_set_id
+
+
+def test_get_eval_set_not_found(manager, app_name):
+  eval_set = manager.get_eval_set(app_name, "nonexistent_set")
+  assert eval_set is None
+
+
+def test_get_eval_set_wrong_app(manager, app_name, eval_set_id):
+  manager.create_eval_set(app_name, eval_set_id)
+  eval_set = manager.get_eval_set("wrong_app", eval_set_id)
+  assert eval_set is None
+
+
+def test_list_eval_sets(manager, app_name):
+  manager.create_eval_set(app_name, "set1")
+  manager.create_eval_set(app_name, "set2")
+  eval_sets = manager.list_eval_sets(app_name)
+  assert len(eval_sets) == 2
+  assert "set1" in eval_sets
+  assert "set2" in eval_sets
+
+
+def test_list_eval_sets_wrong_app(manager, app_name):
+  manager.create_eval_set(app_name, "set1")
+  eval_sets = manager.list_eval_sets("wrong_app")
+  assert len(eval_sets) == 0
+
+
+def test_add_eval_case(manager, app_name, eval_set_id, eval_case_id):
+  manager.create_eval_set(app_name, eval_set_id)
+  eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+  manager.add_eval_case(app_name, eval_set_id, eval_case)
+
+  retrieved_case = manager.get_eval_case(app_name, eval_set_id, eval_case_id)
+  assert retrieved_case is not None
+  assert retrieved_case.eval_id == eval_case_id
+
+  eval_set = manager.get_eval_set(app_name, eval_set_id)
+  assert len(eval_set.eval_cases) == 1
+  assert eval_set.eval_cases[0].eval_id == eval_case_id
+
+
+def test_add_eval_case_set_not_found(
+    manager, app_name, eval_set_id, eval_case_id
+):
+  eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+  with pytest.raises(NotFoundError):
+    manager.add_eval_case(app_name, eval_set_id, eval_case)
+
+
+def test_add_eval_case_already_exists(
+    manager, app_name, eval_set_id, eval_case_id
+):
+  manager.create_eval_set(app_name, eval_set_id)
+  eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+  manager.add_eval_case(app_name, eval_set_id, eval_case)
+  with pytest.raises(ValueError):
+    manager.add_eval_case(app_name, eval_set_id, eval_case)
+
+
+def test_get_eval_case(manager, app_name, eval_set_id, eval_case_id):
+  manager.create_eval_set(app_name, eval_set_id)
+  eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+  manager.add_eval_case(app_name, eval_set_id, eval_case)
+  retrieved_case = manager.get_eval_case(app_name, eval_set_id, eval_case_id)
+  assert retrieved_case is not None
+  assert retrieved_case.eval_id == eval_case_id
+
+
+def test_get_eval_case_not_found(manager, app_name, eval_set_id):
+  manager.create_eval_set(app_name, eval_set_id)
+  retrieved_case = manager.get_eval_case(
+      app_name, eval_set_id, "nonexistent_case"
+  )
+  assert retrieved_case is None
+
+
+def test_get_eval_case_set_not_found(manager, app_name, eval_case_id):
+  retrieved_case = manager.get_eval_case(
+      app_name, "nonexistent_set", eval_case_id
+  )
+  assert retrieved_case is None
+
+
+def test_update_eval_case(manager, app_name, eval_set_id, eval_case_id):
+  manager.create_eval_set(app_name, eval_set_id)
+  eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+  manager.add_eval_case(app_name, eval_set_id, eval_case)
+
+  updated_eval_case = EvalCase(
+      eval_id=eval_case_id, conversation=[], creation_timestamp=time.time()
+  )
+  manager.update_eval_case(app_name, eval_set_id, updated_eval_case)
+
+  retrieved_case = manager.get_eval_case(app_name, eval_set_id, eval_case_id)
+  assert retrieved_case is not None
+  assert retrieved_case.creation_timestamp != 0.0
+  assert (
+      retrieved_case.creation_timestamp == updated_eval_case.creation_timestamp
+  )
+
+  eval_set = manager.get_eval_set(app_name, eval_set_id)
+  assert len(eval_set.eval_cases) == 1
+  assert (
+      eval_set.eval_cases[0].creation_timestamp
+      == updated_eval_case.creation_timestamp
+  )
+
+
+def test_update_eval_case_not_found(
+    manager, app_name, eval_set_id, eval_case_id
+):
+  manager.create_eval_set(app_name, eval_set_id)
+  updated_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+  with pytest.raises(NotFoundError):
+    manager.update_eval_case(app_name, eval_set_id, updated_eval_case)
+
+
+def test_delete_eval_case(manager, app_name, eval_set_id, eval_case_id):
+  manager.create_eval_set(app_name, eval_set_id)
+  eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+  manager.add_eval_case(app_name, eval_set_id, eval_case)
+
+  manager.delete_eval_case(app_name, eval_set_id, eval_case_id)
+
+  retrieved_case = manager.get_eval_case(app_name, eval_set_id, eval_case_id)
+  assert retrieved_case is None
+
+  eval_set = manager.get_eval_set(app_name, eval_set_id)
+  assert len(eval_set.eval_cases) == 0
+
+
+def test_delete_eval_case_not_found(
+    manager, app_name, eval_set_id, eval_case_id
+):
+  manager.create_eval_set(app_name, eval_set_id)
+  with pytest.raises(NotFoundError):
+    manager.delete_eval_case(app_name, eval_set_id, eval_case_id)

tests/unittests/evaluation/test_llm_as_judge.py

@@ -0,0 +1,239 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import Optional
+
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.eval_metrics import JudgeModelOptions
+from google.adk.evaluation.eval_metrics import LlmAsAJudgeCriterion
+from google.adk.evaluation.eval_rubrics import Rubric
+from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.evaluator import EvaluationResult
+from google.adk.evaluation.evaluator import PerInvocationResult
+from google.adk.evaluation.llm_as_judge import AutoRaterScore
+from google.adk.evaluation.llm_as_judge import LlmAsJudge
+from google.adk.evaluation.llm_as_judge_utils import get_eval_status
+from google.adk.evaluation.llm_as_judge_utils import get_text_from_content
+from google.adk.models.llm_response import LlmResponse
+from google.genai import types as genai_types
+import pytest
+
+
+class MockLlmAsJudge(LlmAsJudge):
+
+  def format_auto_rater_prompt(
+      self,
+      actual_invocation: Invocation,
+      expected_invocation: Optional[Invocation],
+      rubrics: Optional[list[Rubric]] = None,
+  ) -> str:
+    return "formatted prompt"
+
+  def convert_auto_rater_response_to_score(
+      self,
+      llm_response: LlmResponse,
+      rubrics: Optional[list[Rubric]] = None,
+  ) -> AutoRaterScore:
+    return AutoRaterScore(score=1.0)
+
+  def aggregate_per_invocation_samples(
+      self,
+      per_invocation_samples: list[PerInvocationResult],
+  ) -> PerInvocationResult:
+    return per_invocation_samples[0]
+
+  def aggregate_invocation_results(
+      self, per_invocation_results: list[PerInvocationResult]
+  ) -> EvaluationResult:
+    return EvaluationResult(
+        overall_score=1.0, overall_eval_status=EvalStatus.PASSED
+    )
+
+
+@pytest.fixture
+def mock_llm_as_judge():
+  return MockLlmAsJudge(
+      eval_metric=EvalMetric(
+          metric_name="test_metric",
+          threshold=0.5,
+          criterion=LlmAsAJudgeCriterion(
+              threshold=0.5,
+              judge_model_options=JudgeModelOptions(
+                  judge_model="gemini-2.5-flash",
+                  judge_model_config=genai_types.GenerateContentConfig(),
+                  num_samples=3,
+              ),
+          ),
+      ),
+      criterion_type=LlmAsAJudgeCriterion,
+  )
+
+
+def test_get_text_from_content():
+  content = genai_types.Content(
+      parts=[
+          genai_types.Part(text="This is a test text."),
+          genai_types.Part(text="This is another test text."),
+      ],
+      role="model",
+  )
+  assert (
+      get_text_from_content(content)
+      == "This is a test text.\nThis is another test text."
+  )
+
+
+def test_get_eval_status():
+  assert get_eval_status(score=0.8, threshold=0.8) == EvalStatus.PASSED
+  assert get_eval_status(score=0.7, threshold=0.8) == EvalStatus.FAILED
+  assert get_eval_status(score=0.8, threshold=0.9) == EvalStatus.FAILED
+  assert get_eval_status(score=0.9, threshold=0.8) == EvalStatus.PASSED
+  assert get_eval_status(score=None, threshold=0.8) == EvalStatus.NOT_EVALUATED
+
+
+def test_llm_as_judge_init_missing_criterion():
+  with pytest.raises(ValueError):
+    MockLlmAsJudge(
+        EvalMetric(metric_name="test_metric", threshold=0.8),
+        criterion_type=LlmAsAJudgeCriterion,
+    )
+
+
+def test_llm_as_judge_init_unregistered_model():
+  with pytest.raises(ValueError):
+    MockLlmAsJudge(
+        EvalMetric(
+            metric_name="test_metric",
+            threshold=0.8,
+            criterion=LlmAsAJudgeCriterion(
+                threshold=0.5,
+                judge_model_options=JudgeModelOptions(
+                    judge_model="unregistered_model",
+                    judge_model_config=genai_types.GenerateContentConfig(),
+                    num_samples=3,
+                ),
+            ),
+        ),
+        criterion_type=LlmAsAJudgeCriterion,
+    )
+
+
+@pytest.fixture
+def mock_judge_model(mocker):
+  mock_judge_model = mocker.MagicMock()
+
+  async def mock_generate_content_async(llm_request):
+    yield LlmResponse(
+        content=genai_types.Content(
+            parts=[genai_types.Part(text="auto rater response")],
+        )
+    )
+
+  mock_judge_model.generate_content_async = mock_generate_content_async
+  return mock_judge_model
+
+
+@pytest.mark.asyncio
+async def test_evaluate_invocations_with_mock(
+    mock_llm_as_judge, mock_judge_model, mocker
+):
+  mock_llm_as_judge._judge_model = mock_judge_model
+
+  mock_format_auto_rater_prompt = mocker.MagicMock(
+      wraps=mock_llm_as_judge.format_auto_rater_prompt
+  )
+  mock_llm_as_judge.format_auto_rater_prompt = mock_format_auto_rater_prompt
+
+  mock_convert_auto_rater_response_to_score = mocker.MagicMock(
+      wraps=mock_llm_as_judge.convert_auto_rater_response_to_score
+  )
+  mock_llm_as_judge.convert_auto_rater_response_to_score = (
+      mock_convert_auto_rater_response_to_score
+  )
+
+  mock_aggregate_per_invocation_samples = mocker.MagicMock(
+      wraps=mock_llm_as_judge.aggregate_per_invocation_samples
+  )
+  mock_llm_as_judge.aggregate_per_invocation_samples = (
+      mock_aggregate_per_invocation_samples
+  )
+
+  mock_aggregate_invocation_results = mocker.MagicMock(
+      wraps=mock_llm_as_judge.aggregate_invocation_results
+  )
+  mock_llm_as_judge.aggregate_invocation_results = (
+      mock_aggregate_invocation_results
+  )
+
+  actual_invocations = [
+      Invocation(
+          invocation_id="id1",
+          user_content=genai_types.Content(
+              parts=[genai_types.Part(text="user content 1")],
+              role="user",
+          ),
+          final_response=genai_types.Content(
+              parts=[genai_types.Part(text="final response 1")],
+              role="model",
+          ),
+      ),
+      Invocation(
+          invocation_id="id2",
+          user_content=genai_types.Content(
+              parts=[genai_types.Part(text="user content 2")],
+              role="user",
+          ),
+          final_response=genai_types.Content(
+              parts=[genai_types.Part(text="final response 2")],
+              role="model",
+          ),
+      ),
+  ]
+  expected_invocations = [
+      Invocation(
+          invocation_id="id1",
+          user_content=genai_types.Content(
+              parts=[genai_types.Part(text="user content 1")],
+              role="user",
+          ),
+          final_response=genai_types.Content(
+              parts=[genai_types.Part(text="expected response 1")],
+              role="model",
+          ),
+      ),
+      Invocation(
+          invocation_id="id2",
+          user_content=genai_types.Content(
+              parts=[genai_types.Part(text="user content 2")],
+              role="user",
+          ),
+          final_response=genai_types.Content(
+              parts=[genai_types.Part(text="expected response 2")],
+              role="model",
+          ),
+      ),
+  ]
+
+  result = await mock_llm_as_judge.evaluate_invocations(
+      actual_invocations, expected_invocations
+  )
+
+  # Assertions
+  assert result.overall_score == 1.0
+  assert mock_llm_as_judge.format_auto_rater_prompt.call_count == 2
+  assert mock_llm_as_judge.convert_auto_rater_response_to_score.call_count == 6
+  assert mock_llm_as_judge.aggregate_invocation_results.call_count == 1

tests/unittests/evaluation/test_llm_as_judge_utils.py

@@ -0,0 +1,364 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import json
+
+from google.adk.evaluation.app_details import AgentDetails
+from google.adk.evaluation.app_details import AppDetails
+from google.adk.evaluation.eval_case import IntermediateData
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_case import InvocationEvent
+from google.adk.evaluation.eval_case import InvocationEvents
+from google.adk.evaluation.eval_rubrics import RubricScore
+from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.llm_as_judge_utils import get_average_rubric_score
+from google.adk.evaluation.llm_as_judge_utils import get_eval_status
+from google.adk.evaluation.llm_as_judge_utils import get_text_from_content
+from google.adk.evaluation.llm_as_judge_utils import get_tool_calls_and_responses_as_json_str
+from google.adk.evaluation.llm_as_judge_utils import get_tool_declarations_as_json_str
+from google.genai import types as genai_types
+
+
+def test_get_text_from_content_with_none():
+  """Tests get_text_from_content with None as input."""
+  assert get_text_from_content(None) is None
+
+
+def test_get_text_from_content_with_content_and_none_parts():
+  """Tests get_text_from_content with Content that has None for parts."""
+  content = genai_types.Content(parts=None)
+  assert get_text_from_content(content) is None
+
+
+def test_get_text_from_content_with_empty_parts():
+  """Tests get_text_from_content with an empty parts list."""
+  content = genai_types.Content(parts=[])
+  assert get_text_from_content(content) == None
+
+
+def test_get_text_from_content_with_parts_but_no_text():
+  """Tests get_text_from_content with parts that do not contain text."""
+  content = genai_types.Content(
+      parts=[
+          genai_types.Part(
+              function_call=genai_types.FunctionCall(name="test_func")
+          )
+      ]
+  )
+  assert get_text_from_content(content) == ""
+
+
+def test_get_text_from_content_with_single_text_part():
+  """Tests get_text_from_content with a single text part."""
+  content = genai_types.Content(parts=[genai_types.Part(text="Hello")])
+  assert get_text_from_content(content) == "Hello"
+
+
+def test_get_text_from_content_with_multiple_text_parts():
+  """Tests get_text_from_content with multiple text parts."""
+  content = genai_types.Content(
+      parts=[genai_types.Part(text="Hello"), genai_types.Part(text="World")]
+  )
+  assert get_text_from_content(content) == "Hello\nWorld"
+
+
+def test_get_text_from_content_with_mixed_parts():
+  """Tests get_text_from_content with a mix of text and non-text parts."""
+  content = genai_types.Content(
+      parts=[
+          genai_types.Part(text="Hello"),
+          genai_types.Part(
+              function_call=genai_types.FunctionCall(name="test_func")
+          ),
+          genai_types.Part(text="World"),
+      ]
+  )
+  assert get_text_from_content(content) == "Hello\nWorld"
+
+
+def test_get_text_from_content_with_invocation_include_intermediate_responses_in_final():
+  """Tests get_text_from_content on an Invocation with and without the flag."""
+  intermediate_text = "Let me check."
+  final_response_text = "Done."
+  invocation = Invocation(
+      user_content=genai_types.Content(parts=[genai_types.Part(text="user")]),
+      intermediate_data=InvocationEvents(
+          invocation_events=[
+              InvocationEvent(
+                  author="agent",
+                  content=genai_types.Content(
+                      parts=[genai_types.Part(text=intermediate_text)]
+                  ),
+              ),
+              InvocationEvent(
+                  author="tool",
+                  content=genai_types.Content(
+                      parts=[
+                          genai_types.Part(
+                              function_call=genai_types.FunctionCall(name="t")
+                          )
+                      ]
+                  ),
+              ),
+          ]
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text=final_response_text)]
+      ),
+  )
+
+  # Flag off (default): only the final response text is returned.
+  assert get_text_from_content(invocation) == final_response_text
+
+  # Flag on: intermediate text is concatenated before the final response.
+  assert (
+      get_text_from_content(
+          invocation, include_intermediate_responses_in_final=True
+      )
+      == f"{intermediate_text}\n{final_response_text}"
+  )
+
+
+def test_get_text_from_content_with_intermediate_data_full_response():
+  invocation = Invocation(
+      user_content=genai_types.Content(parts=[genai_types.Part(text="user")]),
+      intermediate_data=IntermediateData(
+          intermediate_responses=[
+              ("agent", [genai_types.Part(text="legacy intro")]),
+              (
+                  "tool",
+                  [
+                      genai_types.Part(
+                          function_call=genai_types.FunctionCall(name="lookup")
+                      )
+                  ],
+              ),
+          ]
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text="final answer")]
+      ),
+  )
+
+  assert get_text_from_content(invocation) == "final answer"
+  assert (
+      get_text_from_content(
+          invocation, include_intermediate_responses_in_final=True
+      )
+      == "legacy intro\nfinal answer"
+  )
+
+
+def test_get_eval_status_with_none_score():
+  """Tests get_eval_status returns NOT_EVALUATED for a None score."""
+  assert get_eval_status(score=None, threshold=0.5) == EvalStatus.NOT_EVALUATED
+
+
+def test_get_eval_status_when_score_is_greater_than_threshold():
+  """Tests get_eval_status returns PASSED when score > threshold."""
+  assert get_eval_status(score=0.8, threshold=0.5) == EvalStatus.PASSED
+
+
+def test_get_eval_status_when_score_is_equal_to_threshold():
+  """Tests get_eval_status returns PASSED when score == threshold."""
+  assert get_eval_status(score=0.5, threshold=0.5) == EvalStatus.PASSED
+
+
+def test_get_eval_status_when_score_is_less_than_threshold():
+  """Tests get_eval_status returns FAILED when score < threshold."""
+  assert get_eval_status(score=0.4, threshold=0.5) == EvalStatus.FAILED
+
+
+def test_get_average_rubric_score_with_empty_list():
+  """Tests get_average_rubric_score returns None for an empty list."""
+  assert get_average_rubric_score([]) is None
+
+
+def test_get_average_rubric_score_with_all_none_scores():
+  """Tests get_average_rubric_score returns None when all scores are None."""
+  rubric_scores = [
+      RubricScore(rubric_id="1", score=None),
+      RubricScore(rubric_id="2", score=None),
+  ]
+  assert get_average_rubric_score(rubric_scores) is None
+
+
+def test_get_average_rubric_score_with_single_score():
+  """Tests get_average_rubric_score with a single valid score."""
+  rubric_scores = [RubricScore(rubric_id="1", score=0.8)]
+  assert get_average_rubric_score(rubric_scores) == 0.8
+
+
+def test_get_average_rubric_score_with_multiple_scores():
+  """Tests get_average_rubric_score with multiple valid scores."""
+  rubric_scores = [
+      RubricScore(rubric_id="1", score=0.8),
+      RubricScore(rubric_id="2", score=0.6),
+  ]
+  assert get_average_rubric_score(rubric_scores) == 0.7
+
+
+def test_get_average_rubric_score_with_mixed_scores():
+  """Tests get_average_rubric_score with a mix of valid and None scores."""
+  rubric_scores = [
+      RubricScore(rubric_id="1", score=0.8),
+      RubricScore(rubric_id="2", score=None),
+      RubricScore(rubric_id="3", score=0.6),
+  ]
+  assert get_average_rubric_score(rubric_scores) == 0.7
+
+
+def test_get_tool_declarations_as_json_str_with_no_agents():
+  """Tests get_tool_declarations_as_json_str with no agents."""
+  app_details = AppDetails(agent_details={})
+  expected_json = {"tool_declarations": {}}
+  actual_json_str = get_tool_declarations_as_json_str(app_details)
+  assert json.loads(actual_json_str) == expected_json
+
+
+def test_get_tool_declarations_as_json_str_with_agent_no_tools():
+  """Tests get_tool_declarations_as_json_str with an agent that has no tools."""
+  agent_details = {"agent1": AgentDetails(name="agent1", tool_declarations=[])}
+  app_details = AppDetails(agent_details=agent_details)
+  expected_json = {"tool_declarations": {"agent1": []}}
+  actual_json_str = get_tool_declarations_as_json_str(app_details)
+  assert json.loads(actual_json_str) == expected_json
+
+
+def test_get_tool_declarations_as_json_str_with_agent_with_tools():
+  """Tests get_tool_declarations_as_json_str with an agent that has tools."""
+  tool1 = genai_types.Tool(
+      function_declarations=[
+          genai_types.FunctionDeclaration(
+              name="test_func", description="A test function."
+          )
+      ]
+  )
+  agent_details = {
+      "agent1": AgentDetails(name="agent1", tool_declarations=[tool1])
+  }
+  app_details = AppDetails(agent_details=agent_details)
+  expected_json = {
+      "tool_declarations": {
+          "agent1": [{
+              "function_declarations": [{
+                  "name": "test_func",
+                  "description": "A test function.",
+              }]
+          }]
+      }
+  }
+  actual_json_str = get_tool_declarations_as_json_str(app_details)
+  assert json.loads(actual_json_str) == expected_json
+
+
+def test_get_tool_declarations_as_json_str_with_multiple_agents():
+  """Tests get_tool_declarations_as_json_str with multiple agents."""
+  tool1 = genai_types.Tool(
+      function_declarations=[
+          genai_types.FunctionDeclaration(
+              name="test_func1", description="A test function 1."
+          )
+      ]
+  )
+  agent_details = {
+      "agent1": AgentDetails(name="agent1", tool_declarations=[tool1]),
+      "agent2": AgentDetails(name="agent2", tool_declarations=[]),
+  }
+  app_details = AppDetails(agent_details=agent_details)
+  expected_json = {
+      "tool_declarations": {
+          "agent1": [{
+              "function_declarations": [{
+                  "name": "test_func1",
+                  "description": "A test function 1.",
+              }]
+          }],
+          "agent2": [],
+      }
+  }
+  actual_json_str = get_tool_declarations_as_json_str(app_details)
+  assert json.loads(actual_json_str) == expected_json
+
+
+def test_get_tool_calls_and_responses_as_json_str_with_none():
+  """Tests get_tool_calls_and_responses_as_json_str with None."""
+  assert (
+      get_tool_calls_and_responses_as_json_str(None)
+      == "No intermediate steps were taken."
+  )
+
+
+def test_get_tool_calls_and_responses_as_json_str_with_intermediate_data_no_tools():
+  """Tests get_tool_calls_and_responses_as_json_str with IntermediateData and no tools."""
+  intermediate_data = IntermediateData(tool_uses=[], tool_responses=[])
+  assert (
+      get_tool_calls_and_responses_as_json_str(intermediate_data)
+      == "No intermediate steps were taken."
+  )
+
+  intermediate_data = InvocationEvents(invocation_events=[])
+  assert (
+      get_tool_calls_and_responses_as_json_str(intermediate_data)
+      == "No intermediate steps were taken."
+  )
+
+
+def test_get_tool_calls_and_responses_as_json_str_with_invocation_events_multiple_calls():
+  """Tests get_tool_calls_and_responses_as_json_str with multiple calls in InvocationEvents."""
+  tool_call1 = genai_types.FunctionCall(name="func1", args={}, id="call1")
+  tool_call2 = genai_types.FunctionCall(name="func2", args={}, id="call2")
+  tool_response1 = genai_types.FunctionResponse(
+      name="func1", response={"status": "ok"}, id="call1"
+  )
+  invocation_event1 = InvocationEvent(
+      author="agent",
+      content=genai_types.Content(
+          parts=[
+              genai_types.Part(function_call=tool_call1),
+              genai_types.Part(function_call=tool_call2),
+          ]
+      ),
+  )
+  invocation_event2 = InvocationEvent(
+      author="tool",
+      content=genai_types.Content(
+          parts=[genai_types.Part(function_response=tool_response1)]
+      ),
+  )
+  intermediate_data = InvocationEvents(
+      invocation_events=[invocation_event1, invocation_event2]
+  )
+  json_str = get_tool_calls_and_responses_as_json_str(intermediate_data)
+  expected_json = {
+      "tool_calls_and_response": [
+          {
+              "step": 0,
+              "tool_call": {"name": "func1", "args": {}, "id": "call1"},
+              "tool_response": {
+                  "name": "func1",
+                  "response": {"status": "ok"},
+                  "id": "call1",
+              },
+          },
+          {
+              "step": 1,
+              "tool_call": {"name": "func2", "args": {}, "id": "call2"},
+              "tool_response": "None",
+          },
+      ]
+  }
+  assert json.loads(json_str) == expected_json

tests/unittests/evaluation/test_local_eval_service.py

@@ -0,0 +1,949 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import asyncio
+from typing import Optional
+
+from google.adk.agents.llm_agent import LlmAgent
+from google.adk.errors.not_found_error import NotFoundError
+from google.adk.evaluation.base_eval_service import EvaluateConfig
+from google.adk.evaluation.base_eval_service import EvaluateRequest
+from google.adk.evaluation.base_eval_service import InferenceConfig
+from google.adk.evaluation.base_eval_service import InferenceRequest
+from google.adk.evaluation.base_eval_service import InferenceResult
+from google.adk.evaluation.base_eval_service import InferenceStatus
+from google.adk.evaluation.conversation_scenarios import ConversationScenario
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.eval_metrics import EvalMetricResult
+from google.adk.evaluation.eval_metrics import Interval
+from google.adk.evaluation.eval_metrics import MetricInfo
+from google.adk.evaluation.eval_metrics import MetricValueInfo
+from google.adk.evaluation.eval_result import EvalCaseResult
+from google.adk.evaluation.eval_rubrics import Rubric
+from google.adk.evaluation.eval_rubrics import RubricContent
+from google.adk.evaluation.eval_set import EvalCase
+from google.adk.evaluation.eval_set import EvalSet
+from google.adk.evaluation.eval_set_results_manager import EvalSetResultsManager
+from google.adk.evaluation.eval_sets_manager import EvalSetsManager
+from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.evaluator import EvaluationResult
+from google.adk.evaluation.evaluator import Evaluator
+from google.adk.evaluation.evaluator import PerInvocationResult
+from google.adk.evaluation.local_eval_service import _add_rubrics_to_invocation
+from google.adk.evaluation.local_eval_service import _copy_eval_case_rubrics_to_actual_invocations
+from google.adk.evaluation.local_eval_service import _copy_invocation_rubrics_to_actual_invocations
+from google.adk.evaluation.local_eval_service import LocalEvalService
+from google.adk.evaluation.metric_evaluator_registry import DEFAULT_METRIC_EVALUATOR_REGISTRY
+from google.adk.models.registry import LLMRegistry
+from google.genai import types as genai_types
+import pytest
+from typing_extensions import override
+
+
+@pytest.fixture
+def mock_eval_sets_manager(mocker):
+  return mocker.create_autospec(EvalSetsManager)
+
+
+@pytest.fixture
+def dummy_agent():
+  llm = LLMRegistry.new_llm("gemini-pro")
+  return LlmAgent(name="test_agent", model=llm)
+
+
+@pytest.fixture
+def mock_eval_set_results_manager(mocker):
+  return mocker.create_autospec(EvalSetResultsManager)
+
+
+@pytest.fixture
+def eval_service(
+    dummy_agent, mock_eval_sets_manager, mock_eval_set_results_manager
+):
+  DEFAULT_METRIC_EVALUATOR_REGISTRY.register_evaluator(
+      metric_info=FakeEvaluator.get_metric_info(), evaluator=FakeEvaluator
+  )
+  DEFAULT_METRIC_EVALUATOR_REGISTRY.register_evaluator(
+      metric_info=FakeSingleSidedEvaluator.get_metric_info(),
+      evaluator=FakeSingleSidedEvaluator,
+  )
+  return LocalEvalService(
+      root_agent=dummy_agent,
+      eval_sets_manager=mock_eval_sets_manager,
+      eval_set_results_manager=mock_eval_set_results_manager,
+  )
+
+
+class FakeEvaluator(Evaluator):
+
+  def __init__(self, eval_metric: EvalMetric):
+    self._eval_metric = eval_metric
+
+  @staticmethod
+  def get_metric_info() -> MetricInfo:
+    return MetricInfo(
+        metric_name="fake_metric",
+        description="Fake metric description",
+        metric_value_info=MetricValueInfo(
+            interval=Interval(min_value=0.0, max_value=1.0)
+        ),
+    )
+
+  @override
+  def evaluate_invocations(
+      self,
+      actual_invocations: list[Invocation],
+      expected_invocations: Optional[list[Invocation]] = None,
+      conversation_scenario: Optional[ConversationScenario] = None,
+  ) -> EvaluationResult:
+    if expected_invocations is None:
+      raise ValueError("expected_invocations is required for this metric.")
+    per_invocation_results = []
+    for actual, expected in zip(actual_invocations, expected_invocations):
+      per_invocation_results.append(
+          PerInvocationResult(
+              actual_invocation=actual,
+              expected_invocation=expected,
+              score=0.9,
+              eval_status=EvalStatus.PASSED,
+          )
+      )
+    return EvaluationResult(
+        overall_score=0.9,
+        overall_eval_status=EvalStatus.PASSED,
+        per_invocation_results=per_invocation_results,
+    )
+
+
+class FakeSingleSidedEvaluator(Evaluator):
+
+  def __init__(self, eval_metric: EvalMetric):
+    self._eval_metric = eval_metric
+
+  @staticmethod
+  def get_metric_info() -> MetricInfo:
+    return MetricInfo(
+        metric_name="fake_single_sided_metric",
+        description="Fake single sided metric description",
+        metric_value_info=MetricValueInfo(
+            interval=Interval(min_value=0.0, max_value=1.0)
+        ),
+    )
+
+  @override
+  def evaluate_invocations(
+      self,
+      actual_invocations: list[Invocation],
+      expected_invocations: Optional[list[Invocation]] = None,
+      conversation_scenario: Optional[ConversationScenario] = None,
+  ) -> EvaluationResult:
+    per_invocation_results = []
+    for actual in actual_invocations:
+      per_invocation_results.append(
+          PerInvocationResult(
+              actual_invocation=actual,
+              score=0.995,
+              eval_status=EvalStatus.PASSED,
+          )
+      )
+    return EvaluationResult(
+        overall_score=0.95,
+        overall_eval_status=EvalStatus.PASSED,
+        per_invocation_results=per_invocation_results,
+    )
+
+
+@pytest.mark.asyncio
+async def test_perform_inference_success(
+    eval_service,
+    dummy_agent,
+    mock_eval_sets_manager,
+    mocker,
+):
+  eval_set = EvalSet(
+      eval_set_id="test_eval_set",
+      eval_cases=[
+          EvalCase(eval_id="case1", conversation=[], session_input=None),
+          EvalCase(eval_id="case2", conversation=[], session_input=None),
+      ],
+  )
+  mock_eval_sets_manager.get_eval_set.return_value = eval_set
+
+  mock_inference_result = mocker.MagicMock()
+  eval_service._perform_inference_single_eval_item = mocker.AsyncMock(
+      return_value=mock_inference_result
+  )
+
+  inference_request = InferenceRequest(
+      app_name="test_app",
+      eval_set_id="test_eval_set",
+      inference_config=InferenceConfig(parallelism=2),
+  )
+
+  results = []
+  async for result in eval_service.perform_inference(inference_request):
+    results.append(result)
+
+  assert len(results) == 2
+  assert results[0] == mock_inference_result
+  assert results[1] == mock_inference_result
+  mock_eval_sets_manager.get_eval_set.assert_called_once_with(
+      app_name="test_app", eval_set_id="test_eval_set"
+  )
+  assert eval_service._perform_inference_single_eval_item.call_count == 2
+
+
+@pytest.mark.asyncio
+async def test_perform_inference_with_case_ids(
+    eval_service,
+    dummy_agent,
+    mock_eval_sets_manager,
+    mocker,
+):
+  eval_set = EvalSet(
+      eval_set_id="test_eval_set",
+      eval_cases=[
+          EvalCase(eval_id="case1", conversation=[], session_input=None),
+          EvalCase(eval_id="case2", conversation=[], session_input=None),
+          EvalCase(eval_id="case3", conversation=[], session_input=None),
+      ],
+  )
+  mock_eval_sets_manager.get_eval_set.return_value = eval_set
+
+  mock_inference_result = mocker.MagicMock()
+  eval_service._perform_inference_single_eval_item = mocker.AsyncMock(
+      return_value=mock_inference_result
+  )
+
+  inference_request = InferenceRequest(
+      app_name="test_app",
+      eval_set_id="test_eval_set",
+      eval_case_ids=["case1", "case3"],
+      inference_config=InferenceConfig(parallelism=1),
+  )
+
+  results = []
+  async for result in eval_service.perform_inference(inference_request):
+    results.append(result)
+
+  assert len(results) == 2
+  eval_service._perform_inference_single_eval_item.assert_any_call(
+      app_name="test_app",
+      eval_set_id="test_eval_set",
+      eval_case=eval_set.eval_cases[0],
+      root_agent=dummy_agent,
+      use_live=False,
+      live_timeout_seconds=300,
+  )
+  eval_service._perform_inference_single_eval_item.assert_any_call(
+      app_name="test_app",
+      eval_set_id="test_eval_set",
+      eval_case=eval_set.eval_cases[2],
+      root_agent=dummy_agent,
+      use_live=False,
+      live_timeout_seconds=300,
+  )
+
+
+@pytest.mark.asyncio
+async def test_perform_inference_with_use_live(
+    eval_service,
+    dummy_agent,
+    mock_eval_sets_manager,
+    mocker,
+):
+  eval_set = EvalSet(
+      eval_set_id="test_eval_set",
+      eval_cases=[
+          EvalCase(eval_id="case1", conversation=[], session_input=None),
+      ],
+  )
+  mock_eval_sets_manager.get_eval_set.return_value = eval_set
+
+  mock_inference_result = mocker.MagicMock()
+  eval_service._perform_inference_single_eval_item = mocker.AsyncMock(
+      return_value=mock_inference_result
+  )
+
+  inference_request = InferenceRequest(
+      app_name="test_app",
+      eval_set_id="test_eval_set",
+      inference_config=InferenceConfig(
+          parallelism=1, use_live=True, live_timeout_seconds=600
+      ),
+  )
+
+  results = []
+  async for result in eval_service.perform_inference(inference_request):
+    results.append(result)
+
+  assert len(results) == 1
+  eval_service._perform_inference_single_eval_item.assert_called_once_with(
+      app_name="test_app",
+      eval_set_id="test_eval_set",
+      eval_case=eval_set.eval_cases[0],
+      root_agent=dummy_agent,
+      use_live=True,
+      live_timeout_seconds=600,
+  )
+
+
+@pytest.mark.asyncio
+async def test_perform_inference_eval_set_not_found(
+    eval_service,
+    mock_eval_sets_manager,
+):
+  mock_eval_sets_manager.get_eval_set.return_value = None
+
+  inference_request = InferenceRequest(
+      app_name="test_app",
+      eval_set_id="not_found_set",
+      inference_config=InferenceConfig(parallelism=1),
+  )
+
+  with pytest.raises(NotFoundError):
+    async for _ in eval_service.perform_inference(inference_request):
+      pass
+
+
+@pytest.mark.asyncio
+async def test_evaluate_success(
+    eval_service, mock_eval_sets_manager, mock_eval_set_results_manager, mocker
+):
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="test user content.")]
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text="test final response.")]
+      ),
+  )
+  inference_results = [
+      InferenceResult(
+          app_name="test_app",
+          eval_set_id="test_eval_set",
+          eval_case_id="case1",
+          inferences=[invocation.model_copy(deep=True)],
+          session_id="session1",
+      ),
+      InferenceResult(
+          app_name="test_app",
+          eval_set_id="test_eval_set",
+          eval_case_id="case2",
+          inferences=[invocation.model_copy(deep=True)],
+          session_id="session2",
+      ),
+  ]
+  eval_metric = EvalMetric(metric_name="fake_metric", threshold=0.5)
+  evaluate_request = EvaluateRequest(
+      inference_results=inference_results,
+      evaluate_config=EvaluateConfig(eval_metrics=[eval_metric], parallelism=2),
+  )
+
+  mock_eval_case = mocker.MagicMock(spec=EvalCase)
+  mock_eval_case.conversation = [invocation.model_copy(deep=True)]
+  mock_eval_case.conversation_scenario = None
+  mock_eval_case.session_input = None
+  mock_eval_sets_manager.get_eval_case.return_value = mock_eval_case
+
+  results = []
+  async for result in eval_service.evaluate(evaluate_request):
+    results.append(result)
+
+  assert len(results) == 2
+  assert isinstance(results[0], EvalCaseResult)
+  assert isinstance(results[1], EvalCaseResult)
+  assert mock_eval_sets_manager.get_eval_case.call_count == 2
+  assert mock_eval_set_results_manager.save_eval_set_result.call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_evaluate_eval_case_not_found(
+    eval_service,
+    mock_eval_sets_manager,
+):
+  inference_results = [
+      InferenceResult(
+          app_name="test_app",
+          eval_set_id="test_eval_set",
+          eval_case_id="case1",
+          inferences=[],
+          session_id="session1",
+      ),
+  ]
+  eval_metric = EvalMetric(metric_name="fake_metric", threshold=0.5)
+  evaluate_request = EvaluateRequest(
+      inference_results=inference_results,
+      evaluate_config=EvaluateConfig(eval_metrics=[eval_metric], parallelism=1),
+  )
+
+  mock_eval_sets_manager.get_eval_case.return_value = None
+
+  with pytest.raises(NotFoundError):
+    async for _ in eval_service.evaluate(evaluate_request):
+      pass
+
+  mock_eval_sets_manager.get_eval_case.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_evaluate_single_inference_result(
+    eval_service, mock_eval_sets_manager, mock_eval_set_results_manager, mocker
+):
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="test user content.")]
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text="test final response.")]
+      ),
+  )
+  inference_result = InferenceResult(
+      app_name="test_app",
+      eval_set_id="test_eval_set",
+      eval_case_id="case1",
+      inferences=[
+          invocation.model_copy(deep=True),
+          invocation.model_copy(deep=True),
+          invocation.model_copy(deep=True),
+      ],
+      session_id="session1",
+  )
+  eval_metric = EvalMetric(metric_name="fake_metric", threshold=0.5)
+  evaluate_config = EvaluateConfig(eval_metrics=[eval_metric], parallelism=1)
+
+  mock_eval_case = mocker.MagicMock(spec=EvalCase)
+  mock_eval_case.conversation = [
+      invocation.model_copy(deep=True),
+      invocation.model_copy(deep=True),
+      invocation.model_copy(deep=True),
+  ]
+  mock_eval_case.conversation_scenario = None
+  mock_eval_case.session_input = None
+  mock_eval_sets_manager.get_eval_case.return_value = mock_eval_case
+
+  _, result = await eval_service._evaluate_single_inference_result(
+      inference_result=inference_result, evaluate_config=evaluate_config
+  )
+
+  assert isinstance(result, EvalCaseResult)
+  assert result.eval_id == "case1"
+  assert result.session_id == "session1"
+  assert len(result.overall_eval_metric_results) == 1
+  assert result.overall_eval_metric_results[0].metric_name == "fake_metric"
+  assert result.overall_eval_metric_results[0].score == 0.9
+  mock_eval_sets_manager.get_eval_case.assert_called_once_with(
+      app_name="test_app", eval_set_id="test_eval_set", eval_case_id="case1"
+  )
+
+  assert len(result.eval_metric_result_per_invocation) == 3
+  for i in range(3):
+    invocation_result = result.eval_metric_result_per_invocation[i]
+    assert invocation_result.actual_invocation == inference_result.inferences[i]
+    assert (
+        invocation_result.expected_invocation == mock_eval_case.conversation[i]
+    )
+    assert len(invocation_result.eval_metric_results) == 1
+    metric_result = invocation_result.eval_metric_results[0]
+    assert metric_result.metric_name == "fake_metric"
+    assert metric_result.score == 0.9
+    assert metric_result.eval_status == EvalStatus.PASSED
+
+
+@pytest.mark.asyncio
+async def test_evaluate_single_inference_result_failed_without_inferences(
+    eval_service, mock_eval_sets_manager, mocker
+):
+  inference_result = InferenceResult(
+      app_name="test_app",
+      eval_set_id="test_eval_set",
+      eval_case_id="case1",
+      inferences=None,
+      session_id="session1",
+      status=InferenceStatus.FAILURE,
+      error_message="auth failed",
+  )
+  eval_metric = EvalMetric(metric_name="fake_metric", threshold=0.5)
+  evaluate_config = EvaluateConfig(eval_metrics=[eval_metric], parallelism=1)
+
+  mock_eval_case = mocker.MagicMock(spec=EvalCase)
+  mock_eval_case.conversation = []
+  mock_eval_case.conversation_scenario = None
+  mock_eval_case.session_input = None
+  mock_eval_sets_manager.get_eval_case.return_value = mock_eval_case
+
+  _, result = await eval_service._evaluate_single_inference_result(
+      inference_result=inference_result, evaluate_config=evaluate_config
+  )
+
+  assert result.eval_id == "case1"
+  assert result.session_id == "session1"
+  assert result.final_eval_status == EvalStatus.FAILED
+  assert result.overall_eval_metric_results == []
+  assert result.eval_metric_result_per_invocation == []
+
+
+@pytest.mark.asyncio
+async def test_evaluate_single_inference_result_for_conversation_scenario(
+    eval_service, mock_eval_sets_manager, mocker
+):
+  """To be removed once evaluation is implemented for conversation scenarios."""
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="test user content.")]
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text="test final response.")]
+      ),
+  )
+  inference_result = InferenceResult(
+      app_name="test_app",
+      eval_set_id="test_eval_set",
+      eval_case_id="case1",
+      inferences=[
+          invocation.model_copy(deep=True),
+          invocation.model_copy(deep=True),
+          invocation.model_copy(deep=True),
+      ],
+      session_id="session1",
+  )
+  eval_metric = EvalMetric(
+      metric_name="fake_single_sided_metric", threshold=0.5
+  )
+  evaluate_config = EvaluateConfig(eval_metrics=[eval_metric], parallelism=1)
+
+  mock_eval_case = mocker.MagicMock(spec=EvalCase)
+  mock_eval_case.conversation = None
+  mock_eval_case.conversation_scenario = mocker.MagicMock()
+  mock_eval_case.session_input = None
+  mock_eval_sets_manager.get_eval_case.return_value = mock_eval_case
+
+  _, result = await eval_service._evaluate_single_inference_result(
+      inference_result=inference_result, evaluate_config=evaluate_config
+  )
+  assert isinstance(result, EvalCaseResult)
+  assert result.eval_id == "case1"
+  assert result.final_eval_status == EvalStatus.PASSED
+  assert len(result.overall_eval_metric_results) == 1
+  assert (
+      result.overall_eval_metric_results[0].metric_name
+      == "fake_single_sided_metric"
+  )
+  assert result.overall_eval_metric_results[0].score == 0.95
+  mock_eval_sets_manager.get_eval_case.assert_called_once_with(
+      app_name="test_app", eval_set_id="test_eval_set", eval_case_id="case1"
+  )
+
+  assert len(result.eval_metric_result_per_invocation) == 3
+  for i in range(3):
+    invocation_result = result.eval_metric_result_per_invocation[i]
+    assert invocation_result.actual_invocation == inference_result.inferences[i]
+    assert invocation_result.expected_invocation is None
+    assert len(invocation_result.eval_metric_results) == 1
+    metric_result = invocation_result.eval_metric_results[0]
+    assert metric_result.metric_name == "fake_single_sided_metric"
+    assert metric_result.score == 0.995
+    assert metric_result.eval_status == EvalStatus.PASSED
+
+
+@pytest.mark.asyncio
+async def test_evaluate_single_inference_result_for_conversation_scenario_with_unsupported_metric(
+    eval_service, mock_eval_sets_manager, mocker
+):
+  """To be removed once evaluation is implemented for conversation scenarios."""
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="test user content.")]
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text="test final response.")]
+      ),
+  )
+  inference_result = InferenceResult(
+      app_name="test_app",
+      eval_set_id="test_eval_set",
+      eval_case_id="case1",
+      inferences=[
+          invocation.model_copy(deep=True),
+          invocation.model_copy(deep=True),
+          invocation.model_copy(deep=True),
+      ],
+      session_id="session1",
+  )
+  eval_metric = EvalMetric(metric_name="fake_metric", threshold=0.5)
+  evaluate_config = EvaluateConfig(eval_metrics=[eval_metric], parallelism=1)
+
+  mock_eval_case = mocker.MagicMock(spec=EvalCase)
+  mock_eval_case.eval_id = "case1"
+  mock_eval_case.conversation = None
+  mock_eval_case.conversation_scenario = mocker.MagicMock()
+  mock_eval_case.session_input = None
+  mock_eval_sets_manager.get_eval_case.return_value = mock_eval_case
+
+  _, result = await eval_service._evaluate_single_inference_result(
+      inference_result=inference_result, evaluate_config=evaluate_config
+  )
+  assert isinstance(result, EvalCaseResult)
+  assert result.eval_id == "case1"
+  assert result.final_eval_status == EvalStatus.NOT_EVALUATED
+  assert len(result.overall_eval_metric_results) == 1
+  assert result.overall_eval_metric_results[0].metric_name == "fake_metric"
+  assert result.overall_eval_metric_results[0].score is None
+  mock_eval_sets_manager.get_eval_case.assert_called_once_with(
+      app_name="test_app", eval_set_id="test_eval_set", eval_case_id="case1"
+  )
+
+  assert len(result.eval_metric_result_per_invocation) == 3
+
+
+def test_generate_final_eval_status_doesn_t_throw_on(eval_service):
+  # How to fix if this test case fails?
+  # This test case has failed mainly because a new EvalStatus got added. You
+  # mostly need to update _generate_final_eval_status method to handle the new
+  # eval case.
+
+  # We go over all the possible values of EvalStatus one by one and expect
+  # the _generate_final_eval_status to handle it without throwing an exception.
+  for status in EvalStatus:
+    eval_metric_result = EvalMetricResult(
+        metric_name="metric1", threshold=0.5, eval_status=status
+    )
+    eval_service._generate_final_eval_status([eval_metric_result])
+
+
+@pytest.mark.asyncio
+async def test_mcp_stdio_agent_no_runtime_error(mocker):
+  """Test that LocalEvalService can handle MCP stdio agents without RuntimeError.
+
+  This is a regression test for GitHub issue #2196:
+  "RuntimeError: Attempted to exit cancel scope in a different task than it was
+  entered in"
+
+  The fix ensures that Runner.close() is called to properly cleanup MCP
+  connections.
+  """
+  import tempfile
+
+  from google.adk.evaluation.local_eval_service import LocalEvalService
+  from google.adk.tools.mcp_tool.mcp_session_manager import StdioConnectionParams
+  from google.adk.tools.mcp_tool.mcp_toolset import MCPToolset
+  from mcp import StdioServerParameters
+
+  # Mock LLM responses to avoid real API calls
+  from tests.unittests.testing_utils import MockModel
+
+  mock_responses = [
+      genai_types.Content(
+          parts=[genai_types.Part(text="Mocked response from test agent")]
+      )
+  ]
+  mock_model = MockModel.create(responses=mock_responses)
+
+  # Create a test agent with MCP stdio toolset and mocked model
+  test_dir = tempfile.mkdtemp()
+  try:
+    agent = LlmAgent(
+        model=mock_model,
+        name="test_mcp_agent",
+        instruction="Test agent for MCP stdio regression test.",
+        tools=[
+            MCPToolset(
+                connection_params=StdioConnectionParams(
+                    server_params=StdioServerParameters(
+                        command="npx",
+                        args=[
+                            "-y",
+                            "@modelcontextprotocol/server-filesystem",
+                            test_dir,
+                        ],
+                    ),
+                    timeout=5,
+                ),
+                tool_filter=["read_file", "list_directory"],
+            )
+        ],
+    )
+
+    # Create a mock eval sets manager that returns an eval case
+    mock_eval_sets_manager = mocker.create_autospec(EvalSetsManager)
+    test_eval_case = EvalCase(
+        eval_id="test_mcp_case",
+        conversation=[
+            Invocation(
+                user_content=genai_types.Content(
+                    parts=[genai_types.Part(text="List directory contents")]
+                ),
+            )
+        ],
+    )
+    mock_eval_sets_manager.get_eval_case.return_value = test_eval_case
+    eval_set = EvalSet(
+        eval_set_id="test_set",
+        eval_cases=[test_eval_case],
+    )
+    mock_eval_sets_manager.get_eval_set.return_value = eval_set
+
+    # Create LocalEvalService with MCP agent
+    eval_service = LocalEvalService(
+        root_agent=agent,
+        eval_sets_manager=mock_eval_sets_manager,
+    )
+
+    # Create inference request to actually trigger the code path with the fix
+    inference_request = InferenceRequest(
+        app_name="test_app",
+        eval_set_id="test_set",
+        inference_config=InferenceConfig(parallelism=1),
+    )
+
+    # The main test: actually call perform_inference which will trigger
+    # _generate_inferences_from_root_agent where the fix is located
+
+    # Note: In Python 3.10 and 3.11, there may be asyncio.CancelledError during cleanup
+    # due to anyio cancel scope context violations when MCP toolsets are cleaned up
+    # via asyncio.wait_for() in different task contexts. Python 3.12+ enhanced task
+    # context management (Task.get_context(), improved context propagation) resolves this.
+
+    try:
+      results = []
+      async for result in eval_service.perform_inference(inference_request):
+        results.append(result)
+        # We should get at least one result since we mocked the LLM
+        break
+
+      # Test passes if we get here without the cancel scope RuntimeError
+      # With mocked model, we should get successful inference results
+      assert len(results) >= 1
+
+    except RuntimeError as e:
+      # If we get a RuntimeError about cancel scope, the fix isn't working
+      if "cancel scope" in str(e) and "different task" in str(e):
+        pytest.fail(f"MCP stdio RuntimeError regression detected: {e}")
+      else:
+        # Other RuntimeErrors might be acceptable
+        pass
+    except asyncio.CancelledError as e:
+      # In Python 3.10 and 3.11, anyio cancel scope context violations may manifest as CancelledError
+      # when MCP RequestResponder.__exit__() is called in a different task than __enter__()
+      if (
+          hasattr(e, "args")
+          and len(e.args) > 0
+          and "cancel scope" in str(e.args[0])
+      ):
+        pytest.fail(f"MCP stdio cancel scope error regression detected: {e}")
+      else:
+        # Re-raise other CancelledErrors
+        raise
+    except Exception as e:
+      # Check if this is the specific cancel scope error we're testing for
+      if "cancel scope" in str(e) and "different task" in str(e):
+        pytest.fail(f"MCP stdio RuntimeError regression detected: {e}")
+      # Other exceptions are acceptable for this test
+
+    # The main goal is to ensure the test completes without the specific
+    # RuntimeError about cancel scopes. If we reach here, the fix is working.
+
+  finally:
+    # Cleanup
+    import shutil
+
+    shutil.rmtree(test_dir, ignore_errors=True)
+
+
+def test_add_rubrics_to_invocation_initializes_rubrics_list():
+  invocation = Invocation(user_content=genai_types.Content())
+  rubric = Rubric(
+      rubric_id="r1", rubric_content=RubricContent(text_property="p1")
+  )
+  _add_rubrics_to_invocation(invocation, [rubric])
+  assert invocation.rubrics == [rubric]
+
+
+def test_add_rubrics_to_invocation_adds_to_existing_list():
+  rubric1 = Rubric(
+      rubric_id="r1", rubric_content=RubricContent(text_property="p1")
+  )
+  rubric2 = Rubric(
+      rubric_id="r2", rubric_content=RubricContent(text_property="p2")
+  )
+  invocation = Invocation(user_content=genai_types.Content(), rubrics=[rubric1])
+  _add_rubrics_to_invocation(invocation, [rubric2])
+  assert invocation.rubrics == [rubric1, rubric2]
+
+
+def test_add_rubrics_to_invocation_errors_on_duplicate_id():
+  rubric1 = Rubric(
+      rubric_id="r1", rubric_content=RubricContent(text_property="p1")
+  )
+  rubric2 = Rubric(
+      rubric_id="r1", rubric_content=RubricContent(text_property="p2")
+  )
+  invocation = Invocation(user_content=genai_types.Content(), rubrics=[rubric1])
+  with pytest.raises(ValueError):
+    _add_rubrics_to_invocation(invocation, [rubric2])
+
+
+def test_copy_eval_case_rubrics_to_actual_invocations():
+  rubric1 = Rubric(
+      rubric_id="r1", rubric_content=RubricContent(text_property="p1")
+  )
+  eval_case = EvalCase(
+      eval_id="case1",
+      conversation=[
+          Invocation(
+              user_content=genai_types.Content(
+                  parts=[genai_types.Part(text="expected invocation 1.")]
+              )
+          ),
+          Invocation(
+              user_content=genai_types.Content(
+                  parts=[genai_types.Part(text="expected invocation 2.")]
+              )
+          ),
+      ],
+      rubrics=[rubric1],
+  )
+  invocations = [
+      Invocation(
+          user_content=genai_types.Content(
+              parts=[genai_types.Part(text="actual invocation 1.")]
+          )
+      ),
+      Invocation(
+          user_content=genai_types.Content(
+              parts=[genai_types.Part(text="actual invocation 2.")]
+          )
+      ),
+  ]
+  _copy_eval_case_rubrics_to_actual_invocations(eval_case, invocations)
+  assert invocations[0].rubrics == [rubric1]
+  assert invocations[1].rubrics == [rubric1]
+
+
+def test_copy_invocation_rubrics_to_actual_invocations():
+  rubric1 = Rubric(
+      rubric_id="r1", rubric_content=RubricContent(text_property="p1")
+  )
+  rubric2 = Rubric(
+      rubric_id="r2", rubric_content=RubricContent(text_property="p2")
+  )
+  expected = [
+      Invocation(
+          user_content=genai_types.Content(
+              parts=[genai_types.Part(text="expected invocation 1.")]
+          ),
+          rubrics=[rubric1],
+      ),
+      Invocation(
+          user_content=genai_types.Content(
+              parts=[genai_types.Part(text="expected invocation 2.")]
+          ),
+          rubrics=[rubric2],
+      ),
+  ]
+  actual = [
+      Invocation(
+          user_content=genai_types.Content(
+              parts=[genai_types.Part(text="actual invocation 1.")]
+          )
+      ),
+      Invocation(
+          user_content=genai_types.Content(
+              parts=[genai_types.Part(text="actual invocation 2.")]
+          )
+      ),
+  ]
+  _copy_invocation_rubrics_to_actual_invocations(expected, actual)
+  assert actual[0].rubrics == [rubric1]
+  assert actual[1].rubrics == [rubric2]
+
+
+@pytest.mark.asyncio
+async def test_perform_inference_single_eval_item_live(
+    eval_service, dummy_agent, mocker
+):
+  eval_case = EvalCase(eval_id="case1", conversation=[], session_input=None)
+  mock_generate_live = mocker.patch(
+      "google.adk.evaluation.evaluation_generator.EvaluationGenerator._generate_inferences_from_root_agent_live"
+  )
+  mock_generate_live.return_value = []
+
+  eval_service._session_id_supplier = mocker.MagicMock(
+      return_value="test_session_id"
+  )
+  mock_user_sim = mocker.MagicMock()
+  eval_service._user_simulator_provider.provide = mocker.MagicMock(
+      return_value=mock_user_sim
+  )
+
+  await eval_service._perform_inference_single_eval_item(
+      app_name="test_app",
+      eval_set_id="test_eval_set",
+      eval_case=eval_case,
+      root_agent=dummy_agent,
+      use_live=True,
+      live_timeout_seconds=600,
+  )
+
+  mock_generate_live.assert_called_once_with(
+      root_agent=dummy_agent,
+      user_simulator=mock_user_sim,
+      initial_session=None,
+      session_id="test_session_id",
+      session_service=eval_service._session_service,
+      artifact_service=eval_service._artifact_service,
+      memory_service=eval_service._memory_service,
+      live_timeout_seconds=600,
+  )
+
+
+@pytest.mark.asyncio
+async def test_perform_inference_single_eval_item_non_live(
+    eval_service, dummy_agent, mocker
+):
+  eval_case = EvalCase(eval_id="case1", conversation=[], session_input=None)
+  mock_generate = mocker.patch(
+      "google.adk.evaluation.evaluation_generator.EvaluationGenerator._generate_inferences_from_root_agent"
+  )
+  mock_generate.return_value = []
+
+  eval_service._session_id_supplier = mocker.MagicMock(
+      return_value="test_session_id"
+  )
+  mock_user_sim = mocker.MagicMock()
+  eval_service._user_simulator_provider.provide = mocker.MagicMock(
+      return_value=mock_user_sim
+  )
+
+  await eval_service._perform_inference_single_eval_item(
+      app_name="test_app",
+      eval_set_id="test_eval_set",
+      eval_case=eval_case,
+      root_agent=dummy_agent,
+      use_live=False,
+      live_timeout_seconds=300,
+  )
+
+  mock_generate.assert_called_once_with(
+      root_agent=dummy_agent,
+      user_simulator=mock_user_sim,
+      initial_session=None,
+      session_id="test_session_id",
+      session_service=eval_service._session_service,
+      artifact_service=eval_service._artifact_service,
+      memory_service=eval_service._memory_service,
+  )

tests/unittests/evaluation/test_local_eval_set_results_manager.py

@@ -0,0 +1,206 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import tempfile
+import time
+
+from google.adk.errors.not_found_error import NotFoundError
+from google.adk.evaluation._eval_set_results_manager_utils import _sanitize_eval_set_result_name
+from google.adk.evaluation.eval_result import EvalCaseResult
+from google.adk.evaluation.eval_result import EvalSetResult
+from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.local_eval_set_results_manager import _ADK_EVAL_HISTORY_DIR
+from google.adk.evaluation.local_eval_set_results_manager import _EVAL_SET_RESULT_FILE_EXTENSION
+from google.adk.evaluation.local_eval_set_results_manager import LocalEvalSetResultsManager
+import pytest
+
+
+class TestLocalEvalSetResultsManager:
+
+  @pytest.fixture(autouse=True)
+  def setup(self):
+    self.temp_dir = tempfile.mkdtemp()
+    self.agents_dir = os.path.join(self.temp_dir, "agents")
+    os.makedirs(self.agents_dir)
+    self.manager = LocalEvalSetResultsManager(self.agents_dir)
+    self.app_name = "test_app"
+    self.eval_set_id = "test_eval_set"
+    self.eval_case_results = [
+        EvalCaseResult(
+            eval_set_file="test_file",
+            eval_set_id=self.eval_set_id,
+            eval_id="case1",
+            final_eval_status=EvalStatus.PASSED,
+            eval_metric_results=[],
+            overall_eval_metric_results=[],
+            eval_metric_result_per_invocation=[],
+            session_id="session1",
+        )
+    ]
+    self.timestamp = time.time()  # Store the timestamp
+    self.eval_set_result_id = (
+        self.app_name + "_" + self.eval_set_id + "_" + str(self.timestamp)
+    )
+    self.eval_set_result_name = _sanitize_eval_set_result_name(
+        self.eval_set_result_id
+    )
+    self.eval_set_result = EvalSetResult(
+        eval_set_result_id=self.eval_set_result_id,
+        eval_set_result_name=self.eval_set_result_name,
+        eval_set_id=self.eval_set_id,
+        eval_case_results=self.eval_case_results,
+        creation_timestamp=self.timestamp,
+    )
+    yield
+    shutil.rmtree(self.temp_dir)
+
+  def test_save_eval_set_result(self, mocker):
+    mock_time = mocker.patch("time.time")
+    mock_time.return_value = self.timestamp
+    self.manager.save_eval_set_result(
+        self.app_name, self.eval_set_id, self.eval_case_results
+    )
+    eval_history_dir = os.path.join(
+        self.agents_dir, self.app_name, _ADK_EVAL_HISTORY_DIR
+    )
+    expected_file_path = os.path.join(
+        eval_history_dir,
+        self.eval_set_result_name + _EVAL_SET_RESULT_FILE_EXTENSION,
+    )
+    assert os.path.exists(expected_file_path)
+    with open(expected_file_path, "r") as f:
+      actual_eval_set_result_data = json.load(f)
+
+    # Verify the file contains a proper JSON object (not double-encoded)
+    # Use mode='json' to serialize enums to their values for comparison
+    expected_eval_set_result_data = self.eval_set_result.model_dump(mode="json")
+    assert expected_eval_set_result_data == actual_eval_set_result_data
+
+  @pytest.mark.parametrize("app_name", ["", ".", "..", "foo/bar", "foo\\bar"])
+  def test_save_eval_set_result_rejects_invalid_app_name(self, app_name):
+    with pytest.raises(ValueError):
+      self.manager.save_eval_set_result(
+          app_name, self.eval_set_id, self.eval_case_results
+      )
+
+  @pytest.mark.parametrize(
+      "eval_set_id", ["", ".", "..", "foo/bar", "foo\\bar"]
+  )
+  def test_save_eval_set_result_rejects_invalid_eval_set_id(self, eval_set_id):
+    with pytest.raises(ValueError):
+      self.manager.save_eval_set_result(
+          self.app_name, eval_set_id, self.eval_case_results
+      )
+
+  def test_get_eval_set_result(self, mocker):
+    mock_time = mocker.patch("time.time")
+    mock_time.return_value = self.timestamp
+    self.manager.save_eval_set_result(
+        self.app_name, self.eval_set_id, self.eval_case_results
+    )
+    retrieved_result = self.manager.get_eval_set_result(
+        self.app_name, self.eval_set_result_name
+    )
+    assert retrieved_result == self.eval_set_result
+
+  @pytest.mark.parametrize("app_name", ["", ".", "..", "foo/bar", "foo\\bar"])
+  def test_get_eval_set_result_rejects_invalid_app_name(self, app_name):
+    with pytest.raises(ValueError):
+      self.manager.get_eval_set_result(app_name, self.eval_set_result_name)
+
+  @pytest.mark.parametrize(
+      "eval_set_result_id", ["", ".", "..", "foo/bar", "foo\\bar"]
+  )
+  def test_get_eval_set_result_rejects_invalid_eval_set_result_id(
+      self, eval_set_result_id
+  ):
+    with pytest.raises(ValueError):
+      self.manager.get_eval_set_result(self.app_name, eval_set_result_id)
+
+  def test_get_eval_set_result_double_encoded_legacy(self):
+    eval_history_dir = os.path.join(
+        self.agents_dir, self.app_name, _ADK_EVAL_HISTORY_DIR
+    )
+    os.makedirs(eval_history_dir, exist_ok=True)
+    eval_set_result_file_path = os.path.join(
+        eval_history_dir,
+        self.eval_set_result_name + _EVAL_SET_RESULT_FILE_EXTENSION,
+    )
+    double_encoded_json = json.dumps(self.eval_set_result.model_dump_json())
+    with open(eval_set_result_file_path, "w", encoding="utf-8") as f:
+      f.write(double_encoded_json)
+
+    retrieved_result = self.manager.get_eval_set_result(
+        self.app_name, self.eval_set_result_name
+    )
+    assert retrieved_result == self.eval_set_result
+
+  def test_get_eval_set_result_not_found(self, mocker):
+    mock_time = mocker.patch("time.time")
+    mock_time.return_value = self.timestamp
+
+    with pytest.raises(NotFoundError) as e:
+      self.manager.get_eval_set_result(self.app_name, "non_existent_id")
+
+  def test_list_eval_set_results(self, mocker):
+    mock_time = mocker.patch("time.time")
+    mock_time.return_value = self.timestamp
+    # Save two eval set results for the same app
+    self.manager.save_eval_set_result(
+        self.app_name, self.eval_set_id, self.eval_case_results
+    )
+    timestamp2 = time.time() + 1
+    mock_time.return_value = timestamp2
+    eval_set_result_id2 = (
+        self.app_name + "_" + self.eval_set_id + "_" + str(timestamp2)
+    )
+    eval_set_result_name2 = _sanitize_eval_set_result_name(eval_set_result_id2)
+    eval_case_results2 = [
+        EvalCaseResult(
+            eval_set_file="test_file",
+            eval_set_id=self.eval_set_id,
+            eval_id="case2",
+            final_eval_status=EvalStatus.FAILED,
+            eval_metric_results=[],
+            overall_eval_metric_results=[],
+            eval_metric_result_per_invocation=[],
+            session_id="session2",
+        )
+    ]
+    self.manager.save_eval_set_result(
+        self.app_name, self.eval_set_id, eval_case_results2
+    )
+
+    # Save one eval set result for a different app
+    app_name2 = "another_app"
+    timestamp3 = time.time() + 2
+    mock_time.return_value = timestamp3
+
+    self.manager.save_eval_set_result(
+        app_name2, self.eval_set_id, self.eval_case_results
+    )
+
+    results = self.manager.list_eval_set_results(self.app_name)
+    expected_result = [self.eval_set_result_name, eval_set_result_name2]
+    assert set(results) == set(expected_result)
+
+  def test_list_eval_set_results_empty(self):
+    # No eval set results saved for the app
+    results = self.manager.list_eval_set_results(self.app_name)
+    assert results == []

tests/unittests/evaluation/test_local_eval_sets_manager.py

@@ -0,0 +1,774 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import json
+import os
+import uuid
+
+from google.adk.errors.not_found_error import NotFoundError
+from google.adk.evaluation.eval_case import EvalCase
+from google.adk.evaluation.eval_case import IntermediateData
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_set import EvalSet
+from google.adk.evaluation.local_eval_sets_manager import _EVAL_SET_FILE_EXTENSION
+from google.adk.evaluation.local_eval_sets_manager import convert_eval_set_to_pydantic_schema
+from google.adk.evaluation.local_eval_sets_manager import load_eval_set_from_file
+from google.adk.evaluation.local_eval_sets_manager import LocalEvalSetsManager
+from google.genai import types as genai_types
+from pydantic import ValidationError
+import pytest
+
+
+class TestConvertEvalSetToPydanticSchema:
+  """Tests convert_eval_set_to_pydantic_schema method."""
+
+  def test_convert_eval_set_to_pydantic_schema_complete(self):
+    eval_set_id = "test_eval_set"
+    eval_set_in_json_format = [{
+        "name": "roll_17_sided_dice_twice",
+        "data": [
+            {
+                "query": "What can you do?",
+                "expected_tool_use": [],
+                "expected_intermediate_agent_responses": [],
+                "reference": (
+                    "I can roll dice of different sizes and check if a number"
+                    " is prime. I can also use multiple tools in parallel.\n"
+                ),
+            },
+            {
+                "query": "Roll a 17 sided dice twice for me",
+                "expected_tool_use": [
+                    {"tool_name": "roll_die", "tool_input": {"sides": 17}},
+                    {"tool_name": "roll_die", "tool_input": {"sides": 17}},
+                ],
+                "expected_intermediate_agent_responses": [
+                    {"author": "agent1", "text": "thought1"}
+                ],
+                "reference": (
+                    "I have rolled a 17 sided die twice. The first roll was 13"
+                    " and the second roll was 4.\n"
+                ),
+            },
+        ],
+        "initial_session": {
+            "state": {},
+            "app_name": "hello_world",
+            "user_id": "user",
+        },
+    }]
+
+    eval_set = convert_eval_set_to_pydantic_schema(
+        eval_set_id, eval_set_in_json_format
+    )
+
+    assert eval_set.eval_set_id == eval_set_id
+    assert len(eval_set.eval_cases) == 1
+    assert eval_set.eval_cases[0].eval_id == "roll_17_sided_dice_twice"
+    assert len(eval_set.eval_cases[0].conversation) == 2
+    assert eval_set.eval_cases[0].session_input.app_name == "hello_world"
+    assert (
+        len(eval_set.eval_cases[0].conversation[1].intermediate_data.tool_uses)
+        == 2
+    )
+    assert (
+        len(
+            eval_set.eval_cases[0]
+            .conversation[1]
+            .intermediate_data.intermediate_responses
+        )
+        == 1
+    )
+
+  def test_convert_eval_set_to_pydantic_schema_minimal(self):
+    eval_set_id = "test_eval_set"
+    eval_set_in_json_format = [{
+        "name": "minimal_case",
+        "data": [{"query": "Hello", "reference": "World"}],
+    }]
+
+    eval_set = convert_eval_set_to_pydantic_schema(
+        eval_set_id, eval_set_in_json_format
+    )
+
+    assert eval_set.eval_set_id == eval_set_id
+    assert len(eval_set.eval_cases) == 1
+    assert eval_set.eval_cases[0].eval_id == "minimal_case"
+    assert len(eval_set.eval_cases[0].conversation) == 1
+    assert (
+        eval_set.eval_cases[0].conversation[0].user_content.parts[0].text
+        == "Hello"
+    )
+    assert (
+        eval_set.eval_cases[0].conversation[0].final_response.parts[0].text
+        == "World"
+    )
+
+  def test_convert_eval_set_to_pydantic_schema_empty_tool_use_and_intermediate_responses(
+      self,
+  ):
+    eval_set_id = "test_eval_set"
+    eval_set_in_json_format = [{
+        "name": "empty_lists",
+        "data": [{
+            "query": "Test",
+            "reference": "Test Ref",
+            "expected_tool_use": [],
+            "expected_intermediate_agent_responses": [],
+        }],
+    }]
+
+    eval_set = convert_eval_set_to_pydantic_schema(
+        eval_set_id, eval_set_in_json_format
+    )
+
+    assert eval_set.eval_set_id == eval_set_id
+    assert len(eval_set.eval_cases) == 1
+    assert (
+        len(eval_set.eval_cases[0].conversation[0].intermediate_data.tool_uses)
+        == 0
+    )
+    assert (
+        len(
+            eval_set.eval_cases[0]
+            .conversation[0]
+            .intermediate_data.intermediate_responses
+        )
+        == 0
+    )
+
+  def test_convert_eval_set_to_pydantic_schema_empty_initial_session(self):
+    eval_set_id = "test_eval_set"
+    eval_set_in_json_format = [{
+        "name": "empty_session",
+        "data": [{"query": "Test", "reference": "Test Ref"}],
+        "initial_session": {},
+    }]
+
+    eval_set = convert_eval_set_to_pydantic_schema(
+        eval_set_id, eval_set_in_json_format
+    )
+
+    assert eval_set.eval_set_id == eval_set_id
+    assert eval_set.eval_cases[0].session_input is None
+
+  def test_convert_eval_set_to_pydantic_schema_invalid_data(self):
+    # This test implicitly checks for potential validation errors during Pydantic
+    # object creation
+    eval_set_id = "test_eval_set"
+    eval_set_in_json_format = [{
+        "name": 123,  # Invalid name type
+        "data": [{
+            "query": 456,  # Invalid query type
+            "reference": 789,  # Invalid reference type
+            "expected_tool_use": [{
+                "tool_name": 123,
+                "tool_input": 456,
+            }],  # Invalid tool name and input
+            "expected_intermediate_agent_responses": [
+                {"author": 123, "text": 456}  # Invalid author and text
+            ],
+        }],
+        "initial_session": {
+            "state": "invalid",  # Invalid state type
+            "app_name": 123,  # Invalid app_name type
+            "user_id": 456,  # Invalid user_id type
+        },
+    }]
+
+    with pytest.raises(ValidationError):
+      convert_eval_set_to_pydantic_schema(eval_set_id, eval_set_in_json_format)
+
+
+class TestLoadEvalSetFromFile:
+  """Tests for load_eval_set_from_file method."""
+
+  def test_load_eval_set_from_file_new_format(self, tmp_path):
+    # Create a dummy file with EvalSet in the new Pydantic JSON format
+    eval_set = EvalSet(
+        eval_set_id="new_format_eval_set",
+        eval_cases=[
+            EvalCase(
+                eval_id="new_format_case",
+                conversation=[
+                    Invocation(
+                        invocation_id=str(uuid.uuid4()),
+                        user_content=genai_types.Content(
+                            parts=[genai_types.Part(text="New Format Query")]
+                        ),
+                        final_response=genai_types.Content(
+                            parts=[
+                                genai_types.Part(text="New Format Reference")
+                            ]
+                        ),
+                    )
+                ],
+            )
+        ],
+    )
+    file_path = tmp_path / "new_format.json"
+    with open(file_path, "w", encoding="utf-8") as f:
+      f.write(eval_set.model_dump_json())
+
+    loaded_eval_set = load_eval_set_from_file(
+        str(file_path), "new_format_eval_set"
+    )
+
+    assert loaded_eval_set == eval_set
+
+  def test_load_eval_set_from_file_old_format(self, tmp_path, mocker):
+    mocked_time = 12345678
+    mocked_invocation_id = "15061953"
+    mocker.patch("time.time", return_value=mocked_time)
+    mocker.patch("uuid.uuid4", return_value=mocked_invocation_id)
+
+    # Create a dummy file with EvalSet in the old JSON format
+    old_format_json = [{
+        "name": "old_format_case",
+        "data": [
+            {"query": "Old Format Query", "reference": "Old Format Reference"}
+        ],
+    }]
+    file_path = tmp_path / "old_format.json"
+    with open(file_path, "w", encoding="utf-8") as f:
+      json.dump(old_format_json, f)
+
+    loaded_eval_set = load_eval_set_from_file(
+        str(file_path), "old_format_eval_set"
+    )
+
+    expected_eval_set = EvalSet(
+        eval_set_id="old_format_eval_set",
+        name="old_format_eval_set",
+        creation_timestamp=mocked_time,
+        eval_cases=[
+            EvalCase(
+                eval_id="old_format_case",
+                creation_timestamp=mocked_time,
+                conversation=[
+                    Invocation(
+                        invocation_id=mocked_invocation_id,
+                        user_content=genai_types.Content(
+                            parts=[genai_types.Part(text="Old Format Query")],
+                            role="user",
+                        ),
+                        final_response=genai_types.Content(
+                            parts=[
+                                genai_types.Part(text="Old Format Reference")
+                            ],
+                            role="model",
+                        ),
+                        intermediate_data=IntermediateData(
+                            tool_uses=[],
+                            intermediate_responses=[],
+                        ),
+                        creation_timestamp=mocked_time,
+                    )
+                ],
+            )
+        ],
+    )
+
+    assert loaded_eval_set == expected_eval_set
+
+  def test_load_eval_set_from_file_nonexistent_file(self):
+    with pytest.raises(FileNotFoundError):
+      load_eval_set_from_file("nonexistent_file.json", "test_eval_set")
+
+  def test_load_eval_set_from_file_invalid_json(self, tmp_path):
+    # Create a dummy file with invalid JSON
+    file_path = tmp_path / "invalid.json"
+    with open(file_path, "w", encoding="utf-8") as f:
+      f.write("invalid json")
+
+    with pytest.raises(json.JSONDecodeError):
+      load_eval_set_from_file(str(file_path), "test_eval_set")
+
+  def test_load_eval_set_from_file_invalid_data(self, tmp_path, mocker):
+    # Create a dummy file with invalid data that fails both Pydantic validation
+    # and the old format conversion.  We mock the
+    # convert_eval_set_to_pydantic_schema function to raise a ValueError
+    # so that we can assert that the exception is raised.
+    file_path = tmp_path / "invalid_data.json"
+    with open(file_path, "w", encoding="utf-8") as f:
+      f.write('{"invalid": "data"}')
+
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.convert_eval_set_to_pydantic_schema",
+        side_effect=ValueError(),
+    )
+
+    with pytest.raises(ValueError):
+      load_eval_set_from_file(str(file_path), "test_eval_set")
+
+
+class TestLocalEvalSetsManager:
+  """Tests for LocalEvalSetsManager."""
+
+  @pytest.fixture
+  def local_eval_sets_manager(tmp_path):
+    agents_dir = str(tmp_path)
+    return LocalEvalSetsManager(agents_dir=agents_dir)
+
+  def test_local_eval_sets_manager_get_eval_set_success(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    mock_eval_set = EvalSet(eval_set_id=eval_set_id, eval_cases=[])
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.load_eval_set_from_file",
+        return_value=mock_eval_set,
+    )
+    mocker.patch("os.path.exists", return_value=True)
+
+    eval_set = local_eval_sets_manager.get_eval_set(app_name, eval_set_id)
+
+    assert eval_set == mock_eval_set
+
+  def test_local_eval_sets_manager_get_eval_set_not_found(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.load_eval_set_from_file",
+        side_effect=FileNotFoundError,
+    )
+
+    eval_set = local_eval_sets_manager.get_eval_set(app_name, eval_set_id)
+
+    assert eval_set is None
+
+  def test_local_eval_sets_manager_create_eval_set_success(
+      self, local_eval_sets_manager, mocker
+  ):
+    mocked_time = 12345678
+    mocker.patch("time.time", return_value=mocked_time)
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    mocker.patch("os.path.exists", return_value=False)
+    mock_write_eval_set_to_path = mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set_to_path"
+    )
+    eval_set_file_path = os.path.join(
+        local_eval_sets_manager._agents_dir,
+        app_name,
+        eval_set_id + _EVAL_SET_FILE_EXTENSION,
+    )
+
+    created_eval_set = local_eval_sets_manager.create_eval_set(
+        app_name, eval_set_id
+    )
+
+    expected_eval_set = EvalSet(
+        eval_set_id=eval_set_id,
+        name=eval_set_id,
+        eval_cases=[],
+        creation_timestamp=mocked_time,
+    )
+    mock_write_eval_set_to_path.assert_called_once_with(
+        eval_set_file_path,
+        expected_eval_set,
+    )
+    assert created_eval_set == expected_eval_set
+
+  def test_local_eval_sets_manager_create_eval_set_invalid_id(
+      self, local_eval_sets_manager
+  ):
+    app_name = "test_app"
+    eval_set_id = "invalid-id"
+
+    with pytest.raises(ValueError, match="Invalid Eval Set ID"):
+      local_eval_sets_manager.create_eval_set(app_name, eval_set_id)
+
+  @pytest.mark.parametrize("app_name", ["", ".", "..", "foo/bar", "foo\\bar"])
+  def test_local_eval_sets_manager_create_eval_set_rejects_invalid_app_name(
+      self, local_eval_sets_manager, app_name
+  ):
+    with pytest.raises(ValueError):
+      local_eval_sets_manager.create_eval_set(app_name, "test_eval_set")
+
+  @pytest.mark.parametrize("app_name", ["", ".", "..", "foo/bar", "foo\\bar"])
+  def test_local_eval_sets_manager_list_eval_sets_rejects_invalid_app_name(
+      self, local_eval_sets_manager, app_name
+  ):
+    with pytest.raises(ValueError):
+      local_eval_sets_manager.list_eval_sets(app_name)
+
+  @pytest.mark.parametrize(
+      "eval_set_id", ["", ".", "..", "foo/bar", "foo\\bar"]
+  )
+  def test_local_eval_sets_manager_get_eval_set_rejects_invalid_eval_set_id(
+      self, local_eval_sets_manager, eval_set_id
+  ):
+    with pytest.raises(ValueError):
+      local_eval_sets_manager.get_eval_set("test_app", eval_set_id)
+
+  def test_local_eval_sets_manager_create_eval_set_already_exists(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "existing_eval_set_id"
+    mocker.patch("os.path.exists", return_value=True)
+
+    with pytest.raises(
+        ValueError,
+        match="EvalSet existing_eval_set_id already exists for app test_app.",
+    ):
+      local_eval_sets_manager.create_eval_set(app_name, eval_set_id)
+
+  def test_local_eval_sets_manager_list_eval_sets_success(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    mock_listdir_return = [
+        "eval_set_1.evalset.json",
+        "eval_set_2.evalset.json",
+        "not_an_eval_set.txt",
+    ]
+    mocker.patch("os.listdir", return_value=mock_listdir_return)
+    mocker.patch("os.path.join", return_value="dummy_path")
+    mocker.patch("os.path.basename", side_effect=lambda x: x)
+
+    eval_sets = local_eval_sets_manager.list_eval_sets(app_name)
+
+    assert eval_sets == ["eval_set_1", "eval_set_2"]
+
+  def test_local_eval_sets_manager_list_eval_sets_not_found(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    mocker.patch("os.listdir", side_effect=FileNotFoundError)
+
+    with pytest.raises(NotFoundError):
+      local_eval_sets_manager.list_eval_sets(app_name)
+
+  def test_local_eval_sets_manager_add_eval_case_success(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+    mock_eval_set = EvalSet(eval_set_id=eval_set_id, eval_cases=[])
+
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
+        return_value=mock_eval_set,
+    )
+    mock_write_eval_set_to_path = mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set_to_path"
+    )
+
+    local_eval_sets_manager.add_eval_case(app_name, eval_set_id, mock_eval_case)
+
+    assert len(mock_eval_set.eval_cases) == 1
+    assert mock_eval_set.eval_cases[0] == mock_eval_case
+    expected_eval_set_file_path = os.path.join(
+        local_eval_sets_manager._agents_dir,
+        app_name,
+        eval_set_id + _EVAL_SET_FILE_EXTENSION,
+    )
+    mock_eval_set.eval_cases.append(mock_eval_case)
+    mock_write_eval_set_to_path.assert_called_once_with(
+        expected_eval_set_file_path, mock_eval_set
+    )
+
+  def test_local_eval_sets_manager_add_eval_case_eval_set_not_found(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
+        return_value=None,
+    )
+
+    with pytest.raises(
+        NotFoundError, match="Eval set `test_eval_set` not found."
+    ):
+      local_eval_sets_manager.add_eval_case(
+          app_name, eval_set_id, mock_eval_case
+      )
+
+  def test_local_eval_sets_manager_add_eval_case_eval_case_id_exists(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+    mock_eval_set = EvalSet(
+        eval_set_id=eval_set_id, eval_cases=[mock_eval_case]
+    )
+
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
+        return_value=mock_eval_set,
+    )
+
+    with pytest.raises(
+        ValueError,
+        match=(
+            f"Eval id `{eval_case_id}` already exists in `{eval_set_id}` eval"
+            " set."
+        ),
+    ):
+      local_eval_sets_manager.add_eval_case(
+          app_name, eval_set_id, mock_eval_case
+      )
+
+  def test_local_eval_sets_manager_get_eval_case_success(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+    mock_eval_set = EvalSet(
+        eval_set_id=eval_set_id, eval_cases=[mock_eval_case]
+    )
+
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
+        return_value=mock_eval_set,
+    )
+
+    eval_case = local_eval_sets_manager.get_eval_case(
+        app_name, eval_set_id, eval_case_id
+    )
+
+    assert eval_case == mock_eval_case
+
+  def test_local_eval_sets_manager_get_eval_case_eval_set_not_found(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
+        return_value=None,
+    )
+
+    eval_case = local_eval_sets_manager.get_eval_case(
+        app_name, eval_set_id, eval_case_id
+    )
+
+    assert eval_case is None
+
+  def test_local_eval_sets_manager_get_eval_case_eval_case_not_found(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_set = EvalSet(eval_set_id=eval_set_id, eval_cases=[])
+
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
+        return_value=mock_eval_set,
+    )
+
+    eval_case = local_eval_sets_manager.get_eval_case(
+        app_name, eval_set_id, eval_case_id
+    )
+
+    assert eval_case is None
+
+  def test_local_eval_sets_manager_update_eval_case_success(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_case = EvalCase(
+        eval_id=eval_case_id, conversation=[], creation_timestamp=456
+    )
+    updated_eval_case = EvalCase(
+        eval_id=eval_case_id, conversation=[], creation_timestamp=123
+    )
+    mock_eval_set = EvalSet(
+        eval_set_id=eval_set_id, eval_cases=[mock_eval_case]
+    )
+
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
+        return_value=mock_eval_set,
+    )
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
+        return_value=mock_eval_case,
+    )
+    mock_write_eval_set_to_path = mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set_to_path"
+    )
+
+    local_eval_sets_manager.update_eval_case(
+        app_name, eval_set_id, updated_eval_case
+    )
+
+    assert len(mock_eval_set.eval_cases) == 1
+    assert mock_eval_set.eval_cases[0] == updated_eval_case
+    expected_eval_set_file_path = os.path.join(
+        local_eval_sets_manager._agents_dir,
+        app_name,
+        eval_set_id + _EVAL_SET_FILE_EXTENSION,
+    )
+    mock_write_eval_set_to_path.assert_called_once_with(
+        expected_eval_set_file_path,
+        EvalSet(eval_set_id=eval_set_id, eval_cases=[updated_eval_case]),
+    )
+
+  def test_local_eval_sets_manager_update_eval_case_eval_set_not_found(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    updated_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
+        return_value=None,
+    )
+
+    with pytest.raises(
+        NotFoundError,
+        match=f"Eval set `{eval_set_id}` not found.",
+    ):
+      local_eval_sets_manager.update_eval_case(
+          app_name, eval_set_id, updated_eval_case
+      )
+
+  def test_local_eval_sets_manager_update_eval_case_eval_case_not_found(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    updated_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+    mock_eval_set = EvalSet(eval_set_id=eval_set_id, eval_cases=[])
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
+        return_value=mock_eval_set,
+    )
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
+        return_value=None,
+    )
+    with pytest.raises(
+        NotFoundError,
+        match=(
+            f"Eval case `{eval_case_id}` not found in eval set `{eval_set_id}`."
+        ),
+    ):
+      local_eval_sets_manager.update_eval_case(
+          app_name, eval_set_id, updated_eval_case
+      )
+
+  def test_local_eval_sets_manager_delete_eval_case_success(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_case = EvalCase(eval_id=eval_case_id, conversation=[])
+    mock_eval_set = EvalSet(
+        eval_set_id=eval_set_id, eval_cases=[mock_eval_case]
+    )
+
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
+        return_value=mock_eval_set,
+    )
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
+        return_value=mock_eval_case,
+    )
+    mock_write_eval_set_to_path = mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set_to_path"
+    )
+
+    local_eval_sets_manager.delete_eval_case(
+        app_name, eval_set_id, eval_case_id
+    )
+
+    assert len(mock_eval_set.eval_cases) == 0
+    expected_eval_set_file_path = os.path.join(
+        local_eval_sets_manager._agents_dir,
+        app_name,
+        eval_set_id + _EVAL_SET_FILE_EXTENSION,
+    )
+    mock_write_eval_set_to_path.assert_called_once_with(
+        expected_eval_set_file_path,
+        EvalSet(eval_set_id=eval_set_id, eval_cases=[]),
+    )
+
+  def test_local_eval_sets_manager_delete_eval_case_eval_set_not_found(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
+        return_value=None,
+    )
+    mock_write_eval_set_to_path = mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager._write_eval_set_to_path"
+    )
+
+    with pytest.raises(
+        NotFoundError,
+        match=f"Eval set `{eval_set_id}` not found.",
+    ):
+      local_eval_sets_manager.delete_eval_case(
+          app_name, eval_set_id, eval_case_id
+      )
+
+    mock_write_eval_set_to_path.assert_not_called()
+
+  def test_local_eval_sets_manager_delete_eval_case_eval_case_not_found(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    eval_set_id = "test_eval_set"
+    eval_case_id = "test_eval_case"
+    mock_eval_set = EvalSet(eval_set_id=eval_set_id, eval_cases=[])
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_set",
+        return_value=mock_eval_set,
+    )
+    mocker.patch(
+        "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager.get_eval_case",
+        return_value=None,
+    )
+    with pytest.raises(
+        NotFoundError,
+        match=(
+            f"Eval case `{eval_case_id}` not found in eval set `{eval_set_id}`."
+        ),
+    ):
+      local_eval_sets_manager.delete_eval_case(
+          app_name, eval_set_id, eval_case_id
+      )

tests/unittests/evaluation/test_metric_evaluator_registry.py

@@ -0,0 +1,222 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.errors.not_found_error import NotFoundError
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.eval_metrics import Interval
+from google.adk.evaluation.eval_metrics import MetricInfo
+from google.adk.evaluation.eval_metrics import MetricValueInfo
+from google.adk.evaluation.eval_metrics import PrebuiltMetrics
+from google.adk.evaluation.evaluator import Evaluator
+from google.adk.evaluation.metric_evaluator_registry import FinalResponseMatchV2EvaluatorMetricInfoProvider
+from google.adk.evaluation.metric_evaluator_registry import HallucinationsV1EvaluatorMetricInfoProvider
+from google.adk.evaluation.metric_evaluator_registry import MetricEvaluatorRegistry
+from google.adk.evaluation.metric_evaluator_registry import PerTurnUserSimulatorQualityV1MetricInfoProvider
+from google.adk.evaluation.metric_evaluator_registry import ResponseEvaluatorMetricInfoProvider
+from google.adk.evaluation.metric_evaluator_registry import RubricBasedFinalResponseQualityV1EvaluatorMetricInfoProvider
+from google.adk.evaluation.metric_evaluator_registry import RubricBasedMultiTurnTrajectoryMetricInfoProvider
+from google.adk.evaluation.metric_evaluator_registry import RubricBasedToolUseV1EvaluatorMetricInfoProvider
+from google.adk.evaluation.metric_evaluator_registry import SafetyEvaluatorV1MetricInfoProvider
+from google.adk.evaluation.metric_evaluator_registry import TrajectoryEvaluatorMetricInfoProvider
+import pytest
+
+_DUMMY_METRIC_NAME = "dummy_metric_name"
+_DUMMY_METRIC_INFO = MetricInfo(
+    metric_name=_DUMMY_METRIC_NAME,
+    description="Dummy metric description",
+    metric_value_info=MetricValueInfo(
+        interval=Interval(min_value=0.0, max_value=1.0)
+    ),
+)
+_ANOTHER_DUMMY_METRIC_INFO = MetricInfo(
+    metric_name=_DUMMY_METRIC_NAME,
+    description="Another dummy metric description",
+    metric_value_info=MetricValueInfo(
+        interval=Interval(min_value=0.0, max_value=1.0)
+    ),
+)
+
+
+class DummyEvaluator(Evaluator):
+
+  def __init__(self, eval_metric: EvalMetric):
+    self._eval_metric = eval_metric
+
+  def evaluate_invocations(self, actual_invocations, expected_invocations):
+    return "dummy_result"
+
+
+class AnotherDummyEvaluator(Evaluator):
+
+  def __init__(self, eval_metric: EvalMetric):
+    self._eval_metric = eval_metric
+
+  def evaluate_invocations(self, actual_invocations, expected_invocations):
+    return "another_dummy_result"
+
+
+class TestMetricEvaluatorRegistry:
+  """Test cases for MetricEvaluatorRegistry."""
+
+  @pytest.fixture
+  def registry(self):
+    return MetricEvaluatorRegistry()
+
+  def test_register_evaluator(self, registry):
+    registry.register_evaluator(
+        _DUMMY_METRIC_INFO,
+        DummyEvaluator,
+    )
+    assert _DUMMY_METRIC_NAME in registry._registry
+    assert registry._registry[_DUMMY_METRIC_NAME] == (
+        DummyEvaluator,
+        _DUMMY_METRIC_INFO,
+    )
+
+  def test_register_evaluator_updates_existing(self, registry):
+    registry.register_evaluator(
+        _DUMMY_METRIC_INFO,
+        DummyEvaluator,
+    )
+
+    assert registry._registry[_DUMMY_METRIC_NAME] == (
+        DummyEvaluator,
+        _DUMMY_METRIC_INFO,
+    )
+
+    registry.register_evaluator(
+        _ANOTHER_DUMMY_METRIC_INFO, AnotherDummyEvaluator
+    )
+    assert registry._registry[_DUMMY_METRIC_NAME] == (
+        AnotherDummyEvaluator,
+        _ANOTHER_DUMMY_METRIC_INFO,
+    )
+
+  def test_get_evaluator(self, registry):
+    registry.register_evaluator(
+        _DUMMY_METRIC_INFO,
+        DummyEvaluator,
+    )
+    eval_metric = EvalMetric(metric_name=_DUMMY_METRIC_NAME, threshold=0.5)
+    evaluator = registry.get_evaluator(eval_metric)
+    assert isinstance(evaluator, DummyEvaluator)
+
+  def test_get_evaluator_not_found(self, registry):
+    eval_metric = EvalMetric(metric_name="non_existent_metric", threshold=0.5)
+    with pytest.raises(NotFoundError):
+      registry.get_evaluator(eval_metric)
+
+
+class TestMetricInfoProviders:
+  """Test cases for MetricInfoProviders."""
+
+  def test_trajectory_evaluator_metric_info_provider(self):
+    metric_info = TrajectoryEvaluatorMetricInfoProvider().get_metric_info()
+    assert (
+        metric_info.metric_name
+        == PrebuiltMetrics.TOOL_TRAJECTORY_AVG_SCORE.value
+    )
+    assert metric_info.metric_value_info.interval.min_value == 0.0
+    assert metric_info.metric_value_info.interval.max_value == 1.0
+
+  def test_response_evaluator_metric_info_provider_eval_score(self):
+    metric_info = ResponseEvaluatorMetricInfoProvider(
+        PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value
+    ).get_metric_info()
+    assert (
+        metric_info.metric_name
+        == PrebuiltMetrics.RESPONSE_EVALUATION_SCORE.value
+    )
+    assert metric_info.metric_value_info.interval.min_value == 1.0
+    assert metric_info.metric_value_info.interval.max_value == 5.0
+
+  def test_response_evaluator_metric_info_provider_match_score(self):
+    metric_info = ResponseEvaluatorMetricInfoProvider(
+        PrebuiltMetrics.RESPONSE_MATCH_SCORE.value
+    ).get_metric_info()
+    assert metric_info.metric_name == PrebuiltMetrics.RESPONSE_MATCH_SCORE.value
+    assert metric_info.metric_value_info.interval.min_value == 0.0
+    assert metric_info.metric_value_info.interval.max_value == 1.0
+
+  def test_safety_evaluator_v1_metric_info_provider(self):
+    metric_info = SafetyEvaluatorV1MetricInfoProvider().get_metric_info()
+    assert metric_info.metric_name == PrebuiltMetrics.SAFETY_V1.value
+    assert metric_info.metric_value_info.interval.min_value == 0.0
+    assert metric_info.metric_value_info.interval.max_value == 1.0
+
+  def test_final_response_match_v2_evaluator_metric_info_provider(self):
+    metric_info = (
+        FinalResponseMatchV2EvaluatorMetricInfoProvider().get_metric_info()
+    )
+    assert (
+        metric_info.metric_name == PrebuiltMetrics.FINAL_RESPONSE_MATCH_V2.value
+    )
+    assert metric_info.metric_value_info.interval.min_value == 0.0
+    assert metric_info.metric_value_info.interval.max_value == 1.0
+
+  def test_rubric_based_final_response_quality_v1_evaluator_metric_info_provider(
+      self,
+  ):
+    metric_info = (
+        RubricBasedFinalResponseQualityV1EvaluatorMetricInfoProvider().get_metric_info()
+    )
+    assert (
+        metric_info.metric_name
+        == PrebuiltMetrics.RUBRIC_BASED_FINAL_RESPONSE_QUALITY_V1.value
+    )
+    assert metric_info.metric_value_info.interval.min_value == 0.0
+    assert metric_info.metric_value_info.interval.max_value == 1.0
+
+  def test_hallucinations_v1_evaluator_metric_info_provider(self):
+    metric_info = (
+        HallucinationsV1EvaluatorMetricInfoProvider().get_metric_info()
+    )
+    assert metric_info.metric_name == PrebuiltMetrics.HALLUCINATIONS_V1.value
+    assert metric_info.metric_value_info.interval.min_value == 0.0
+    assert metric_info.metric_value_info.interval.max_value == 1.0
+
+  def test_rubric_based_tool_use_v1_evaluator_metric_info_provider(self):
+    metric_info = (
+        RubricBasedToolUseV1EvaluatorMetricInfoProvider().get_metric_info()
+    )
+    assert (
+        metric_info.metric_name
+        == PrebuiltMetrics.RUBRIC_BASED_TOOL_USE_QUALITY_V1.value
+    )
+    assert metric_info.metric_value_info.interval.min_value == 0.0
+    assert metric_info.metric_value_info.interval.max_value == 1.0
+
+  def test_per_turn_user_simulator_quality_v1_metric_info_provider(self):
+    metric_info = (
+        PerTurnUserSimulatorQualityV1MetricInfoProvider().get_metric_info()
+    )
+    assert (
+        metric_info.metric_name
+        == PrebuiltMetrics.PER_TURN_USER_SIMULATOR_QUALITY_V1.value
+    )
+    assert metric_info.metric_value_info.interval.min_value == 0.0
+    assert metric_info.metric_value_info.interval.max_value == 1.0
+
+  def test_rubric_based_multi_turn_trajectory_metric_info_provider(self):
+    metric_info = (
+        RubricBasedMultiTurnTrajectoryMetricInfoProvider().get_metric_info()
+    )
+    assert (
+        metric_info.metric_name
+        == PrebuiltMetrics.RUBRIC_BASED_MULTI_TURN_TRAJECTORY_QUALITY_V1.value
+    )
+    assert metric_info.metric_value_info.interval.min_value == 0.0
+    assert metric_info.metric_value_info.interval.max_value == 1.0

tests/unittests/evaluation/test_multi_turn_task_success_evaluator.py

@@ -0,0 +1,106 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for the Multi Turn Task Success Evaluator."""
+
+from google.adk.dependencies.vertexai import vertexai
+from google.adk.evaluation.app_details import AgentDetails
+from google.adk.evaluation.app_details import AppDetails
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_case import InvocationEvent
+from google.adk.evaluation.eval_case import InvocationEvents
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.multi_turn_task_success_evaluator import MultiTurnTaskSuccessV1Evaluator
+from google.genai import types as genai_types
+
+vertexai_types = vertexai.types
+
+
+class TestMultiTurnTaskSuccessV1Evaluator:
+  """A class to help organize "patch" that are applicable to all tests."""
+
+  def test_evaluate_invocations_metric_passed(self, mocker):
+    """Test evaluate_invocations function for multi-turn task success metric."""
+    mock_perform_eval = mocker.patch(
+        "google.adk.evaluation.vertex_ai_eval_facade._VertexAiEvalFacade._perform_eval"
+    )
+    actual_invocations = [
+        Invocation(
+            invocation_id="inv1",
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="q1")]
+            ),
+            intermediate_data=InvocationEvents(invocation_events=[]),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="r1")]
+            ),
+            app_details=AppDetails(
+                agent_details={
+                    "agent1": AgentDetails(
+                        name="agent1", instructions="instructions1"
+                    )
+                }
+            ),
+        ),
+        Invocation(
+            invocation_id="inv2",
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="q2")]
+            ),
+            intermediate_data=InvocationEvents(
+                invocation_events=[
+                    InvocationEvent(
+                        author="agent1",
+                        content=genai_types.Content(
+                            parts=[genai_types.Part(text="intermediate")]
+                        ),
+                    )
+                ]
+            ),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="r2")]
+            ),
+            app_details=AppDetails(
+                agent_details={
+                    "agent1": AgentDetails(
+                        name="agent1", instructions="instructions1"
+                    )
+                }
+            ),
+        ),
+    ]
+    evaluator = MultiTurnTaskSuccessV1Evaluator(
+        eval_metric=EvalMetric(
+            threshold=0.8, metric_name="multi_turn_task_success"
+        )
+    )
+    # Mock the return value of _perform_eval
+    mock_perform_eval.return_value = vertexai_types.EvaluationResult(
+        summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.9)],
+        eval_case_results=[],
+    )
+
+    evaluation_result = evaluator.evaluate_invocations(
+        actual_invocations,
+    )
+
+    assert evaluation_result.overall_score == 0.9
+    assert evaluation_result.overall_eval_status == EvalStatus.PASSED
+    mock_perform_eval.assert_called_once()
+    _, mock_kwargs = mock_perform_eval.call_args
+    # Compare the names of the metrics.
+    assert [m.name for m in mock_kwargs["metrics"]] == [
+        vertexai_types.RubricMetric.MULTI_TURN_TASK_SUCCESS.name
+    ]

tests/unittests/evaluation/test_multi_turn_tool_use_quality_evaluator.py

@@ -0,0 +1,106 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for the Multi Turn Tool Use Quality Evaluator."""
+
+from google.adk.dependencies.vertexai import vertexai
+from google.adk.evaluation.app_details import AgentDetails
+from google.adk.evaluation.app_details import AppDetails
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_case import InvocationEvent
+from google.adk.evaluation.eval_case import InvocationEvents
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.multi_turn_tool_use_quality_evaluator import MultiTurnToolUseQualityV1Evaluator
+from google.genai import types as genai_types
+
+vertexai_types = vertexai.types
+
+
+class TestMultiTurnToolUseQualityV1Evaluator:
+  """A class to help organize "patch" that are applicable to all tests."""
+
+  def test_evaluate_invocations_metric_passed(self, mocker):
+    """Test evaluate_invocations function for multi-turn tool use quality metric."""
+    mock_perform_eval = mocker.patch(
+        "google.adk.evaluation.vertex_ai_eval_facade._VertexAiEvalFacade._perform_eval"
+    )
+    actual_invocations = [
+        Invocation(
+            invocation_id="inv1",
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="q1")]
+            ),
+            intermediate_data=InvocationEvents(invocation_events=[]),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="r1")]
+            ),
+            app_details=AppDetails(
+                agent_details={
+                    "agent1": AgentDetails(
+                        name="agent1", instructions="instructions1"
+                    )
+                }
+            ),
+        ),
+        Invocation(
+            invocation_id="inv2",
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="q2")]
+            ),
+            intermediate_data=InvocationEvents(
+                invocation_events=[
+                    InvocationEvent(
+                        author="agent1",
+                        content=genai_types.Content(
+                            parts=[genai_types.Part(text="intermediate")]
+                        ),
+                    )
+                ]
+            ),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="r2")]
+            ),
+            app_details=AppDetails(
+                agent_details={
+                    "agent1": AgentDetails(
+                        name="agent1", instructions="instructions1"
+                    )
+                }
+            ),
+        ),
+    ]
+    evaluator = MultiTurnToolUseQualityV1Evaluator(
+        eval_metric=EvalMetric(
+            threshold=0.8, metric_name="multi_turn_tool_use_quality"
+        )
+    )
+    # Mock the return value of _perform_eval
+    mock_perform_eval.return_value = vertexai_types.EvaluationResult(
+        summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.9)],
+        eval_case_results=[],
+    )
+
+    evaluation_result = evaluator.evaluate_invocations(
+        actual_invocations,
+    )
+
+    assert evaluation_result.overall_score == 0.9
+    assert evaluation_result.overall_eval_status == EvalStatus.PASSED
+    mock_perform_eval.assert_called_once()
+    _, mock_kwargs = mock_perform_eval.call_args
+    # Compare the names of the metrics.
+    assert [m.name for m in mock_kwargs["metrics"]] == [
+        vertexai_types.RubricMetric.MULTI_TURN_TOOL_USE_QUALITY.name
+    ]

tests/unittests/evaluation/test_multi_turn_trajectory_quality_evaluator.py

@@ -0,0 +1,106 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for the Multi Turn Trajectory Quality Evaluator."""
+
+from google.adk.dependencies.vertexai import vertexai
+from google.adk.evaluation.app_details import AgentDetails
+from google.adk.evaluation.app_details import AppDetails
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_case import InvocationEvent
+from google.adk.evaluation.eval_case import InvocationEvents
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.multi_turn_trajectory_quality_evaluator import MultiTurnTrajectoryQualityV1Evaluator
+from google.genai import types as genai_types
+
+vertexai_types = vertexai.types
+
+
+class TestMultiTurnTrajectoryQualityV1Evaluator:
+  """A class to help organize "patch" that are applicable to all tests."""
+
+  def test_evaluate_invocations_metric_passed(self, mocker):
+    """Test evaluate_invocations function for multi-turn trajectory quality metric."""
+    mock_perform_eval = mocker.patch(
+        "google.adk.evaluation.vertex_ai_eval_facade._VertexAiEvalFacade._perform_eval"
+    )
+    actual_invocations = [
+        Invocation(
+            invocation_id="inv1",
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="q1")]
+            ),
+            intermediate_data=InvocationEvents(invocation_events=[]),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="r1")]
+            ),
+            app_details=AppDetails(
+                agent_details={
+                    "agent1": AgentDetails(
+                        name="agent1", instructions="instructions1"
+                    )
+                }
+            ),
+        ),
+        Invocation(
+            invocation_id="inv2",
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="q2")]
+            ),
+            intermediate_data=InvocationEvents(
+                invocation_events=[
+                    InvocationEvent(
+                        author="agent1",
+                        content=genai_types.Content(
+                            parts=[genai_types.Part(text="intermediate")]
+                        ),
+                    )
+                ]
+            ),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="r2")]
+            ),
+            app_details=AppDetails(
+                agent_details={
+                    "agent1": AgentDetails(
+                        name="agent1", instructions="instructions1"
+                    )
+                }
+            ),
+        ),
+    ]
+    evaluator = MultiTurnTrajectoryQualityV1Evaluator(
+        eval_metric=EvalMetric(
+            threshold=0.8, metric_name="multi_turn_trajectory_quality"
+        )
+    )
+    # Mock the return value of _perform_eval
+    mock_perform_eval.return_value = vertexai_types.EvaluationResult(
+        summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.9)],
+        eval_case_results=[],
+    )
+
+    evaluation_result = evaluator.evaluate_invocations(
+        actual_invocations,
+    )
+
+    assert evaluation_result.overall_score == 0.9
+    assert evaluation_result.overall_eval_status == EvalStatus.PASSED
+    mock_perform_eval.assert_called_once()
+    _, mock_kwargs = mock_perform_eval.call_args
+    # Compare the names of the metrics.
+    assert [m.name for m in mock_kwargs["metrics"]] == [
+        vertexai_types.RubricMetric.MULTI_TURN_TRAJECTORY_QUALITY.name
+    ]

tests/unittests/evaluation/test_request_intercepter_plugin.py

@@ -0,0 +1,72 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.agents.callback_context import CallbackContext
+from google.adk.evaluation.request_intercepter_plugin import _LLM_REQUEST_ID_KEY
+from google.adk.evaluation.request_intercepter_plugin import _RequestIntercepterPlugin
+from google.adk.models.llm_request import LlmRequest
+from google.adk.models.llm_response import LlmResponse
+from google.genai import types
+
+
+class TestRequestIntercepterPlugin:
+
+  async def test_intercept_request_and_response(self, mocker):
+    plugin = _RequestIntercepterPlugin(name="test_plugin")
+    llm_request = LlmRequest(
+        model="test_model",
+        contents=[
+            types.Content(
+                role="user",
+                parts=[types.Part(text="hello")],
+            )
+        ],
+    )
+    mock_invocation_context = mocker.MagicMock()
+    mock_invocation_context.session.state = {}
+    mock_invocation_context._state_schema = None
+    callback_context = CallbackContext(mock_invocation_context)
+    llm_response = LlmResponse()
+
+    # Test before_model_callback
+    await plugin.before_model_callback(
+        callback_context=callback_context, llm_request=llm_request
+    )
+    assert _LLM_REQUEST_ID_KEY in callback_context.state
+    request_id = callback_context.state[_LLM_REQUEST_ID_KEY]
+    assert isinstance(request_id, str)
+
+    # Test after_model_callback
+    await plugin.after_model_callback(
+        callback_context=callback_context, llm_response=llm_response
+    )
+    assert llm_response.custom_metadata is not None
+    assert _LLM_REQUEST_ID_KEY in llm_response.custom_metadata
+    assert llm_response.custom_metadata[_LLM_REQUEST_ID_KEY] == request_id
+
+    # Test get_model_request
+    retrieved_request = plugin.get_model_request(llm_response)
+    assert retrieved_request == llm_request
+
+  def test_get_model_request_not_found(self):
+    plugin = _RequestIntercepterPlugin(name="test_plugin")
+    llm_response = LlmResponse()
+    assert plugin.get_model_request(llm_response) is None
+
+    llm_response_with_metadata = LlmResponse(
+        custom_metadata={_LLM_REQUEST_ID_KEY: "non_existent_id"}
+    )
+    assert plugin.get_model_request(llm_response_with_metadata) is None

tests/unittests/evaluation/test_response_evaluator.py

@@ -0,0 +1,120 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+"""Tests for the Response Evaluator."""
+
+from google.adk.dependencies.vertexai import vertexai
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_metrics import PrebuiltMetrics
+from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.response_evaluator import ResponseEvaluator
+from google.genai import types as genai_types
+import pytest
+
+vertexai_types = vertexai.types
+
+
+class TestResponseEvaluator:
+  """A class to help organize "patch" that are applicable to all tests."""
+
+  def test_evaluate_invocations_rouge_metric(self, mocker):
+    """Test evaluate_invocations function for Rouge metric."""
+    mock_perform_eval = mocker.patch(
+        "google.adk.evaluation.vertex_ai_eval_facade._VertexAiEvalFacade._perform_eval"
+    )
+    actual_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[
+                    genai_types.Part(text="This is a test candidate response.")
+                ]
+            ),
+        )
+    ]
+    expected_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test reference.")]
+            ),
+        )
+    ]
+    evaluator = ResponseEvaluator(
+        threshold=0.8, metric_name="response_match_score"
+    )
+
+    evaluation_result = evaluator.evaluate_invocations(
+        actual_invocations, expected_invocations
+    )
+
+    assert evaluation_result.overall_score == pytest.approx(8 / 11)
+    # ROUGE-1 F1 is approx. 0.73 < 0.8 threshold, so eval status is FAILED.
+    assert evaluation_result.overall_eval_status == EvalStatus.FAILED
+    mock_perform_eval.assert_not_called()  # Ensure _perform_eval was not called
+
+  def test_evaluate_invocations_coherence_metric_passed(self, mocker):
+    """Test evaluate_invocations function for Coherence metric."""
+    mock_perform_eval = mocker.patch(
+        "google.adk.evaluation.vertex_ai_eval_facade._VertexAiEvalFacade._perform_eval"
+    )
+    actual_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[
+                    genai_types.Part(text="This is a test candidate response.")
+                ]
+            ),
+        )
+    ]
+    expected_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test reference.")]
+            ),
+        )
+    ]
+    evaluator = ResponseEvaluator(
+        threshold=0.8, metric_name="response_evaluation_score"
+    )
+    # Mock the return value of _perform_eval
+    mock_perform_eval.return_value = vertexai_types.EvaluationResult(
+        summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.9)],
+        eval_case_results=[],
+    )
+
+    evaluation_result = evaluator.evaluate_invocations(
+        actual_invocations, expected_invocations
+    )
+
+    assert evaluation_result.overall_score == 0.9
+    assert evaluation_result.overall_eval_status == EvalStatus.PASSED
+    mock_perform_eval.assert_called_once()
+    _, mock_kwargs = mock_perform_eval.call_args
+    # Compare the names of the metrics.
+    assert [m.name for m in mock_kwargs["metrics"]] == [
+        vertexai_types.PrebuiltMetric.COHERENCE.name
+    ]

tests/unittests/evaluation/test_retry_options_utils.py

@@ -0,0 +1,78 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.adk.agents.callback_context import CallbackContext
+from google.adk.evaluation import _retry_options_utils
+from google.adk.models.llm_request import LlmRequest
+from google.genai import types
+import pytest
+
+
+def test_add_retry_options_with_default_request():
+  request = LlmRequest()
+  _retry_options_utils.add_default_retry_options_if_not_present(request)
+  assert request.config.http_options is not None
+  assert (
+      request.config.http_options.retry_options
+      == _retry_options_utils._DEFAULT_HTTP_RETRY_OPTIONS
+  )
+
+
+def test_add_retry_options_when_retry_options_is_none():
+  request = LlmRequest()
+  request.config.http_options = types.HttpOptions(retry_options=None)
+  _retry_options_utils.add_default_retry_options_if_not_present(request)
+  assert (
+      request.config.http_options.retry_options
+      == _retry_options_utils._DEFAULT_HTTP_RETRY_OPTIONS
+  )
+
+
+def test_add_retry_options_does_not_override_existing_options():
+  my_retry_options = types.HttpRetryOptions(attempts=1)
+  request = LlmRequest()
+  request.config.http_options = types.HttpOptions(
+      retry_options=my_retry_options
+  )
+  _retry_options_utils.add_default_retry_options_if_not_present(request)
+  assert request.config.http_options.retry_options == my_retry_options
+
+
+def test_add_retry_options_when_config_is_none():
+  request = LlmRequest()
+  request.config = None
+  _retry_options_utils.add_default_retry_options_if_not_present(request)
+  assert request.config is not None
+  assert request.config.http_options is not None
+  assert (
+      request.config.http_options.retry_options
+      == _retry_options_utils._DEFAULT_HTTP_RETRY_OPTIONS
+  )
+
+
+@pytest.mark.asyncio
+async def test_ensure_retry_options_plugin(mocker):
+  request = LlmRequest()
+  plugin = _retry_options_utils.EnsureRetryOptionsPlugin(name="test_plugin")
+  mock_invocation_context = mocker.MagicMock()
+  mock_invocation_context.session.state = {}
+  callback_context = CallbackContext(mock_invocation_context)
+  await plugin.before_model_callback(
+      callback_context=callback_context, llm_request=request
+  )
+  assert request.config.http_options is not None
+  assert (
+      request.config.http_options.retry_options
+      == _retry_options_utils._DEFAULT_HTTP_RETRY_OPTIONS
+  )

tests/unittests/evaluation/test_rubric_based_evaluator.py

@@ -0,0 +1,697 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import logging
+
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.eval_metrics import JudgeModelOptions
+from google.adk.evaluation.eval_metrics import PrebuiltMetrics
+from google.adk.evaluation.eval_metrics import RubricsBasedCriterion
+from google.adk.evaluation.eval_rubrics import Rubric
+from google.adk.evaluation.eval_rubrics import RubricContent
+from google.adk.evaluation.eval_rubrics import RubricScore
+from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.evaluator import PerInvocationResult
+from google.adk.evaluation.llm_as_judge_utils import get_average_rubric_score
+from google.adk.evaluation.rubric_based_evaluator import DefaultAutoRaterResponseParser
+from google.adk.evaluation.rubric_based_evaluator import MajorityVotePerInvocationResultsAggregator
+from google.adk.evaluation.rubric_based_evaluator import MeanInvocationResultsSummarizer
+from google.adk.evaluation.rubric_based_evaluator import RubricBasedEvaluator
+from google.adk.models.llm_response import LlmResponse
+from google.genai import types as genai_types
+import pytest
+
+
+class FakeRubricBasedEvaluator(RubricBasedEvaluator):
+  """A fake implementation of RubricBasedEvaluator intended for testing."""
+
+  def __init__(
+      self,
+      eval_metric: EvalMetric,
+      rubric_type: str | None = None,
+  ):
+    super().__init__(
+        eval_metric,
+        criterion_type=RubricsBasedCriterion,
+        rubric_type=rubric_type,
+    )
+
+  def format_auto_rater_prompt(
+      self, actual: Invocation, expected: Invocation
+  ) -> str:
+    return "fake response"
+
+
+def _create_per_invocation_result(
+    rubric_scores: list[RubricScore],
+) -> PerInvocationResult:
+  """Helper to create a PerInvocationResult."""
+  return PerInvocationResult(
+      actual_invocation=Invocation(
+          user_content=genai_types.Content(
+              parts=[genai_types.Part(text="part_1")]
+          )
+      ),
+      expected_invocation=Invocation(
+          user_content=genai_types.Content(
+              parts=[genai_types.Part(text="part_2")]
+          )
+      ),
+      score=get_average_rubric_score(rubric_scores),
+      rubric_scores=rubric_scores,
+      eval_status=EvalStatus.NOT_EVALUATED,
+  )
+
+
+class TestDefaultAutoRaterResponseParser:
+  """Test cases for DefaultAutoRaterResponseParser."""
+
+  def test_parse_auto_rater_response_with_empty_string(self):
+    """Tests _parse_auto_rater_response with an empty string."""
+    assert DefaultAutoRaterResponseParser().parse("") == []
+
+  def test_parse_auto_rater_response_with_malformed_string(self):
+    """Tests _parse_auto_rater_response with a malformed string."""
+    response = "This is just some random text without the expected format."
+    assert DefaultAutoRaterResponseParser().parse(response) == []
+
+  def test_parse_auto_rater_response_with_single_yes_verdict(self):
+    """Tests _parse_auto_rater_response with a single 'yes' verdict."""
+    response = """
+      Property: Is the response good?
+      Rationale: It was good.
+      Verdict: yes
+      """
+    parsed = DefaultAutoRaterResponseParser().parse(response)
+    assert len(parsed) == 1
+    assert parsed[0].property_text == "Is the response good?"
+    assert parsed[0].rationale == "It was good."
+    assert parsed[0].score == 1.0
+
+  def test_parse_auto_rater_response_with_single_no_verdict(self):
+    """Tests _parse_auto_rater_response with a single 'no' verdict."""
+    response = """
+      Property: Is the response bad?
+      Rationale: It was bad.
+      Verdict: no
+      """
+    parsed = DefaultAutoRaterResponseParser().parse(response)
+    assert len(parsed) == 1
+    assert parsed[0].property_text == "Is the response bad?"
+    assert parsed[0].rationale == "It was bad."
+    assert parsed[0].score == 0.0
+
+  def test_parse_auto_rater_response_with_invalid_verdict(self):
+    """Tests _parse_auto_rater_response with an invalid verdict."""
+    response = """
+      Property: Is it unclear?
+      Rationale: I cannot tell.
+      Verdict: maybe
+      """
+    parsed = DefaultAutoRaterResponseParser().parse(response)
+    assert len(parsed) == 1
+    assert parsed[0].property_text == "Is it unclear?"
+    assert parsed[0].rationale == "I cannot tell."
+    assert parsed[0].score is None
+
+  def test_parse_auto_rater_response_with_multiple_verdicts(self):
+    """Tests _parse_auto_rater_response with multiple verdicts."""
+    response = """
+      Property: Is the response good?
+      Rationale: It was good.
+      Verdict: yes
+
+      Property: Is the response bad?
+      Rationale: It was not bad.
+      Verdict: no
+      """
+    parsed = DefaultAutoRaterResponseParser().parse(response)
+    assert len(parsed) == 2
+    assert parsed[0].property_text == "Is the response good?"
+    assert parsed[0].rationale == "It was good."
+    assert parsed[0].score == 1.0
+    assert parsed[1].property_text == "Is the response bad?"
+    assert parsed[1].rationale == "It was not bad."
+    assert parsed[1].score == 0.0
+
+  def test_parse_auto_rater_response_with_incomplete_entry(self):
+    """Tests _parse_auto_rater_response with an incomplete entry."""
+    response = """
+      Property: Is the response good?
+      Rationale: It was good.
+      Verdict: yes
+
+      Property: Is the response bad?
+      Rationale: It was not bad.
+      """  # Missing Verdict
+    parsed = DefaultAutoRaterResponseParser().parse(response)
+    assert len(parsed) == 1  # zip will only create one item
+    assert parsed[0].property_text == "Is the response good?"
+
+  def test_parse_auto_rater_response_with_case_insensitive_verdict(self):
+    """Tests _parse_auto_rater_response is case-insensitive for verdicts."""
+    response = """
+      Property: Is the response good?
+      Rationale: It was good.
+      Verdict: Yes
+      Property: Is the response bad?
+      Rationale: It was bad.
+      Verdict: NO
+      """
+    parsed = DefaultAutoRaterResponseParser().parse(response)
+    assert len(parsed) == 2
+    assert parsed[0].score == 1.0
+    assert parsed[1].score == 0.0
+
+
+class TestMajorityVotePerInvocationResultsAggregator:
+
+  def test_aggregate_per_invocation_samples_with_no_rubric_scores(
+      self,
+  ):
+    """Tests aggregation when samples have no rubric scores."""
+    samples = [
+        _create_per_invocation_result([]),
+        _create_per_invocation_result([]),
+    ]
+
+    result = MajorityVotePerInvocationResultsAggregator().aggregate(
+        samples, threshold=0.5
+    )
+
+    assert result.score is None
+    assert result.rubric_scores == []
+
+  def test_aggregate_per_invocation_samples_with_majority_positive(
+      self,
+  ):
+    """Tests aggregation with a majority of positive scores."""
+    samples = [
+        _create_per_invocation_result([RubricScore(rubric_id="1", score=1.0)]),
+        _create_per_invocation_result([RubricScore(rubric_id="1", score=1.0)]),
+        _create_per_invocation_result([RubricScore(rubric_id="1", score=0.0)]),
+    ]
+
+    result = MajorityVotePerInvocationResultsAggregator().aggregate(
+        samples, threshold=0.5
+    )
+
+    assert result.score == 1.0
+    assert len(result.rubric_scores) == 1
+    assert result.rubric_scores[0].rubric_id == "1"
+    assert result.rubric_scores[0].score == 1.0
+
+  def test_aggregate_per_invocation_samples_with_majority_negative(
+      self,
+  ):
+    """Tests aggregation with a majority of negative scores."""
+    samples = [
+        _create_per_invocation_result([RubricScore(rubric_id="1", score=1.0)]),
+        _create_per_invocation_result([RubricScore(rubric_id="1", score=0.0)]),
+        _create_per_invocation_result([RubricScore(rubric_id="1", score=0.0)]),
+    ]
+
+    result = MajorityVotePerInvocationResultsAggregator().aggregate(
+        samples, threshold=0.5
+    )
+
+    assert result.score == 0.0
+    assert len(result.rubric_scores) == 1
+    assert result.rubric_scores[0].rubric_id == "1"
+    assert result.rubric_scores[0].score == 0.0
+
+  def test_aggregate_per_invocation_samples_with_tie_verdicts(
+      self,
+  ):
+    """Tests aggregation with a tie, where negative should win."""
+    samples = [
+        _create_per_invocation_result([RubricScore(rubric_id="1", score=1.0)]),
+        _create_per_invocation_result([RubricScore(rubric_id="1", score=0.0)]),
+    ]
+
+    result = MajorityVotePerInvocationResultsAggregator().aggregate(
+        samples, threshold=0.5
+    )
+
+    assert result.score == 0.0
+    assert len(result.rubric_scores) == 1
+    assert result.rubric_scores[0].rubric_id == "1"
+    assert result.rubric_scores[0].score == 0.0
+
+  def test_aggregate_per_invocation_samples_with_all_none_scores(
+      self,
+  ):
+    """Tests aggregation when all samples have a score of None."""
+    samples = [
+        _create_per_invocation_result(
+            [RubricScore(rubric_id="1", score=None, rationale="r1")]
+        ),
+        _create_per_invocation_result(
+            [RubricScore(rubric_id="1", score=None, rationale="r2")]
+        ),
+    ]
+
+    result = MajorityVotePerInvocationResultsAggregator().aggregate(
+        samples, threshold=0.5
+    )
+
+    assert result.score is None
+    assert len(result.rubric_scores) == 1
+    assert result.rubric_scores[0].rubric_id == "1"
+    assert result.rubric_scores[0].score is None
+    assert result.rubric_scores[0].rationale == "r1"
+
+  def test_aggregate_per_invocation_samples_with_multiple_rubrics(
+      self,
+  ):
+    """Tests aggregation with multiple rubrics."""
+    samples = [
+        _create_per_invocation_result([
+            RubricScore(rubric_id="1", score=1.0),
+            RubricScore(rubric_id="2", score=0.0),
+        ]),
+        _create_per_invocation_result([
+            RubricScore(rubric_id="1", score=1.0),
+            RubricScore(rubric_id="2", score=0.0),
+        ]),
+        _create_per_invocation_result([
+            RubricScore(rubric_id="1", score=0.0),
+            RubricScore(rubric_id="2", score=1.0),
+        ]),
+    ]
+
+    result = MajorityVotePerInvocationResultsAggregator().aggregate(
+        samples, threshold=0.5
+    )
+
+    assert result.score == 0.5
+    assert len(result.rubric_scores) == 2
+    rubric1_score = next(
+        (s for s in result.rubric_scores if s.rubric_id == "1"), None
+    )
+    rubric2_score = next(
+        (s for s in result.rubric_scores if s.rubric_id == "2"), None
+    )
+    assert rubric1_score is not None
+    assert rubric1_score.score == 1.0
+    assert rubric2_score is not None
+    assert rubric2_score.score == 0.0
+
+
+class TestMeanInvocationResultsSummarizer:
+  """Test cases for MeanInvocationResultsSummarizer."""
+
+  def test_summarize_with_empty_list(
+      self,
+  ):
+    """Tests aggregate_invocation_results with an empty list."""
+    result = MeanInvocationResultsSummarizer().summarize([], threshold=0.5)
+    assert result.overall_score is None
+    assert result.overall_rubric_scores == []
+    assert result.per_invocation_results == []
+
+  def test_summarize_with_no_rubric_scores(
+      self,
+  ):
+    """Tests aggregate_invocation_results with samples that have no rubric scores."""
+    invocations = [
+        _create_per_invocation_result([]),
+        _create_per_invocation_result([]),
+    ]
+    result = MeanInvocationResultsSummarizer().summarize(
+        invocations, threshold=0.5
+    )
+    assert result.overall_score is None
+    assert result.overall_rubric_scores == []
+    assert result.per_invocation_results == invocations
+
+  def test_summarize_with_single_invocation(
+      self,
+  ):
+    """Tests aggregate_invocation_results with a single invocation result."""
+    invocations = [
+        _create_per_invocation_result([
+            RubricScore(rubric_id="1", score=1.0),
+            RubricScore(rubric_id="2", score=0.0),
+        ])
+    ]
+    result = MeanInvocationResultsSummarizer().summarize(
+        invocations, threshold=0.5
+    )
+    assert result.overall_score == 0.5
+    assert len(result.overall_rubric_scores) == 2
+    rubric1_score = next(
+        s for s in result.overall_rubric_scores if s.rubric_id == "1"
+    )
+    rubric2_score = next(
+        s for s in result.overall_rubric_scores if s.rubric_id == "2"
+    )
+    assert rubric1_score.score == 1.0
+    assert rubric2_score.score == 0.0
+
+  def test_summarize_with_multiple_invocations_single_rubric(
+      self,
+  ):
+    """Tests aggregate_invocation_results with multiple invocations for a single rubric."""
+    invocations = [
+        _create_per_invocation_result([RubricScore(rubric_id="1", score=1.0)]),
+        _create_per_invocation_result([RubricScore(rubric_id="1", score=0.0)]),
+        _create_per_invocation_result([RubricScore(rubric_id="1", score=1.0)]),
+    ]
+    result = MeanInvocationResultsSummarizer().summarize(
+        invocations, threshold=0.5
+    )
+    assert result.overall_score == pytest.approx(2 / 3)
+    assert len(result.overall_rubric_scores) == 1
+    assert result.overall_rubric_scores[0].rubric_id == "1"
+    assert result.overall_rubric_scores[0].score == pytest.approx(2 / 3)
+
+  def test_summarize_with_multiple_invocations_and_rubrics(
+      self,
+  ):
+    """Tests aggregate_invocation_results with multiple invocations and rubrics."""
+    invocations = [
+        _create_per_invocation_result([
+            RubricScore(rubric_id="1", score=1.0),
+            RubricScore(rubric_id="2", score=0.0),
+        ]),
+        _create_per_invocation_result([
+            RubricScore(rubric_id="1", score=0.0),
+            RubricScore(rubric_id="2", score=1.0),
+        ]),
+    ]
+    result = MeanInvocationResultsSummarizer().summarize(
+        invocations, threshold=0.5
+    )
+    assert result.overall_score == 0.5
+    assert len(result.overall_rubric_scores) == 2
+    rubric1_score = next(
+        s for s in result.overall_rubric_scores if s.rubric_id == "1"
+    )
+    rubric2_score = next(
+        s for s in result.overall_rubric_scores if s.rubric_id == "2"
+    )
+    assert rubric1_score.score == 0.5
+    assert rubric2_score.score == 0.5
+
+  def test_summarize_with_none_scores(
+      self,
+  ):
+    """Tests aggregate_invocation_results with some None scores."""
+    invocations = [
+        _create_per_invocation_result([
+            RubricScore(rubric_id="1", score=1.0),
+            RubricScore(rubric_id="2", score=None),
+        ]),
+        _create_per_invocation_result([
+            RubricScore(rubric_id="1", score=0.0),
+            RubricScore(rubric_id="2", score=1.0),
+        ]),
+    ]
+    result = MeanInvocationResultsSummarizer().summarize(
+        invocations, threshold=0.5
+    )
+    assert result.overall_score == pytest.approx(2 / 3)
+    assert len(result.overall_rubric_scores) == 2
+    rubric1_score = next(
+        s for s in result.overall_rubric_scores if s.rubric_id == "1"
+    )
+    rubric2_score = next(
+        s for s in result.overall_rubric_scores if s.rubric_id == "2"
+    )
+    assert rubric1_score.score == 0.5
+    assert rubric2_score.score == 1.0
+
+
+class TestRubricBasedEvaluator:
+  """Tests for RubricBasedEvaluator."""
+
+  @pytest.fixture
+  def evaluator(self) -> FakeRubricBasedEvaluator:
+    """Returns a RubricBasedFinalResponseQualityV1Evaluator."""
+    rubrics = [
+        Rubric(
+            rubric_id="1",
+            rubric_content=RubricContent(text_property="Is the response good?"),
+        ),
+        Rubric(
+            rubric_id="2",
+            rubric_content=RubricContent(text_property="Is the response bad?"),
+        ),
+    ]
+    judge_model_options = JudgeModelOptions(
+        judge_model_config=None,
+        num_samples=3,
+    )
+    criterion = RubricsBasedCriterion(
+        threshold=0.5, rubrics=rubrics, judge_model_options=judge_model_options
+    )
+    metric = EvalMetric(
+        metric_name=PrebuiltMetrics.RUBRIC_BASED_FINAL_RESPONSE_QUALITY_V1.value,
+        threshold=0.5,
+        criterion=criterion,
+    )
+    return FakeRubricBasedEvaluator(metric)
+
+  def test_convert_auto_rater_response_to_score_with_empty_response(
+      self,
+      evaluator: RubricBasedEvaluator,
+  ):
+    """Tests convert_auto_rater_response_to_score with an empty response."""
+    evaluator.create_effective_rubrics_list(None)
+    response = LlmResponse(
+        content=genai_types.Content(parts=[genai_types.Part(text="")])
+    )
+    auto_rater_score = evaluator.convert_auto_rater_response_to_score(response)
+    assert auto_rater_score.score is None
+    assert auto_rater_score.rubric_scores == []
+
+  def test_convert_auto_rater_response_to_score_with_malformed_response(
+      self,
+      evaluator: RubricBasedEvaluator,
+  ):
+    """Tests convert_auto_rater_response_to_score with a malformed response."""
+    evaluator.create_effective_rubrics_list(None)
+    response = LlmResponse(
+        content=genai_types.Content(
+            parts=[genai_types.Part(text="This is not a valid format.")]
+        )
+    )
+    auto_rater_score = evaluator.convert_auto_rater_response_to_score(response)
+    assert auto_rater_score.score is None
+    assert auto_rater_score.rubric_scores == []
+
+  def test_convert_auto_rater_response_to_score_with_none_content(
+      self,
+      evaluator: RubricBasedEvaluator,
+      caplog: pytest.LogCaptureFixture,
+  ):
+    """An empty auto-rater response is scored as empty, not crashed on."""
+    evaluator.create_effective_rubrics_list(None)
+    response = LlmResponse(content=None)
+    with caplog.at_level(logging.WARNING):
+      auto_rater_score = evaluator.convert_auto_rater_response_to_score(
+          response
+      )
+    assert auto_rater_score.score is None
+    assert auto_rater_score.rubric_scores == []
+    assert "empty response" in caplog.text
+
+  def test_convert_auto_rater_response_to_score_warns_on_unparseable(
+      self,
+      evaluator: RubricBasedEvaluator,
+      caplog: pytest.LogCaptureFixture,
+  ):
+    """Auto-rater output that misses the expected format logs a diagnostic."""
+    evaluator.create_effective_rubrics_list(None)
+    response = LlmResponse(
+        content=genai_types.Content(
+            parts=[genai_types.Part(text="**Verdict**: Yes")]
+        )
+    )
+    with caplog.at_level(logging.WARNING):
+      auto_rater_score = evaluator.convert_auto_rater_response_to_score(
+          response
+      )
+    assert auto_rater_score.rubric_scores == []
+    assert "did not match the expected" in caplog.text
+
+  def test_convert_auto_rater_response_to_score_with_mixed_verdicts(
+      self,
+      evaluator: RubricBasedEvaluator,
+  ):
+    """Tests convert_auto_rater_response_to_score with mixed verdicts."""
+    evaluator.create_effective_rubrics_list(None)
+    response_text = """
+    Property: Is the response good?
+    Rationale: It was good.
+    Verdict: yes
+    Property: Is the response bad?
+    Rationale: It was bad.
+    Verdict: no
+    """
+    response = LlmResponse(
+        content=genai_types.Content(
+            parts=[genai_types.Part(text=response_text)]
+        )
+    )
+    auto_rater_score = evaluator.convert_auto_rater_response_to_score(response)
+    assert auto_rater_score.score == 0.5
+    assert len(auto_rater_score.rubric_scores) == 2
+    assert auto_rater_score.rubric_scores[0].score == 1.0
+    assert auto_rater_score.rubric_scores[1].score == 0.0
+
+  def test_convert_auto_rater_response_to_score_with_invalid_verdict(
+      self,
+      evaluator: RubricBasedEvaluator,
+  ):
+    """Tests convert_auto_rater_response_to_score with an invalid verdict."""
+    evaluator.create_effective_rubrics_list(None)
+    response_text = """
+    Property: Is the response good?
+    Rationale: It was good.
+    Verdict: yes
+    Property: Is the response bad?
+    Rationale: I cannot tell.
+    Verdict: invalid
+    """
+    response = LlmResponse(
+        content=genai_types.Content(
+            parts=[genai_types.Part(text=response_text)]
+        )
+    )
+    auto_rater_score = evaluator.convert_auto_rater_response_to_score(response)
+    assert auto_rater_score.score == 1.0
+    assert len(auto_rater_score.rubric_scores) == 2
+    assert auto_rater_score.rubric_scores[0].score == 1.0
+    assert auto_rater_score.rubric_scores[1].score is None
+
+  def test_convert_auto_rater_response_to_score_with_unknown_property(
+      self,
+      evaluator: RubricBasedEvaluator,
+  ):
+    """Tests convert_auto_rater_response_to_score with an unknown property."""
+    evaluator.create_effective_rubrics_list(None)
+    response_text = """
+    Property: Is the response amazing?
+    Rationale: It was amazing.
+    Verdict: yes
+    """
+    response = LlmResponse(
+        content=genai_types.Content(
+            parts=[genai_types.Part(text=response_text)]
+        )
+    )
+    auto_rater_score = evaluator.convert_auto_rater_response_to_score(response)
+    assert auto_rater_score.score is None
+    assert auto_rater_score.rubric_scores == []
+
+  def test_create_effective_rubrics_list_with_invocation_rubrics(
+      self, evaluator: RubricBasedEvaluator
+  ):
+    invocation_rubrics = [
+        Rubric(
+            rubric_id="3",
+            rubric_content=RubricContent(text_property="Invocation rubric"),
+        )
+    ]
+    evaluator.create_effective_rubrics_list(invocation_rubrics)
+    effective_rubrics = evaluator.get_effective_rubrics_list()
+    assert len(effective_rubrics) == 3
+    assert {r.rubric_id for r in effective_rubrics} == {"1", "2", "3"}
+
+  def test_create_effective_rubrics_list_with_duplicate_invocation_rubric_id(
+      self, evaluator: RubricBasedEvaluator
+  ):
+    invocation_rubrics = [
+        Rubric(
+            rubric_id="1",
+            rubric_content=RubricContent(text_property="Invocation rubric"),
+        )
+    ]
+    with pytest.raises(
+        ValueError, match="Rubric with rubric_id '1' already exists."
+    ):
+      evaluator.create_effective_rubrics_list(invocation_rubrics)
+
+  def test_create_effective_rubrics_list_with_no_invocation_rubrics(
+      self, evaluator: RubricBasedEvaluator
+  ):
+    evaluator.create_effective_rubrics_list(None)
+    effective_rubrics = evaluator.get_effective_rubrics_list()
+    assert len(effective_rubrics) == 2
+    assert {r.rubric_id for r in effective_rubrics} == {"1", "2"}
+
+  def test_get_effective_rubrics_list_before_creation_raises_error(
+      self, evaluator: RubricBasedEvaluator
+  ):
+    with pytest.raises(
+        ValueError, match="Effective rubrics list not initialized."
+    ):
+      evaluator.get_effective_rubrics_list()
+
+  def test_create_effective_rubrics_list_multiple_calls(
+      self, evaluator: RubricBasedEvaluator
+  ):
+    invocation_rubrics1 = [
+        Rubric(
+            rubric_id="3",
+            rubric_content=RubricContent(text_property="Invocation rubric 1"),
+        )
+    ]
+    evaluator.create_effective_rubrics_list(invocation_rubrics1)
+    effective_rubrics1 = evaluator.get_effective_rubrics_list()
+    assert len(effective_rubrics1) == 3
+    assert {r.rubric_id for r in effective_rubrics1} == {"1", "2", "3"}
+
+    invocation_rubrics2 = [
+        Rubric(
+            rubric_id="4",
+            rubric_content=RubricContent(text_property="Invocation rubric 2"),
+        )
+    ]
+    evaluator.create_effective_rubrics_list(invocation_rubrics2)
+    effective_rubrics2 = evaluator.get_effective_rubrics_list()
+    assert len(effective_rubrics2) == 3
+    assert {r.rubric_id for r in effective_rubrics2} == {"1", "2", "4"}
+
+  def test_create_effective_rubrics_filters_by_rubric_type(
+      self, evaluator: RubricBasedEvaluator
+  ):
+    evaluator_with_type = FakeRubricBasedEvaluator(
+        evaluator._eval_metric, rubric_type="TEST_TYPE"
+    )
+    invocation_rubrics = [
+        Rubric(
+            rubric_id="test_type_rubric",
+            rubric_content=RubricContent(text_property="Invocation rubric 1"),
+            type="TEST_TYPE",
+        ),
+        Rubric(
+            rubric_id="other_type_rubric",
+            rubric_content=RubricContent(text_property="Invocation rubric 2"),
+            type="OTHER_TYPE",
+        ),
+    ]
+    evaluator_with_type.create_effective_rubrics_list(invocation_rubrics)
+    effective_rubrics = evaluator_with_type.get_effective_rubrics_list()
+    assert len(effective_rubrics) == 3
+    assert {r.rubric_id for r in effective_rubrics} == {
+        "1",
+        "2",
+        "test_type_rubric",
+    }

tests/unittests/evaluation/test_rubric_based_final_response_quality_v1.py

@@ -0,0 +1,224 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation.app_details import AgentDetails
+from google.adk.evaluation.app_details import AppDetails
+from google.adk.evaluation.eval_case import IntermediateData
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_case import InvocationEvent
+from google.adk.evaluation.eval_case import InvocationEvents
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.eval_metrics import JudgeModelOptions
+from google.adk.evaluation.eval_metrics import PrebuiltMetrics
+from google.adk.evaluation.eval_metrics import RubricsBasedCriterion
+from google.adk.evaluation.eval_rubrics import Rubric
+from google.adk.evaluation.eval_rubrics import RubricContent
+from google.adk.evaluation.eval_rubrics import RubricScore
+from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.evaluator import PerInvocationResult
+from google.adk.evaluation.llm_as_judge_utils import get_average_rubric_score
+from google.adk.evaluation.rubric_based_final_response_quality_v1 import RubricBasedFinalResponseQualityV1Evaluator
+from google.genai import types as genai_types
+import pytest
+
+
+@pytest.fixture
+def evaluator() -> RubricBasedFinalResponseQualityV1Evaluator:
+  """Returns a RubricBasedFinalResponseQualityV1Evaluator."""
+  rubrics = [
+      Rubric(
+          rubric_id="1",
+          rubric_content=RubricContent(text_property="Is the response good?"),
+      ),
+      Rubric(
+          rubric_id="2",
+          rubric_content=RubricContent(text_property="Is the response bad?"),
+      ),
+  ]
+  judge_model_options = JudgeModelOptions(
+      judge_model_config=None,
+      num_samples=3,
+  )
+  criterion = RubricsBasedCriterion(
+      threshold=0.5, rubrics=rubrics, judge_model_options=judge_model_options
+  )
+  metric = EvalMetric(
+      metric_name=PrebuiltMetrics.RUBRIC_BASED_FINAL_RESPONSE_QUALITY_V1.value,
+      threshold=0.5,
+      criterion=criterion,
+  )
+  return RubricBasedFinalResponseQualityV1Evaluator(metric)
+
+
+def _create_per_invocation_result(
+    rubric_scores: list[RubricScore],
+) -> PerInvocationResult:
+  """Helper to create a PerInvocationResult."""
+  return PerInvocationResult(
+      actual_invocation=Invocation(
+          user_content=genai_types.Content(
+              parts=[genai_types.Part(text="part_1")]
+          )
+      ),
+      expected_invocation=Invocation(
+          user_content=genai_types.Content(
+              parts=[genai_types.Part(text="part_2")]
+          )
+      ),
+      score=get_average_rubric_score(rubric_scores),
+      rubric_scores=rubric_scores,
+      eval_status=EvalStatus.NOT_EVALUATED,
+  )
+
+
+def test_format_auto_rater_prompt_with_basic_invocation(
+    evaluator: RubricBasedFinalResponseQualityV1Evaluator,
+):
+  """Tests format_auto_rater_prompt with a basic invocation."""
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="User input here.")]
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text="Final agent response.")]
+      ),
+  )
+  prompt = evaluator.format_auto_rater_prompt(invocation, None)
+
+  assert "User input here." in prompt
+  assert "Final agent response." in prompt
+  assert "Is the response good?" in prompt
+  assert "Is the response bad?" in prompt
+  assert "<developer_instructions>\n  \n  </developer_instructions>" in prompt
+  assert (
+      "<available_tools>\n  Agent has no tools.\n  </available_tools>" in prompt
+  )
+  assert (
+      "<response_steps>\n  No intermediate steps were taken.\n "
+      " </response_steps>"
+  ) in prompt
+
+
+def test_format_auto_rater_prompt_with_app_details(
+    evaluator: RubricBasedFinalResponseQualityV1Evaluator,
+):
+  """Tests format_auto_rater_prompt with app_details in invocation."""
+  tool = genai_types.Tool(
+      function_declarations=[
+          genai_types.FunctionDeclaration(
+              name="test_func", description="A test function."
+          )
+      ]
+  )
+  app_details = AppDetails(
+      agent_details={
+          "agent1": AgentDetails(
+              name="agent1",
+              instructions="This is an agent instruction.",
+              tool_declarations=[tool],
+          )
+      },
+  )
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="User input here.")]
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text="Final agent response.")]
+      ),
+      app_details=app_details,
+      intermediate_data=InvocationEvents(
+          invocation_events=[InvocationEvent(author="agent1", content=None)]
+      ),
+  )
+  prompt = evaluator.format_auto_rater_prompt(invocation, None)
+
+  assert "This is an agent instruction." in prompt
+  assert '"name": "test_func"' in prompt
+  assert '"description": "A test function."' in prompt
+
+
+def test_format_auto_rater_prompt_with_intermediate_data(
+    evaluator: RubricBasedFinalResponseQualityV1Evaluator,
+):
+  """Tests format_auto_rater_prompt with intermediate_data in invocation."""
+  tool_call = genai_types.FunctionCall(
+      name="test_func", args={"arg1": "val1"}, id="call1"
+  )
+  tool_response = genai_types.FunctionResponse(
+      name="test_func", response={"result": "ok"}, id="call1"
+  )
+  intermediate_data = IntermediateData(
+      tool_uses=[tool_call], tool_responses=[tool_response]
+  )
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="User input here.")]
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text="Final agent response.")]
+      ),
+      intermediate_data=intermediate_data,
+  )
+  prompt = evaluator.format_auto_rater_prompt(invocation, None)
+
+  assert '"step": 0' in prompt
+  assert '"tool_call":' in prompt
+  assert '"name": "test_func"' in prompt
+  assert '"tool_response":' in prompt
+  assert '"result": "ok"' in prompt
+
+
+def test_format_auto_rater_prompt_with_app_details_no_tools(
+    evaluator: RubricBasedFinalResponseQualityV1Evaluator,
+):
+  """Tests format_auto_rater_prompt with app_details but no tools."""
+  app_details = AppDetails(
+      agent_details={
+          "agent1": AgentDetails(name="agent1", tool_declarations=[])
+      },
+  )
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="User input here.")]
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text="Final agent response.")]
+      ),
+      app_details=app_details,
+  )
+  prompt = evaluator.format_auto_rater_prompt(invocation, None)
+
+  assert '"tool_declarations": {\n    "agent1": []\n  }' in prompt
+
+
+def test_format_auto_rater_prompt_with_intermediate_data_no_tools(
+    evaluator: RubricBasedFinalResponseQualityV1Evaluator,
+):
+  """Tests format_auto_rater_prompt with intermediate_data but no tool calls."""
+  intermediate_data = IntermediateData(tool_uses=[], tool_responses=[])
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="User input here.")]
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text="Final agent response.")]
+      ),
+      intermediate_data=intermediate_data,
+  )
+  prompt = evaluator.format_auto_rater_prompt(invocation, None)
+
+  assert "No intermediate steps were taken." in prompt

tests/unittests/evaluation/test_rubric_based_multi_turn_trajectory_evaluator.py

@@ -0,0 +1,298 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation.app_details import AgentDetails
+from google.adk.evaluation.app_details import AppDetails
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_case import InvocationEvent
+from google.adk.evaluation.eval_case import InvocationEvents
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.eval_metrics import JudgeModelOptions
+from google.adk.evaluation.eval_metrics import PrebuiltMetrics
+from google.adk.evaluation.eval_metrics import RubricsBasedCriterion
+from google.adk.evaluation.eval_rubrics import Rubric
+from google.adk.evaluation.eval_rubrics import RubricContent
+from google.adk.evaluation.rubric_based_multi_turn_trajectory_evaluator import RubricBasedMultiTurnTrajectoryEvaluator
+from google.genai import types as genai_types
+import pytest
+
+_RUBRICS = [
+    Rubric(
+        rubric_id="1",
+        rubric_content=RubricContent(
+            text_property="The agent uses the correct tool."
+        ),
+        type="TOOL_USAGE",
+    ),
+    Rubric(
+        rubric_id="2",
+        rubric_content=RubricContent(
+            text_property="The agent fulfills the user intent."
+        ),
+        type="FULFILL_USER_INTENT",
+    ),
+]
+
+
+def _make_evaluator(
+    rubrics: list[Rubric] | None = None,
+) -> RubricBasedMultiTurnTrajectoryEvaluator:
+  """Helper to build an evaluator with the given rubrics."""
+  rubrics = rubrics or _RUBRICS
+  criterion = RubricsBasedCriterion(
+      threshold=0.5,
+      rubrics=rubrics,
+      judge_model_options=JudgeModelOptions(
+          judge_model_config=None,
+          num_samples=3,
+      ),
+  )
+  metric = EvalMetric(
+      metric_name=PrebuiltMetrics.RUBRIC_BASED_MULTI_TURN_TRAJECTORY_QUALITY_V1.value,
+      threshold=0.5,
+      criterion=criterion,
+  )
+  return RubricBasedMultiTurnTrajectoryEvaluator(metric)
+
+
+def _make_invocation(
+    user_text: str,
+    agent_text: str | None = None,
+    invocation_id: str = "",
+    rubrics: list[Rubric] | None = None,
+    app_details: AppDetails | None = None,
+    intermediate_data: InvocationEvents | None = None,
+) -> Invocation:
+  """Helper to build an Invocation."""
+  return Invocation(
+      invocation_id=invocation_id,
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text=user_text)]
+      ),
+      final_response=(
+          genai_types.Content(parts=[genai_types.Part(text=agent_text)])
+          if agent_text
+          else None
+      ),
+      rubrics=rubrics,
+      app_details=app_details,
+      intermediate_data=intermediate_data,
+  )
+
+
+class TestFormatAutoRaterPrompt:
+  """Tests for format_auto_rater_prompt."""
+
+  def test_basic_dialogue_and_rubrics_in_prompt(self):
+    """Tests that user dialogue and rubrics appear in the generated prompt."""
+    evaluator = _make_evaluator()
+    invocation = _make_invocation(
+        user_text="What is the balance?",
+        agent_text="Your balance is $100.",
+        rubrics=_RUBRICS,
+    )
+    # Simulate evaluate_invocations dialogue assembly by setting internal state.
+    evaluator._formatted_dialogue = "USER TURN 1: What is the balance?"
+    evaluator._formatted_instructions = ""
+    evaluator._formatted_tools = ""
+
+    prompt = evaluator.format_auto_rater_prompt(invocation, None)
+
+    assert "USER TURN 1: What is the balance?" in prompt
+    assert "The agent uses the correct tool." in prompt
+    assert "The agent fulfills the user intent." in prompt
+    assert "TOOL_USAGE" in prompt
+    assert "FULFILL_USER_INTENT" in prompt
+
+  def test_prompt_includes_agent_instructions_and_tools(self):
+    """Tests that agent instructions and tools are inserted into the prompt."""
+    evaluator = _make_evaluator()
+    invocation = _make_invocation(
+        user_text="Transfer funds",
+        rubrics=_RUBRICS,
+    )
+    evaluator._formatted_dialogue = "USER TURN 1: Transfer funds"
+    evaluator._formatted_instructions = (
+        "Agent banking_agent Instructions:\nYou are a banking assistant."
+    )
+    evaluator._formatted_tools = (
+        "Agent: banking_agent\n- transfer_funds: Transfer money between"
+        " accounts."
+    )
+
+    prompt = evaluator.format_auto_rater_prompt(invocation, None)
+
+    assert "You are a banking assistant." in prompt
+    assert "transfer_funds" in prompt
+
+
+class TestDialogueAssembly:
+  """Tests for the dialogue assembly logic in evaluate_invocations.
+
+  These test the internal dialogue construction by calling evaluate_invocations
+  and inspecting self._formatted_dialogue.
+  """
+
+  @pytest.fixture
+  def evaluator(self):
+    return _make_evaluator()
+
+  @pytest.mark.asyncio
+  async def test_single_turn_user_and_agent(self, evaluator):
+    """Tests that a single turn assembles user and agent dialogue."""
+    invocations = [
+        _make_invocation(
+            user_text="Hello",
+            agent_text="Hi there!",
+            invocation_id="agent1",
+            rubrics=_RUBRICS,
+        ),
+    ]
+    # We need to mock the super().evaluate_invocations call since it calls
+    # the LLM. Instead, we just test the dialogue assembly part directly.
+    evaluator._formatted_dialogue = None
+
+    # Manually run the dialogue assembly portion
+    evaluator._assemble_dialogue_history(invocations)
+
+    assert "USER TURN 1: Hello" in evaluator._formatted_dialogue
+    assert "AGENT (agent) TURN 1: Hi there!" in evaluator._formatted_dialogue
+
+  @pytest.mark.asyncio
+  async def test_multi_turn_dialogue(self, evaluator):
+    """Tests dialogue assembly across multiple turns."""
+    invocations = [
+        _make_invocation(
+            user_text="Check my balance",
+            agent_text="Your balance is $100.",
+            invocation_id="agent1",
+            rubrics=_RUBRICS,
+        ),
+        _make_invocation(
+            user_text="Transfer $50",
+            agent_text="Transfer complete.",
+            invocation_id="agent1",
+            rubrics=_RUBRICS,
+        ),
+    ]
+    evaluator._assemble_dialogue_history(invocations)
+
+    assert "USER TURN 1: Check my balance" in evaluator._formatted_dialogue
+    assert (
+        "AGENT (agent) TURN 1: Your balance is $100."
+        in evaluator._formatted_dialogue
+    )
+    assert "USER TURN 2: Transfer $50" in evaluator._formatted_dialogue
+    assert (
+        "AGENT (agent) TURN 2: Transfer complete."
+        in evaluator._formatted_dialogue
+    )
+
+  @pytest.mark.asyncio
+  async def test_intermediate_events_with_function_calls(self, evaluator):
+    """Tests that intermediate function calls and responses appear in dialogue."""
+    tool_call_part = genai_types.Part(
+        function_call=genai_types.FunctionCall(
+            name="get_balance", args={"account_id": "123"}
+        )
+    )
+    tool_response_part = genai_types.Part(
+        function_response=genai_types.FunctionResponse(
+            name="get_balance", response={"balance": 100}
+        )
+    )
+    intermediate_data = InvocationEvents(
+        invocation_events=[
+            InvocationEvent(
+                author="banking_agent",
+                content=genai_types.Content(parts=[tool_call_part]),
+            ),
+            InvocationEvent(
+                author="banking_agent",
+                content=genai_types.Content(parts=[tool_response_part]),
+            ),
+        ]
+    )
+    invocations = [
+        _make_invocation(
+            user_text="What is my balance?",
+            agent_text="Your balance is $100.",
+            invocation_id="banking_agent",
+            rubrics=_RUBRICS,
+            intermediate_data=intermediate_data,
+        ),
+    ]
+    evaluator._assemble_dialogue_history(invocations)
+
+    assert "get_balance" in evaluator._formatted_dialogue
+    assert '"account_id": "123"' in evaluator._formatted_dialogue
+    assert '"balance": 100' in evaluator._formatted_dialogue
+
+  @pytest.mark.asyncio
+  async def test_app_details_instructions_and_tools(self, evaluator):
+    """Tests that app_details instructions and tools are captured."""
+    tool = genai_types.Tool(
+        function_declarations=[
+            genai_types.FunctionDeclaration(
+                name="transfer_funds",
+                description="Transfer money between accounts.",
+            )
+        ]
+    )
+    app_details = AppDetails(
+        agent_details={
+            "banking_agent": AgentDetails(
+                name="banking_agent",
+                instructions="You are a banking assistant.",
+                tool_declarations=[tool],
+            )
+        },
+    )
+    invocations = [
+        _make_invocation(
+            user_text="Transfer $50",
+            agent_text="Done.",
+            invocation_id="banking_agent",
+            rubrics=_RUBRICS,
+            app_details=app_details,
+        ),
+    ]
+    evaluator._assemble_dialogue_history(invocations)
+
+    assert "You are a banking assistant." in evaluator._formatted_instructions
+    assert "transfer_funds" in evaluator._formatted_tools
+    assert "Transfer money between accounts." in evaluator._formatted_tools
+
+  @pytest.mark.asyncio
+  async def test_invocation_without_user_content(self, evaluator):
+    """Tests that invocations with no user text parts are handled gracefully."""
+    invocations = [
+        Invocation(
+            user_content=genai_types.Content(parts=[]),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="Agent response.")]
+            ),
+            invocation_id="agent1",
+            rubrics=_RUBRICS,
+        ),
+    ]
+    evaluator._assemble_dialogue_history(invocations)
+
+    # No user turn should appear, but agent turn should
+    assert "USER TURN" not in evaluator._formatted_dialogue
+    assert (
+        "AGENT (agent) TURN 1: Agent response." in evaluator._formatted_dialogue
+    )

tests/unittests/evaluation/test_rubric_based_tool_use_quality_v1.py

@@ -0,0 +1,138 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.evaluation.app_details import AgentDetails
+from google.adk.evaluation.app_details import AppDetails
+from google.adk.evaluation.eval_case import IntermediateData
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.eval_metrics import JudgeModelOptions
+from google.adk.evaluation.eval_metrics import PrebuiltMetrics
+from google.adk.evaluation.eval_metrics import RubricsBasedCriterion
+from google.adk.evaluation.eval_rubrics import Rubric
+from google.adk.evaluation.eval_rubrics import RubricContent
+from google.adk.evaluation.rubric_based_tool_use_quality_v1 import RubricBasedToolUseV1Evaluator
+from google.genai import types as genai_types
+import pytest
+
+
+@pytest.fixture
+def evaluator() -> RubricBasedToolUseV1Evaluator:
+  """Returns a RubricBasedToolUseV1Evaluator."""
+  rubrics = [
+      Rubric(
+          rubric_id="1",
+          rubric_content=RubricContent(
+              text_property="Did the agent use the correct tool?"
+          ),
+      ),
+      Rubric(
+          rubric_id="2",
+          rubric_content=RubricContent(
+              text_property="Were the tool parameters correct?"
+          ),
+      ),
+  ]
+  judge_model_options = JudgeModelOptions(
+      judge_model_config=None,
+      num_samples=3,
+  )
+  criterion = RubricsBasedCriterion(
+      threshold=0.5, rubrics=rubrics, judge_model_options=judge_model_options
+  )
+  metric = EvalMetric(
+      metric_name=PrebuiltMetrics.RUBRIC_BASED_TOOL_USE_QUALITY_V1.value,
+      threshold=0.5,
+      criterion=criterion,
+  )
+  return RubricBasedToolUseV1Evaluator(metric)
+
+
+def test_format_auto_rater_prompt_with_basic_invocation(
+    evaluator: RubricBasedToolUseV1Evaluator,
+):
+  """Tests format_auto_rater_prompt with a basic invocation."""
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="User input here.")]
+      ),
+  )
+  prompt = evaluator.format_auto_rater_prompt(invocation, None)
+
+  assert "User input here." in prompt
+  assert "Did the agent use the correct tool?" in prompt
+  assert "Were the tool parameters correct?" in prompt
+  assert "<available_tools>\nAgent has no tools.\n</available_tools>" in prompt
+  assert "<response>\nNo intermediate steps were taken.\n</response>" in prompt
+
+
+def test_format_auto_rater_prompt_with_app_details(
+    evaluator: RubricBasedToolUseV1Evaluator,
+):
+  """Tests format_auto_rater_prompt with app_details in invocation."""
+  tool = genai_types.Tool(
+      function_declarations=[
+          genai_types.FunctionDeclaration(
+              name="test_func", description="A test function."
+          )
+      ]
+  )
+  app_details = AppDetails(
+      agent_details={
+          "agent1": AgentDetails(
+              name="agent1",
+              tool_declarations=[tool],
+          )
+      },
+  )
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="User input here.")]
+      ),
+      app_details=app_details,
+  )
+  prompt = evaluator.format_auto_rater_prompt(invocation, None)
+
+  assert '"name": "test_func"' in prompt
+  assert '"description": "A test function."' in prompt
+
+
+def test_format_auto_rater_prompt_with_intermediate_data(
+    evaluator: RubricBasedToolUseV1Evaluator,
+):
+  """Tests format_auto_rater_prompt with intermediate_data in invocation."""
+  tool_call = genai_types.FunctionCall(
+      name="test_func", args={"arg1": "val1"}, id="call1"
+  )
+  tool_response = genai_types.FunctionResponse(
+      name="test_func", response={"result": "ok"}, id="call1"
+  )
+  intermediate_data = IntermediateData(
+      tool_uses=[tool_call], tool_responses=[tool_response]
+  )
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="User input here.")]
+      ),
+      intermediate_data=intermediate_data,
+  )
+  prompt = evaluator.format_auto_rater_prompt(invocation, None)
+
+  assert '"step": 0' in prompt
+  assert '"tool_call":' in prompt
+  assert '"name": "test_func"' in prompt
+  assert '"tool_response":' in prompt
+  assert '"result": "ok"' in prompt

tests/unittests/evaluation/test_safety_evaluator.py

@@ -0,0 +1,78 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for the Response Evaluator."""
+
+from google.adk.dependencies.vertexai import vertexai
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.eval_metrics import PrebuiltMetrics
+from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.safety_evaluator import SafetyEvaluatorV1
+from google.genai import types as genai_types
+
+vertexai_types = vertexai.types
+
+
+class TestSafetyEvaluatorV1:
+  """A class to help organize "patch" that are applicable to all tests."""
+
+  def test_evaluate_invocations_coherence_metric_passed(self, mocker):
+    """Test evaluate_invocations function for Coherence metric."""
+    mock_perform_eval = mocker.patch(
+        "google.adk.evaluation.vertex_ai_eval_facade._VertexAiEvalFacade._perform_eval"
+    )
+    actual_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[
+                    genai_types.Part(text="This is a test candidate response.")
+                ]
+            ),
+        )
+    ]
+    expected_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test reference.")]
+            ),
+        )
+    ]
+    evaluator = SafetyEvaluatorV1(
+        eval_metric=EvalMetric(threshold=0.8, metric_name="safety")
+    )
+    # Mock the return value of _perform_eval
+    mock_perform_eval.return_value = vertexai_types.EvaluationResult(
+        summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.9)],
+        eval_case_results=[],
+    )
+
+    evaluation_result = evaluator.evaluate_invocations(
+        actual_invocations, expected_invocations
+    )
+
+    assert evaluation_result.overall_score == 0.9
+    assert evaluation_result.overall_eval_status == EvalStatus.PASSED
+    mock_perform_eval.assert_called_once()
+    _, mock_kwargs = mock_perform_eval.call_args
+    # Compare the names of the metrics.
+    assert [m.name for m in mock_kwargs["metrics"]] == [
+        vertexai_types.PrebuiltMetric.SAFETY.name
+    ]

tests/unittests/evaluation/test_trajectory_evaluator.py

@@ -0,0 +1,525 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Testings for the Trajectory Evaluator."""
+
+from google.adk.evaluation.eval_case import IntermediateData
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_case import InvocationEvent
+from google.adk.evaluation.eval_case import InvocationEvents
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.eval_metrics import PrebuiltMetrics
+from google.adk.evaluation.eval_metrics import ToolTrajectoryCriterion
+from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.trajectory_evaluator import TrajectoryEvaluator
+from google.genai import types as genai_types
+from pydantic import ValidationError
+import pytest
+
+_USER_CONTENT = genai_types.Content(
+    parts=[genai_types.Part(text="User input here.")]
+)
+
+
+def test_tool_trajectory_criterion_accepts_string_match_type():
+  criterion = ToolTrajectoryCriterion(threshold=0.5, match_type="in_order")
+  assert criterion.match_type == ToolTrajectoryCriterion.MatchType.IN_ORDER
+
+
+@pytest.mark.parametrize(
+    ("match_type", "expected"),
+    [
+        ("exact", ToolTrajectoryCriterion.MatchType.EXACT),
+        ("EXACT", ToolTrajectoryCriterion.MatchType.EXACT),
+        (" exact ", ToolTrajectoryCriterion.MatchType.EXACT),
+        ("in order", ToolTrajectoryCriterion.MatchType.IN_ORDER),
+        ("IN ORDER", ToolTrajectoryCriterion.MatchType.IN_ORDER),
+        ("In OrDeR", ToolTrajectoryCriterion.MatchType.IN_ORDER),
+        ("in-order", ToolTrajectoryCriterion.MatchType.IN_ORDER),
+        ("IN-ORDER", ToolTrajectoryCriterion.MatchType.IN_ORDER),
+        ("in_order", ToolTrajectoryCriterion.MatchType.IN_ORDER),
+        ("any order", ToolTrajectoryCriterion.MatchType.ANY_ORDER),
+        ("ANY ORDER", ToolTrajectoryCriterion.MatchType.ANY_ORDER),
+        ("any-order", ToolTrajectoryCriterion.MatchType.ANY_ORDER),
+        ("ANY-ORDER", ToolTrajectoryCriterion.MatchType.ANY_ORDER),
+        ("any_order", ToolTrajectoryCriterion.MatchType.ANY_ORDER),
+    ],
+)
+def test_tool_trajectory_criterion_normalizes_string_match_type(
+    match_type: str, expected: ToolTrajectoryCriterion.MatchType
+):
+  criterion = ToolTrajectoryCriterion(threshold=0.5, match_type=match_type)
+  assert criterion.match_type == expected
+
+
+def test_tool_trajectory_criterion_rejects_unknown_string_match_type():
+  with pytest.raises(ValidationError):
+    ToolTrajectoryCriterion(threshold=0.5, match_type="random string")
+
+
+def test_trajectory_evaluator_accepts_string_match_type_from_eval_metric_dict():
+  eval_metric = EvalMetric(
+      threshold=0.5,
+      metric_name=PrebuiltMetrics.TOOL_TRAJECTORY_AVG_SCORE.value,
+      criterion={
+          "threshold": 0.5,
+          "match_type": "ANY_ORDER",
+      },
+  )
+  evaluator = TrajectoryEvaluator(eval_metric=eval_metric)
+
+  tool_call1 = genai_types.FunctionCall(name="test_func1", args={})
+  tool_call2 = genai_types.FunctionCall(name="test_func2", args={})
+
+  actual_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[tool_call1, tool_call2]),
+  )
+  expected_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[tool_call2, tool_call1]),
+  )
+
+  result = evaluator.evaluate_invocations(
+      [actual_invocation], [expected_invocation]
+  )
+  assert result.overall_score == 1.0
+
+
+@pytest.fixture
+def evaluator() -> TrajectoryEvaluator:
+  """Returns a TrajectoryEvaluator."""
+  return TrajectoryEvaluator(
+      eval_metric=EvalMetric(
+          threshold=0.5,
+          metric_name=PrebuiltMetrics.TOOL_TRAJECTORY_AVG_SCORE.value,
+          criterion=ToolTrajectoryCriterion(
+              threshold=0.5,
+              match_type=ToolTrajectoryCriterion.MatchType.EXACT,
+          ),
+      )
+  )
+
+
+def test_evaluate_invocations_equal_tool_calls(evaluator: TrajectoryEvaluator):
+  """Tests evaluate_invocations with equal tool calls."""
+  tool_call = genai_types.FunctionCall(name="test_func", args={"arg1": "val1"})
+  intermediate_data = IntermediateData(tool_uses=[tool_call])
+  invocation = Invocation(
+      user_content=_USER_CONTENT, intermediate_data=intermediate_data
+  )
+  result = evaluator.evaluate_invocations([invocation], [invocation])
+  assert result.overall_score == 1.0
+  assert result.overall_eval_status == EvalStatus.PASSED
+  assert len(result.per_invocation_results) == 1
+  assert result.per_invocation_results[0].score == 1.0
+  assert result.per_invocation_results[0].eval_status == EvalStatus.PASSED
+
+
+def test_evaluate_invocations_different_tool_call_names(
+    evaluator: TrajectoryEvaluator,
+):
+  """Tests evaluate_invocations with different tool call names."""
+  tool_call1 = genai_types.FunctionCall(
+      name="test_func1", args={"arg1": "val1"}
+  )
+  tool_call2 = genai_types.FunctionCall(
+      name="test_func2", args={"arg1": "val1"}
+  )
+  invocation1 = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[tool_call1]),
+  )
+  invocation2 = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[tool_call2]),
+  )
+  result = evaluator.evaluate_invocations([invocation1], [invocation2])
+  assert result.overall_score == 0.0
+  assert result.overall_eval_status == EvalStatus.FAILED
+  assert result.per_invocation_results[0].score == 0.0
+  assert result.per_invocation_results[0].eval_status == EvalStatus.FAILED
+
+
+def test_evaluate_invocations_different_tool_call_args(
+    evaluator: TrajectoryEvaluator,
+):
+  """Tests evaluate_invocations with different tool call args."""
+  tool_call1 = genai_types.FunctionCall(name="test_func", args={"arg1": "val1"})
+  tool_call2 = genai_types.FunctionCall(name="test_func", args={"arg1": "val2"})
+  invocation1 = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[tool_call1]),
+  )
+  invocation2 = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[tool_call2]),
+  )
+  result = evaluator.evaluate_invocations([invocation1], [invocation2])
+  assert result.overall_score == 0.0
+  assert result.overall_eval_status == EvalStatus.FAILED
+  assert result.per_invocation_results[0].score == 0.0
+  assert result.per_invocation_results[0].eval_status == EvalStatus.FAILED
+
+
+def test_evaluate_invocations_different_number_of_tool_calls(
+    evaluator: TrajectoryEvaluator,
+):
+  """Tests evaluate_invocations with different number of tool calls."""
+  tool_call1 = genai_types.FunctionCall(name="test_func", args={"arg1": "val1"})
+  tool_call2 = genai_types.FunctionCall(name="test_func", args={"arg1": "val1"})
+  invocation1 = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[tool_call1]),
+  )
+  invocation2 = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[tool_call1, tool_call2]),
+  )
+  result = evaluator.evaluate_invocations([invocation1], [invocation2])
+  assert result.overall_score == 0.0
+  assert result.overall_eval_status == EvalStatus.FAILED
+  assert result.per_invocation_results[0].score == 0.0
+  assert result.per_invocation_results[0].eval_status == EvalStatus.FAILED
+
+
+def test_evaluate_invocations_no_tool_calls(evaluator: TrajectoryEvaluator):
+  """Tests evaluate_invocations with no tool calls."""
+  invocation = Invocation(
+      user_content=_USER_CONTENT, intermediate_data=IntermediateData()
+  )
+  result = evaluator.evaluate_invocations([invocation], [invocation])
+  assert result.overall_score == 1.0
+  assert result.overall_eval_status == EvalStatus.PASSED
+  assert result.per_invocation_results[0].score == 1.0
+  assert result.per_invocation_results[0].eval_status == EvalStatus.PASSED
+
+
+def test_evaluate_invocations_multiple_invocations(
+    evaluator: TrajectoryEvaluator,
+):
+  """Tests evaluate_invocations with multiple invocations."""
+  tool_call1 = genai_types.FunctionCall(
+      name="test_func1", args={"arg1": "val1"}
+  )
+  tool_call2 = genai_types.FunctionCall(
+      name="test_func2", args={"arg1": "val1"}
+  )
+  inv1_actual = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[tool_call1]),
+  )
+  inv1_expected = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[tool_call1]),
+  )
+  inv2_actual = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[tool_call1]),
+  )
+  inv2_expected = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[tool_call2]),
+  )
+  result = evaluator.evaluate_invocations(
+      [inv1_actual, inv2_actual], [inv1_expected, inv2_expected]
+  )
+  assert result.overall_score == 0.5
+  assert result.overall_eval_status == EvalStatus.PASSED
+  assert len(result.per_invocation_results) == 2
+  assert result.per_invocation_results[0].score == 1.0
+  assert result.per_invocation_results[0].eval_status == EvalStatus.PASSED
+  assert result.per_invocation_results[1].score == 0.0
+  assert result.per_invocation_results[1].eval_status == EvalStatus.FAILED
+
+
+@pytest.fixture
+def in_order_evaluator() -> TrajectoryEvaluator:
+  """Returns a TrajectoryEvaluator for IN_ORDER match."""
+  return TrajectoryEvaluator(
+      eval_metric=EvalMetric(
+          threshold=0.5,
+          metric_name=PrebuiltMetrics.TOOL_TRAJECTORY_AVG_SCORE.value,
+          criterion=ToolTrajectoryCriterion(
+              threshold=0.5,
+              match_type=ToolTrajectoryCriterion.MatchType.IN_ORDER,
+          ),
+      )
+  )
+
+
+def test_evaluate_invocations_in_order_match_with_extra_tool_calls(
+    in_order_evaluator: TrajectoryEvaluator,
+):
+  """Tests evaluate_invocations with IN_ORDER match type and extra tool calls."""
+  t1 = genai_types.FunctionCall(name="t1", args={})
+  t1_1 = genai_types.FunctionCall(name="t1_1", args={})
+  t2 = genai_types.FunctionCall(name="t2", args={})
+  t2_1 = genai_types.FunctionCall(name="t2_1", args={})
+  t3 = genai_types.FunctionCall(name="t3", args={})
+  t3_1 = genai_types.FunctionCall(name="t3_1", args={})
+  actual_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(
+          tool_uses=[t1, t1_1, t2, t2_1, t3, t3_1]
+      ),
+  )
+  expected_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[t1, t2, t3]),
+  )
+  result = in_order_evaluator.evaluate_invocations(
+      [actual_invocation], [expected_invocation]
+  )
+  assert result.overall_score == 1.0
+  assert result.overall_eval_status == EvalStatus.PASSED
+  assert result.per_invocation_results[0].score == 1.0
+  assert result.per_invocation_results[0].eval_status == EvalStatus.PASSED
+
+
+def test_evaluate_invocations_in_order_match_fails_with_missing_tool_call(
+    in_order_evaluator: TrajectoryEvaluator,
+):
+  """Tests evaluate_invocations with IN_ORDER match type and missing tool call."""
+  t1 = genai_types.FunctionCall(name="t1", args={})
+  t1_1 = genai_types.FunctionCall(name="t1_1", args={})
+  t2 = genai_types.FunctionCall(name="t2", args={})
+  t2_1 = genai_types.FunctionCall(name="t2_1", args={})
+  t3_1 = genai_types.FunctionCall(name="t3_1", args={})
+  t4 = genai_types.FunctionCall(name="t4", args={})
+  actual_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[t1, t1_1, t2, t2_1, t3_1]),
+  )
+  expected_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[t1, t2, t4]),
+  )
+  result = in_order_evaluator.evaluate_invocations(
+      [actual_invocation], [expected_invocation]
+  )
+  assert result.overall_score == 0.0
+  assert result.overall_eval_status == EvalStatus.FAILED
+  assert result.per_invocation_results[0].score == 0.0
+  assert result.per_invocation_results[0].eval_status == EvalStatus.FAILED
+
+
+def test_evaluate_invocations_in_order_match_fails_with_wrong_order(
+    in_order_evaluator: TrajectoryEvaluator,
+):
+  """Tests evaluate_invocations with IN_ORDER match type and wrong order."""
+  t1 = genai_types.FunctionCall(name="t1", args={})
+  t2 = genai_types.FunctionCall(name="t2", args={})
+  t3 = genai_types.FunctionCall(name="t3", args={})
+  actual_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[t1, t3, t2]),
+  )
+  expected_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[t1, t2, t3]),
+  )
+  result = in_order_evaluator.evaluate_invocations(
+      [actual_invocation], [expected_invocation]
+  )
+  assert result.overall_score == 0.0
+  assert result.overall_eval_status == EvalStatus.FAILED
+  assert result.per_invocation_results[0].score == 0.0
+  assert result.per_invocation_results[0].eval_status == EvalStatus.FAILED
+
+
+@pytest.fixture
+def any_order_evaluator() -> TrajectoryEvaluator:
+  """Returns a TrajectoryEvaluator for ANY_ORDER match."""
+  return TrajectoryEvaluator(
+      eval_metric=EvalMetric(
+          threshold=0.5,
+          metric_name=PrebuiltMetrics.TOOL_TRAJECTORY_AVG_SCORE.value,
+          criterion=ToolTrajectoryCriterion(
+              threshold=0.5,
+              match_type=ToolTrajectoryCriterion.MatchType.ANY_ORDER,
+          ),
+      )
+  )
+
+
+def test_evaluate_invocations_any_order_match_with_extra_tool_calls_different_order(
+    any_order_evaluator: TrajectoryEvaluator,
+):
+  """Tests evaluate_invocations with ANY_ORDER match type and extra tool calls."""
+  t1 = genai_types.FunctionCall(name="t1", args={})
+  t1_1 = genai_types.FunctionCall(name="t1_1", args={})
+  t2 = genai_types.FunctionCall(name="t2", args={})
+  t2_1 = genai_types.FunctionCall(name="t2_1", args={})
+  t3 = genai_types.FunctionCall(name="t3", args={})
+  t3_1 = genai_types.FunctionCall(name="t3_1", args={})
+  actual_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(
+          tool_uses=[t2, t2_1, t1, t1_1, t3, t3_1]
+      ),
+  )
+  expected_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[t1, t2, t3]),
+  )
+  result = any_order_evaluator.evaluate_invocations(
+      [actual_invocation], [expected_invocation]
+  )
+  assert result.overall_score == 1.0
+  assert result.overall_eval_status == EvalStatus.PASSED
+  assert result.per_invocation_results[0].score == 1.0
+  assert result.per_invocation_results[0].eval_status == EvalStatus.PASSED
+
+
+def test_evaluate_invocations_any_order_match_fails_with_missing_tool_call(
+    any_order_evaluator: TrajectoryEvaluator,
+):
+  """Tests evaluate_invocations with ANY_ORDER match type and missing tool call."""
+  t1 = genai_types.FunctionCall(name="t1", args={})
+  t1_1 = genai_types.FunctionCall(name="t1_1", args={})
+  t2 = genai_types.FunctionCall(name="t2", args={})
+  t2_1 = genai_types.FunctionCall(name="t2_1", args={})
+  t3_1 = genai_types.FunctionCall(name="t3_1", args={})
+  t4 = genai_types.FunctionCall(name="t4", args={})
+  actual_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[t1, t1_1, t2, t2_1, t3_1]),
+  )
+  expected_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[t1, t2, t4]),
+  )
+  result = any_order_evaluator.evaluate_invocations(
+      [actual_invocation], [expected_invocation]
+  )
+  assert result.overall_score == 0.0
+  assert result.overall_eval_status == EvalStatus.FAILED
+  assert result.per_invocation_results[0].score == 0.0
+  assert result.per_invocation_results[0].eval_status == EvalStatus.FAILED
+
+
+def test_evaluate_invocations_any_order_match_with_duplicates(
+    any_order_evaluator: TrajectoryEvaluator,
+):
+  """Tests evaluate_invocations with ANY_ORDER match type with duplicates."""
+  t1 = genai_types.FunctionCall(name="t1", args={})
+  t2 = genai_types.FunctionCall(name="t2", args={})
+  t3 = genai_types.FunctionCall(name="t3", args={})
+  actual_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[t1, t2, t3, t1]),
+  )
+  expected_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[t1, t2, t1]),
+  )
+  result = any_order_evaluator.evaluate_invocations(
+      [actual_invocation], [expected_invocation]
+  )
+  assert result.overall_score == 1.0
+  assert result.overall_eval_status == EvalStatus.PASSED
+  assert result.per_invocation_results[0].score == 1.0
+  assert result.per_invocation_results[0].eval_status == EvalStatus.PASSED
+
+
+def test_evaluate_invocations_any_order_match_fails_with_duplicates_missing(
+    any_order_evaluator: TrajectoryEvaluator,
+):
+  """Tests evaluate_invocations with ANY_ORDER match type with missing duplicates."""
+  t1 = genai_types.FunctionCall(name="t1", args={})
+  t2 = genai_types.FunctionCall(name="t2", args={})
+  t3 = genai_types.FunctionCall(name="t3", args={})
+  actual_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[t1, t2, t3]),
+  )
+  expected_invocation = Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=IntermediateData(tool_uses=[t1, t2, t1]),
+  )
+  result = any_order_evaluator.evaluate_invocations(
+      [actual_invocation], [expected_invocation]
+  )
+  assert result.overall_score == 0.0
+  assert result.overall_eval_status == EvalStatus.FAILED
+  assert result.per_invocation_results[0].score == 0.0
+  assert result.per_invocation_results[0].eval_status == EvalStatus.FAILED
+
+
+def test_evaluate_invocations_no_invocations(evaluator: TrajectoryEvaluator):
+  """Tests evaluate_invocations with no invocations."""
+  result = evaluator.evaluate_invocations([], [])
+  assert result.overall_score is None
+  assert result.overall_eval_status == EvalStatus.NOT_EVALUATED
+  assert not result.per_invocation_results
+
+
+def _make_invocation_events(
+    *tool_calls: genai_types.FunctionCall,
+) -> Invocation:
+  """Returns an Invocation using InvocationEvents intermediate_data format."""
+  return Invocation(
+      user_content=_USER_CONTENT,
+      intermediate_data=InvocationEvents(
+          invocation_events=[
+              InvocationEvent(
+                  author="agent",
+                  content=genai_types.Content(
+                      parts=[genai_types.Part(function_call=tc)]
+                  ),
+              )
+              for tc in tool_calls
+          ]
+      ),
+  )
+
+
+def test_evaluate_invocations_invocation_events_format_exact_match(
+    evaluator: TrajectoryEvaluator,
+):
+  """InvocationEvents intermediate_data format should score 1.0 on exact match.
+
+  Regression test for #5410: tool_trajectory_avg_score returned 0.0 even when
+  tool name and args were identical because function-call events with
+  skip_summarization=True were incorrectly excluded from invocation_events.
+  """
+  tool_call = genai_types.FunctionCall(
+      id="toolu_01", name="execute_sql", args={"query": "SELECT 1"}
+  )
+  expected_tool_call = genai_types.FunctionCall(
+      name="execute_sql", args={"query": "SELECT 1"}
+  )
+  actual = _make_invocation_events(tool_call)
+  expected = _make_invocation_events(expected_tool_call)
+
+  result = evaluator.evaluate_invocations([actual], [expected])
+  assert result.overall_score == 1.0
+  assert result.overall_eval_status == EvalStatus.PASSED
+
+
+def test_evaluate_invocations_invocation_events_format_mismatch(
+    evaluator: TrajectoryEvaluator,
+):
+  """InvocationEvents format should score 0.0 when tool calls differ."""
+  actual = _make_invocation_events(
+      genai_types.FunctionCall(name="tool_a", args={"x": "1"})
+  )
+  expected = _make_invocation_events(
+      genai_types.FunctionCall(name="tool_b", args={"x": "1"})
+  )
+
+  result = evaluator.evaluate_invocations([actual], [expected])
+  assert result.overall_score == 0.0
+  assert result.overall_eval_status == EvalStatus.FAILED

tests/unittests/evaluation/test_vertex_ai_eval_facade.py

@@ -0,0 +1,594 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+"""Tests for the Response Evaluator."""
+import math
+import os
+import random
+
+from google.adk.dependencies.vertexai import vertexai
+from google.adk.evaluation.app_details import AgentDetails
+from google.adk.evaluation.app_details import AppDetails
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_case import InvocationEvent
+from google.adk.evaluation.eval_case import InvocationEvents
+from google.adk.evaluation.evaluator import EvalStatus
+from google.adk.evaluation.vertex_ai_eval_facade import _MultiTurnVertexiAiEvalFacade
+from google.adk.evaluation.vertex_ai_eval_facade import _SingleTurnVertexAiEvalFacade
+from google.adk.evaluation.vertex_ai_eval_facade import _VertexAiEvalFacade
+from google.genai import types as genai_types
+import pandas as pd
+import pytest
+
+vertexai_types = vertexai.types
+
+
+class TestSingleTurnVertexAiEvalFacade:
+  """A class to help organize "patch" that are applicable to all tests."""
+
+  def test_evaluate_invocations_metric_passed(self, mocker):
+    """Test evaluate_invocations function for a metric."""
+    mocker.patch("google.adk.dependencies.vertexai.vertexai.Client")
+    mock_perform_eval = mocker.patch(
+        "google.adk.evaluation.vertex_ai_eval_facade._VertexAiEvalFacade._perform_eval"
+    )
+    actual_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[
+                    genai_types.Part(text="This is a test candidate response.")
+                ]
+            ),
+        )
+    ]
+    expected_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test reference.")]
+            ),
+        )
+    ]
+    evaluator = _SingleTurnVertexAiEvalFacade(
+        threshold=0.8, metric_name=vertexai_types.PrebuiltMetric.COHERENCE
+    )
+    # Mock the return value of _perform_eval
+    mock_perform_eval.return_value = vertexai_types.EvaluationResult(
+        summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.9)],
+        eval_case_results=[],
+    )
+
+    evaluation_result = evaluator.evaluate_invocations(
+        actual_invocations, expected_invocations
+    )
+
+    assert evaluation_result.overall_score == 0.9
+    assert evaluation_result.overall_eval_status == EvalStatus.PASSED
+    mock_perform_eval.assert_called_once()
+    _, mock_kwargs = mock_perform_eval.call_args
+    # Compare the names of the metrics.
+    assert [m.name for m in mock_kwargs["metrics"]] == [
+        vertexai_types.PrebuiltMetric.COHERENCE.name
+    ]
+
+  def test_evaluate_invocations_metric_failed(self, mocker):
+    """Test evaluate_invocations function for a metric."""
+    mocker.patch("google.adk.dependencies.vertexai.vertexai.Client")
+    mock_perform_eval = mocker.patch(
+        "google.adk.evaluation.vertex_ai_eval_facade._VertexAiEvalFacade._perform_eval"
+    )
+    actual_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[
+                    genai_types.Part(text="This is a test candidate response.")
+                ]
+            ),
+        )
+    ]
+    expected_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test reference.")]
+            ),
+        )
+    ]
+    evaluator = _SingleTurnVertexAiEvalFacade(
+        threshold=0.8, metric_name=vertexai_types.PrebuiltMetric.COHERENCE
+    )
+    # Mock the return value of _perform_eval
+    mock_perform_eval.return_value = vertexai_types.EvaluationResult(
+        summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.7)],
+        eval_case_results=[],
+    )
+
+    evaluation_result = evaluator.evaluate_invocations(
+        actual_invocations, expected_invocations
+    )
+
+    assert evaluation_result.overall_score == 0.7
+    assert evaluation_result.overall_eval_status == EvalStatus.FAILED
+    mock_perform_eval.assert_called_once()
+    _, mock_kwargs = mock_perform_eval.call_args
+    # Compare the names of the metrics.
+    assert [m.name for m in mock_kwargs["metrics"]] == [
+        vertexai_types.PrebuiltMetric.COHERENCE.name
+    ]
+
+  @pytest.mark.parametrize(
+      "summary_metric_with_no_score",
+      [
+          ([]),
+          ([vertexai_types.AggregatedMetricResult(mean_score=float("nan"))]),
+          ([vertexai_types.AggregatedMetricResult(mean_score=None)]),
+          ([vertexai_types.AggregatedMetricResult(mean_score=math.nan)]),
+      ],
+  )
+  def test_evaluate_invocations_metric_no_score(
+      self, mocker, summary_metric_with_no_score
+  ):
+    """Test evaluate_invocations function for a metric."""
+    mocker.patch("google.adk.dependencies.vertexai.vertexai.Client")
+    mock_perform_eval = mocker.patch(
+        "google.adk.evaluation.vertex_ai_eval_facade._VertexAiEvalFacade._perform_eval"
+    )
+    actual_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[
+                    genai_types.Part(text="This is a test candidate response.")
+                ]
+            ),
+        )
+    ]
+    expected_invocations = [
+        Invocation(
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test query.")]
+            ),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="This is a test reference.")]
+            ),
+        )
+    ]
+    evaluator = _SingleTurnVertexAiEvalFacade(
+        threshold=0.8, metric_name=vertexai_types.PrebuiltMetric.COHERENCE
+    )
+    # Mock the return value of _perform_eval
+    mock_perform_eval.return_value = vertexai_types.EvaluationResult(
+        summary_metrics=summary_metric_with_no_score,
+        eval_case_results=[],
+    )
+
+    evaluation_result = evaluator.evaluate_invocations(
+        actual_invocations, expected_invocations
+    )
+
+    assert evaluation_result.overall_score is None
+    assert evaluation_result.overall_eval_status == EvalStatus.NOT_EVALUATED
+    mock_perform_eval.assert_called_once()
+    _, mock_kwargs = mock_perform_eval.call_args
+    # Compare the names of the metrics.
+    assert [m.name for m in mock_kwargs["metrics"]] == [
+        vertexai_types.PrebuiltMetric.COHERENCE.name
+    ]
+
+  def test_evaluate_invocations_metric_multiple_invocations(self, mocker):
+    """Test evaluate_invocations function for a metric with multiple invocations."""
+    mocker.patch("google.adk.dependencies.vertexai.vertexai.Client")
+    mock_perform_eval = mocker.patch(
+        "google.adk.evaluation.vertex_ai_eval_facade._VertexAiEvalFacade._perform_eval"
+    )
+    num_invocations = 6
+    actual_invocations = []
+    expected_invocations = []
+    mock_eval_results = []
+    random.seed(61553)
+    scores = [random.random() for _ in range(num_invocations)]
+
+    for i in range(num_invocations):
+      actual_invocations.append(
+          Invocation(
+              user_content=genai_types.Content(
+                  parts=[genai_types.Part(text=f"Query {i+1}")]
+              ),
+              final_response=genai_types.Content(
+                  parts=[genai_types.Part(text=f"Response {i+1}")]
+              ),
+          )
+      )
+      expected_invocations.append(
+          Invocation(
+              user_content=genai_types.Content(
+                  parts=[genai_types.Part(text=f"Query {i+1}")]
+              ),
+              final_response=genai_types.Content(
+                  parts=[genai_types.Part(text=f"Reference {i+1}")]
+              ),
+          )
+      )
+      mock_eval_results.append(
+          vertexai_types.EvaluationResult(
+              summary_metrics=[
+                  vertexai_types.AggregatedMetricResult(mean_score=scores[i])
+              ],
+              eval_case_results=[],
+          )
+      )
+
+    evaluator = _SingleTurnVertexAiEvalFacade(
+        threshold=0.8, metric_name=vertexai_types.PrebuiltMetric.COHERENCE
+    )
+    # Mock the return value of _perform_eval
+    mock_perform_eval.side_effect = mock_eval_results
+
+    evaluation_result = evaluator.evaluate_invocations(
+        actual_invocations, expected_invocations
+    )
+
+    assert evaluation_result.overall_score == pytest.approx(
+        sum(scores) / num_invocations
+    )
+    assert evaluation_result.overall_eval_status == EvalStatus.FAILED
+    assert mock_perform_eval.call_count == num_invocations
+
+
+class TestVertexAiEvalFacade:
+  """A class to help organize "patch" that are applicable to all tests."""
+
+  def test_constructor_with_api_key(self, mocker):
+    mocker.patch.dict(
+        os.environ, {"GOOGLE_API_KEY": "test_api_key"}, clear=True
+    )
+    mock_client_cls = mocker.patch(
+        "google.adk.dependencies.vertexai.vertexai.Client"
+    )
+    _SingleTurnVertexAiEvalFacade(
+        threshold=0.8, metric_name=vertexai_types.PrebuiltMetric.COHERENCE
+    )
+
+    mock_client_cls.assert_called_once_with(api_key="test_api_key")
+
+  def test_constructor_with_project_and_location(self, mocker):
+
+    mocker.patch.dict(
+        os.environ,
+        {
+            "GOOGLE_CLOUD_PROJECT": "test_project",
+            "GOOGLE_CLOUD_LOCATION": "test_location",
+        },
+        clear=True,
+    )
+    mock_client_cls = mocker.patch(
+        "google.adk.dependencies.vertexai.vertexai.Client"
+    )
+    _SingleTurnVertexAiEvalFacade(
+        threshold=0.8, metric_name=vertexai_types.PrebuiltMetric.COHERENCE
+    )
+
+    mock_client_cls.assert_called_once_with(
+        project="test_project", location="test_location"
+    )
+
+  def test_constructor_with_project_only_raises_error(self, mocker):
+    mocker.patch.dict(
+        os.environ, {"GOOGLE_CLOUD_PROJECT": "test_project"}, clear=True
+    )
+    mocker.patch("google.adk.dependencies.vertexai.vertexai.Client")
+
+    with pytest.raises(ValueError, match="Missing location."):
+      _SingleTurnVertexAiEvalFacade(
+          threshold=0.8, metric_name=vertexai_types.PrebuiltMetric.COHERENCE
+      )
+
+  def test_constructor_with_location_only_raises_error(self, mocker):
+    mocker.patch.dict(
+        os.environ, {"GOOGLE_CLOUD_LOCATION": "test_location"}, clear=True
+    )
+    mocker.patch("google.adk.dependencies.vertexai.vertexai.Client")
+
+    with pytest.raises(ValueError, match="Missing project id."):
+      _SingleTurnVertexAiEvalFacade(
+          threshold=0.8, metric_name=vertexai_types.PrebuiltMetric.COHERENCE
+      )
+
+  def test_constructor_with_no_env_vars_raises_error(self, mocker):
+    mocker.patch.dict(os.environ, {}, clear=True)
+    mocker.patch("google.adk.dependencies.vertexai.vertexai.Client")
+
+    with pytest.raises(
+        ValueError,
+        match=(
+            "Either API Key or Google cloud Project id and location should be"
+            " specified."
+        ),
+    ):
+      _SingleTurnVertexAiEvalFacade(
+          threshold=0.8, metric_name=vertexai_types.PrebuiltMetric.COHERENCE
+      )
+
+
+class TestMultiTurnVertexAiEvalFacade:
+  """Tests for _MultiTurnVertexiAiEvalFacade."""
+
+  def test_map_agent_details_to_agent_config(self):
+    tool_declarations = [
+        genai_types.Tool(
+            function_declarations=[
+                genai_types.FunctionDeclaration(
+                    name="tool_1",
+                    description="this is tool 1",
+                )
+            ]
+        )
+    ]
+    agent_details = AgentDetails(
+        name="test_agent",
+        instructions="test_instructions",
+        tool_declarations=tool_declarations,
+    )
+    agent_config = (
+        _MultiTurnVertexiAiEvalFacade._map_agent_details_to_agent_config(
+            agent_details
+        )
+    )
+    assert agent_config.agent_id == "test_agent"
+    assert agent_config.instruction == "test_instructions"
+    assert agent_config.tools == tool_declarations
+
+  def test_get_agent_details(self):
+    invocations = [
+        Invocation(
+            user_content=genai_types.Content(),
+            app_details=AppDetails(
+                agent_details={
+                    "agent1": AgentDetails(
+                        name="agent1", instructions="instructions1"
+                    ),
+                    "agent2": AgentDetails(
+                        name="agent2", instructions="instructions2"
+                    ),
+                }
+            ),
+        ),
+        Invocation(
+            user_content=genai_types.Content(),
+            app_details=AppDetails(
+                agent_details={
+                    "agent1": AgentDetails(
+                        name="agent1", instructions="instructions1"
+                    ),
+                    "agent3": AgentDetails(
+                        name="agent3", instructions="instructions3"
+                    ),
+                }
+            ),
+        ),
+    ]
+    agent_configs = _MultiTurnVertexiAiEvalFacade._get_agent_details(
+        invocations
+    )
+    assert len(agent_configs) == 3
+    assert "agent1" in agent_configs
+    assert "agent2" in agent_configs
+    assert "agent3" in agent_configs
+    assert agent_configs["agent1"].instruction == "instructions1"
+    assert agent_configs["agent2"].instruction == "instructions2"
+    assert agent_configs["agent3"].instruction == "instructions3"
+
+  def test_map_invocation_event_to_agent_event(self):
+    invocation_event = InvocationEvent(
+        author="test_author",
+        content=genai_types.Content(
+            parts=[genai_types.Part(text="test_content")]
+        ),
+    )
+    agent_event = (
+        _MultiTurnVertexiAiEvalFacade._map_inovcation_event_to_agent_event(
+            invocation_event
+        )
+    )
+    assert agent_event.author == "test_author"
+    assert agent_event.content.parts[0].text == "test_content"
+
+  def test_map_invocation_turn(self):
+    invocation = Invocation(
+        invocation_id="inv1",
+        user_content=genai_types.Content(
+            parts=[genai_types.Part(text="user query")]
+        ),
+        intermediate_data=InvocationEvents(
+            invocation_events=[
+                InvocationEvent(
+                    author="agent1",
+                    content=genai_types.Content(
+                        parts=[genai_types.Part(text="intermediate content")]
+                    ),
+                )
+            ]
+        ),
+        final_response=genai_types.Content(
+            parts=[genai_types.Part(text="final response")]
+        ),
+    )
+    conversation_turn = _MultiTurnVertexiAiEvalFacade._map_invocation_turn(
+        0, invocation
+    )
+    assert conversation_turn.turn_index == 0
+    assert conversation_turn.turn_id == "inv1"
+    assert len(conversation_turn.events) == 3
+    assert conversation_turn.events[0].author == "user"
+    assert conversation_turn.events[0].content.parts[0].text == "user query"
+    assert conversation_turn.events[1].author == "agent1"
+    assert (
+        conversation_turn.events[1].content.parts[0].text
+        == "intermediate content"
+    )
+    assert conversation_turn.events[2].author == "agent"
+    assert conversation_turn.events[2].content.parts[0].text == "final response"
+
+  def test_get_turns(self):
+    invocations = [
+        Invocation(
+            invocation_id="inv1",
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="q1")]
+            ),
+            intermediate_data=InvocationEvents(invocation_events=[]),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="r1")]
+            ),
+        ),
+        Invocation(
+            invocation_id="inv2",
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="q2")]
+            ),
+            intermediate_data=InvocationEvents(invocation_events=[]),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="r2")]
+            ),
+        ),
+    ]
+    turns = _MultiTurnVertexiAiEvalFacade._get_turns(invocations)
+    assert len(turns) == 2
+    assert turns[0].turn_id == "inv1"
+    assert turns[1].turn_id == "inv2"
+
+  def test_get_agent_data(self):
+    invocations = [
+        Invocation(
+            invocation_id="inv1",
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="q1")]
+            ),
+            intermediate_data=InvocationEvents(invocation_events=[]),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="r1")]
+            ),
+            app_details=AppDetails(
+                agent_details={
+                    "agent1": AgentDetails(
+                        name="agent1", instructions="instructions1"
+                    )
+                }
+            ),
+        )
+    ]
+    agent_data = _MultiTurnVertexiAiEvalFacade._get_agent_data(invocations)
+    assert "agent1" in agent_data.agents
+    assert len(agent_data.turns) == 1
+
+  def test_evaluate_invocations_multi_turn_metric_passed(self, mocker):
+    """Test evaluate_invocations function for a multi-turn metric."""
+    mock_perform_eval = mocker.patch(
+        "google.adk.evaluation.vertex_ai_eval_facade._VertexAiEvalFacade._perform_eval"
+    )
+    actual_invocations = [
+        Invocation(
+            invocation_id="inv1",
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="q1")]
+            ),
+            intermediate_data=InvocationEvents(invocation_events=[]),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="r1")]
+            ),
+            app_details=AppDetails(
+                agent_details={
+                    "agent1": AgentDetails(
+                        name="agent1", instructions="instructions1"
+                    )
+                }
+            ),
+        ),
+        Invocation(
+            invocation_id="inv2",
+            user_content=genai_types.Content(
+                parts=[genai_types.Part(text="q2")]
+            ),
+            intermediate_data=InvocationEvents(
+                invocation_events=[
+                    InvocationEvent(
+                        author="agent1",
+                        content=genai_types.Content(
+                            parts=[genai_types.Part(text="intermediate")]
+                        ),
+                    )
+                ]
+            ),
+            final_response=genai_types.Content(
+                parts=[genai_types.Part(text="r2")]
+            ),
+            app_details=AppDetails(
+                agent_details={
+                    "agent1": AgentDetails(
+                        name="agent1", instructions="instructions1"
+                    )
+                }
+            ),
+        ),
+    ]
+    evaluator = _MultiTurnVertexiAiEvalFacade(
+        threshold=0.8,
+        metric_name=vertexai_types.PrebuiltMetric.CONVERSATIONAL_COHERENCE,
+    )
+    # Mock the return value of _perform_eval
+    mock_perform_eval.return_value = vertexai_types.EvaluationResult(
+        summary_metrics=[vertexai_types.AggregatedMetricResult(mean_score=0.9)],
+        eval_case_results=[],
+    )
+
+    evaluation_result = evaluator.evaluate_invocations(actual_invocations)
+
+    assert evaluation_result.overall_score == 0.9
+    assert evaluation_result.overall_eval_status == EvalStatus.PASSED
+    assert len(evaluation_result.per_invocation_results) == 2
+    assert (
+        evaluation_result.per_invocation_results[0].eval_status
+        == EvalStatus.NOT_EVALUATED
+    )
+    assert (
+        evaluation_result.per_invocation_results[1].eval_status
+        == EvalStatus.PASSED
+    )
+    mock_perform_eval.assert_called_once()
+    _, mock_kwargs = mock_perform_eval.call_args
+    assert [m.name for m in mock_kwargs["metrics"]] == [
+        vertexai_types.PrebuiltMetric.CONVERSATIONAL_COHERENCE.name
+    ]
+    dataset = mock_kwargs["dataset"]
+    assert len(dataset.eval_cases) == 1
+    agent_data = dataset.eval_cases[0].agent_data
+    assert "agent1" in agent_data.agents
+    assert len(agent_data.turns) == 2
+    assert agent_data.turns[0].turn_id == "inv1"
+    assert agent_data.turns[1].turn_id == "inv2"
+    assert len(agent_data.turns[1].events) == 3  # user, intermediate, agent

tests/unittests/evaluation/test_vertex_ai_scenario_generation_facade.py

@@ -0,0 +1,155 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for the Vertex AI Scenario Generation Facade."""
+
+from __future__ import annotations
+
+import os
+
+from google.adk.agents.base_agent import BaseAgent
+from google.adk.dependencies.vertexai import vertexai
+from google.adk.evaluation._vertex_ai_scenario_generation_facade import ScenarioGenerator
+from google.adk.evaluation.conversation_scenarios import ConversationGenerationConfig
+import pytest
+
+vertexai_types = vertexai.types
+
+
+class TestScenarioGenerator:
+  """Unit tests for ScenarioGenerator."""
+
+  def test_constructor_with_api_key(self, mocker):
+    mocker.patch.dict(
+        os.environ, {"GOOGLE_API_KEY": "test_api_key"}, clear=True
+    )
+    mock_client_cls = mocker.patch(
+        "google.adk.dependencies.vertexai.vertexai.Client"
+    )
+    ScenarioGenerator()
+
+    mock_client_cls.assert_called_once_with(api_key="test_api_key")
+
+  def test_constructor_with_project_and_location(self, mocker):
+    """Test constructor with project and location in env."""
+    mocker.patch.dict(
+        os.environ,
+        {
+            "GOOGLE_CLOUD_PROJECT": "test_project",
+            "GOOGLE_CLOUD_LOCATION": "test_location",
+        },
+        clear=True,
+    )
+    mock_client_cls = mocker.patch(
+        "google.adk.dependencies.vertexai.vertexai.Client"
+    )
+    ScenarioGenerator()
+
+    mock_client_cls.assert_called_once_with(
+        project="test_project", location="test_location"
+    )
+
+  def test_constructor_with_project_only_raises_error(self, mocker):
+    mocker.patch.dict(
+        os.environ, {"GOOGLE_CLOUD_PROJECT": "test_project"}, clear=True
+    )
+    mocker.patch("google.adk.dependencies.vertexai.vertexai.Client")
+
+    with pytest.raises(ValueError, match="Missing location."):
+      ScenarioGenerator()
+
+  def test_constructor_with_location_only_raises_error(self, mocker):
+    mocker.patch.dict(
+        os.environ, {"GOOGLE_CLOUD_LOCATION": "test_location"}, clear=True
+    )
+    mocker.patch("google.adk.dependencies.vertexai.vertexai.Client")
+
+    with pytest.raises(ValueError, match="Missing project id."):
+      ScenarioGenerator()
+
+  def test_constructor_with_no_env_vars_raises_error(self, mocker):
+    mocker.patch.dict(os.environ, {}, clear=True)
+    mocker.patch("google.adk.dependencies.vertexai.vertexai.Client")
+
+    with pytest.raises(
+        ValueError,
+        match=(
+            "Either API Key or Google cloud Project id and location should be"
+            " specified."
+        ),
+    ):
+      ScenarioGenerator()
+
+  def test_generate_scenarios(self, mocker):
+    """Test scenario generation with mocked components."""
+    mocker.patch.dict(
+        os.environ, {"GOOGLE_API_KEY": "test_api_key"}, clear=True
+    )
+    mock_client_cls = mocker.patch(
+        "google.adk.dependencies.vertexai.vertexai.Client"
+    )
+    mock_client = mock_client_cls.return_value
+
+    # I need to mock AgentInfo.load_from_agent(agent=agent)
+    mock_agent_info_cls = mocker.patch(
+        "google.adk.dependencies.vertexai.vertexai.types.evals.AgentInfo"
+    )
+    mock_agent_info_cls.load_from_agent.return_value = "mock_agent_info"
+
+    mock_generate = mocker.patch.object(
+        mock_client.evals, "generate_conversation_scenarios"
+    )
+
+    mock_eval_cases = [
+        mocker.Mock(
+            user_scenario=mocker.Mock(
+                starting_prompt="Hello", conversation_plan="Say hello"
+            )
+        ),
+        mocker.Mock(user_scenario=None),  # testing handling of None
+        mocker.Mock(
+            user_scenario=mocker.Mock(
+                starting_prompt="Bye", conversation_plan="Say bye"
+            )
+        ),
+    ]
+    mock_generate.return_value = mocker.Mock(eval_cases=mock_eval_cases)
+
+    generator = ScenarioGenerator()
+
+    mock_agent = mocker.Mock(spec=BaseAgent)
+    config = ConversationGenerationConfig(
+        count=2,
+        generation_instruction="Test generation",
+        model_name="gemini-2.5-flash",
+    )
+
+    scenarios = generator.generate_scenarios(mock_agent, config)
+
+    assert len(scenarios) == 2
+    assert scenarios[0].starting_prompt == "Hello"
+    assert scenarios[0].conversation_plan == "Say hello"
+    assert scenarios[1].starting_prompt == "Bye"
+    assert scenarios[1].conversation_plan == "Say bye"
+
+    mock_agent_info_cls.load_from_agent.assert_called_once_with(
+        agent=mock_agent
+    )
+
+    mock_generate.assert_called_once()
+    _, kwargs = mock_generate.call_args
+    assert kwargs["agent_info"] == "mock_agent_info"
+    passed_config = kwargs["config"]
+    assert passed_config.count == 2
+    assert passed_config.generation_instruction == "Test generation"