chore: import upstream snapshot with attribution

sdks/python/tests/unit/simulation/test_simulated_user.py

@@ -0,0 +1,129 @@
+"""Tests for SimulatedUser class."""
+
+from unittest.mock import Mock, patch
+from opik.simulation.simulated_user import SimulatedUser
+
+
+class TestSimulatedUser:
+    """Test cases for SimulatedUser class."""
+
+    def test_init_with_persona_and_model(self):
+        """Test SimulatedUser initialization with persona and model."""
+        user = SimulatedUser(persona="You are a helpful assistant", model="gpt-4o-mini")
+
+        assert user.persona == "You are a helpful assistant"
+        assert user.model == "gpt-4o-mini"
+        assert user.fixed_responses == []
+        assert user._response_index == 0
+        assert user._llm is not None
+
+    def test_init_with_fixed_responses(self):
+        """Test SimulatedUser initialization with fixed responses."""
+        fixed_responses = ["Hello!", "How are you?", "Goodbye!"]
+        user = SimulatedUser(persona="Test persona", fixed_responses=fixed_responses)
+
+        assert user.fixed_responses == fixed_responses
+        assert user._response_index == 0
+
+    def test_generate_response_with_fixed_responses(self):
+        """Test response generation using fixed responses."""
+        fixed_responses = ["Response 1", "Response 2", "Response 3"]
+        user = SimulatedUser(persona="Test persona", fixed_responses=fixed_responses)
+
+        # First call
+        response1 = user.generate_response([])
+        assert response1 == "Response 1"
+        assert user._response_index == 1
+
+        # Second call
+        response2 = user.generate_response([])
+        assert response2 == "Response 2"
+        assert user._response_index == 2
+
+        # Third call
+        response3 = user.generate_response([])
+        assert response3 == "Response 3"
+        assert user._response_index == 3
+
+        # Fourth call (cycles back)
+        response4 = user.generate_response([])
+        assert response4 == "Response 1"
+        assert user._response_index == 4
+
+    @patch("opik.simulation.simulated_user.get_model")
+    def test_generate_response_with_llm(self, mock_get_model):
+        """Test response generation using LLM when no fixed responses."""
+        # Mock the LLM response
+        mock_llm_instance = Mock()
+        mock_llm_instance.generate_string.return_value = "LLM generated response"
+        mock_get_model.return_value = mock_llm_instance
+
+        user = SimulatedUser(persona="You are a helpful assistant", model="gpt-4o-mini")
+
+        conversation_history = [
+            {"role": "user", "content": "Hello"},
+            {"role": "assistant", "content": "Hi there!"},
+        ]
+
+        response = user.generate_response(conversation_history)
+
+        assert response == "LLM generated response"
+
+        # Verify LLM was called with correct input
+        mock_llm_instance.generate_string.assert_called_once()
+        call_args = mock_llm_instance.generate_string.call_args[1]["input"]
+        assert (
+            "You are a simulated user with the following persona: You are a helpful assistant"
+            in call_args
+        )
+        assert "User: Hello" in call_args
+        assert "Assistant: Hi there!" in call_args
+
+    @patch("opik.simulation.simulated_user.get_model")
+    def test_generate_response_with_llm_error(self, mock_get_model):
+        """Test response generation when LLM raises an exception."""
+        # Mock the LLM to raise an exception
+        mock_llm_instance = Mock()
+        mock_llm_instance.generate_string.side_effect = Exception("LLM error")
+        mock_get_model.return_value = mock_llm_instance
+
+        user = SimulatedUser(persona="Test persona", model="gpt-4o-mini")
+
+        response = user.generate_response([])
+
+        assert "I'm having trouble responding right now. (LLM error)" in response
+
+    @patch("opik.simulation.simulated_user.get_model")
+    def test_generate_response_with_long_history(self, mock_get_model):
+        """Test that long conversation history is handled correctly."""
+        mock_llm_instance = Mock()
+        mock_llm_instance.generate_string.return_value = "Response"
+        mock_get_model.return_value = mock_llm_instance
+
+        user = SimulatedUser(persona="Test persona", model="gpt-4o-mini")
+
+        # Create a long conversation history (15 messages)
+        long_history = []
+        for i in range(15):
+            long_history.extend(
+                [
+                    {"role": "user", "content": f"Message {i}"},
+                    {"role": "assistant", "content": f"Response {i}"},
+                ]
+            )
+
+        user.generate_response(long_history)
+
+        # Verify LLM was called
+        mock_llm_instance.generate_string.assert_called_once()
+        call_args = mock_llm_instance.generate_string.call_args[1]["input"]
+
+        # Should contain system message and all conversation messages
+        assert (
+            "You are a simulated user with the following persona: Test persona"
+            in call_args
+        )
+        assert "User: Message 0" in call_args
+        assert "Assistant: Response 0" in call_args
+        assert "User: Message 14" in call_args
+        assert "Assistant: Response 14" in call_args

sdks/python/tests/unit/simulation/test_simulator.py

@@ -0,0 +1,188 @@
+"""Tests for run_simulation function."""
+
+from unittest.mock import patch
+from opik.simulation.simulator import run_simulation
+from opik.simulation.simulated_user import SimulatedUser
+
+
+class TestRunSimulation:
+    """Test cases for run_simulation function."""
+
+    def test_run_simulation_basic(self, fake_backend):
+        """Test basic simulation functionality."""
+
+        # Mock app that returns simple responses
+        def mock_app(message, *, thread_id, **kwargs):
+            return {"role": "assistant", "content": f"Response to: {message}"}
+
+        # Mock user simulator with fixed responses
+        user_simulator = SimulatedUser(
+            persona="Test user", fixed_responses=["Hello", "How are you?", "Goodbye"]
+        )
+
+        result = run_simulation(
+            app=mock_app, user_simulator=user_simulator, max_turns=3
+        )
+
+        assert "thread_id" in result
+        assert "conversation_history" in result
+        assert len(result["conversation_history"]) == 6  # 3 turns * 2 messages each
+
+        # Check conversation structure
+        history = result["conversation_history"]
+        assert history[0]["role"] == "user"
+        assert history[0]["content"] == "Hello"
+        assert history[1]["role"] == "assistant"
+        assert "Response to: Hello" in history[1]["content"]
+
+    def test_run_simulation_with_initial_message(self, fake_backend):
+        """Test simulation with provided initial message."""
+
+        def mock_app(message, *, thread_id, **kwargs):
+            return {"role": "assistant", "content": "Got it"}
+
+        user_simulator = SimulatedUser(
+            persona="Test user", fixed_responses=["Follow up message"]
+        )
+
+        result = run_simulation(
+            app=mock_app,
+            user_simulator=user_simulator,
+            initial_message="Custom initial message",
+            max_turns=2,
+        )
+
+        history = result["conversation_history"]
+        assert history[0]["content"] == "Custom initial message"
+        assert history[2]["content"] == "Follow up message"
+
+    def test_run_simulation_with_thread_id(self, fake_backend):
+        """Test simulation with provided thread_id."""
+
+        def mock_app(message, *, thread_id, **kwargs):
+            return {"role": "assistant", "content": "Response"}
+
+        user_simulator = SimulatedUser(persona="Test user", fixed_responses=["Message"])
+
+        custom_thread_id = "custom-thread-123"
+        result = run_simulation(
+            app=mock_app,
+            user_simulator=user_simulator,
+            thread_id=custom_thread_id,
+            max_turns=1,
+        )
+
+        assert result["thread_id"] == custom_thread_id
+
+    def test_run_simulation_with_app_kwargs(self, fake_backend):
+        """Test simulation with additional app kwargs."""
+
+        def mock_app(message, *, thread_id, custom_param=None, **kwargs):
+            return {"role": "assistant", "content": f"Custom: {custom_param}"}
+
+        user_simulator = SimulatedUser(persona="Test user", fixed_responses=["Message"])
+
+        result = run_simulation(
+            app=mock_app,
+            user_simulator=user_simulator,
+            custom_param="test_value",
+            max_turns=1,
+        )
+
+        history = result["conversation_history"]
+        assert "Custom: test_value" in history[1]["content"]
+
+    def test_run_simulation_app_error_handling(self, fake_backend):
+        """Test simulation handles app errors gracefully."""
+
+        def failing_app(message, *, thread_id, **kwargs):
+            raise Exception("App error")
+
+        user_simulator = SimulatedUser(persona="Test user", fixed_responses=["Message"])
+
+        result = run_simulation(
+            app=failing_app, user_simulator=user_simulator, max_turns=1
+        )
+
+        history = result["conversation_history"]
+        assert "Error processing message: App error" in history[1]["content"]
+
+    def test_run_simulation_invalid_app_response(self, fake_backend):
+        """Test simulation handles invalid app responses."""
+
+        def invalid_app(message, *, thread_id, **kwargs):
+            return "Not a dict"  # Invalid response format
+
+        user_simulator = SimulatedUser(persona="Test user", fixed_responses=["Message"])
+
+        result = run_simulation(
+            app=invalid_app, user_simulator=user_simulator, max_turns=1
+        )
+
+        history = result["conversation_history"]
+        assert history[1]["role"] == "assistant"
+        assert history[1]["content"] == "Not a dict"
+
+    def test_run_simulation_app_returns_none(self, fake_backend):
+        """Test simulation handles None app responses."""
+
+        def none_app(message, *, thread_id, **kwargs):
+            return None
+
+        user_simulator = SimulatedUser(persona="Test user", fixed_responses=["Message"])
+
+        result = run_simulation(
+            app=none_app, user_simulator=user_simulator, max_turns=1
+        )
+
+        history = result["conversation_history"]
+        assert history[1]["role"] == "assistant"
+        assert history[1]["content"] == "No response"
+
+    @patch("opik.simulation.simulator.id_helpers.generate_id")
+    def test_run_simulation_generates_thread_id(self, mock_generate_id, fake_backend):
+        """Test that thread_id is generated when not provided."""
+        mock_generate_id.return_value = "generated-thread-456"
+
+        def mock_app(message, *, thread_id, **kwargs):
+            return {"role": "assistant", "content": "Response"}
+
+        user_simulator = SimulatedUser(persona="Test user", fixed_responses=["Message"])
+
+        result = run_simulation(
+            app=mock_app, user_simulator=user_simulator, max_turns=1
+        )
+
+        assert result["thread_id"] == "generated-thread-456"
+        mock_generate_id.assert_called_once()
+
+    def test_run_simulation_with_project_name(self, fake_backend):
+        """Test simulation includes project_name in result."""
+
+        def mock_app(message, *, thread_id, **kwargs):
+            return {"role": "assistant", "content": "Response"}
+
+        user_simulator = SimulatedUser(persona="Test user", fixed_responses=["Message"])
+
+        result = run_simulation(
+            app=mock_app,
+            user_simulator=user_simulator,
+            project_name="test_project",
+            max_turns=1,
+        )
+
+        assert result["project_name"] == "test_project"
+
+    def test_run_simulation_max_turns_zero(self, fake_backend):
+        """Test simulation with zero max_turns."""
+
+        def mock_app(message, *, thread_id, **kwargs):
+            return {"role": "assistant", "content": "Response"}
+
+        user_simulator = SimulatedUser(persona="Test user", fixed_responses=["Message"])
+
+        result = run_simulation(
+            app=mock_app, user_simulator=user_simulator, max_turns=0
+        )
+
+        assert len(result["conversation_history"]) == 0