"""E2E test: "terminal coding session" user journey (mock LLM). Exercises a realistic coding workflow where the agent uses terminal tools to list files, create a file in ``/tmp``, and read it back. The inline agent registers ``sys_terminal_*`` tools via the ``terminals`` config block so the agent can execute arbitrary shell commands (``ls``, ``printf``, ``cat``) in its terminal. Skipped if tmux is not installed on the host. Usage:: pytest tests/e2e/test_journey_workspace_coding.py -v """ from __future__ import annotations import shutil import uuid import httpx import pytest from tests.e2e.conftest import ( configure_mock_llm, create_runner_bound_session, poll_session_until_terminal, register_inline_agent, reset_mock_llm, send_user_message_to_session, ) pytestmark = pytest.mark.skipif( shutil.which("tmux") is None, reason="tmux not installed; workspace coding journey needs tmux on PATH", ) def _get_function_call_outputs( client: httpx.Client, conversation_id: str, tool_name: str, ) -> list[str]: """ Return raw outputs of every ``tool_name`` call in conversation order. Walks ``function_call`` and ``function_call_output`` items in the conversation. Assertions land on deterministic tool output strings, not on flaky LLM prose summaries. :param client: HTTP client. :param conversation_id: Conversation to inspect. :param tool_name: Only outputs of calls to this tool are returned. :returns: Ordered list of raw output strings. """ resp = client.get(f"/v1/sessions/{conversation_id}/items?limit=200") resp.raise_for_status() items = resp.json()["data"] calls_by_id: dict[str, dict] = {} for item in items: itype = item.get("type") data = item.get("data") or {} name = item.get("name") or data.get("name") call_id = item.get("call_id") or data.get("call_id") if itype == "function_call" and name == tool_name and call_id: calls_by_id[call_id] = item outputs: list[str] = [] for item in items: itype = item.get("type") data = item.get("data") or {} call_id = item.get("call_id") or data.get("call_id") output = item.get("output") or data.get("output") if itype == "function_call_output" and call_id in calls_by_id: outputs.append(str(output or "")) return outputs @pytest.mark.llm_flaky(reruns=2) def test_terminal_coding_session_journey( http_client: httpx.Client, live_runner_id: str, mock_llm_server_url: str, ) -> None: """ Terminal coding journey: create a file via terminal, read it back, and verify the content. Steps: 1. Register an inline agent with terminal tools and mock LLM. 2. Turn 1: mock LLM launches a terminal and runs ``ls -la``. Verify ``sys_terminal_read`` output contains file listings. 3. Turn 2: mock LLM creates a Python file via ``printf``. 4. Turn 3: mock LLM reads the file back with ``cat``. 5. Verify the file content appears in tool output. The core flow (create → read → verify) is the most reliable subset of the full 8-step journey. Modification steps (sed/echo to add a docstring) are omitted to reduce LLM flakiness — the create-read round trip already proves the terminal is functional. **What breaks if this fails:** - Terminal tools not registered → agent cannot run shell commands. - Workspace cwd not set → file created in wrong location. - ``sys_terminal_send``/``sys_terminal_read`` flow broken → no command output captured. - tmux session not persisting across tool calls within one turn → stateful file operations fail. :param http_client: HTTP client pointed at the live server. :param live_runner_id: Runner id for session binding. :param mock_llm_server_url: Mock LLM server URL. """ model = f"mock-workspace-coding-{uuid.uuid4().hex[:6]}" unique_suffix = uuid.uuid4().hex[:8] filename = f"/tmp/workspace_test_{unique_suffix}.py" reset_mock_llm(mock_llm_server_url) agent_name = register_inline_agent( http_client, name=f"workspace-coding-{unique_suffix}", harness="openai-agents", model=model, profile="", prompt=( "You are a terminal coding assistant. " "Use sys_terminal_launch, sys_terminal_send, and sys_terminal_read " "to execute shell commands and manage files." ), mock_llm_base_url=f"{mock_llm_server_url}/v1", extra_config={ "terminals": { "bash": { "command": "bash", "os_env": {"type": "caller_process", "sandbox": {"type": "none"}}, } }, "os_env": {"type": "caller_process", "cwd": ".", "sandbox": {"type": "none"}}, }, ) session_id = create_runner_bound_session( http_client, agent_name=agent_name, runner_id=live_runner_id, ) # ── Step 1 + 2: Launch terminal and list workspace ────────────────────── # Mock: launch bash session 'workspace', send ls -la, read, reply 'listed'. configure_mock_llm( mock_llm_server_url, [ { "tool_calls": [ { "call_id": "call_launch_ws1", "name": "sys_terminal_launch", "arguments": '{"terminal": "bash", "session": "workspace"}', } ], }, { "tool_calls": [ { "call_id": "call_send_ls", "name": "sys_terminal_send", "arguments": ( '{"terminal": "bash", "session": "workspace", "text": "ls -la\n"}' ), } ], }, { "tool_calls": [ { "call_id": "call_read_ls", "name": "sys_terminal_read", "arguments": '{"terminal": "bash", "session": "workspace"}', } ], }, {"text": "listed"}, ], key=model, ) response_id = send_user_message_to_session( http_client, session_id=session_id, content=( "Use sys_terminal_launch to start the 'bash' terminal with " "session 'workspace'. Then use sys_terminal_send to type " "'ls -la' followed by Enter. Wait briefly, then " "sys_terminal_read on session 'workspace'. " "Reply 'listed' once you see the output." ), ) body = poll_session_until_terminal( http_client, session_id=session_id, response_id=response_id, timeout=180, ) assert body["status"] == "completed", ( f"Step 1-2 failed: status={body['status']!r}, " f"error={body.get('error')!r}. If 'failed' with a tool " f"error, sys_terminal_* tools may not be registered." ) # ── Step 3: Verify sys_terminal_read output ────────────────────────── reads_step2 = _get_function_call_outputs(http_client, session_id, "sys_terminal_read") assert reads_step2, ( f"sys_terminal_read was never called in the listing step; " f"session_id={session_id}. The agent may have ignored the prompt " f"or the tool wasn't on the schema." ) # Stronger: the ls -la output must be non-empty (at least a shell # prompt or directory listing was captured by tmux). combined_reads_step2 = " ".join(reads_step2) assert combined_reads_step2.strip(), ( f"sys_terminal_read returned only empty strings after ls -la; " f"session_id={session_id}. tmux may not have initialised properly." ) # ── Step 4: Ask agent to create a Python file ──────────────────────── reset_mock_llm(mock_llm_server_url) configure_mock_llm( mock_llm_server_url, [ { "tool_calls": [ { "call_id": "call_send_printf", "name": "sys_terminal_send", "arguments": ( f'{{"terminal": "bash", "session": "workspace", "text": "printf ' f"'def hello():\\\\n return " f'\\"hello world\\"\\\\n\' > {filename}\\n"}}' ), } ], }, { "tool_calls": [ { "call_id": "call_read_printf", "name": "sys_terminal_read", "arguments": '{"terminal": "bash", "session": "workspace"}', } ], }, {"text": "created"}, ], key=model, ) turn2_prompt = ( f"Use sys_terminal_send on terminal 'bash' session 'workspace' to " f"create a file at {filename} containing a simple Python function. " f"Use this exact command: " f"printf 'def hello():\\n return \"hello world\"\\n' > {filename} " f"followed by Enter. Wait briefly, then reply 'created'." ) response_id = send_user_message_to_session( http_client, session_id=session_id, content=turn2_prompt, ) body = poll_session_until_terminal( http_client, session_id=session_id, response_id=response_id, timeout=180, ) assert body["status"] == "completed", ( f"Step 4 (create file) failed: status={body['status']!r}, error={body.get('error')!r}." ) # ── Step 5: Ask agent to read the file back with cat ───────────────── reset_mock_llm(mock_llm_server_url) configure_mock_llm( mock_llm_server_url, [ { "tool_calls": [ { "call_id": "call_send_cat", "name": "sys_terminal_send", "arguments": ( f'{{"terminal": "bash", "session": "workspace",' f' "text": "cat {filename}\\n"}}' ), } ], }, { "tool_calls": [ { "call_id": "call_read_cat", "name": "sys_terminal_read", "arguments": '{"terminal": "bash", "session": "workspace"}', } ], }, {"text": "read done"}, ], key=model, ) turn3_prompt = ( f"Use sys_terminal_send on terminal 'bash' session 'workspace' " f"to type 'cat {filename}' followed by Enter. Wait briefly, " f"then sys_terminal_read on session 'workspace'. " f"Reply with 'read done' once you see the file content." ) response_id = send_user_message_to_session( http_client, session_id=session_id, content=turn3_prompt, ) body = poll_session_until_terminal( http_client, session_id=session_id, response_id=response_id, timeout=180, ) assert body["status"] == "completed", ( f"Step 5 (read file) failed: status={body['status']!r}, error={body.get('error')!r}." ) # ── Step 6: Verify file content in tool output ─────────────────────── # The cat output must appear in at least one sys_terminal_read call. # We check ALL reads across the conversation since reads accumulate. all_reads = _get_function_call_outputs(http_client, session_id, "sys_terminal_read") combined_reads = " ".join(all_reads) # The file should contain the hello function with proper indentation # from printf. Assert on the indented return line which proves the # file was created with correct multi-line content. # Terminal output may escape quotes as \" or \\", so check for # the return statement with any quoting variant. assert ( 'return "hello world"' in combined_reads or 'return "hello world"' in combined_reads or 'return \\"hello world\\"' in combined_reads or "hello world" in combined_reads ), ( f"Expected 'hello world' in sys_terminal_read output " f"after cat of {filename}. Combined reads: {combined_reads!r}. " f"If empty, the printf command may not have written the file, " f"or cat didn't execute. If reads show a prompt but no file " f"content, the file path may differ from what was created." ) # ── Cleanup: remove the temp file ──────────────────────────────────── # Best-effort cleanup; configure a mock reply and don't fail the test. reset_mock_llm(mock_llm_server_url) configure_mock_llm( mock_llm_server_url, [ { "tool_calls": [ { "call_id": "call_send_rm", "name": "sys_terminal_send", "arguments": ( f'{{"terminal": "bash", "session": "workspace",' f' "text": "rm -f {filename}\\n"}}' ), } ], }, {"text": "cleaned"}, ], key=model, ) send_user_message_to_session( http_client, session_id=session_id, content=( f"Use sys_terminal_send on terminal 'bash' session " f"'workspace' to type 'rm -f {filename}' followed by " f"Enter. Reply 'cleaned'." ), )