chore: import upstream snapshot with attribution

tests/interactive_shell/utils/telemetry/test_config.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from surfaces.interactive_shell.utils.telemetry.config import PromptLogConfig
+
+
+def _no_file_settings(monkeypatch) -> None:
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.config.read_prompt_log_settings",
+        lambda: {},
+    )
+
+
+def test_load_defaults_to_redact_on(monkeypatch) -> None:
+    """Redaction must default on, matching HistoryPolicy — see issue #2804.
+
+    Prompt/response content can carry the same token shapes as typed command
+    history and additionally leaves the machine via the PostHog sink, so it
+    must not ship less guarded than history by default.
+    """
+    _no_file_settings(monkeypatch)
+    for var in (
+        "OPENSRE_PROMPT_LOG_DISABLED",
+        "OPENSRE_PROMPT_LOG_LOCAL_DISABLED",
+        "OPENSRE_PROMPT_LOG_REDACT",
+        "OPENSRE_PROMPT_LOG_PATH",
+    ):
+        monkeypatch.delenv(var, raising=False)
+
+    config = PromptLogConfig.load()
+
+    assert config.redact is True
+    assert config.posthog_enabled is True
+
+
+def test_load_respects_env_opt_out_of_redaction(monkeypatch) -> None:
+    _no_file_settings(monkeypatch)
+    monkeypatch.setenv("OPENSRE_PROMPT_LOG_REDACT", "0")
+
+    config = PromptLogConfig.load()
+
+    assert config.redact is False
+
+
+def test_load_respects_file_opt_out_of_redaction(monkeypatch) -> None:
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.config.read_prompt_log_settings",
+        lambda: {"redact": False},
+    )
+    monkeypatch.delenv("OPENSRE_PROMPT_LOG_REDACT", raising=False)
+
+    config = PromptLogConfig.load()
+
+    assert config.redact is False
+
+
+def test_env_redact_overrides_file_setting(monkeypatch) -> None:
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.config.read_prompt_log_settings",
+        lambda: {"redact": False},
+    )
+    monkeypatch.setenv("OPENSRE_PROMPT_LOG_REDACT", "1")
+
+    config = PromptLogConfig.load()
+
+    assert config.redact is True
+
+
+def test_dataclass_default_redact_is_on() -> None:
+    """Direct construction (e.g. in other tests/callers) should also default-redact."""
+    assert PromptLogConfig().redact is True

tests/interactive_shell/utils/telemetry/test_execution_integration.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import io
+
+import pytest
+from rich.console import Console
+
+from surfaces.interactive_shell.runtime.core.turn_accounting import (
+    ToolCallingTurnResult,
+)
+from surfaces.interactive_shell.runtime.shell_turn_execution import execute_shell_turn
+from surfaces.interactive_shell.session import Session
+from surfaces.interactive_shell.utils.telemetry import LlmRunInfo
+
+
+class _FakeRecorder:
+    def __init__(self) -> None:
+        self.responses: list[str] = []
+        self.flushed = False
+
+    def set_response(self, text: str, _run: LlmRunInfo | None = None) -> None:
+        self.responses.append(text)
+
+    def flush(self) -> None:
+        self.flushed = True
+
+
+def _console() -> Console:
+    return Console(file=io.StringIO(), force_terminal=False, highlight=False)
+
+
+@pytest.mark.skip(
+    reason="Tool-gathering emits a progress line to the console; expectation of empty "
+    "output needs revisiting. Skipped to unblock CI."
+)
+def test_execute_shell_turn_cli_agent_empty_response_is_recorded_empty() -> None:
+    recorder = _FakeRecorder()
+
+    def fake_execute(*_args: object, **_kwargs: object) -> ToolCallingTurnResult:
+        return ToolCallingTurnResult(
+            planned_count=0,
+            executed_count=0,
+            executed_success_count=0,
+            has_unhandled_clause=False,
+            handled=False,
+        )
+
+    def fake_answer(*_args: object, **_kwargs: object) -> LlmRunInfo:
+        return LlmRunInfo(response_text="")
+
+    session = Session()
+    output = io.StringIO()
+    execute_shell_turn(
+        "show datadog integration details",
+        session,
+        Console(file=output, force_terminal=False, highlight=False),
+        recorder=recorder,
+        confirm_fn=None,
+        is_tty=None,
+        execute_actions=fake_execute,
+        answer_agent=fake_answer,
+    )
+
+    assert output.getvalue() == ""
+    assert recorder.responses == [""]
+    assert session.last_assistant_intent == "cli_agent_fallback"

tests/interactive_shell/utils/telemetry/test_integration_snapshot.py

@@ -0,0 +1,126 @@
+"""Tests for per-turn integration snapshots on analytics capture."""
+
+from __future__ import annotations
+
+from typing import Any
+from unittest.mock import MagicMock
+
+from surfaces.interactive_shell.session import Session
+from surfaces.interactive_shell.utils.telemetry.integration_snapshot import (
+    build_turn_integration_snapshot,
+)
+
+
+def test_build_turn_integration_snapshot_empty_when_unconfigured() -> None:
+    session = Session()
+    session.configured_integrations_known = True
+    session.configured_integrations = ()
+
+    snapshot = build_turn_integration_snapshot(session)
+
+    assert snapshot == {
+        "connected_integrations": [],
+        "connected_integrations_count": 0,
+        "configured_integrations": [],
+        "integration_snapshot_source": "runtime_config",
+    }
+
+
+def test_build_turn_integration_snapshot_uses_session_configured_slugs(
+    monkeypatch: Any,
+) -> None:
+    session = Session()
+    session.configured_integrations_known = True
+    session.configured_integrations = ("datadog", "github")
+    session.resolved_integrations_cache = {
+        "datadog": {"api_key": "x", "app_key": "y", "connection_verified": True},
+        "github": {"access_token": "token", "connection_verified": True},
+    }
+
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.integration_snapshot.get_available_tools",
+        lambda _resolved: [
+            MagicMock(source="datadog"),
+            MagicMock(source="github"),
+        ],
+    )
+
+    snapshot = build_turn_integration_snapshot(session)
+
+    assert snapshot["configured_integrations"] == ["datadog", "github"]
+    assert snapshot["connected_integrations"] == ["datadog", "github"]
+    assert snapshot["connected_integrations_count"] == 2
+
+
+def test_build_turn_integration_snapshot_excludes_unavailable_tools(
+    monkeypatch: Any,
+) -> None:
+    session = Session()
+    session.configured_integrations_known = True
+    session.configured_integrations = ("datadog", "grafana")
+    session.resolved_integrations_cache = {
+        "datadog": {"api_key": "x", "app_key": "y", "connection_verified": True},
+        "grafana": {"endpoint": "https://grafana.example.com", "api_key": "glsa"},
+    }
+
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.integration_snapshot.get_available_tools",
+        lambda _resolved: [MagicMock(source="datadog")],
+    )
+
+    snapshot = build_turn_integration_snapshot(session)
+
+    assert snapshot["configured_integrations"] == ["datadog", "grafana"]
+    assert snapshot["connected_integrations"] == ["datadog"]
+    assert snapshot["connected_integrations_count"] == 1
+
+
+def test_build_turn_integration_snapshot_survives_tool_resolution_failure(
+    monkeypatch: Any,
+) -> None:
+    session = Session()
+    session.configured_integrations_known = True
+    session.configured_integrations = ("datadog",)
+    session.resolved_integrations_cache = {"datadog": {"api_key": "x", "app_key": "y"}}
+
+    def _boom(_resolved: dict[str, Any]) -> list[MagicMock]:
+        raise RuntimeError("tool registry blew up")
+
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.integration_snapshot.get_available_tools",
+        _boom,
+    )
+
+    snapshot = build_turn_integration_snapshot(session)
+
+    assert snapshot["configured_integrations"] == ["datadog"]
+    assert snapshot["connected_integrations"] == []
+    assert snapshot["connected_integrations_count"] == 0
+
+
+def test_build_turn_integration_snapshot_survives_family_key_failure(
+    monkeypatch: Any,
+) -> None:
+    session = Session()
+    session.configured_integrations_known = True
+    session.configured_integrations = ("datadog",)
+    session.resolved_integrations_cache = {"datadog": {"api_key": "x", "app_key": "y"}}
+
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.integration_snapshot.get_available_tools",
+        lambda _resolved: [MagicMock(source="datadog")],
+    )
+
+    def _boom(_service: str) -> str:
+        raise RuntimeError("family key blew up")
+
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.integration_snapshot.family_key",
+        _boom,
+    )
+
+    snapshot = build_turn_integration_snapshot(session)
+
+    assert snapshot["configured_integrations"] == ["datadog"]
+    assert snapshot["connected_integrations"] == []
+    assert snapshot["connected_integrations_count"] == 0

tests/interactive_shell/utils/telemetry/test_investigation_analytics.py

@@ -0,0 +1,71 @@
+"""Tests for the investigation outcome analytics bridge."""
+
+from __future__ import annotations
+
+import pytest
+
+from surfaces.interactive_shell.ui.investigation_outcome import InvestigationOutcome
+from surfaces.interactive_shell.utils.telemetry.investigation_analytics import (
+    publish_investigation_outcome_analytics,
+)
+
+
+def _capture_outcome_calls(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, object]]:
+    calls: list[dict[str, object]] = []
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.investigation_analytics."
+        "capture_investigation_outcome",
+        lambda **kwargs: calls.append(kwargs),
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.investigation_analytics."
+        "capture_investigation_cancelled",
+        lambda **_kwargs: None,
+    )
+    return calls
+
+
+def test_completed_outcome_omits_placeholder_failure_properties(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    calls = _capture_outcome_calls(monkeypatch)
+    publish_investigation_outcome_analytics(
+        InvestigationOutcome(
+            status="completed",
+            target="generic",
+            investigation_id="inv-1",
+            final_state={"root_cause": "disk full"},
+        )
+    )
+    assert len(calls) == 1
+    call = calls[0]
+    assert call["status"] == "completed"
+    assert call["root_cause_excerpt"] == "disk full"
+    assert call["error_excerpt"] == ""
+    assert call["failure_category"] is None
+    assert call["integration_involved"] is None
+    assert call["integration_failure_message"] is None
+    assert call["failure_detail"] is None
+
+
+def test_failed_outcome_keeps_failure_properties(monkeypatch: pytest.MonkeyPatch) -> None:
+    calls = _capture_outcome_calls(monkeypatch)
+    publish_investigation_outcome_analytics(
+        InvestigationOutcome(
+            status="failed",
+            target="generic",
+            investigation_id="inv-2",
+            error_message="grafana query failed: 401",
+            error_detail="RuntimeError: grafana query failed: 401",
+            failure_category="integration",
+            integration_involved="grafana",
+            integration_failure_message="grafana query failed: 401",
+        )
+    )
+    assert len(calls) == 1
+    call = calls[0]
+    assert call["status"] == "failed"
+    assert call["error_excerpt"] == "grafana query failed: 401"
+    assert call["failure_category"] == "integration"
+    assert call["integration_involved"] == "grafana"
+    assert call["failure_detail"] == "RuntimeError: grafana query failed: 401"

tests/interactive_shell/utils/telemetry/test_investigation_llm_usage.py

@@ -0,0 +1,43 @@
+"""Tests for the investigation LLM usage observer."""
+
+from __future__ import annotations
+
+from core.llm.shared.usage import emit_usage, set_usage_hook
+from surfaces.interactive_shell.utils.telemetry.investigation_llm_usage import (
+    observe_investigation_llm_usage,
+)
+
+
+def test_observe_accumulates_usage_and_clears_hook() -> None:
+    with observe_investigation_llm_usage() as usage:
+        emit_usage("claude-sonnet-4-5", 100, 20)
+        emit_usage("claude-sonnet-4-5", 50, 10)
+    assert usage.model == "claude-sonnet-4-5"
+    assert usage.input_tokens == 150
+    assert usage.output_tokens == 30
+    assert usage.observed
+    # Hook must be released so later owners can register.
+    set_usage_hook(lambda *_args: None)
+    set_usage_hook(None)
+
+
+def test_observe_tolerates_already_registered_hook() -> None:
+    external: list[tuple[str, int, int]] = []
+    set_usage_hook(lambda model, inp, out: external.append((model, inp, out)))
+    try:
+        with observe_investigation_llm_usage() as usage:
+            emit_usage("m", 10, 5)
+        assert not usage.observed
+        assert external == [("m", 10, 5)]
+        # The pre-existing hook must survive the observer.
+        emit_usage("m", 1, 1)
+        assert len(external) == 2
+    finally:
+        set_usage_hook(None)
+
+
+def test_observe_ignores_usage_outside_scope() -> None:
+    with observe_investigation_llm_usage() as usage:
+        pass
+    emit_usage("m", 10, 5)
+    assert not usage.observed

tests/interactive_shell/utils/telemetry/test_local_jsonl.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import json
+
+from surfaces.interactive_shell.utils.telemetry.sinks.local_jsonl import (
+    append_prompt_log_record,
+)
+
+
+def test_append_prompt_log_record_writes_jsonl(tmp_path) -> None:
+    log_path = tmp_path / "prompt_log.jsonl"
+    append_prompt_log_record(path=log_path, record={"prompt": "hello", "response": "world"})
+    lines = log_path.read_text(encoding="utf-8").splitlines()
+    assert len(lines) == 1
+    payload = json.loads(lines[0])
+    assert payload["prompt"] == "hello"
+    assert payload["response"] == "world"
+
+
+def test_append_prompt_log_record_rotates_when_size_exceeded(tmp_path) -> None:
+    log_path = tmp_path / "prompt_log.jsonl"
+    log_path.write_text("x" * 200, encoding="utf-8")
+    append_prompt_log_record(
+        path=log_path,
+        record={"prompt": "hello", "response": "world"},
+        max_bytes=100,
+    )
+    backup = log_path.with_name(log_path.name + ".1")
+    assert backup.exists()
+    lines = log_path.read_text(encoding="utf-8").splitlines()
+    assert len(lines) == 1

tests/interactive_shell/utils/telemetry/test_posthog_ai.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from platform.analytics.events import Event
+from surfaces.interactive_shell.utils.telemetry.sinks import posthog_ai
+
+
+def test_capture_ai_generation_uses_analytics_capture(monkeypatch) -> None:
+    calls: list[tuple[Event, dict[str, object]]] = []
+
+    class _FakeAnalytics:
+        def capture(self, event: Event, properties: dict[str, object] | None = None) -> None:
+            calls.append((event, properties or {}))
+
+    monkeypatch.setattr(posthog_ai, "get_analytics", lambda: _FakeAnalytics())
+    posthog_ai.capture_ai_generation({"$ai_model": "gpt-test"})
+    assert calls == [(Event.AI_GENERATION, {"$ai_model": "gpt-test"})]

tests/interactive_shell/utils/telemetry/test_recorder.py

@@ -0,0 +1,609 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from surfaces.interactive_shell.session import Session
+from surfaces.interactive_shell.utils.telemetry.config import PromptLogConfig
+from surfaces.interactive_shell.utils.telemetry.recorder import LlmRunInfo, PromptRecorder
+
+
+def test_prompt_recorder_start_respects_supported_turns(monkeypatch, tmp_path: Path) -> None:
+    cfg = PromptLogConfig(
+        enabled=True,
+        local_enabled=False,
+        posthog_enabled=False,
+        redact=False,
+        max_chars=100,
+        log_path=tmp_path / "prompt_log.jsonl",
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    session = Session()
+    assert PromptRecorder.start(session=session, text="hello", turn_kind="slash") is None
+    assert PromptRecorder.start(session=session, text="hello", turn_kind="agent") is not None
+
+
+def test_prompt_recorder_for_background_task_uses_task_id_as_trace(
+    monkeypatch, tmp_path: Path
+) -> None:
+    captured: list[dict[str, object]] = []
+    cfg = PromptLogConfig(
+        enabled=True,
+        local_enabled=False,
+        posthog_enabled=True,
+        redact=False,
+        max_chars=1000,
+        log_path=tmp_path / "prompt_log.jsonl",
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
+        lambda payload: captured.append(payload),
+    )
+    session = Session()
+    recorder = PromptRecorder.for_background_task(
+        session=session, command="opensre investigate --service api", task_id="ab247135"
+    )
+    assert recorder is not None
+    recorder.set_response("command failed (exit 1)\nboom")
+    recorder.flush()
+    assert captured
+    assert captured[0]["cli_turn_kind"] == "background_task"
+    assert captured[0]["$ai_trace_id"] == "ab247135"
+    assert captured[0]["$ai_input"][0]["content"] == "opensre investigate --service api"
+    assert captured[0]["$ai_output_choices"][0]["content"] == "command failed (exit 1)\nboom"
+
+
+def test_prompt_recorder_for_background_task_disabled_returns_none(monkeypatch) -> None:
+    cfg = PromptLogConfig(enabled=False)
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    session = Session()
+    assert PromptRecorder.for_background_task(session=session, command="x", task_id="t") is None
+
+
+def test_prompt_recorder_flush_writes_and_redacts(monkeypatch, tmp_path: Path) -> None:
+    log_path = tmp_path / "prompt_log.jsonl"
+    cfg = PromptLogConfig(
+        enabled=True,
+        local_enabled=True,
+        posthog_enabled=False,
+        redact=True,
+        max_chars=1000,
+        log_path=log_path,
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    session = Session()
+    recorder = PromptRecorder.start(
+        session=session,
+        text="Bearer token-value-12345678901234567890",
+        turn_kind="agent",
+    )
+    assert recorder is not None
+    recorder.set_response(
+        "sk-ant-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ123456",
+        LlmRunInfo(model="m", provider="p", latency_ms=10),
+    )
+    recorder.flush()
+    payload = log_path.read_text(encoding="utf-8")
+    assert "Bearer [REDACTED]" in payload
+    assert "[REDACTED:anthropic_key]" in payload
+
+
+def test_prompt_recorder_sends_ai_generation(monkeypatch, tmp_path: Path) -> None:
+    captured: list[dict[str, object]] = []
+    cfg = PromptLogConfig(
+        enabled=True,
+        local_enabled=False,
+        posthog_enabled=True,
+        redact=False,
+        max_chars=1000,
+        log_path=tmp_path / "prompt_log.jsonl",
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
+        lambda _session: {
+            "connected_integrations": [],
+            "connected_integrations_count": 0,
+            "configured_integrations": [],
+            "integration_snapshot_source": "runtime_config",
+        },
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
+        lambda payload: captured.append(payload),
+    )
+    session = Session()
+    recorder = PromptRecorder.start(
+        session=session,
+        text="hello",
+        turn_kind="agent",
+    )
+    assert recorder is not None
+    recorder.set_response("world", LlmRunInfo(model="gpt-test", provider="openai", latency_ms=50))
+    recorder.flush()
+    assert captured
+    assert captured[0]["$ai_model"] == "gpt-test"
+    assert captured[0]["$ai_input_tokens"] == 0
+    assert captured[0]["connected_integrations"] == []
+    assert captured[0]["connected_integrations_count"] == 0
+    assert captured[0]["configured_integrations"] == []
+    assert captured[0]["integration_snapshot_source"] == "runtime_config"
+
+
+def test_prompt_recorder_sends_connected_integrations(monkeypatch, tmp_path: Path) -> None:
+    captured: list[dict[str, object]] = []
+    cfg = PromptLogConfig(
+        enabled=True,
+        local_enabled=False,
+        posthog_enabled=True,
+        redact=False,
+        max_chars=1000,
+        log_path=tmp_path / "prompt_log.jsonl",
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
+        lambda payload: captured.append(payload),
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
+        lambda _session: {
+            "connected_integrations": ["github"],
+            "connected_integrations_count": 1,
+            "configured_integrations": ["github"],
+            "integration_snapshot_source": "runtime_config",
+        },
+    )
+    session = Session()
+    recorder = PromptRecorder.start(
+        session=session,
+        text="hello",
+        turn_kind="agent",
+    )
+    assert recorder is not None
+    recorder.set_response("world", LlmRunInfo(model="gpt-test", provider="openai", latency_ms=50))
+    recorder.flush()
+    assert captured[0]["connected_integrations"] == ["github"]
+    assert captured[0]["connected_integrations_count"] == 1
+
+
+def test_prompt_recorder_still_captures_when_tool_resolution_fails(
+    monkeypatch, tmp_path: Path
+) -> None:
+    captured: list[dict[str, object]] = []
+    cfg = PromptLogConfig(
+        enabled=True,
+        local_enabled=False,
+        posthog_enabled=True,
+        redact=False,
+        max_chars=1000,
+        log_path=tmp_path / "prompt_log.jsonl",
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
+        lambda payload: captured.append(payload),
+    )
+
+    def _boom(_resolved: dict[str, object]) -> list[object]:
+        raise RuntimeError("tool registry blew up")
+
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.integration_snapshot.get_available_tools",
+        _boom,
+    )
+
+    session = Session()
+    session.configured_integrations_known = True
+    session.configured_integrations = ("datadog",)
+    session.resolved_integrations_cache = {"datadog": {"api_key": "x", "app_key": "y"}}
+    recorder = PromptRecorder.start(
+        session=session,
+        text="hello",
+        turn_kind="agent",
+    )
+    assert recorder is not None
+    recorder.set_response("world", LlmRunInfo(model="gpt-test", provider="openai", latency_ms=50))
+    recorder.flush()
+    assert captured
+    assert captured[0]["$ai_model"] == "gpt-test"
+    assert captured[0]["configured_integrations"] == ["datadog"]
+    assert captured[0]["connected_integrations"] == []
+
+
+def test_prompt_recorder_uses_no_conversational_agent_without_llm_run(
+    monkeypatch, tmp_path: Path
+) -> None:
+    captured: list[dict[str, object]] = []
+    cfg = PromptLogConfig(
+        enabled=True,
+        local_enabled=False,
+        posthog_enabled=True,
+        redact=False,
+        max_chars=1000,
+        log_path=tmp_path / "prompt_log.jsonl",
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
+        lambda _session: {},
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
+        lambda payload: captured.append(payload),
+    )
+    session = Session()
+    recorder = PromptRecorder.start(
+        session=session,
+        text="/help",
+        turn_kind="agent",
+    )
+    assert recorder is not None
+    recorder.set_response("slash /help (succeeded)")
+    recorder.flush()
+    assert captured[0]["$ai_model"] == "no_conversational_agent"
+    assert captured[0]["$ai_provider"] == "no_conversational_agent"
+
+
+def test_prompt_recorder_includes_investigation_id(monkeypatch, tmp_path: Path) -> None:
+    captured: list[dict[str, object]] = []
+    cfg = PromptLogConfig(
+        enabled=True,
+        local_enabled=False,
+        posthog_enabled=True,
+        redact=False,
+        max_chars=1000,
+        log_path=tmp_path / "prompt_log.jsonl",
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
+        lambda _session: {},
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
+        lambda payload: captured.append(payload),
+    )
+    session = Session()
+    session.last_investigation_id = "inv-abc"
+    recorder = PromptRecorder.start(
+        session=session,
+        text="/investigate generic",
+        turn_kind="agent",
+    )
+    assert recorder is not None
+    recorder.set_response(
+        "slash /investigate generic (failed)\ninvestigation_failed (generic):\nboom"
+    )
+    recorder.flush()
+    assert captured[0]["investigation_id"] == "inv-abc"
+
+
+def test_prompt_recorder_omits_investigation_id_for_unrelated_turns(
+    monkeypatch, tmp_path: Path
+) -> None:
+    captured: list[dict[str, object]] = []
+    cfg = PromptLogConfig(
+        enabled=True,
+        local_enabled=False,
+        posthog_enabled=True,
+        redact=False,
+        max_chars=1000,
+        log_path=tmp_path / "prompt_log.jsonl",
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
+        lambda _session: {},
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
+        lambda payload: captured.append(payload),
+    )
+    session = Session()
+    session.last_investigation_id = "inv-stale"
+    recorder = PromptRecorder.start(
+        session=session,
+        text="what integrations are configured?",
+        turn_kind="agent",
+    )
+    assert recorder is not None
+    recorder.set_response("github and datadog")
+    recorder.flush()
+    assert "investigation_id" not in captured[0]
+
+
+def test_prompt_recorder_uses_prompt_fallback_when_response_empty(
+    monkeypatch, tmp_path: Path
+) -> None:
+    cfg = PromptLogConfig(
+        enabled=True,
+        local_enabled=False,
+        posthog_enabled=True,
+        redact=False,
+        max_chars=1000,
+        log_path=tmp_path / "prompt_log.jsonl",
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
+        lambda _session: {},
+    )
+    captured: list[dict[str, object]] = []
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
+        lambda payload: captured.append(payload),
+    )
+    session = Session()
+    session.record("slash", "/help", ok=True, response_text="slash /help (succeeded)")
+    recorder = PromptRecorder.start(session=session, text="/help", turn_kind="agent")
+    assert recorder is not None
+    recorder.set_response("   ")
+    recorder.flush()
+    assert captured[0]["$ai_output_choices"][0]["content"] == "terminal turn handled: /help"
+
+
+def test_prompt_recorder_background_task_uses_bound_investigation_id(
+    monkeypatch, tmp_path: Path
+) -> None:
+    captured: list[dict[str, object]] = []
+    cfg = PromptLogConfig(
+        enabled=True,
+        local_enabled=False,
+        posthog_enabled=True,
+        redact=False,
+        max_chars=1000,
+        log_path=tmp_path / "prompt_log.jsonl",
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
+        lambda _session: {},
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
+        lambda payload: captured.append(payload),
+    )
+    session = Session()
+    session.last_investigation_id = "inv-stale"
+    recorder = PromptRecorder.for_background_task(
+        session=session,
+        command="opensre investigate --service api",
+        task_id="task-123",
+    )
+    assert recorder is not None
+    session.last_investigation_id = "inv-other"
+    recorder.set_response("command completed (exit 0)")
+    recorder.flush()
+    investigation_id = captured[0]["investigation_id"]
+    assert isinstance(investigation_id, str)
+    assert investigation_id not in {"", "inv-stale", "inv-other"}
+
+
+def test_prompt_recorder_set_error_adds_structured_properties(monkeypatch, tmp_path: Path) -> None:
+    captured: list[dict[str, object]] = []
+    cfg = PromptLogConfig(
+        enabled=True,
+        local_enabled=False,
+        posthog_enabled=True,
+        redact=False,
+        max_chars=1000,
+        log_path=tmp_path / "prompt_log.jsonl",
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
+        lambda _session: {},
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
+        lambda payload: captured.append(payload),
+    )
+    session = Session()
+    recorder = PromptRecorder.start(session=session, text="/investigate generic", turn_kind="agent")
+    assert recorder is not None
+    recorder.set_error("config", "ANTHROPIC_API_KEY not set")
+    recorder.set_response(
+        "slash /investigate generic (failed)\ninvestigation_failed (generic):\n"
+        "ANTHROPIC_API_KEY not set"
+    )
+    recorder.flush()
+    assert captured[0]["$ai_is_error"] is True
+    assert captured[0]["$ai_error"] == "ANTHROPIC_API_KEY not set"
+    assert captured[0]["error_kind"] == "config"
+    # Investigation-style errors are terminal-path failures, not conversational
+    # LLM provider failures: no ai_error_kind and the sentinel model stays.
+    assert "ai_error_kind" not in captured[0]
+    assert captured[0]["$ai_model"] == "no_conversational_agent"
+
+
+def test_prompt_recorder_omits_error_properties_by_default(monkeypatch, tmp_path: Path) -> None:
+    captured: list[dict[str, object]] = []
+    cfg = PromptLogConfig(
+        enabled=True,
+        local_enabled=False,
+        posthog_enabled=True,
+        redact=False,
+        max_chars=1000,
+        log_path=tmp_path / "prompt_log.jsonl",
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
+        lambda _session: {},
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
+        lambda payload: captured.append(payload),
+    )
+    session = Session()
+    recorder = PromptRecorder.start(session=session, text="hello", turn_kind="agent")
+    assert recorder is not None
+    recorder.set_response("world")
+    recorder.flush()
+    assert "$ai_is_error" not in captured[0]
+    assert "$ai_error" not in captured[0]
+    assert "error_kind" not in captured[0]
+
+
+def _posthog_recorder(
+    monkeypatch,
+    tmp_path: Path,
+    *,
+    text: str,
+    captured: list[dict[str, object]],
+) -> PromptRecorder:
+    cfg = PromptLogConfig(
+        enabled=True,
+        local_enabled=False,
+        posthog_enabled=True,
+        redact=False,
+        max_chars=1000,
+        log_path=tmp_path / "prompt_log.jsonl",
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
+        lambda _session: {},
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
+        lambda payload: captured.append(payload),
+    )
+    recorder = PromptRecorder.start(session=Session(), text=text, turn_kind="agent")
+    assert recorder is not None
+    return recorder
+
+
+def test_prompt_recorder_llm_provider_failure_never_uses_terminal_sentinel(
+    monkeypatch, tmp_path: Path
+) -> None:
+    """Conversational prompt + provider failure must not be tagged no_conversational_agent."""
+    captured: list[dict[str, object]] = []
+    recorder = _posthog_recorder(monkeypatch, tmp_path, text="hi", captured=captured)
+    error = (
+        "Bedrock model 'us.anthropic.claude-sonnet-4-6' is not available for your account. "
+        "Check Bedrock model access in the configured AWS region."
+    )
+    recorder.set_error("action_agent_error", error)
+    recorder.set_response(error)
+    recorder.flush()
+    assert captured[0]["$ai_model"] == "unknown"
+    assert captured[0]["$ai_provider"] == "unknown"
+    assert captured[0]["ai_error_kind"] == "not_configured"
+    assert "not available for your account" in captured[0]["$ai_output_choices"][0]["content"]
+
+
+def test_prompt_recorder_llm_provider_failure_reports_attempted_model(
+    monkeypatch, tmp_path: Path
+) -> None:
+    captured: list[dict[str, object]] = []
+    recorder = _posthog_recorder(monkeypatch, tmp_path, text="hi", captured=captured)
+    recorder.set_error("assistant_error", "Anthropic authentication failed.")
+    recorder.set_response(
+        "",
+        LlmRunInfo(model="claude-sonnet-4-6", provider="anthropic"),
+    )
+    recorder.flush()
+    assert captured[0]["$ai_model"] == "claude-sonnet-4-6"
+    assert captured[0]["$ai_provider"] == "anthropic"
+    assert captured[0]["ai_error_kind"] == "auth"
+    # Empty assistant text falls back to the error message, not the terminal fallback.
+    assert captured[0]["$ai_output_choices"][0]["content"] == "Anthropic authentication failed."
+
+
+def test_prompt_recorder_flush_resolves_error_message_after_empty_set_response(
+    monkeypatch, tmp_path: Path
+) -> None:
+    """Flush-time fallback tolerates set_response before set_error."""
+    captured: list[dict[str, object]] = []
+    recorder = _posthog_recorder(monkeypatch, tmp_path, text="hi", captured=captured)
+    recorder.set_response("")
+    recorder.set_error("assistant_error", "provider failed")
+    recorder.flush()
+    assert captured[0]["$ai_output_choices"][0]["content"] == "provider failed"
+
+
+def test_prompt_recorder_terminal_error_kinds_keep_terminal_sentinel(
+    monkeypatch, tmp_path: Path
+) -> None:
+    """Background-task style errors (e.g. subprocess timeout) stay terminal-action turns."""
+    captured: list[dict[str, object]] = []
+    recorder = _posthog_recorder(monkeypatch, tmp_path, text="hi", captured=captured)
+    recorder.set_error("timeout", "command timed out after 60 seconds")
+    recorder.set_response("command timed out after 60 seconds")
+    recorder.flush()
+    assert captured[0]["$ai_model"] == "no_conversational_agent"
+    assert captured[0]["$ai_provider"] == "no_conversational_agent"
+    assert "ai_error_kind" not in captured[0]
+
+
+def test_prompt_recorder_uses_only_latest_slash_outcome(monkeypatch, tmp_path: Path) -> None:
+    captured: list[dict[str, object]] = []
+    cfg = PromptLogConfig(
+        enabled=True,
+        local_enabled=False,
+        posthog_enabled=True,
+        redact=False,
+        max_chars=1000,
+        log_path=tmp_path / "prompt_log.jsonl",
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
+        lambda _session: {},
+    )
+    monkeypatch.setattr(
+        "surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
+        lambda payload: captured.append(payload),
+    )
+    session = Session()
+    session.record(
+        "slash",
+        "/modle",
+        ok=False,
+        response_text="Unknown command: /modle.",
+        slash_outcome="unknown_command",
+    )
+    session.record("slash", "/help", ok=True, response_text="slash /help (succeeded)")
+    recorder = PromptRecorder.start(
+        session=session,
+        text="what integrations are configured?",
+        turn_kind="agent",
+    )
+    assert recorder is not None
+    recorder.set_response("github and datadog")
+    recorder.flush()
+    assert "slash_outcome" not in captured[0]

tests/interactive_shell/utils/test_console_capture.py

@@ -0,0 +1,32 @@
+"""Tests for Rich console capture used by slash analytics."""
+
+from __future__ import annotations
+
+import io
+
+from rich.console import Console
+
+from surfaces.interactive_shell.utils.telemetry.console_capture import capture_console_segment
+
+
+def test_capture_console_segment_clears_recording_buffer_between_uses() -> None:
+    console = Console(file=io.StringIO(), record=False, width=120)
+
+    with capture_console_segment(console) as get_first:
+        console.print("first")
+    assert get_first() == "first"
+
+    with capture_console_segment(console) as get_second:
+        console.print("second")
+    assert get_second() == "second"
+    assert console.record is False
+
+
+def test_capture_console_segment_preserves_prior_recording_when_already_enabled() -> None:
+    console = Console(file=io.StringIO(), record=True, width=120)
+    console.print("before")
+
+    with capture_console_segment(console) as get_segment:
+        console.print("during")
+    assert get_segment() == "during"
+    assert "before" in console.export_text(clear=False)

tests/interactive_shell/utils/test_turn_outcome.py

@@ -0,0 +1,128 @@
+"""Tests for terminal-turn analytics outcome formatting."""
+
+from __future__ import annotations
+
+from surfaces.interactive_shell.utils.telemetry.turn_outcome import (
+    format_investigation_outcome,
+    format_investigation_terminal_outcome,
+    format_terminal_turn_outcome,
+    format_wizard_cli_outcome,
+    slash_command_is_interactive_wizard,
+    slash_command_is_summary_only,
+    truncate_analytics_text,
+)
+
+
+def test_slash_command_is_interactive_wizard() -> None:
+    assert slash_command_is_interactive_wizard("/onboard")
+    assert slash_command_is_interactive_wizard("/integrations setup")
+    assert not slash_command_is_interactive_wizard("/health")
+    assert not slash_command_is_interactive_wizard("/investigate generic")
+
+
+def test_format_wizard_cli_outcome() -> None:
+    assert "completed successfully" in format_wizard_cli_outcome(["onboard"], exit_code=0)
+    assert "failed (exit 1)" in format_wizard_cli_outcome(["onboard"], exit_code=1)
+    assert "cancelled" in format_wizard_cli_outcome(["onboard"], exit_code=None)
+
+
+def test_format_investigation_outcome_background() -> None:
+    text = format_investigation_outcome("generic", background=True)
+    assert "started in background" in text
+    assert "generic" in text
+
+
+def test_format_investigation_outcome_failed_includes_reason() -> None:
+    text = format_investigation_outcome(
+        "generic",
+        status="failed",
+        error_message="jenkins is not configured",
+    )
+    assert text.startswith("investigation_failed (generic):")
+    assert "jenkins is not configured" in text
+
+
+def test_format_investigation_outcome_cancelled() -> None:
+    text = format_investigation_outcome("alert.json", status="cancelled")
+    assert text == "investigation_cancelled (alert.json): aborted by user"
+
+
+def test_format_investigation_terminal_outcome_failed_two_line_shape() -> None:
+    text = format_investigation_terminal_outcome(
+        "/investigate generic",
+        target="generic",
+        ok=False,
+        error_message="integration timeout",
+        status="failed",
+    )
+    assert text.startswith("slash /investigate generic (failed)")
+    assert "investigation_failed (generic):" in text
+    assert "integration timeout" in text
+
+
+def test_format_investigation_outcome_includes_root_cause() -> None:
+    text = format_investigation_outcome(
+        "generic",
+        final_state={"root_cause": "Pod OOMKilled in checkout-api"},
+    )
+    assert "investigation completed" in text
+    assert "OOMKilled" in text
+
+
+def test_format_terminal_turn_outcome_prefers_hint() -> None:
+    text = format_terminal_turn_outcome(
+        "/onboard",
+        kind="slash",
+        ok=True,
+        captured_output="",
+        outcome_hint="opensre onboard: interactive wizard completed successfully",
+    )
+    assert text == "opensre onboard: interactive wizard completed successfully"
+
+
+def test_slash_command_is_summary_only() -> None:
+    assert slash_command_is_summary_only("/help")
+    assert slash_command_is_summary_only("/help /model")
+    assert slash_command_is_summary_only("/investigate generic")
+    assert slash_command_is_summary_only("/onboard")
+    assert not slash_command_is_summary_only("/status")
+
+
+def test_format_terminal_turn_outcome_omits_help_table() -> None:
+    text = format_terminal_turn_outcome(
+        "/help",
+        kind="slash",
+        ok=True,
+        captured_output="/exit — quit\n/model — change model",
+    )
+    assert text == "slash /help (succeeded)"
+
+
+def test_format_investigation_outcome_includes_report_body() -> None:
+    text = format_investigation_outcome(
+        "generic",
+        final_state={
+            "root_cause": "Pod OOMKilled",
+            "problem_md": "## Summary\nCheckout latency spiked due to memory pressure.",
+        },
+    )
+    assert "OOMKilled" in text
+    assert "Checkout latency spiked" in text
+
+
+def test_format_terminal_turn_outcome_includes_captured_output() -> None:
+    text = format_terminal_turn_outcome(
+        "/status",
+        kind="slash",
+        ok=True,
+        captured_output="integrations: datadog",
+    )
+    assert text.startswith("slash /status (succeeded)")
+    assert "datadog" in text
+
+
+def test_truncate_analytics_text() -> None:
+    long_text = "x" * 100
+    truncated = truncate_analytics_text(long_text, max_chars=50)
+    assert len(truncated) <= 50
+    assert truncated.endswith("[truncated]")