chore: import upstream snapshot with attribution
CI (OpenClaw E2E) / openclaw test (push) Has been cancelled
CI / coverage-report (push) Has been cancelled
CI / test-kubernetes (push) Has been cancelled
CI / should-run-thorough (push) Has been cancelled
CI / test-thorough (cloudwatch-demo) (push) Has been cancelled
CI / test-thorough (flink-ecs) (push) Has been cancelled
CI / test-thorough (upstream-lambda) (push) Has been cancelled
CI / test-thorough (prefect-ecs-fargate) (push) Has been cancelled
Release / build-binaries (zip, opensre.exe, onefile, windows-latest, windows-x64) (push) Has been cancelled
Benchmark image — build + push to ECR (any adapter) / build + push (push) Has been cancelled
CI / quality (ubuntu-latest) (push) Has been cancelled
CI / test (tools-runtime) (push) Has been cancelled
CI / test (e2e-general) (push) Has been cancelled
CI / test (cli-runtime) (push) Has been cancelled
CI / test (e2e-provider-and-openclaw) (push) Has been cancelled
CI / test (integrations-and-misc) (push) Has been cancelled
Release / verify (push) Has been cancelled
Release / build-python-dist (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-15-intel, darwin-x64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-latest, darwin-arm64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04, linux-x64) (push) Has been cancelled
Release / publish-release (push) Has been cancelled
Release / publish-main-release (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-checks (no-LLM) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-live shard ${{ matrix.shard_index }} (push) Has been cancelled
Release / prepare (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04-arm, linux-arm64) (push) Has been cancelled
Synthetic Deterministic Tests / Synthetic offline (deterministic) (push) Has been cancelled
CI (OpenClaw E2E) / openclaw test (push) Has been cancelled
CI / coverage-report (push) Has been cancelled
CI / test-kubernetes (push) Has been cancelled
CI / should-run-thorough (push) Has been cancelled
CI / test-thorough (cloudwatch-demo) (push) Has been cancelled
CI / test-thorough (flink-ecs) (push) Has been cancelled
CI / test-thorough (upstream-lambda) (push) Has been cancelled
CI / test-thorough (prefect-ecs-fargate) (push) Has been cancelled
Release / build-binaries (zip, opensre.exe, onefile, windows-latest, windows-x64) (push) Has been cancelled
Benchmark image — build + push to ECR (any adapter) / build + push (push) Has been cancelled
CI / quality (ubuntu-latest) (push) Has been cancelled
CI / test (tools-runtime) (push) Has been cancelled
CI / test (e2e-general) (push) Has been cancelled
CI / test (cli-runtime) (push) Has been cancelled
CI / test (e2e-provider-and-openclaw) (push) Has been cancelled
CI / test (integrations-and-misc) (push) Has been cancelled
Release / verify (push) Has been cancelled
Release / build-python-dist (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-15-intel, darwin-x64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-latest, darwin-arm64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04, linux-x64) (push) Has been cancelled
Release / publish-release (push) Has been cancelled
Release / publish-main-release (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-checks (no-LLM) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-live shard ${{ matrix.shard_index }} (push) Has been cancelled
Release / prepare (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04-arm, linux-arm64) (push) Has been cancelled
Synthetic Deterministic Tests / Synthetic offline (deterministic) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Shared interactive-shell utilities."""
|
||||
@@ -0,0 +1 @@
|
||||
"""Interactive shell error mapping, rendering, and reporting helpers."""
|
||||
@@ -0,0 +1,66 @@
|
||||
"""CLI rendering for structured OpenSRE errors.
|
||||
|
||||
The frontend-agnostic error contract lives in :mod:`platform.common.errors`.
|
||||
This module adds the CLI presentation layer: a ``click.ClickException``
|
||||
subclass whose :meth:`show` renders a clean, traceback-free panel via
|
||||
:func:`render_error`. CLI code raises this subclass so Click's error path
|
||||
renders it; non-CLI code (tools, integrations) raises the platform base, and
|
||||
:mod:`cli.__main__` renders that. Catch ``platform.common.errors.OpenSREError``
|
||||
to handle both.
|
||||
|
||||
render_error()
|
||||
--------------
|
||||
Catches any exception and displays a clean, terminal-safe error panel without
|
||||
ever surfacing a raw Python traceback. Format:
|
||||
|
||||
✗ ExceptionType ← ERROR
|
||||
message text ← TEXT
|
||||
path/to/file.py:42 in fn_name ← DIM
|
||||
Run opensre doctor to diagnose ← SECONDARY hint
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import typing as t
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
|
||||
from platform.common.errors import OpenSREError as _OpenSREError
|
||||
from platform.terminal.errors import render_error
|
||||
|
||||
|
||||
# ClickException.message is Final in newer Click; the platform base owns ``message``.
|
||||
class OpenSREError(_OpenSREError, click.ClickException): # type: ignore[misc]
|
||||
"""A CLI error that renders with an optional suggestion and docs URL."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
suggestion: str | None = None,
|
||||
docs_url: str | None = None,
|
||||
exit_code: int = 1,
|
||||
) -> None:
|
||||
# Click 8.2+ marks ``message`` final on ``ClickException``; initialize
|
||||
# through Click and set the platform fields without reassigning ``message``.
|
||||
click.ClickException.__init__(self, message)
|
||||
self.suggestion = suggestion
|
||||
self.docs_url = docs_url
|
||||
self.exit_code = exit_code
|
||||
|
||||
def format_message(self) -> str:
|
||||
return _OpenSREError.format_message(self)
|
||||
|
||||
def show(self, file: t.IO[t.Any] | None = None) -> None:
|
||||
_file = file if file is not None else sys.stderr
|
||||
console = Console(stderr=(_file is sys.stderr), highlight=False)
|
||||
# Prefer the structured suggestion over the generic doctor hint.
|
||||
custom_hint: str | None = None
|
||||
if self.suggestion:
|
||||
parts = [self.suggestion]
|
||||
if self.docs_url:
|
||||
parts.append(f"Docs: {self.docs_url}")
|
||||
custom_hint = " ".join(parts)
|
||||
render_error(self, console=console, hint=custom_hint)
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Shared exception reporting policy for CLI and interactive-shell boundaries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import click
|
||||
|
||||
from platform.common.errors import OpenSREError
|
||||
from platform.observability.errors.sentry import capture_exception
|
||||
|
||||
|
||||
def should_report_exception(exc: BaseException, *, expected: bool = False) -> bool:
|
||||
"""Return whether a caught exception should be reported to Sentry."""
|
||||
if expected:
|
||||
return False
|
||||
if isinstance(exc, (KeyboardInterrupt, EOFError, OpenSREError, click.Abort)):
|
||||
return False
|
||||
return not isinstance(exc, click.UsageError)
|
||||
|
||||
|
||||
def report_exception(
|
||||
exc: BaseException,
|
||||
*,
|
||||
context: str,
|
||||
extra: Mapping[str, Any] | None = None,
|
||||
expected: bool = False,
|
||||
) -> bool:
|
||||
"""Best-effort Sentry report for swallowed boundary exceptions."""
|
||||
if not should_report_exception(exc, expected=expected):
|
||||
return False
|
||||
capture_exception(exc, context=context, extra=extra)
|
||||
return True
|
||||
|
||||
|
||||
__all__ = ["report_exception", "should_report_exception"]
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Interactive-shell telemetry helpers."""
|
||||
|
||||
from surfaces.interactive_shell.utils.telemetry.config import PromptLogConfig
|
||||
from surfaces.interactive_shell.utils.telemetry.recorder import (
|
||||
NO_CONVERSATIONAL_AGENT,
|
||||
LlmRunInfo,
|
||||
PromptRecorder,
|
||||
)
|
||||
|
||||
__all__ = ["LlmRunInfo", "NO_CONVERSATIONAL_AGENT", "PromptLogConfig", "PromptRecorder"]
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Configuration helpers for interactive-shell prompt logging."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from config.constants import OPENSRE_HOME_DIR
|
||||
from config.repl_config import read_prompt_log_settings
|
||||
|
||||
_FALSE_VALUES = {"", "0", "false", "off", "no"}
|
||||
_DEFAULT_MAX_CHARS = 32_000
|
||||
_DEFAULT_LOG_PATH = OPENSRE_HOME_DIR / "prompt_log.jsonl"
|
||||
|
||||
|
||||
def _coerce_bool(value: Any, *, default: bool) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if value is None:
|
||||
return default
|
||||
return str(value).strip().lower() not in _FALSE_VALUES
|
||||
|
||||
|
||||
def _coerce_int(value: Any, *, default: int) -> int:
|
||||
try:
|
||||
parsed = int(str(value).strip())
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
return parsed if parsed > 0 else default
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PromptLogConfig:
|
||||
enabled: bool = True
|
||||
local_enabled: bool = True
|
||||
posthog_enabled: bool = True
|
||||
redact: bool = True
|
||||
max_chars: int = _DEFAULT_MAX_CHARS
|
||||
log_path: Path = _DEFAULT_LOG_PATH
|
||||
|
||||
@classmethod
|
||||
def load(cls) -> PromptLogConfig:
|
||||
file_conf = read_prompt_log_settings()
|
||||
disabled = os.getenv("OPENSRE_PROMPT_LOG_DISABLED")
|
||||
local_disabled = os.getenv("OPENSRE_PROMPT_LOG_LOCAL_DISABLED")
|
||||
redact_env = os.getenv("OPENSRE_PROMPT_LOG_REDACT")
|
||||
path_env = os.getenv("OPENSRE_PROMPT_LOG_PATH")
|
||||
|
||||
enabled = not _coerce_bool(disabled, default=False)
|
||||
local_enabled = not _coerce_bool(local_disabled, default=False)
|
||||
posthog_enabled = _coerce_bool(file_conf.get("posthog_enabled"), default=True)
|
||||
# Default on, matching HistoryPolicy.redact — prompt/response content can
|
||||
# carry the same token shapes as typed history and additionally leaves the
|
||||
# machine via the PostHog sink, so it should not be less guarded by default
|
||||
# than command history is. See docs/interactive-shell-privacy.mdx.
|
||||
redact = _coerce_bool(
|
||||
redact_env, default=_coerce_bool(file_conf.get("redact"), default=True)
|
||||
)
|
||||
max_chars = _coerce_int(file_conf.get("max_chars"), default=_DEFAULT_MAX_CHARS)
|
||||
|
||||
raw_path = path_env or file_conf.get("path")
|
||||
log_path = Path(raw_path).expanduser() if raw_path else _DEFAULT_LOG_PATH
|
||||
|
||||
return cls(
|
||||
enabled=enabled,
|
||||
local_enabled=local_enabled,
|
||||
posthog_enabled=posthog_enabled,
|
||||
redact=redact,
|
||||
max_chars=max_chars,
|
||||
log_path=log_path,
|
||||
)
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Capture Rich console output without suppressing on-screen rendering."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
|
||||
@contextmanager
|
||||
def capture_console_segment(console: Console) -> Iterator[Callable[[], str]]:
|
||||
"""Record console output printed inside the block (tee to the real console).
|
||||
|
||||
Uses Rich's ``record`` mode with ``export_text(clear=False)`` so output still
|
||||
appears in the REPL while a plain-text slice is available for analytics. The
|
||||
recording buffer is cleared on exit when this context enabled recording, so
|
||||
long REPL sessions do not accumulate unbounded ``export_text`` history.
|
||||
"""
|
||||
was_recording = console.record
|
||||
enabled_recording = not was_recording
|
||||
if enabled_recording:
|
||||
console.record = True
|
||||
start = len(console.export_text(clear=False))
|
||||
captured: list[str] = []
|
||||
|
||||
def get_captured() -> str:
|
||||
if captured:
|
||||
return captured[0]
|
||||
return console.export_text(clear=False)[start:].strip()
|
||||
|
||||
try:
|
||||
yield get_captured
|
||||
finally:
|
||||
captured.append(console.export_text(clear=False)[start:].strip())
|
||||
if enabled_recording:
|
||||
console.export_text(clear=True)
|
||||
console.record = False
|
||||
else:
|
||||
console.record = was_recording
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Per-turn integration snapshots for analytics capture."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Protocol
|
||||
|
||||
from core.domain.alerts.alert_source import SECONDARY_TOOL_SOURCES
|
||||
from integrations.registry import family_key
|
||||
from tools.investigation.stages.gather_evidence.tools import get_available_tools
|
||||
|
||||
|
||||
class _IntegrationSession(Protocol):
|
||||
configured_integrations: tuple[str, ...]
|
||||
configured_integrations_known: bool
|
||||
resolved_integrations_cache: dict[str, Any] | None
|
||||
|
||||
|
||||
def build_turn_integration_snapshot(session: _IntegrationSession | None) -> dict[str, Any]:
|
||||
"""Return analytics-friendly integration state for one LLM generation turn."""
|
||||
configured = _configured_slugs(session)
|
||||
resolved = _resolved_integrations(session)
|
||||
connected = _connected_slugs(configured, resolved)
|
||||
return {
|
||||
"connected_integrations": connected,
|
||||
"connected_integrations_count": len(connected),
|
||||
"configured_integrations": configured,
|
||||
"integration_snapshot_source": "runtime_config",
|
||||
}
|
||||
|
||||
|
||||
def _configured_slugs(session: _IntegrationSession | None) -> list[str]:
|
||||
if session is not None and session.configured_integrations_known:
|
||||
return sorted(session.configured_integrations)
|
||||
try:
|
||||
from integrations.verify import resolve_effective_integrations
|
||||
|
||||
return sorted(resolve_effective_integrations())
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _resolved_integrations(session: _IntegrationSession | None) -> dict[str, Any]:
|
||||
if session is not None and session.resolved_integrations_cache is not None:
|
||||
return session.resolved_integrations_cache
|
||||
try:
|
||||
from core.agent_harness.session.integration_resolution import resolve_integrations
|
||||
|
||||
return resolve_integrations()
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _connected_slugs(configured: list[str], resolved: dict[str, Any]) -> list[str]:
|
||||
if not configured or not resolved:
|
||||
return []
|
||||
try:
|
||||
tools = get_available_tools(resolved)
|
||||
active_families = {
|
||||
family_key(str(tool.source))
|
||||
for tool in tools
|
||||
if str(tool.source) not in SECONDARY_TOOL_SOURCES
|
||||
}
|
||||
if not active_families:
|
||||
return []
|
||||
return sorted(svc for svc in configured if family_key(svc) in active_families)
|
||||
except Exception:
|
||||
return []
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Bridge investigation terminal outcomes to PostHog lifecycle analytics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from platform.analytics.cli import (
|
||||
capture_investigation_cancelled,
|
||||
capture_investigation_outcome,
|
||||
)
|
||||
from surfaces.interactive_shell.ui.investigation_outcome import InvestigationOutcome
|
||||
|
||||
|
||||
def _root_cause_excerpt(final_state: dict[str, Any] | None) -> str:
|
||||
if not final_state:
|
||||
return ""
|
||||
root = final_state.get("root_cause")
|
||||
if isinstance(root, str) and root.strip():
|
||||
return root.strip()
|
||||
return ""
|
||||
|
||||
|
||||
def publish_investigation_outcome_analytics(outcome: InvestigationOutcome) -> None:
|
||||
"""Emit structured investigation analytics for one terminal run."""
|
||||
if outcome.status == "cancelled":
|
||||
capture_investigation_cancelled(
|
||||
investigation_id=outcome.investigation_id,
|
||||
investigation_target=outcome.target,
|
||||
state=outcome.final_state,
|
||||
)
|
||||
# Completed runs must not carry placeholder failure properties; consumers
|
||||
# would otherwise have to special-case failure_category == "unknown".
|
||||
not_completed = outcome.status != "completed"
|
||||
capture_investigation_outcome(
|
||||
investigation_id=outcome.investigation_id,
|
||||
status=outcome.status,
|
||||
investigation_target=outcome.target,
|
||||
root_cause_excerpt=_root_cause_excerpt(outcome.final_state),
|
||||
error_excerpt=outcome.error_message if not_completed else "",
|
||||
failure_category=(outcome.failure_category or None) if not_completed else None,
|
||||
integration_involved=(outcome.integration_involved or None) if not_completed else None,
|
||||
integration_failure_message=(
|
||||
(outcome.integration_failure_message or None) if not_completed else None
|
||||
),
|
||||
failure_detail=(outcome.error_detail or None) if not_completed else None,
|
||||
state=outcome.final_state,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["publish_investigation_outcome_analytics"]
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Observe LLM usage during a foreground investigation for turn telemetry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
|
||||
from core.llm.shared.usage import set_usage_hook
|
||||
|
||||
|
||||
@dataclass
|
||||
class InvestigationLlmUsage:
|
||||
"""Accumulated provider-reported LLM usage for one investigation run."""
|
||||
|
||||
model: str = ""
|
||||
input_tokens: int = 0
|
||||
output_tokens: int = 0
|
||||
|
||||
@property
|
||||
def observed(self) -> bool:
|
||||
return bool(self.model) or self.input_tokens > 0 or self.output_tokens > 0
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def observe_investigation_llm_usage() -> Iterator[InvestigationLlmUsage]:
|
||||
"""Accumulate provider-reported token usage while the body runs.
|
||||
|
||||
Registration is best-effort: if another owner already holds the process-wide
|
||||
usage hook (`core.llm.shared.usage.set_usage_hook`), the investigation proceeds
|
||||
without usage observation rather than failing.
|
||||
"""
|
||||
usage = InvestigationLlmUsage()
|
||||
|
||||
def _hook(model: str, tokens_in: int, tokens_out: int) -> None:
|
||||
if model:
|
||||
usage.model = model
|
||||
usage.input_tokens += tokens_in
|
||||
usage.output_tokens += tokens_out
|
||||
|
||||
registered = False
|
||||
with contextlib.suppress(RuntimeError):
|
||||
set_usage_hook(_hook)
|
||||
registered = True
|
||||
try:
|
||||
yield usage
|
||||
finally:
|
||||
if registered:
|
||||
set_usage_hook(None)
|
||||
|
||||
|
||||
def resolve_configured_llm_identity() -> tuple[str, str]:
|
||||
"""Best-effort ``(provider, model)`` from the configured LLM settings."""
|
||||
try:
|
||||
from config.config import resolve_llm_settings
|
||||
from config.llm_auth.auth_method import (
|
||||
effective_llm_provider,
|
||||
get_configured_llm_auth_method,
|
||||
)
|
||||
|
||||
settings = resolve_llm_settings()
|
||||
provider = effective_llm_provider(
|
||||
settings.provider, get_configured_llm_auth_method(settings.provider)
|
||||
)
|
||||
model = str(getattr(settings, f"{provider}_reasoning_model", "") or "")
|
||||
return provider, model
|
||||
except Exception:
|
||||
return "", ""
|
||||
|
||||
|
||||
__all__ = [
|
||||
"InvestigationLlmUsage",
|
||||
"observe_investigation_llm_usage",
|
||||
"resolve_configured_llm_identity",
|
||||
]
|
||||
@@ -0,0 +1,330 @@
|
||||
"""Prompt/response recorder for interactive-shell turns."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import time
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from config.version import get_opensre_version
|
||||
from core.agent_harness.accounting.token_accounting import LlmRunInfo
|
||||
from core.llm_invoke_errors import LLM_PROVIDER_FAILURE_KINDS, classify_provider_error_kind
|
||||
from platform.analytics.provider import JsonValue
|
||||
from surfaces.interactive_shell.prompt_history.policy import redact_text
|
||||
from surfaces.interactive_shell.utils.telemetry.config import PromptLogConfig
|
||||
from surfaces.interactive_shell.utils.telemetry.integration_snapshot import (
|
||||
build_turn_integration_snapshot,
|
||||
)
|
||||
from surfaces.interactive_shell.utils.telemetry.sinks.local_jsonl import (
|
||||
append_prompt_log_record,
|
||||
)
|
||||
from surfaces.interactive_shell.utils.telemetry.sinks.posthog_ai import capture_ai_generation
|
||||
|
||||
_SUPPORTED_TURN_KINDS = frozenset({"agent", "follow_up", "new_alert", "background_task"})
|
||||
|
||||
# Sentinel for turns handled by terminal tools/slash commands without the
|
||||
# conversational assistant LLM (PostHog ``$ai_model`` / ``$ai_provider``).
|
||||
NO_CONVERSATIONAL_AGENT = "no_conversational_agent"
|
||||
|
||||
# Sentinel for turns where the conversational LLM was attempted but failed
|
||||
# before the model/provider identity could be resolved (missing key, bad
|
||||
# config). Distinct from ``NO_CONVERSATIONAL_AGENT`` so provider failures are
|
||||
# never mislabeled as terminal-action turns in analytics.
|
||||
UNKNOWN_LLM = "unknown"
|
||||
|
||||
# Maps PromptRecorder turn_kind to session turn kind stored in turn_detail records.
|
||||
_TURN_TO_SESSION_KIND: dict[str, str] = {
|
||||
"agent": "chat",
|
||||
"follow_up": "follow_up",
|
||||
"new_alert": "alert",
|
||||
"background_task": "cli_command",
|
||||
}
|
||||
|
||||
|
||||
def _latest_slash_outcome(session: Any) -> str | None:
|
||||
history = getattr(session, "history", None)
|
||||
if not isinstance(history, list):
|
||||
return None
|
||||
for entry in reversed(history):
|
||||
if not isinstance(entry, dict) or entry.get("type") != "slash":
|
||||
continue
|
||||
outcome = entry.get("slash_outcome")
|
||||
if isinstance(outcome, str) and outcome:
|
||||
return outcome
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _fallback_terminal_response(*, prompt: str) -> str:
|
||||
stripped = prompt.strip()
|
||||
if stripped:
|
||||
return f"terminal turn handled: {stripped}"
|
||||
return "terminal turn handled"
|
||||
|
||||
|
||||
def _prompt_relates_to_investigation(*, prompt: str, turn_kind: str) -> bool:
|
||||
stripped = prompt.strip().lower()
|
||||
if turn_kind == "background_task":
|
||||
return "investigate" in stripped
|
||||
return stripped.startswith("/investigate")
|
||||
|
||||
|
||||
class PromptRecorder:
|
||||
"""Captures one `(prompt, response)` pair and flushes to configured sinks."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
config: PromptLogConfig,
|
||||
turn_kind: str,
|
||||
session_id: str,
|
||||
turn_id: str,
|
||||
prompt: str,
|
||||
session: Any | None = None,
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._turn_kind = turn_kind
|
||||
self._session_id = session_id
|
||||
self._turn_id = turn_id
|
||||
self._prompt = prompt
|
||||
self._session = session
|
||||
self._response: str = ""
|
||||
self._scoped_investigation_id: str | None = None
|
||||
self._error_kind: str = ""
|
||||
self._error_message: str = ""
|
||||
self._model: str | None = None
|
||||
self._provider: str | None = None
|
||||
self._latency_ms: int | None = None
|
||||
self._input_tokens: int | None = None
|
||||
self._output_tokens: int | None = None
|
||||
self._start = time.monotonic()
|
||||
self._flushed = False
|
||||
|
||||
@property
|
||||
def turn_id(self) -> str:
|
||||
"""Stable correlation id for this prompt turn."""
|
||||
return self._turn_id
|
||||
|
||||
@classmethod
|
||||
def start(
|
||||
cls,
|
||||
*,
|
||||
session: Any,
|
||||
text: str,
|
||||
turn_kind: str,
|
||||
) -> PromptRecorder | None:
|
||||
config = PromptLogConfig.load()
|
||||
if not config.enabled or turn_kind not in _SUPPORTED_TURN_KINDS:
|
||||
# When prompt logging is fully disabled, no recorder is created and
|
||||
# no turn_detail records are written to the session file. This means
|
||||
# the crash-recovery fallback in load_session() will produce empty
|
||||
# cli_agent_messages for sessions that crashed before flush(). The
|
||||
# conversation_snapshot written at clean exit is unaffected.
|
||||
return None
|
||||
return cls(
|
||||
config=config,
|
||||
turn_kind=turn_kind,
|
||||
session_id=_session_id(session),
|
||||
turn_id=str(uuid.uuid4()),
|
||||
prompt=_sanitize_text(text, config=config),
|
||||
session=session,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def for_background_task(
|
||||
cls,
|
||||
*,
|
||||
session: Any,
|
||||
command: str,
|
||||
task_id: str,
|
||||
) -> PromptRecorder | None:
|
||||
"""Create a recorder for an async background task.
|
||||
|
||||
Background CLI tasks (e.g. ``opensre investigate``) finish long after
|
||||
the originating turn has flushed, so their stdout/stderr/exit outcome is
|
||||
not available to the turn-level recorder. This recorder is created at
|
||||
task launch — so its latency clock spans the full task duration — and is
|
||||
flushed by the task watcher once the outcome (including any error text)
|
||||
is known. ``turn_id`` is set to ``task_id`` so the prompt-log event
|
||||
correlates with the task surfaced by ``/tasks``.
|
||||
"""
|
||||
config = PromptLogConfig.load()
|
||||
if not config.enabled:
|
||||
return None
|
||||
recorder = cls(
|
||||
config=config,
|
||||
turn_kind="background_task",
|
||||
session_id=_session_id(session),
|
||||
turn_id=task_id or str(uuid.uuid4()),
|
||||
prompt=_sanitize_text(command, config=config),
|
||||
session=session,
|
||||
)
|
||||
if _prompt_relates_to_investigation(prompt=command, turn_kind="background_task"):
|
||||
investigation_id = str(uuid.uuid4())
|
||||
recorder.bind_investigation_id(investigation_id)
|
||||
session.last_investigation_id = investigation_id
|
||||
return recorder
|
||||
|
||||
def bind_investigation_id(self, investigation_id: str) -> None:
|
||||
cleaned = investigation_id.strip()
|
||||
if cleaned:
|
||||
self._scoped_investigation_id = cleaned
|
||||
|
||||
def set_error(self, kind: str, message: str) -> None:
|
||||
"""Attach a structured turn error emitted as ``$ai_error`` properties.
|
||||
|
||||
The human-readable response text is unaffected; these properties make
|
||||
LLM/provider error detection exact instead of a regex over
|
||||
``$ai_output_choices``.
|
||||
"""
|
||||
kind = kind.strip()
|
||||
message = message.strip()
|
||||
if not (kind or message):
|
||||
return
|
||||
self._error_kind = kind or "error"
|
||||
self._error_message = _sanitize_text(message, config=self._config)
|
||||
|
||||
def _resolve_investigation_id(self) -> str:
|
||||
if self._scoped_investigation_id:
|
||||
return self._scoped_investigation_id
|
||||
if self._session is None or not _prompt_relates_to_investigation(
|
||||
prompt=self._prompt,
|
||||
turn_kind=self._turn_kind,
|
||||
):
|
||||
return ""
|
||||
investigation_id = getattr(self._session, "last_investigation_id", "")
|
||||
if isinstance(investigation_id, str):
|
||||
return investigation_id
|
||||
return ""
|
||||
|
||||
def set_response(self, text: str, run: LlmRunInfo | None = None) -> None:
|
||||
cleaned = _sanitize_text(text, config=self._config)
|
||||
if not cleaned.strip():
|
||||
cleaned = ""
|
||||
self._response = cleaned
|
||||
if run is None:
|
||||
self._latency_ms = int((time.monotonic() - self._start) * 1000)
|
||||
return
|
||||
self._model = run.model
|
||||
self._provider = run.provider
|
||||
self._latency_ms = run.latency_ms or int((time.monotonic() - self._start) * 1000)
|
||||
self._input_tokens = run.input_tokens
|
||||
self._output_tokens = run.output_tokens
|
||||
|
||||
def _response_for_emit(self) -> str:
|
||||
"""Resolve the assistant text written to sinks at flush time."""
|
||||
if self._response.strip():
|
||||
return self._response
|
||||
if self._error_message.strip():
|
||||
return self._error_message
|
||||
return _fallback_terminal_response(prompt=self._prompt)
|
||||
|
||||
def flush(self) -> None:
|
||||
if self._flushed:
|
||||
return
|
||||
self._flushed = True
|
||||
response_text = self._response_for_emit()
|
||||
latency_ms = self._latency_ms or int((time.monotonic() - self._start) * 1000)
|
||||
record = {
|
||||
"ts": datetime.now(UTC).isoformat(),
|
||||
"session_id": self._session_id,
|
||||
"turn_id": self._turn_id,
|
||||
"turn_kind": self._turn_kind,
|
||||
"prompt": self._prompt,
|
||||
"response": response_text,
|
||||
"model": self._model or "",
|
||||
"provider": self._provider or "",
|
||||
"latency_ms": latency_ms,
|
||||
"input_tokens": self._input_tokens,
|
||||
"output_tokens": self._output_tokens,
|
||||
"opensre_version": get_opensre_version(),
|
||||
}
|
||||
if self._config.local_enabled:
|
||||
with contextlib.suppress(OSError):
|
||||
append_prompt_log_record(path=self._config.log_path, record=record)
|
||||
|
||||
# Also write enriched turn to the session file so /resume can restore context.
|
||||
with contextlib.suppress(Exception):
|
||||
from core.agent_harness.session import default_session_storage
|
||||
|
||||
session_kind = _TURN_TO_SESSION_KIND.get(self._turn_kind, self._turn_kind)
|
||||
default_session_storage().append_turn_detail(
|
||||
self._session_id,
|
||||
session_kind,
|
||||
self._prompt,
|
||||
response=response_text or None,
|
||||
turn_id=self._turn_id,
|
||||
model=self._model or None,
|
||||
provider=self._provider or None,
|
||||
latency_ms=latency_ms,
|
||||
)
|
||||
|
||||
if self._config.posthog_enabled:
|
||||
with contextlib.suppress(Exception):
|
||||
# When the conversational LLM was attempted but the provider
|
||||
# failed, the turn is a failed LLM call — never a terminal
|
||||
# action. Fall back to "unknown" instead of the terminal
|
||||
# sentinel when the attempted model could not be resolved.
|
||||
llm_provider_failed = self._error_kind in LLM_PROVIDER_FAILURE_KINDS
|
||||
fallback_label = UNKNOWN_LLM if llm_provider_failed else NO_CONVERSATIONAL_AGENT
|
||||
integration_snapshot = build_turn_integration_snapshot(self._session)
|
||||
posthog_properties: dict[str, JsonValue] = {
|
||||
"$ai_trace_id": self._turn_id,
|
||||
"$ai_session_id": self._session_id,
|
||||
"$ai_span_id": self._turn_id,
|
||||
"$ai_span_name": f"surfaces.interactive_shell.{self._turn_kind}",
|
||||
"$ai_model": self._model or fallback_label,
|
||||
"$ai_provider": self._provider or fallback_label,
|
||||
"$ai_input": [{"role": "user", "content": self._prompt}],
|
||||
"$ai_output_choices": [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": response_text,
|
||||
}
|
||||
],
|
||||
"$ai_latency": (
|
||||
round((self._latency_ms or 0) / 1000.0, 3) if self._latency_ms else 0.0
|
||||
),
|
||||
"$ai_input_tokens": self._input_tokens or 0,
|
||||
"$ai_output_tokens": self._output_tokens or 0,
|
||||
"cli_turn_kind": self._turn_kind,
|
||||
"cli_session_id": self._session_id,
|
||||
"cli_turn_id": self._turn_id,
|
||||
"opensre_version": get_opensre_version(),
|
||||
**integration_snapshot,
|
||||
}
|
||||
slash_outcome = _latest_slash_outcome(self._session)
|
||||
if slash_outcome:
|
||||
posthog_properties["slash_outcome"] = slash_outcome
|
||||
investigation_id = self._resolve_investigation_id()
|
||||
if investigation_id:
|
||||
posthog_properties["investigation_id"] = investigation_id
|
||||
if self._error_kind:
|
||||
posthog_properties["$ai_is_error"] = True
|
||||
posthog_properties["$ai_error"] = self._error_message or self._error_kind
|
||||
posthog_properties["error_kind"] = self._error_kind
|
||||
if llm_provider_failed:
|
||||
posthog_properties["ai_error_kind"] = classify_provider_error_kind(
|
||||
self._error_message or self._error_kind
|
||||
)
|
||||
capture_ai_generation(posthog_properties)
|
||||
|
||||
|
||||
def _sanitize_text(text: str, *, config: PromptLogConfig) -> str:
|
||||
if config.redact:
|
||||
text = redact_text(text)
|
||||
return text[: config.max_chars]
|
||||
|
||||
|
||||
def _session_id(session: Any) -> str:
|
||||
# Prefer the stable first-class field set at Session construction.
|
||||
# Fall back to the legacy side-channel for non-Session callers.
|
||||
sid = getattr(session, "session_id", None) or getattr(session, "_prompt_log_session_id", None)
|
||||
if isinstance(sid, str) and sid:
|
||||
return sid
|
||||
sid = str(uuid.uuid4())
|
||||
with contextlib.suppress(AttributeError):
|
||||
session._prompt_log_session_id = sid
|
||||
return sid
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Prompt logging sinks."""
|
||||
|
||||
from surfaces.interactive_shell.utils.telemetry.sinks.local_jsonl import (
|
||||
append_prompt_log_record,
|
||||
)
|
||||
from surfaces.interactive_shell.utils.telemetry.sinks.posthog_ai import capture_ai_generation
|
||||
|
||||
__all__ = ["append_prompt_log_record", "capture_ai_generation"]
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Local JSONL sink for prompt/response records."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
_DEFAULT_MAX_BYTES = 50 * 1024 * 1024
|
||||
|
||||
|
||||
def append_prompt_log_record(
|
||||
*,
|
||||
path: Path,
|
||||
record: dict[str, Any],
|
||||
max_bytes: int = _DEFAULT_MAX_BYTES,
|
||||
) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
_rotate_if_needed(path, max_bytes=max_bytes)
|
||||
with path.open("a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def _rotate_if_needed(path: Path, *, max_bytes: int) -> None:
|
||||
if max_bytes <= 0 or not path.exists():
|
||||
return
|
||||
if path.stat().st_size <= max_bytes:
|
||||
return
|
||||
backup = path.with_name(path.name + ".1")
|
||||
if backup.exists():
|
||||
backup.unlink()
|
||||
path.rename(backup)
|
||||
@@ -0,0 +1,10 @@
|
||||
"""PostHog sink for `$ai_generation` events."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from platform.analytics.events import Event
|
||||
from platform.analytics.provider import JsonValue, get_analytics
|
||||
|
||||
|
||||
def capture_ai_generation(properties: dict[str, JsonValue]) -> None:
|
||||
get_analytics().capture(Event.AI_GENERATION, properties)
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Format terminal-turn outcomes for prompt-log and PostHog analytics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
_ANALYTICS_OUTPUT_MAX_CHARS = 8_000
|
||||
|
||||
# Slash commands whose handlers attach to the real TTY (wizards, pickers). Analytics
|
||||
# should record structured success/failure, not full interactive transcripts.
|
||||
_INTERACTIVE_WIZARD_SLASH_ROOTS: frozenset[str] = frozenset(
|
||||
{
|
||||
"/onboard",
|
||||
"/auth",
|
||||
"/login",
|
||||
"/integrations",
|
||||
"/mcp",
|
||||
}
|
||||
)
|
||||
_INTERACTIVE_WIZARD_SLASH_PATHS: frozenset[str] = frozenset(
|
||||
{
|
||||
"/integrations setup",
|
||||
"/integrations remove",
|
||||
"/mcp connect",
|
||||
"/mcp disconnect",
|
||||
"/auth login",
|
||||
"/auth logout",
|
||||
}
|
||||
)
|
||||
|
||||
# Slash commands where console capture is noisy or redundant. Substantive output for
|
||||
# investigations is stored on the companion ``alert`` history row instead.
|
||||
_SUMMARY_ONLY_SLASH_ROOTS: frozenset[str] = frozenset(
|
||||
{
|
||||
"/",
|
||||
"/help",
|
||||
"/?",
|
||||
"/investigate",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def slash_command_is_interactive_wizard(command_line: str) -> bool:
|
||||
"""True when ``command_line`` names a multi-step TTY wizard or picker."""
|
||||
stripped = command_line.strip()
|
||||
if not stripped.startswith("/"):
|
||||
return False
|
||||
parts = stripped.split()
|
||||
root = parts[0].lower()
|
||||
if root in _INTERACTIVE_WIZARD_SLASH_ROOTS and len(parts) == 1:
|
||||
return True
|
||||
if len(parts) >= 2:
|
||||
path = f"{root} {parts[1].lower()}"
|
||||
if path in _INTERACTIVE_WIZARD_SLASH_PATHS:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def slash_command_is_summary_only(command_line: str) -> bool:
|
||||
"""True when analytics should omit captured console text for this slash command."""
|
||||
stripped = command_line.strip()
|
||||
if not stripped.startswith("/"):
|
||||
return False
|
||||
parts = stripped.split()
|
||||
root = parts[0].lower()
|
||||
if root in _SUMMARY_ONLY_SLASH_ROOTS:
|
||||
return True
|
||||
return slash_command_is_interactive_wizard(command_line)
|
||||
|
||||
|
||||
def truncate_analytics_text(text: str, *, max_chars: int = _ANALYTICS_OUTPUT_MAX_CHARS) -> str:
|
||||
cleaned = text.strip()
|
||||
if len(cleaned) <= max_chars:
|
||||
return cleaned
|
||||
return f"{cleaned[: max_chars - 20].rstrip()}… [truncated]"
|
||||
|
||||
|
||||
def format_wizard_cli_outcome(args: list[str], *, exit_code: int | None) -> str:
|
||||
"""Structured outcome for delegated interactive CLI wizards (e.g. ``/onboard``)."""
|
||||
command = " ".join(["opensre", *args]).strip()
|
||||
if exit_code is None:
|
||||
return f"{command}: interactive wizard cancelled"
|
||||
if exit_code == 0:
|
||||
return f"{command}: interactive wizard completed successfully"
|
||||
return f"{command}: interactive wizard failed (exit {exit_code})"
|
||||
|
||||
|
||||
def _investigation_report_excerpt(final_state: dict[str, object]) -> str:
|
||||
sections: list[str] = []
|
||||
root = final_state.get("root_cause")
|
||||
if isinstance(root, str) and root.strip():
|
||||
sections.append(f"Root cause: {root.strip()}")
|
||||
for key in ("problem_md", "slack_message"):
|
||||
body = final_state.get(key)
|
||||
if isinstance(body, str) and body.strip():
|
||||
sections.append(body.strip())
|
||||
break
|
||||
return "\n\n".join(sections)
|
||||
|
||||
|
||||
def format_investigation_outcome(
|
||||
target: str,
|
||||
*,
|
||||
final_state: dict[str, object] | None = None,
|
||||
background: bool = False,
|
||||
error_message: str = "",
|
||||
status: str | None = None,
|
||||
) -> str:
|
||||
"""Human-readable investigation outcome body for analytics."""
|
||||
label = target.strip() or "investigation"
|
||||
if background:
|
||||
return f"investigation started in background: {label}"
|
||||
if status == "cancelled":
|
||||
return f"investigation_cancelled ({label}): aborted by user"
|
||||
if status == "failed" or (final_state is None and error_message):
|
||||
reason = error_message.strip() or "investigation failed"
|
||||
return truncate_analytics_text(f"investigation_failed ({label}):\n{reason}")
|
||||
if final_state is None:
|
||||
return f"investigation_failed ({label}): investigation did not complete"
|
||||
excerpt = _investigation_report_excerpt(final_state)
|
||||
if excerpt:
|
||||
return truncate_analytics_text(f"investigation completed ({label}):\n{excerpt}")
|
||||
return f"investigation completed: {label}"
|
||||
|
||||
|
||||
def format_investigation_terminal_outcome(
|
||||
command_line: str,
|
||||
*,
|
||||
target: str,
|
||||
ok: bool,
|
||||
final_state: dict[str, object] | None = None,
|
||||
background: bool = False,
|
||||
error_message: str = "",
|
||||
status: str | None = None,
|
||||
) -> str:
|
||||
"""Two-line terminal analytics payload for ``/investigate`` turns."""
|
||||
if background:
|
||||
return format_investigation_outcome(target, background=True)
|
||||
resolved_status = status or ("succeeded" if ok and final_state is not None else "failed")
|
||||
if resolved_status == "completed":
|
||||
resolved_status = "succeeded"
|
||||
slash_status = {
|
||||
"succeeded": "succeeded",
|
||||
"failed": "failed",
|
||||
"cancelled": "cancelled",
|
||||
}.get(resolved_status, "failed")
|
||||
prefix = f"slash {command_line.strip()} ({slash_status})"
|
||||
body = format_investigation_outcome(
|
||||
target,
|
||||
final_state=final_state,
|
||||
error_message=error_message,
|
||||
status="cancelled"
|
||||
if resolved_status == "cancelled"
|
||||
else ("failed" if resolved_status == "failed" else None),
|
||||
)
|
||||
return truncate_analytics_text(f"{prefix}\n{body}")
|
||||
|
||||
|
||||
def format_terminal_turn_outcome(
|
||||
command_line: str,
|
||||
*,
|
||||
kind: str,
|
||||
ok: bool,
|
||||
captured_output: str = "",
|
||||
outcome_hint: str | None = None,
|
||||
include_captured_on_summary_only: bool = False,
|
||||
) -> str:
|
||||
"""Build the analytics payload for one handled terminal turn."""
|
||||
if outcome_hint and outcome_hint.strip():
|
||||
return truncate_analytics_text(outcome_hint.strip())
|
||||
|
||||
status = "succeeded" if ok else "failed"
|
||||
prefix = f"{kind} {command_line.strip()} ({status})"
|
||||
|
||||
summary_only = kind == "slash" and slash_command_is_summary_only(command_line)
|
||||
if summary_only and not include_captured_on_summary_only:
|
||||
return prefix
|
||||
|
||||
if captured_output:
|
||||
return truncate_analytics_text(f"{prefix}\n{captured_output}")
|
||||
return prefix
|
||||
Reference in New Issue
Block a user