chore: import upstream snapshot with attribution
CI (OpenClaw E2E) / openclaw test (push) Has been cancelled
CI / coverage-report (push) Has been cancelled
CI / test-kubernetes (push) Has been cancelled
CI / should-run-thorough (push) Has been cancelled
CI / test-thorough (cloudwatch-demo) (push) Has been cancelled
CI / test-thorough (flink-ecs) (push) Has been cancelled
CI / test-thorough (upstream-lambda) (push) Has been cancelled
CI / test-thorough (prefect-ecs-fargate) (push) Has been cancelled
Release / build-binaries (zip, opensre.exe, onefile, windows-latest, windows-x64) (push) Has been cancelled
Benchmark image — build + push to ECR (any adapter) / build + push (push) Has been cancelled
CI / quality (ubuntu-latest) (push) Has been cancelled
CI / test (tools-runtime) (push) Has been cancelled
CI / test (e2e-general) (push) Has been cancelled
CI / test (cli-runtime) (push) Has been cancelled
CI / test (e2e-provider-and-openclaw) (push) Has been cancelled
CI / test (integrations-and-misc) (push) Has been cancelled
Release / verify (push) Has been cancelled
Release / build-python-dist (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-15-intel, darwin-x64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-latest, darwin-arm64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04, linux-x64) (push) Has been cancelled
Release / publish-release (push) Has been cancelled
Release / publish-main-release (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-checks (no-LLM) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-live shard ${{ matrix.shard_index }} (push) Has been cancelled
Release / prepare (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04-arm, linux-arm64) (push) Has been cancelled
Synthetic Deterministic Tests / Synthetic offline (deterministic) (push) Has been cancelled
CI (OpenClaw E2E) / openclaw test (push) Has been cancelled
CI / coverage-report (push) Has been cancelled
CI / test-kubernetes (push) Has been cancelled
CI / should-run-thorough (push) Has been cancelled
CI / test-thorough (cloudwatch-demo) (push) Has been cancelled
CI / test-thorough (flink-ecs) (push) Has been cancelled
CI / test-thorough (upstream-lambda) (push) Has been cancelled
CI / test-thorough (prefect-ecs-fargate) (push) Has been cancelled
Release / build-binaries (zip, opensre.exe, onefile, windows-latest, windows-x64) (push) Has been cancelled
Benchmark image — build + push to ECR (any adapter) / build + push (push) Has been cancelled
CI / quality (ubuntu-latest) (push) Has been cancelled
CI / test (tools-runtime) (push) Has been cancelled
CI / test (e2e-general) (push) Has been cancelled
CI / test (cli-runtime) (push) Has been cancelled
CI / test (e2e-provider-and-openclaw) (push) Has been cancelled
CI / test (integrations-and-misc) (push) Has been cancelled
Release / verify (push) Has been cancelled
Release / build-python-dist (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-15-intel, darwin-x64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-latest, darwin-arm64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04, linux-x64) (push) Has been cancelled
Release / publish-release (push) Has been cancelled
Release / publish-main-release (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-checks (no-LLM) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-live shard ${{ matrix.shard_index }} (push) Has been cancelled
Release / prepare (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04-arm, linux-arm64) (push) Has been cancelled
Synthetic Deterministic Tests / Synthetic offline (deterministic) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
"""Slash command: ``/fleet`` (registered local AI agent fleet view)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from surfaces.interactive_shell.command_registry.agents.core import COMMANDS
|
||||
|
||||
__all__ = ["COMMANDS"]
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Rich presentation for fleet file-write conflict detection results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from rich.markup import escape
|
||||
from rich.table import Table
|
||||
|
||||
import platform.terminal.theme as ui_theme
|
||||
from surfaces.interactive_shell.ui.components.rendering import repl_table
|
||||
from tools.system.fleet_monitoring.conflicts import FileWriteConflict
|
||||
|
||||
_EMPTY_STATE = "no conflicts detected"
|
||||
|
||||
|
||||
def _format_timestamp(timestamp: float) -> str:
|
||||
return datetime.fromtimestamp(timestamp, tz=UTC).strftime("%H:%M:%S UTC")
|
||||
|
||||
|
||||
def render_conflicts(conflicts: list[FileWriteConflict]) -> Table | str:
|
||||
"""Render conflicts as a Rich table, or the empty-state string."""
|
||||
if not conflicts:
|
||||
return _EMPTY_STATE
|
||||
|
||||
table = repl_table(
|
||||
title="Agent file-write conflicts",
|
||||
title_style=ui_theme.BOLD_BRAND,
|
||||
)
|
||||
table.add_column("path", style="bold", overflow="fold")
|
||||
table.add_column("agents", overflow="fold")
|
||||
table.add_column("first seen", style=ui_theme.DIM)
|
||||
table.add_column("last seen", style=ui_theme.DIM)
|
||||
|
||||
for conflict in conflicts:
|
||||
table.add_row(
|
||||
escape(conflict.path),
|
||||
escape(", ".join(conflict.agents)),
|
||||
_format_timestamp(conflict.first_seen),
|
||||
_format_timestamp(conflict.last_seen),
|
||||
)
|
||||
return table
|
||||
|
||||
|
||||
__all__ = ["render_conflicts"]
|
||||
@@ -0,0 +1,471 @@
|
||||
"""Slash command: ``/fleet`` (registered local AI agent fleet view).
|
||||
|
||||
Bare ``/fleet`` renders the registered-agents dashboard; subcommands
|
||||
cover ``budget``, ``bus``, ``claim``, ``conflicts``, ``kill``, ``release``,
|
||||
and ``trace`` (with more surfaces planned for monitor-local-agents).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import ValidationError
|
||||
from rich.console import Console
|
||||
from rich.markup import escape
|
||||
from rich.tree import Tree
|
||||
|
||||
from surfaces.interactive_shell.command_registry.agents.conflicts_view import render_conflicts
|
||||
from surfaces.interactive_shell.command_registry.agents.kill import _cmd_agents_kill
|
||||
from surfaces.interactive_shell.command_registry.agents.trace import _cmd_agents_trace
|
||||
from surfaces.interactive_shell.command_registry.types import SlashCommand
|
||||
from surfaces.interactive_shell.runtime import Session
|
||||
from surfaces.interactive_shell.ui import (
|
||||
BOLD_BRAND,
|
||||
DIM,
|
||||
ERROR,
|
||||
HIGHLIGHT,
|
||||
WARNING,
|
||||
print_repl_table,
|
||||
render_agents_table,
|
||||
repl_table,
|
||||
)
|
||||
from tools.system.fleet_monitoring.bus import BusMessage, subscribe
|
||||
from tools.system.fleet_monitoring.config import (
|
||||
agents_config_path,
|
||||
load_agents_config,
|
||||
set_agent_budget,
|
||||
)
|
||||
from tools.system.fleet_monitoring.conflicts import (
|
||||
DEFAULT_WINDOW_SECONDS,
|
||||
WriteEvent,
|
||||
detect_conflicts,
|
||||
)
|
||||
from tools.system.fleet_monitoring.coordination import BranchClaims
|
||||
from tools.system.fleet_monitoring.discovery import registered_and_discovered_agents
|
||||
from tools.system.fleet_monitoring.registry import AgentRegistry
|
||||
|
||||
_AGENTS_FIRST_ARGS: tuple[tuple[str, str], ...] = (
|
||||
("budget", "view or edit per-agent hourly budgets"),
|
||||
("bus", "live-tail the cross-agent context bus"),
|
||||
("claim", "claim a branch for an agent"),
|
||||
("conflicts", "show file-write conflicts between local AI agents"),
|
||||
("kill", "SIGTERM → SIGKILL a local agent by PID"),
|
||||
("release", "release a branch claim"),
|
||||
("trace", "live tail of an agent's stdout by pid"),
|
||||
("graph", "render the wait-on dependency graph as a tree"),
|
||||
("wait", "mark <pid> as waiting on another pid: /fleet wait <pid> --on <other-pid>"),
|
||||
)
|
||||
|
||||
|
||||
def _opensre_agent_id() -> str:
|
||||
return f"opensre:{os.getpid()}"
|
||||
|
||||
|
||||
def _display_path(path: Path) -> str:
|
||||
"""Replace the user's home prefix with ``~`` for cleaner CLI output."""
|
||||
try:
|
||||
return f"~/{path.relative_to(Path.home())}"
|
||||
except ValueError:
|
||||
return str(path)
|
||||
|
||||
|
||||
def _print_config_error(console: Console, exc: ValidationError) -> None:
|
||||
console.print(f"[{ERROR}]agents.yaml has invalid contents:[/] {escape(str(exc))}")
|
||||
|
||||
|
||||
def _cmd_agents_list(console: Console) -> bool:
|
||||
"""Render registered plus read-only discovered agents as a Rich table.
|
||||
|
||||
Bare ``/fleet`` resolves here. Explicit registry rows keep winning
|
||||
on PID collisions; process discovery fills in Cursor, Claude Code,
|
||||
Codex, Aider, and Gemini CLI sessions that the user never registered.
|
||||
"""
|
||||
registry = AgentRegistry()
|
||||
render_agents_table(console, registered_and_discovered_agents(registry))
|
||||
return True
|
||||
|
||||
|
||||
def _format_bus_message(msg: BusMessage) -> str:
|
||||
"""Render one ``BusMessage`` as ``[agent] path — summary`` (path optional)."""
|
||||
parts = [f"[{HIGHLIGHT}]\\[{escape(msg.agent)}][/]"]
|
||||
if msg.path:
|
||||
parts.append(escape(msg.path))
|
||||
parts.append("—")
|
||||
parts.append(escape(msg.summary))
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def _cmd_agents_bus(console: Console) -> bool:
|
||||
"""Live-tail the cross-agent context bus until ``Ctrl-C`` or broker exit.
|
||||
|
||||
Self-elects a broker if none is running, then streams each ``BusMessage``
|
||||
as it arrives. The loop ends in three ways, each with explicit feedback:
|
||||
``KeyboardInterrupt`` (user detached), broker disconnect (e.g. the
|
||||
publishing process exited), or socket error.
|
||||
"""
|
||||
console.print(
|
||||
f"[{DIM}]tailing /fleet bus — Ctrl-C to exit[/]",
|
||||
)
|
||||
try:
|
||||
for msg in subscribe():
|
||||
console.print(_format_bus_message(msg))
|
||||
except KeyboardInterrupt:
|
||||
console.print(f"[{DIM}](detached)[/]")
|
||||
return True
|
||||
except OSError as exc:
|
||||
console.print(f"[{ERROR}]bus error:[/] {escape(str(exc))}")
|
||||
return False
|
||||
# ``subscribe()`` returned cleanly — the broker closed our connection
|
||||
# (e.g. it stopped, or its host process exited). Surface that explicitly
|
||||
# so the user isn't left wondering why the prompt came back.
|
||||
console.print(f"[{DIM}]bus broker disconnected[/]")
|
||||
return True
|
||||
|
||||
|
||||
def _cmd_agents_conflicts(console: Console) -> bool:
|
||||
# Real write-event collection comes from #1500 (filesystem blast-radius
|
||||
# watcher), out of scope for this PR. Until that lands, the event source
|
||||
# is empty and `/fleet conflicts` reports "no conflicts detected".
|
||||
events: list[WriteEvent] = []
|
||||
conflicts = detect_conflicts(
|
||||
events,
|
||||
window_seconds=DEFAULT_WINDOW_SECONDS,
|
||||
opensre_agent_id=_opensre_agent_id(),
|
||||
)
|
||||
console.print(render_conflicts(conflicts))
|
||||
return True
|
||||
|
||||
|
||||
def _cmd_agents_claim(session: Session, console: Console, args: list[str]) -> bool:
|
||||
"""Handle /fleet claim <branch> <agent-name>."""
|
||||
if len(args) < 2:
|
||||
console.print(f"[{ERROR}]Usage:[/] /fleet claim <branch> <agent-name>")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return False
|
||||
|
||||
branch = args[0].strip()
|
||||
agent_name = args[1].strip()
|
||||
|
||||
# Look up the PID from the registry for the given agent name
|
||||
registry = AgentRegistry()
|
||||
pid = None
|
||||
for record in registry.list():
|
||||
if record.name == agent_name:
|
||||
pid = record.pid
|
||||
break
|
||||
|
||||
if pid is None:
|
||||
console.print(
|
||||
f"[{ERROR}]Agent '{escape(agent_name)}' not found in registry. "
|
||||
"Use /fleet to see registered agents."
|
||||
)
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return False
|
||||
|
||||
claims = BranchClaims()
|
||||
claim = claims.claim(branch, agent_name, pid)
|
||||
|
||||
if claim is None:
|
||||
existing = claims.get(branch)
|
||||
assert existing is not None # claim() only returns None when branch is held
|
||||
console.print(
|
||||
f"[{ERROR}]Cannot claim:[/] {escape(branch)} is already held by "
|
||||
f"{escape(existing.agent_name)} (pid {existing.pid}). "
|
||||
"Use /fleet release first."
|
||||
)
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return False
|
||||
|
||||
console.print(
|
||||
f"[{HIGHLIGHT}]Branch {escape(branch)} now held by {escape(agent_name)} (pid {pid}).[/]"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _cmd_agents_release(session: Session, console: Console, args: list[str]) -> bool:
|
||||
"""Handle /fleet release <branch>."""
|
||||
if len(args) < 1:
|
||||
console.print(f"[{ERROR}]Usage:[/] /fleet release <branch>")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return False
|
||||
|
||||
branch = args[0].strip()
|
||||
claims = BranchClaims()
|
||||
|
||||
existing = claims.get(branch)
|
||||
if existing is None:
|
||||
console.print(f"[{ERROR}]{escape(branch)} is not currently held by any agent.")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return False
|
||||
|
||||
# release() cannot return None here because we confirmed existing is not None above
|
||||
removed = claims.release(branch)
|
||||
assert removed is not None
|
||||
console.print(
|
||||
f"[{HIGHLIGHT}]Released {escape(branch)} (was held by {escape(removed.agent_name)}).[/]"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _cmd_agents_budget(session: Session, console: Console, args: list[str]) -> bool:
|
||||
"""View or edit per-agent budgets stored in ``~/.opensre/agents.yaml``.
|
||||
|
||||
No args -> render the current budgets as a table. Two args
|
||||
(``<agent> <usd>``) -> set ``hourly_budget_usd`` for that agent and
|
||||
persist. Anything else -> usage hint.
|
||||
"""
|
||||
if not args:
|
||||
try:
|
||||
config = load_agents_config()
|
||||
except ValidationError as exc:
|
||||
_print_config_error(console, exc)
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
if not config.agents:
|
||||
console.print(
|
||||
f"[{DIM}]no per-agent budgets configured.[/] "
|
||||
"use [bold]/fleet budget <agent> <usd>[/bold] to set one."
|
||||
)
|
||||
return True
|
||||
table = repl_table(title="agent budgets", title_style=BOLD_BRAND)
|
||||
table.add_column("agent", style="bold")
|
||||
table.add_column("hourly $", justify="right")
|
||||
table.add_column("progress min", justify="right")
|
||||
table.add_column("error %", justify="right")
|
||||
for name in sorted(config.agents):
|
||||
budget = config.agents[name]
|
||||
table.add_row(
|
||||
escape(name),
|
||||
f"${budget.hourly_budget_usd:.2f}" if budget.hourly_budget_usd is not None else "-",
|
||||
str(budget.progress_minutes) if budget.progress_minutes is not None else "-",
|
||||
f"{budget.error_rate_pct:.1f}" if budget.error_rate_pct is not None else "-",
|
||||
)
|
||||
print_repl_table(console, table)
|
||||
return True
|
||||
|
||||
if len(args) != 2:
|
||||
console.print(f"[{ERROR}]usage:[/] /fleet budget [<agent> <usd>]")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
name = args[0].strip()
|
||||
raw_usd = args[1]
|
||||
try:
|
||||
usd = float(raw_usd)
|
||||
except ValueError:
|
||||
console.print(f"[{ERROR}]invalid budget:[/] {escape(raw_usd)} is not a number")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
# ``nan`` and ``inf`` slip past ``usd <= 0`` because both
|
||||
# ``float("nan") <= 0`` and ``float("inf") <= 0`` are ``False``.
|
||||
# Without this guard a stored ``nan`` would corrupt agents.yaml
|
||||
# (next load fails Pydantic's ``gt=0`` since ``nan > 0`` is
|
||||
# ``False``) and ``inf`` would render as ``$inf`` in the dashboard.
|
||||
if not math.isfinite(usd) or usd <= 0:
|
||||
console.print(f"[{ERROR}]invalid budget:[/] must be a positive finite number")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
try:
|
||||
set_agent_budget(name, usd)
|
||||
except ValidationError as exc:
|
||||
_print_config_error(console, exc)
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
console.print(
|
||||
f"updated [bold]{escape(name)}[/]: ${usd:.2f}/hr -> {_display_path(agents_config_path())}"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _cmd_agents_wait(session: Session, console: Console, args: list[str]) -> bool:
|
||||
"""Handle ``/fleet wait <pid> --on <other-pid>``.
|
||||
|
||||
Parse the two pids out of ``args``, registers the dependency in the agent registry.
|
||||
"""
|
||||
if len(args) != 3 or args[1] != "--on":
|
||||
console.print(f"[{ERROR}]usage:[/] /fleet wait <pid> --on <other-pid>")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
try:
|
||||
pid = int(args[0])
|
||||
except ValueError:
|
||||
console.print(f"[{ERROR}]invalid pid:[/] {escape(args[0])}")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
try:
|
||||
on_pid = int(args[2])
|
||||
except ValueError:
|
||||
console.print(f"[{ERROR}]invalid other-pid:[/] {escape(args[2])}")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
if pid == on_pid:
|
||||
console.print(f"[{ERROR}]invalid pid:[/] {pid} waiting for itself")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
registry = AgentRegistry()
|
||||
waiter = registry.get(pid)
|
||||
if waiter is None:
|
||||
console.print(f"[{ERROR}]pid {pid} is not in the agent registry[/]")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
target = registry.get(on_pid)
|
||||
if target is None:
|
||||
console.print(f"[{ERROR}]pid {on_pid} is not in the agent registry[/]")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
waiter = waiter.add_waits_on(target)
|
||||
registry.register(waiter)
|
||||
console.print(
|
||||
f"[{HIGHLIGHT}]{escape(waiter.name)} (pid {pid}) now waits on "
|
||||
f"{escape(target.name)} (pid {on_pid}).[/]"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _cmd_agents_graph(console: Console) -> bool:
|
||||
"""Render the ``waits_on`` dependency graph as a Rich tree.
|
||||
|
||||
Single-pass DFS over the inverse ``waits_on`` edges (depended-on
|
||||
-> waiter), building the Rich tree as it descends. A back edge — a
|
||||
pid re-encountered while still in the active path — is the
|
||||
canonical cycle witness for a directed graph; a warning naming the agents
|
||||
in the loop is emitted instead.
|
||||
"""
|
||||
|
||||
def _label(pid: int, ppid: int | None = None) -> str:
|
||||
r = records[pid]
|
||||
if ppid is None:
|
||||
return f"{escape(r.name)} ({pid}) \\[active]"
|
||||
|
||||
pr = records[ppid]
|
||||
return f"{escape(r.name)} ({pid}) \\[waiting on {escape(pr.name)}]"
|
||||
|
||||
def _walk(pid: int, parent: Tree, path: list[int], visited: set[int]) -> list[int] | None:
|
||||
for child in waiters_of.get(pid, []):
|
||||
if child in visited:
|
||||
return path[path.index(child) :] + [child]
|
||||
|
||||
path.append(child)
|
||||
visited.add(child)
|
||||
node = parent.add(_label(child, pid))
|
||||
c = _walk(child, node, path, visited)
|
||||
if c is not None:
|
||||
return c
|
||||
|
||||
path.pop()
|
||||
visited.remove(child)
|
||||
return None
|
||||
|
||||
registry = AgentRegistry()
|
||||
records = {r.pid: r for r in registry.list()}
|
||||
if not records:
|
||||
console.print(f"[{DIM}]no registered agents[/]")
|
||||
return True
|
||||
|
||||
waiters_of: dict[int, list[int]] = defaultdict(list)
|
||||
for record in records.values():
|
||||
for on_pid in record.waits_on:
|
||||
waiters_of[on_pid].append(record.pid)
|
||||
|
||||
# Roots are pids that wait on nothing. If every pid waits on
|
||||
# something the graph is fully covered by a cycle — fall back to
|
||||
# all pids so the walker enters somewhere and surfaces the back
|
||||
# edge instead of silently exiting on an empty root list.
|
||||
roots = [pid for pid, r in records.items() if not r.waits_on] or list(records)
|
||||
|
||||
trees: list[Tree] = []
|
||||
chain: str | None = None
|
||||
for root in roots:
|
||||
tree = Tree(label=_label(root))
|
||||
cycle = _walk(root, tree, [root], {root})
|
||||
if cycle is not None:
|
||||
chain = " -> ".join(f"{records[p].name} ({p})" for p in cycle)
|
||||
break
|
||||
trees.append(tree)
|
||||
|
||||
for i, tree in enumerate(trees):
|
||||
console.print(tree)
|
||||
if i != len(trees) - 1 and chain is None:
|
||||
console.line()
|
||||
|
||||
if chain is not None:
|
||||
console.print(f"[{WARNING}]: agent dependency cycle detected: {escape(chain)}.[/]")
|
||||
return True
|
||||
|
||||
|
||||
def _cmd_agents(session: Session, console: Console, args: list[str]) -> bool:
|
||||
# The sampler is lazy: the first /fleet renders a cold snapshot and starts
|
||||
# the background sampler so CPU/token columns warm up on subsequent views.
|
||||
session.terminal.ensure_fleet_sampler_started()
|
||||
if not args:
|
||||
return _cmd_agents_list(console)
|
||||
|
||||
sub = args[0].lower().strip()
|
||||
if sub == "budget":
|
||||
return _cmd_agents_budget(session, console, args[1:])
|
||||
if sub == "bus":
|
||||
return _cmd_agents_bus(console)
|
||||
if sub == "conflicts":
|
||||
return _cmd_agents_conflicts(console)
|
||||
|
||||
if sub == "claim":
|
||||
return _cmd_agents_claim(session, console, args[1:])
|
||||
|
||||
if sub == "kill":
|
||||
return _cmd_agents_kill(session, console, args[1:])
|
||||
|
||||
if sub == "release":
|
||||
return _cmd_agents_release(session, console, args[1:])
|
||||
|
||||
if sub == "trace":
|
||||
return _cmd_agents_trace(session, console, args[1:])
|
||||
|
||||
if sub == "wait":
|
||||
return _cmd_agents_wait(session, console, args[1:])
|
||||
|
||||
if sub == "graph":
|
||||
return _cmd_agents_graph(console)
|
||||
|
||||
console.print(
|
||||
f"[{ERROR}]unknown subcommand:[/] {escape(sub)} "
|
||||
"(try [bold]/fleet[/bold], [bold]/fleet budget[/bold], "
|
||||
"[bold]/fleet bus[/bold], [bold]/fleet claim[/bold], "
|
||||
"[bold]/fleet conflicts[/bold], [bold]/fleet kill[/bold], "
|
||||
"[bold]/fleet release[/bold], [bold]/fleet trace[/bold], "
|
||||
"[bold]/fleet wait[/bold] or [bold]/fleet graph[/bold])"
|
||||
)
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
|
||||
COMMANDS: list[SlashCommand] = [
|
||||
SlashCommand(
|
||||
"/fleet",
|
||||
"Show and manage registered local AI agents.",
|
||||
_cmd_agents,
|
||||
usage=(
|
||||
"/fleet",
|
||||
"/fleet budget",
|
||||
"/fleet bus",
|
||||
"/fleet claim",
|
||||
"/fleet conflicts",
|
||||
"/fleet kill",
|
||||
"/fleet release",
|
||||
"/fleet trace",
|
||||
"/fleet wait",
|
||||
"/fleet graph",
|
||||
),
|
||||
first_arg_completions=_AGENTS_FIRST_ARGS,
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,111 @@
|
||||
"""/fleet kill subcommand."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
|
||||
from rich.console import Console
|
||||
from rich.markup import escape
|
||||
|
||||
from platform.analytics.events import Event
|
||||
from platform.analytics.provider import get_analytics
|
||||
from surfaces.interactive_shell.runtime import Session
|
||||
from surfaces.interactive_shell.ui import DIM, ERROR, HIGHLIGHT, WARNING
|
||||
from tools.system.fleet_monitoring.lifecycle import TerminateResult, terminate
|
||||
from tools.system.fleet_monitoring.registry import AgentRegistry
|
||||
|
||||
# Type alias for the optional confirmation callback (used for testing).
|
||||
_ConfirmFn = Callable[[str], str]
|
||||
|
||||
|
||||
def _cmd_agents_kill(
|
||||
session: Session,
|
||||
console: Console,
|
||||
args: list[str],
|
||||
*,
|
||||
confirm_fn: _ConfirmFn | None = None,
|
||||
) -> bool:
|
||||
"""Handle ``/fleet kill <pid> [--force]``.
|
||||
|
||||
Sends SIGTERM, waits up to 5 s, then escalates to SIGKILL.
|
||||
Asks for confirmation unless ``--force`` is present.
|
||||
Emits an ``agent_killed`` analytics event on success.
|
||||
"""
|
||||
force = "--force" in args
|
||||
positional = [a for a in args if a != "--force"]
|
||||
|
||||
if not positional:
|
||||
console.print(f"[{ERROR}]usage:[/] /fleet kill <pid> [--force]")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
raw_pid = positional[0]
|
||||
try:
|
||||
pid = int(raw_pid)
|
||||
except ValueError:
|
||||
console.print(f"[{ERROR}]invalid pid:[/] {escape(raw_pid)} is not an integer")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
if pid == os.getpid():
|
||||
console.print(f"[{ERROR}]refusing to kill the opensre process itself[/]")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
# Look up agent name from registry for friendlier output.
|
||||
registry = AgentRegistry()
|
||||
record = registry.get(pid)
|
||||
label = f"{record.name} (pid {pid})" if record else f"pid {pid}"
|
||||
|
||||
if not force:
|
||||
prompt_text = f"About to SIGTERM {label}. Confirm? [y/N] "
|
||||
if confirm_fn is not None:
|
||||
answer = confirm_fn(prompt_text)
|
||||
else:
|
||||
answer = console.input(prompt_text)
|
||||
if answer.strip().lower() not in ("y", "yes"):
|
||||
console.print(f"[{DIM}]aborted.[/]")
|
||||
return True
|
||||
|
||||
try:
|
||||
result: TerminateResult = terminate(pid)
|
||||
except ProcessLookupError:
|
||||
console.print(f"[{ERROR}]no such process:[/] pid {pid}")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
except PermissionError:
|
||||
console.print(f"[{ERROR}]permission denied:[/] cannot signal pid {pid}")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
if result.exited:
|
||||
console.print(
|
||||
f"[{HIGHLIGHT}]Sent {result.signal_sent}. "
|
||||
f"Process exited after {result.elapsed_seconds:.1f}s.[/]"
|
||||
)
|
||||
else:
|
||||
console.print(
|
||||
f"[{WARNING}]Sent {result.signal_sent} but process may still be running "
|
||||
f"after {result.elapsed_seconds:.1f}s.[/]"
|
||||
)
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
|
||||
# Remove from the agent registry so `/fleet` no longer shows the dead PID.
|
||||
# Only forget when the process actually exited — otherwise it stays visible
|
||||
# for further monitoring or another kill attempt.
|
||||
if record is not None and result.exited:
|
||||
registry.forget(pid)
|
||||
|
||||
event = Event.AGENT_KILLED if result.exited else Event.AGENT_KILL_FAILED
|
||||
get_analytics().capture(
|
||||
event,
|
||||
{
|
||||
"pid": str(pid),
|
||||
"agent_name": record.name if record else "unknown",
|
||||
"signal": result.signal_sent,
|
||||
"exited": result.exited,
|
||||
"elapsed_seconds": str(round(result.elapsed_seconds, 2)),
|
||||
},
|
||||
)
|
||||
return True
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Live trace rendering and /fleet trace subcommand."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from contextlib import nullcontext
|
||||
|
||||
from prompt_toolkit.patch_stdout import patch_stdout
|
||||
from rich.console import Console
|
||||
from rich.live import Live
|
||||
from rich.markup import escape
|
||||
from rich.text import Text
|
||||
|
||||
from surfaces.interactive_shell.runtime import Session
|
||||
from surfaces.interactive_shell.ui import BOLD_BRAND, DIM, ERROR
|
||||
from tools.system.fleet_monitoring.registry import AgentRegistry
|
||||
from tools.system.fleet_monitoring.tail import AttachSession, AttachUnsupported, attach
|
||||
|
||||
_TRACE_REFRESH_PER_SECOND = 10
|
||||
# Match the throttle period to ``Live``'s refresh rate: under a 1k-line/sec
|
||||
# agent the reader thread can publish chunks faster than Rich actually
|
||||
# paints, and each ``live.update(Text.from_ansi(...))`` we make in
|
||||
# excess just creates a Renderable Rich will discard at the next paint.
|
||||
# Throttling the *call* to ``Live`` to one period bounds CPU under burst
|
||||
# writers without affecting how fast the screen updates.
|
||||
_TRACE_RENDER_PERIOD_S = 1.0 / _TRACE_REFRESH_PER_SECOND
|
||||
# Cap the on-screen render to the most recent slice of the 4 MiB buffer
|
||||
# so we don't reparse a 4 MiB string through Rich at 10 fps under burst
|
||||
# writers. A few screens of context is plenty for "what is the agent
|
||||
# doing right now"; the full tail is still in ``sess.buffer`` for any
|
||||
# future drill-down view.
|
||||
_TRACE_RENDER_TAIL_BYTES = 64 * 1024
|
||||
|
||||
|
||||
def _render_trace_snapshot(live: Live, sess: AttachSession) -> None:
|
||||
"""Decode the bounded snapshot with UTF-8 boundary safety for ``Live``.
|
||||
|
||||
ANSI sequences are interpreted (Rich); treat traced output like unfiltered ``kubectl logs``.
|
||||
"""
|
||||
snapshot = _slice_to_utf8_boundary(sess.buffer.snapshot(), _TRACE_RENDER_TAIL_BYTES)
|
||||
live.update(Text.from_ansi(snapshot.decode("utf-8", errors="replace")))
|
||||
|
||||
|
||||
def _slice_to_utf8_boundary(data: bytes, max_bytes: int) -> bytes:
|
||||
"""Return the suffix of ``data`` that fits in ``max_bytes`` and starts
|
||||
on a UTF-8 codepoint boundary.
|
||||
|
||||
A plain ``data[-max_bytes:]`` can land mid-codepoint, which decodes to
|
||||
a leading U+FFFD under ``errors="replace"`` — visible as a stray
|
||||
replacement character at the top of the live view. ``TailBuffer``
|
||||
preserves boundaries by dropping whole chunks; we have to do the same
|
||||
once we re-flatten and slice on the render side. UTF-8 continuation
|
||||
bytes match ``10xxxxxx`` (``b & 0xC0 == 0x80``); a codepoint is at
|
||||
most 4 bytes, so we walk forward at most 3 continuation bytes to
|
||||
reach the next start byte.
|
||||
"""
|
||||
if len(data) <= max_bytes:
|
||||
return data
|
||||
sliced = data[-max_bytes:]
|
||||
start = 0
|
||||
while start < 4 and start < len(sliced) and (sliced[start] & 0xC0) == 0x80:
|
||||
start += 1
|
||||
return sliced[start:]
|
||||
|
||||
|
||||
def _render_live_tail(console: Console, label: str, sess: AttachSession) -> None:
|
||||
"""kubectl-logs-style render: a single Ctrl+C returns to the prompt.
|
||||
|
||||
Catches :class:`KeyboardInterrupt` inside the ``Live`` block and
|
||||
swallows it so the REPL doesn't see a traceback. ``stream_to_console``
|
||||
in ``streaming.py`` uses a double-press pattern because it's
|
||||
rendering an LLM response that the user might *not* want to abort
|
||||
on a stray keypress; a logs-style view is the inverse — one press
|
||||
is the canonical "stop" signal.
|
||||
"""
|
||||
console.print(f"[{BOLD_BRAND}]trace {escape(label)}[/] [{DIM}]Ctrl+C to stop[/]")
|
||||
isatty = getattr(console.file, "isatty", None)
|
||||
stdout_context = patch_stdout(raw=True) if callable(isatty) and isatty() else nullcontext()
|
||||
try:
|
||||
with (
|
||||
stdout_context,
|
||||
Live(
|
||||
Text(""),
|
||||
console=console,
|
||||
refresh_per_second=_TRACE_REFRESH_PER_SECOND,
|
||||
transient=False,
|
||||
vertical_overflow="visible",
|
||||
) as live,
|
||||
):
|
||||
# Iterating ``sess`` is what drains the reader queue and
|
||||
# appends to ``sess.buffer`` — the loop body only needs the
|
||||
# *side effect* of advancing, not the chunk value, so the
|
||||
# iteration variable is intentionally discarded.
|
||||
# Seeded at 0.0 so the first iteration always renders (any
|
||||
# ``time.monotonic() - 0.0`` clears the period); the throttle
|
||||
# kicks in from the second iteration onward.
|
||||
last_render = 0.0
|
||||
pending = False
|
||||
for _ in sess:
|
||||
now = time.monotonic()
|
||||
if now - last_render >= _TRACE_RENDER_PERIOD_S:
|
||||
_render_trace_snapshot(live, sess)
|
||||
last_render = now
|
||||
pending = False
|
||||
else:
|
||||
pending = True
|
||||
# Final flush: the last chunk(s) may have arrived inside a
|
||||
# throttle window; render once after the loop so the user
|
||||
# sees the very latest state instead of whatever was on
|
||||
# screen at the last gated update.
|
||||
if pending:
|
||||
_render_trace_snapshot(live, sess)
|
||||
except KeyboardInterrupt:
|
||||
# kubectl-logs-style: a single Ctrl+C ends the trace and returns
|
||||
# to the REPL prompt without propagating a traceback. The
|
||||
# ``with sess:`` in the caller still runs and joins the reader
|
||||
# thread, so this swallow is safe.
|
||||
pass
|
||||
if sess.producer_exited:
|
||||
# Distinguish "the agent died and we noticed" from "the user
|
||||
# asked us to stop" so a long unattended trace doesn't look the
|
||||
# same as a Ctrl+C abort.
|
||||
console.print(f"[{DIM}]· process exited[/]")
|
||||
console.print(f"[{DIM}]· trace ended[/]")
|
||||
|
||||
|
||||
def _cmd_agents_trace(session: Session, console: Console, args: list[str]) -> bool:
|
||||
"""Live-tail an agent's stdout by pid; see :func:`_render_live_tail`.
|
||||
|
||||
Validates eagerly (``attach()`` raises :class:`AttachUnsupported`
|
||||
synchronously on bad pid / unsupported fd type / missing file) so
|
||||
we never enter the ``Live`` block on a target we cannot tail.
|
||||
"""
|
||||
if len(args) != 1:
|
||||
console.print(f"[{ERROR}]usage:[/] /fleet trace <pid>")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
try:
|
||||
pid = int(args[0])
|
||||
except ValueError:
|
||||
console.print(f"[{ERROR}]invalid pid:[/] {escape(args[0])}")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
record = AgentRegistry().get(pid)
|
||||
label = f"{record.name} (pid {pid})" if record else f"pid {pid}"
|
||||
|
||||
try:
|
||||
sess = attach(pid)
|
||||
except AttachUnsupported as exc:
|
||||
console.print(f"[{ERROR}]cannot trace {escape(label)}:[/] {escape(exc.reason)}")
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
return True
|
||||
|
||||
with sess:
|
||||
_render_live_tail(console, label, sess)
|
||||
return True
|
||||
Reference in New Issue
Block a user