from __future__ import annotations import asyncio from datetime import UTC, datetime from pathlib import Path import pytest from tools.system.fleet_monitoring.meters import TokenSample, TokenUsage from tools.system.fleet_monitoring.probe import ProcessSnapshot from tools.system.fleet_monitoring.registry import AgentRecord, AgentRegistry from tools.system.fleet_monitoring.sampler import ( _latest, get_snapshot, get_tokens_per_min, get_usd_per_hour, start_sampler, ) from tools.system.fleet_monitoring.token_rate import TOKEN_RATE_TRACKER @pytest.fixture def registry(tmp_path: Path) -> AgentRegistry: return AgentRegistry(path=tmp_path / "agents.jsonl") @pytest.fixture def fake_snapshot() -> ProcessSnapshot: return ProcessSnapshot( pid=8421, cpu_percent=23.5, rss_mb=128.0, num_fds=42, num_connections=3, status="running", started_at=datetime(2026, 5, 10, 12, 0, 0, tzinfo=UTC), ) @pytest.fixture(autouse=True) def _clear_sampler_state(monkeypatch: pytest.MonkeyPatch) -> None: """Reset module-level state between tests. Also resets the global token rate tracker so a test from #2023 cannot leak per-PID entries into a peer test. """ _latest.clear() # Forget every PID the tracker still knows about so per-test # state stays isolated. for pid in list(TOKEN_RATE_TRACKER.known_pids()): TOKEN_RATE_TRACKER.forget(pid) monkeypatch.setattr( "tools.system.fleet_monitoring.sampler._records_for_tick", lambda registry: registry.list(), ) class _FakeSource: """Test double for :class:`TokenSource`. Returns a queue of chunks. Once the queued chunks are exhausted, subsequent reads repeat the *last* enqueued value indefinitely. This matters because the sampler runs many ticks during a short ``asyncio.sleep``, and the test wants to assert a stable per-source behavior — not a one-shot result that decays into a different code path on later ticks. """ def __init__(self, chunks: list[str | None]) -> None: if not chunks: raise ValueError("at least one chunk required") self._chunks = list(chunks) self.forgotten: list[int] = [] def read_new_chunk(self, pid: int) -> str | None: # noqa: ARG002 if len(self._chunks) > 1: return self._chunks.pop(0) # Last entry sticks — every subsequent tick returns the same # value so the test's assertion observes the steady state. return self._chunks[0] def forget(self, pid: int) -> None: self.forgotten.append(pid) class _FakeMeter: """Test double for :class:`TokenMeter`. Maps chunk → fixed sample.""" def __init__(self, mapping: dict[str, TokenSample]) -> None: self._mapping = mapping def parse_chunk(self, chunk: str) -> int: return self._mapping.get(chunk, TokenSample()).tokens def sample_chunk(self, chunk: str, *, pid: int | None = None) -> TokenSample: # noqa: ARG002 return self._mapping.get(chunk, TokenSample()) def forget(self, _pid: int) -> None: return None def known_pids(self) -> list[int]: return [] @pytest.mark.asyncio async def test_sampler_stores_snapshots( monkeypatch: pytest.MonkeyPatch, registry: AgentRegistry, fake_snapshot: ProcessSnapshot, ) -> None: """Sampler probes registered agents and stores snapshots.""" registry.register( AgentRecord( name="claude-code", pid=8421, command="claude --dangerously-skip-permissions", registered_at="2026-05-07T12:00:00+00:00", ) ) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.AgentRegistry", lambda: registry) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.probe", lambda _pid: fake_snapshot) task = start_sampler(interval=0.01) await asyncio.sleep(0.05) task.cancel() with pytest.raises(asyncio.CancelledError): _ = await task assert get_snapshot(fake_snapshot.pid) == fake_snapshot @pytest.mark.asyncio async def test_none_probe_does_not_store( monkeypatch: pytest.MonkeyPatch, registry: AgentRegistry, ) -> None: """When probe returns None, no snapshot is stored.""" registry.register(AgentRecord(name="dead-agent", pid=9999, command="bin")) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.AgentRegistry", lambda: registry) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.probe", lambda _pid: None) task = start_sampler(interval=0.01) await asyncio.sleep(0.05) task.cancel() with pytest.raises(asyncio.CancelledError): _ = await task assert get_snapshot(9999) is None @pytest.mark.asyncio async def test_one_pid_failure_does_not_crash_loop( monkeypatch: pytest.MonkeyPatch, registry: AgentRegistry, fake_snapshot: ProcessSnapshot, ) -> None: """A failing probe for one PID doesn't prevent probing others.""" registry.register(AgentRecord(name="crasher", pid=1111, command="bin")) registry.register(AgentRecord(name="healthy", pid=8421, command="claude")) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.AgentRegistry", lambda: registry) def mock_probe(pid: int) -> ProcessSnapshot | None: if pid == 1111: raise RuntimeError("simulated psutil failure") return fake_snapshot monkeypatch.setattr("tools.system.fleet_monitoring.sampler.probe", mock_probe) task = start_sampler(interval=0.01) await asyncio.sleep(0.05) task.cancel() with pytest.raises(asyncio.CancelledError): _ = await task # The healthy agent was still probed despite the crasher assert get_snapshot(8421) == fake_snapshot # The crasher has no snapshot assert get_snapshot(1111) is None @pytest.mark.asyncio async def test_sampler_cancels_cleanly( monkeypatch: pytest.MonkeyPatch, registry: AgentRegistry, ) -> None: """Cancelling the sampler task raises CancelledError and nothing else.""" registry.register(AgentRecord(name="agent", pid=1234, command="bin")) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.AgentRegistry", lambda: registry) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.probe", lambda _pid: None) task = start_sampler(interval=0.01) await asyncio.sleep(0.02) task.cancel() with pytest.raises(asyncio.CancelledError): _ = await task @pytest.mark.asyncio async def test_stale_snapshot_evicted_when_probe_returns_none( monkeypatch: pytest.MonkeyPatch, registry: AgentRegistry, fake_snapshot: ProcessSnapshot, ) -> None: """A previously stored snapshot is evicted when probe returns None.""" _latest[8421] = fake_snapshot assert get_snapshot(8421) == fake_snapshot registry.register( AgentRecord( name="claude-code", pid=8421, command="claude --dangerously-skip-permissions", registered_at="2026-05-07T12:00:00+00:00", ) ) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.AgentRegistry", lambda: registry) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.probe", lambda _pid: None) task = start_sampler(interval=0.01) await asyncio.sleep(0.02) task.cancel() with pytest.raises(asyncio.CancelledError): _ = await task assert get_snapshot(8421) is None # ----- #2023: token-source / meter / tracker wiring ------------------------ def _stub_token_pipeline( monkeypatch: pytest.MonkeyPatch, *, provider: str, source: _FakeSource, meter: _FakeMeter, ) -> None: """Wire fakes into the sampler's import-site lookups. Patches the three functions :func:`_sample_tokens` calls so each test can swap in a deterministic source/meter pair without touching the disk or psutil. """ monkeypatch.setattr( "tools.system.fleet_monitoring.sampler.provider_for", lambda _record: provider ) monkeypatch.setattr( "tools.system.fleet_monitoring.sampler.get_token_source", lambda _provider: source ) monkeypatch.setattr( "tools.system.fleet_monitoring.sampler.get_token_meter", lambda _provider: meter ) @pytest.mark.asyncio async def test_sampler_records_tokens_when_meter_returns_positive( monkeypatch: pytest.MonkeyPatch, registry: AgentRegistry, fake_snapshot: ProcessSnapshot, ) -> None: """End-to-end of the token path: source emits a chunk, meter parses tokens + model, tracker records the entry, accessor returns the rate. """ registry.register(AgentRecord(name="claude-code-8421", pid=8421, command="claude")) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.AgentRegistry", lambda: registry) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.probe", lambda _pid: fake_snapshot) source = _FakeSource(chunks=["chunk-1"]) meter = _FakeMeter({"chunk-1": TokenSample.from_tokens(200, model="claude-sonnet-4-5")}) _stub_token_pipeline(monkeypatch, provider="claude-code", source=source, meter=meter) task = start_sampler(interval=0.01) await asyncio.sleep(0.05) task.cancel() with pytest.raises(asyncio.CancelledError): _ = await task # Tracker recorded the 200-token chunk; tokens_per_min reflects # the window-scaled rate (one observation in a 60 s window). assert get_tokens_per_min(8421) is not None assert get_tokens_per_min(8421) >= 200.0 # $/hr derives from tokens_per_min × 60 × blended-rate; a known # model yields a positive figure rather than None. cost = get_usd_per_hour(8421) assert cost is not None assert cost > 0.0 def test_get_usd_per_hour_uses_structured_usage_buckets() -> None: TOKEN_RATE_TRACKER.record( 8421, usage=TokenUsage(input_tokens=1000, cached_input_tokens=250, output_tokens=100), model="gpt-5-codex", ) expected_per_min = (750 * 1.25e-6) + (250 * 0.125e-6) + (100 * 10e-6) assert get_usd_per_hour(8421) == pytest.approx(expected_per_min * 60.0) @pytest.mark.asyncio async def test_sampler_records_tokens_for_auto_discovered_agents( monkeypatch: pytest.MonkeyPatch, registry: AgentRegistry, fake_snapshot: ProcessSnapshot, ) -> None: """The sampler must cover the same discovered rows that ``/fleet`` renders.""" discovered = AgentRecord( name="codex", pid=9999, command="codex exec --ephemeral", source="discovered", provider="codex", ) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.AgentRegistry", lambda: registry) monkeypatch.setattr( "tools.system.fleet_monitoring.sampler._records_for_tick", lambda _registry: [discovered] ) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.probe", lambda _pid: fake_snapshot) source = _FakeSource(chunks=["chunk-1"]) meter = _FakeMeter({"chunk-1": TokenSample.from_tokens(123, model="gpt-5-codex")}) _stub_token_pipeline(monkeypatch, provider="codex", source=source, meter=meter) task = start_sampler(interval=0.01) await asyncio.sleep(0.05) task.cancel() with pytest.raises(asyncio.CancelledError): _ = await task assert get_tokens_per_min(9999) is not None assert get_tokens_per_min(9999) >= 123.0 @pytest.mark.asyncio async def test_sampler_does_not_record_when_source_returns_none( monkeypatch: pytest.MonkeyPatch, registry: AgentRegistry, fake_snapshot: ProcessSnapshot, ) -> None: """``None`` from the source means "cannot observe this PID"; the tracker must stay empty so the view renders ``-``, not ``0``. """ registry.register(AgentRecord(name="claude-code-8421", pid=8421, command="claude")) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.AgentRegistry", lambda: registry) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.probe", lambda _pid: fake_snapshot) source = _FakeSource(chunks=[None]) meter = _FakeMeter({}) _stub_token_pipeline(monkeypatch, provider="claude-code", source=source, meter=meter) task = start_sampler(interval=0.01) await asyncio.sleep(0.05) task.cancel() with pytest.raises(asyncio.CancelledError): _ = await task # Tracker never recorded an entry; view sees None → renders ``-``. assert get_tokens_per_min(8421) is None assert get_usd_per_hour(8421) is None @pytest.mark.asyncio async def test_one_token_source_failure_does_not_crash_loop( monkeypatch: pytest.MonkeyPatch, registry: AgentRegistry, fake_snapshot: ProcessSnapshot, ) -> None: """The #2023 counterpart of ``test_one_pid_failure_does_not_crash_loop``: a raising token-source on one PID must not prevent token sampling for healthy peers. """ registry.register(AgentRecord(name="crasher", pid=1111, command="bin")) registry.register(AgentRecord(name="healthy", pid=8421, command="claude")) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.AgentRegistry", lambda: registry) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.probe", lambda _pid: fake_snapshot) class _CrashingSource: def __init__(self) -> None: self.calls: list[int] = [] def read_new_chunk(self, pid: int) -> str | None: self.calls.append(pid) if pid == 1111: raise RuntimeError("simulated source failure") return "ok" def forget(self, pid: int) -> None: # noqa: ARG002 pass source = _CrashingSource() meter = _FakeMeter({"ok": TokenSample.from_tokens(42, model="claude-sonnet-4-5")}) _stub_token_pipeline(monkeypatch, provider="claude-code", source=source, meter=meter) task = start_sampler(interval=0.01) await asyncio.sleep(0.05) task.cancel() with pytest.raises(asyncio.CancelledError): _ = await task # Healthy PID was observed despite the crasher. assert get_tokens_per_min(8421) is not None # Crasher has no tracker entry (its read failed). assert get_tokens_per_min(1111) is None @pytest.mark.asyncio async def test_disappeared_agent_state_is_garbage_collected( monkeypatch: pytest.MonkeyPatch, registry: AgentRegistry, fake_snapshot: ProcessSnapshot, ) -> None: """Once a PID is removed from the registry, the next sampler tick drops its tracker entry. Bounded to one tick of staleness. """ record = AgentRecord(name="claude-code-8421", pid=8421, command="claude") registry.register(record) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.AgentRegistry", lambda: registry) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.probe", lambda _pid: fake_snapshot) source = _FakeSource(chunks=["chunk-1"]) meter = _FakeMeter({"chunk-1": TokenSample.from_tokens(100, model="claude-sonnet-4-5")}) _stub_token_pipeline(monkeypatch, provider="claude-code", source=source, meter=meter) task = start_sampler(interval=0.01) await asyncio.sleep(0.03) assert get_tokens_per_min(8421) is not None # Forget the agent; the next tick's GC pass should clear its # tracker entry. Give the loop a few iterations to land. registry.forget(8421) await asyncio.sleep(0.05) task.cancel() with pytest.raises(asyncio.CancelledError): _ = await task assert get_tokens_per_min(8421) is None @pytest.mark.asyncio async def test_provider_unknown_skips_token_path_without_crashing( monkeypatch: pytest.MonkeyPatch, registry: AgentRegistry, fake_snapshot: ProcessSnapshot, ) -> None: """An agent whose ``provider`` cannot be resolved must not crash the sampler — it just keeps the resource-snapshot path and leaves tokens/cost as ``-``. """ registry.register(AgentRecord(name="my-custom-bot", pid=8421, command="bin")) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.AgentRegistry", lambda: registry) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.probe", lambda _pid: fake_snapshot) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.provider_for", lambda _record: None) task = start_sampler(interval=0.01) await asyncio.sleep(0.03) task.cancel() with pytest.raises(asyncio.CancelledError): _ = await task # Snapshot path still works. assert get_snapshot(8421) == fake_snapshot # Token path was skipped — no tracker entry. assert get_tokens_per_min(8421) is None assert get_usd_per_hour(8421) is None @pytest.mark.asyncio async def test_schema_invalid_agents_yaml_does_not_crash_loop( monkeypatch: pytest.MonkeyPatch, registry: AgentRegistry, fake_snapshot: ProcessSnapshot, ) -> None: """A hand-edited ``agents.yaml`` with a typo'd field raises ``ValidationError``. The sampler must swallow it; the dashboard stays alive even when the user's config is unparseable. """ from pydantic import ValidationError registry.register(AgentRecord(name="claude-code-8421", pid=8421, command="claude")) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.AgentRegistry", lambda: registry) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.probe", lambda _pid: fake_snapshot) def _raise() -> None: raise ValidationError.from_exception_data("AgentsConfig", []) monkeypatch.setattr("tools.system.fleet_monitoring.sampler.load_agents_config", _raise) task = start_sampler(interval=0.01) await asyncio.sleep(0.03) task.cancel() with pytest.raises(asyncio.CancelledError): _ = await task # Loop didn't tear down; snapshot path still works. assert get_snapshot(8421) == fake_snapshot