chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
"""conftest.py — put ``showcase/integrations`` on ``sys.path`` so the test suite
|
||||
can ``import _shared.*`` exactly as the runtime does (``/app`` on PYTHONPATH,
|
||||
``/app/_shared`` the package).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# showcase/integrations/_shared/tests/conftest.py → showcase/integrations
|
||||
_INTEGRATIONS_DIR = Path(__file__).resolve().parents[2]
|
||||
if str(_INTEGRATIONS_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(_INTEGRATIONS_DIR))
|
||||
@@ -0,0 +1,101 @@
|
||||
"""test_cvdiag_emit_gate_degrade.py — M5 CR R3 functional regression for C1: the
|
||||
fail-closed DEBUG degrade must actually SUPPRESS emission.
|
||||
|
||||
Background: a prior fix made ``cvdiag_bootstrap.setup()`` DEGRADE
|
||||
(``_ENABLED=False``, instrumentation disabled) instead of crashing the backend on
|
||||
a fail-closed DEBUG misconfig. But the emit path never consulted that flag —
|
||||
``emit_cvdiag`` wrote the ``CVDIAG`` envelope to stdout regardless of
|
||||
``is_enabled()``. A degraded backend therefore STILL emitted default-tier
|
||||
boundaries whenever the per-integration gate (``CVDIAG_BACKEND_EMITTER=1``) was
|
||||
on. ``is_enabled()`` was a dead flag.
|
||||
|
||||
RED (pre-fix): a degraded ``setup()`` (``_ENABLED=False``) followed by
|
||||
``emit_cvdiag(...)`` still writes a ``CVDIAG `` line to stdout.
|
||||
GREEN (post-fix): the degraded emit writes NOTHING; the normal (enabled) path
|
||||
still emits, so the gate isn't over-tightened.
|
||||
|
||||
Imports the package as ``_shared.*`` to mirror the runtime layout; conftest.py
|
||||
puts ``showcase/integrations`` on sys.path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from _shared import cvdiag_bootstrap
|
||||
|
||||
# A valid UUIDv7 test_id (version nibble 7, variant 8/9/a/b).
|
||||
_VALID_TEST_ID = "018f8b2a-7c3e-7a1b-9f4d-0123456789ab"
|
||||
_VALID_SPAN_ID = "0123456789abcdef"
|
||||
|
||||
|
||||
def _base_envelope(**overrides):
|
||||
"""A schema-valid default-tier boundary envelope (backend.agent.enter)."""
|
||||
env = {
|
||||
"schema_version": 1,
|
||||
"test_id": _VALID_TEST_ID,
|
||||
"trace_id": _VALID_TEST_ID,
|
||||
"span_id": _VALID_SPAN_ID,
|
||||
"parent_span_id": None,
|
||||
"layer": "backend",
|
||||
"boundary": "backend.agent.enter",
|
||||
"slug": "langgraph-python",
|
||||
"demo": "chat",
|
||||
"ts": "2026-06-18T12:00:00Z",
|
||||
"mono_ns": 123456789,
|
||||
"duration_ms": None,
|
||||
"outcome": "ok",
|
||||
"edge_headers": {
|
||||
"cf-ray": None,
|
||||
"cf-mitigated": None,
|
||||
"cf-cache-status": None,
|
||||
"x-railway-edge": None,
|
||||
"x-railway-request-id": None,
|
||||
"x-hikari-trace": None,
|
||||
"retry-after": None,
|
||||
"via": None,
|
||||
"server": None,
|
||||
},
|
||||
"metadata": {"agent_name": "weather", "model_id": "claude"},
|
||||
}
|
||||
env.update(overrides)
|
||||
return env
|
||||
|
||||
|
||||
def test_degraded_setup_suppresses_emission(capsys):
|
||||
"""A degraded backend (``_ENABLED=False``) must emit NOTHING.
|
||||
|
||||
Reproduces the prod-reachable path ``CVDIAG_BACKEND_EMITTER=1`` +
|
||||
``CVDIAG_DEBUG=1`` (unresolved env → fail-closed degrade): the per-integration
|
||||
gate is armed, but the shared bootstrap degraded instrumentation OFF. The
|
||||
shared ``emit_cvdiag`` is the single chokepoint every integration routes
|
||||
through, so it must honor ``is_enabled()``.
|
||||
|
||||
RED (pre-fix): ``emit_cvdiag`` ignores ``_ENABLED`` and still writes the
|
||||
``CVDIAG `` line.
|
||||
"""
|
||||
cvdiag_bootstrap.reset_for_test()
|
||||
# Degrade: fail-closed DEBUG with an unresolved env disables instrumentation.
|
||||
cvdiag_bootstrap.setup({"CVDIAG_BACKEND_EMITTER": "1", "CVDIAG_DEBUG": "1"})
|
||||
assert cvdiag_bootstrap.is_enabled() is False, "precondition: setup degraded"
|
||||
|
||||
cvdiag_bootstrap.emit_cvdiag(_base_envelope())
|
||||
|
||||
out = capsys.readouterr().out
|
||||
assert "CVDIAG " not in out, (
|
||||
"degraded backend must suppress emission; got stdout:\n" + out
|
||||
)
|
||||
|
||||
|
||||
def test_enabled_setup_still_emits(capsys):
|
||||
"""Sanity / over-tightening guard: the normal enabled path still emits.
|
||||
|
||||
GREEN must not silence a healthy backend — a non-degraded ``setup()`` keeps
|
||||
``emit_cvdiag`` writing the ``CVDIAG `` envelope line.
|
||||
"""
|
||||
cvdiag_bootstrap.reset_for_test()
|
||||
cvdiag_bootstrap.setup({"CVDIAG_BACKEND_EMITTER": "1", "SHOWCASE_ENV": "staging"})
|
||||
assert cvdiag_bootstrap.is_enabled() is True, "precondition: setup enabled"
|
||||
|
||||
cvdiag_bootstrap.emit_cvdiag(_base_envelope())
|
||||
|
||||
out = capsys.readouterr().out
|
||||
assert "CVDIAG " in out, "enabled backend must still emit; got stdout:\n" + out
|
||||
@@ -0,0 +1,133 @@
|
||||
"""test_cvdiag_inert_when_disabled.py — M5 CR R4 functional regression for N3:
|
||||
importing the cvdiag bootstrap must be FULLY INERT when cvdiag is disabled.
|
||||
|
||||
Background (N3): ``cvdiag_bootstrap.setup()`` ran
|
||||
``logging.basicConfig(level=…, format=…, force=True)`` unconditionally at import
|
||||
time (the module calls ``setup()`` at the bottom). ``force=True`` REMOVES and
|
||||
CLOSES every pre-existing root-logger handler, so merely importing the module —
|
||||
which every Python integration backend does whenever it imports
|
||||
``_cvdiag_backend`` — ripped out the HOST application's logging configuration,
|
||||
even with the backend emitter OFF (``CVDIAG_BACKEND_EMITTER`` unset, the
|
||||
canary-safe default). That violates the "byte-for-byte inert when disabled /
|
||||
canary-safe" contract the backend emitters advertise; the canonical TypeScript
|
||||
emitter performs NO equivalent global mutation.
|
||||
|
||||
RED (pre-fix): a pre-configured host root-logger handler is TORN DOWN by the
|
||||
import-time ``basicConfig(force=True)``.
|
||||
GREEN (post-fix): importing the bootstrap with cvdiag disabled is fully inert —
|
||||
the host root handler survives, nothing is written to stdout, and no rogue
|
||||
non-daemon threads are spawned.
|
||||
|
||||
This locks the CLASS of "fully inert when disabled" so future CR rounds stop
|
||||
re-discovering individual instances of host-state mutation at import.
|
||||
|
||||
The assertions run in a FRESH SUBPROCESS so the import-time side effects of
|
||||
``cvdiag_bootstrap`` are actually observable (a same-process import is cached and
|
||||
``basicConfig`` is a no-op the second time).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# showcase/integrations/_shared/tests/ → showcase/integrations
|
||||
_INTEGRATIONS_DIR = Path(__file__).resolve().parents[2]
|
||||
|
||||
# Child program: pre-configure a host root handler, import the bootstrap with
|
||||
# cvdiag DISABLED, then assert the import was fully inert. Printed sentinels are
|
||||
# parsed by the parent so a failure produces a readable diff.
|
||||
_CHILD = r"""
|
||||
import logging
|
||||
import sys
|
||||
import threading
|
||||
|
||||
# (host) Pre-configure the application's own root-logger handler BEFORE importing
|
||||
# cvdiag — this is the state a real host app has when the backend boots.
|
||||
_host_handler = logging.StreamHandler(sys.stderr)
|
||||
_host_handler.set_name("HOST_ROOT_HANDLER")
|
||||
_root = logging.getLogger()
|
||||
_root.addHandler(_host_handler)
|
||||
_root.setLevel(logging.INFO)
|
||||
|
||||
_threads_before = {t.ident for t in threading.enumerate()}
|
||||
|
||||
import io
|
||||
_cap = io.StringIO()
|
||||
_real_stdout = sys.stdout
|
||||
sys.stdout = _cap
|
||||
try:
|
||||
# cvdiag DISABLED: backend emitter OFF (canary-safe default), no debug.
|
||||
import _shared.cvdiag_bootstrap # noqa: F401 (import-time setup() runs here)
|
||||
finally:
|
||||
sys.stdout = _real_stdout
|
||||
|
||||
_stdout_text = _cap.getvalue()
|
||||
|
||||
# (1) The host's pre-existing root handler MUST survive the import. This is the
|
||||
# RED trigger on the force=True code (basicConfig(force=True) removed it).
|
||||
_host_present = any(
|
||||
getattr(h, "get_name", lambda: None)() == "HOST_ROOT_HANDLER"
|
||||
for h in logging.getLogger().handlers
|
||||
)
|
||||
|
||||
# (2) Zero CVDIAG lines written to stdout at import (disabled → nothing emitted).
|
||||
_no_cvdiag = "CVDIAG " not in _stdout_text
|
||||
|
||||
# (3) No rogue non-daemon threads spawned by the import.
|
||||
_new_threads = [
|
||||
t for t in threading.enumerate()
|
||||
if t.ident not in _threads_before and not t.daemon
|
||||
]
|
||||
_no_nondaemon_threads = len(_new_threads) == 0
|
||||
|
||||
print("HOST_PRESENT=" + ("1" if _host_present else "0"))
|
||||
print("NO_CVDIAG=" + ("1" if _no_cvdiag else "0"))
|
||||
print("NO_NONDAEMON_THREADS=" + ("1" if _no_nondaemon_threads else "0"))
|
||||
print("NEW_NONDAEMON_THREADS=" + ",".join(t.name for t in _new_threads))
|
||||
"""
|
||||
|
||||
|
||||
def _run_child() -> dict[str, str]:
|
||||
env_clean = {
|
||||
# Inherit a minimal env but ensure cvdiag is fully disabled.
|
||||
"PATH": "/usr/bin:/bin",
|
||||
"PYTHONPATH": str(_INTEGRATIONS_DIR),
|
||||
# Explicitly NOT setting CVDIAG_BACKEND_EMITTER / CVDIAG_DEBUG /
|
||||
# CVDIAG_VERBOSE — disabled, canary-safe default.
|
||||
}
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "-c", _CHILD],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env_clean,
|
||||
cwd=str(_INTEGRATIONS_DIR),
|
||||
timeout=60,
|
||||
)
|
||||
assert proc.returncode == 0, (
|
||||
"child process failed:\nSTDOUT:\n" + proc.stdout + "\nSTDERR:\n" + proc.stderr
|
||||
)
|
||||
result: dict[str, str] = {}
|
||||
for line in proc.stdout.splitlines():
|
||||
if "=" in line:
|
||||
key, _, val = line.partition("=")
|
||||
result[key] = val
|
||||
return result
|
||||
|
||||
|
||||
def test_import_is_fully_inert():
|
||||
"""Importing cvdiag_bootstrap (disabled) must not mutate host logging state."""
|
||||
r = _run_child()
|
||||
|
||||
assert r.get("HOST_PRESENT") == "1", (
|
||||
"host root-logger handler was torn down by the import-time "
|
||||
"basicConfig(force=True); the disabled cvdiag bootstrap must be inert"
|
||||
)
|
||||
assert r.get("NO_CVDIAG") == "1", (
|
||||
"a disabled cvdiag import wrote a CVDIAG line to stdout"
|
||||
)
|
||||
assert r.get("NO_NONDAEMON_THREADS") == "1", (
|
||||
"a disabled cvdiag import spawned non-daemon thread(s): "
|
||||
+ r.get("NEW_NONDAEMON_THREADS", "")
|
||||
)
|
||||
@@ -0,0 +1,163 @@
|
||||
"""test_cvdiag_schema.py — L0-C unit suite for the Python ``_shared`` CVDIAG
|
||||
bootstrap module (6 tests, spec §5/§6).
|
||||
|
||||
Run from the repo root::
|
||||
|
||||
python3 -m pytest showcase/integrations/_shared/tests/
|
||||
|
||||
These tests import the package as ``_shared.*`` to mirror the runtime layout
|
||||
(``/app`` on PYTHONPATH, ``/app/_shared`` the package). ``conftest.py`` puts
|
||||
``showcase/integrations`` on ``sys.path`` so ``import _shared`` resolves.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from _shared import cvdiag_schema as schema
|
||||
|
||||
# A valid UUIDv7 test_id (version nibble 7, variant 8/9/a/b).
|
||||
_VALID_TEST_ID = "018f8b2a-7c3e-7a1b-9f4d-0123456789ab"
|
||||
_VALID_SPAN_ID = "0123456789abcdef"
|
||||
|
||||
|
||||
def _base_envelope(**overrides):
|
||||
env = {
|
||||
"schema_version": 1,
|
||||
"test_id": _VALID_TEST_ID,
|
||||
"trace_id": _VALID_TEST_ID,
|
||||
"span_id": _VALID_SPAN_ID,
|
||||
"parent_span_id": None,
|
||||
"layer": "backend",
|
||||
"boundary": "backend.agent.enter",
|
||||
"slug": "langgraph-python",
|
||||
"demo": "chat",
|
||||
"ts": "2026-06-18T12:00:00Z",
|
||||
"mono_ns": 123456789,
|
||||
"duration_ms": None,
|
||||
"outcome": "ok",
|
||||
"edge_headers": {
|
||||
"cf-ray": None,
|
||||
"cf-mitigated": None,
|
||||
"cf-cache-status": None,
|
||||
"x-railway-edge": None,
|
||||
"x-railway-request-id": None,
|
||||
"x-hikari-trace": None,
|
||||
"retry-after": None,
|
||||
"via": None,
|
||||
"server": None,
|
||||
},
|
||||
"metadata": {"agent_name": "weather", "model_id": "claude"},
|
||||
}
|
||||
env.update(overrides)
|
||||
return env
|
||||
|
||||
|
||||
def test_valid_envelope_round_trips():
|
||||
"""A well-formed envelope validates and round-trips by alias."""
|
||||
model = schema.CvdiagEnvelope.model_validate(_base_envelope())
|
||||
assert model.boundary is schema.CvdiagBoundary.BACKEND_AGENT_ENTER
|
||||
assert model.layer is schema.CvdiagLayer.BACKEND
|
||||
assert model.outcome is schema.CvdiagOutcome.OK
|
||||
assert model.metadata_dropped is False
|
||||
dumped = model.model_dump(by_alias=True)
|
||||
# The edge-header alias keys survive the round-trip.
|
||||
assert dumped["edge_headers"]["cf-ray"] is None
|
||||
assert dumped["_metadata_dropped"] is False
|
||||
|
||||
|
||||
def test_unknown_metadata_key_stamps_metadata_dropped():
|
||||
"""An unknown metadata key sets ``_metadata_dropped`` on the envelope."""
|
||||
env = _base_envelope(
|
||||
metadata={"agent_name": "weather", "bogus_key": "x"},
|
||||
)
|
||||
model = schema.CvdiagEnvelope.model_validate(env)
|
||||
assert model.metadata_dropped is True
|
||||
# Also true for an unknown TOP-LEVEL key.
|
||||
env2 = _base_envelope()
|
||||
env2["totally_unknown"] = "y"
|
||||
model2 = schema.CvdiagEnvelope.model_validate(env2)
|
||||
assert model2.metadata_dropped is True
|
||||
|
||||
|
||||
def test_forbidden_edge_header_dropped():
|
||||
"""A deny-list edge header (cf-connecting-ip) is rejected by EdgeHeaders.
|
||||
|
||||
``EdgeHeaders`` forbids extra keys, so a forbidden header can never round
|
||||
-trip through the closed model.
|
||||
"""
|
||||
bad = {
|
||||
"cf-ray": "abc",
|
||||
"cf-mitigated": None,
|
||||
"cf-cache-status": None,
|
||||
"x-railway-edge": None,
|
||||
"x-railway-request-id": None,
|
||||
"x-hikari-trace": None,
|
||||
"retry-after": None,
|
||||
"via": None,
|
||||
"server": None,
|
||||
"cf-connecting-ip": "1.2.3.4", # forbidden PII header
|
||||
}
|
||||
with pytest.raises(Exception):
|
||||
schema.EdgeHeaders.model_validate(bad)
|
||||
|
||||
|
||||
def test_uuidv7_test_id_validation():
|
||||
"""UUIDv7 test_id passes; a UUIDv4 / malformed test_id is rejected."""
|
||||
# Pass.
|
||||
schema.CvdiagEnvelope.model_validate(_base_envelope(test_id=_VALID_TEST_ID))
|
||||
# UUIDv4 (version nibble 4) → reject.
|
||||
uuid_v4 = "018f8b2a-7c3e-4a1b-9f4d-0123456789ab"
|
||||
with pytest.raises(Exception):
|
||||
schema.CvdiagEnvelope.model_validate(_base_envelope(test_id=uuid_v4))
|
||||
# Malformed → reject.
|
||||
with pytest.raises(Exception):
|
||||
schema.CvdiagEnvelope.model_validate(_base_envelope(test_id="not-a-uuid"))
|
||||
|
||||
|
||||
def test_debug_in_production_degrades_at_setup():
|
||||
"""``CVDIAG_DEBUG=1`` + ``SHOWCASE_ENV=production`` fails closed at setup.
|
||||
|
||||
Fail-closed intent: instrumentation is DISABLED (tier stays ``default``,
|
||||
``is_enabled()`` False). Degrade-not-crash: ``setup()`` must NOT raise —
|
||||
a misconfig may not abort the backend's module import.
|
||||
"""
|
||||
from _shared import cvdiag_bootstrap
|
||||
|
||||
cvdiag_bootstrap.reset_for_test()
|
||||
cvdiag_bootstrap.setup({"CVDIAG_DEBUG": "1", "SHOWCASE_ENV": "production"})
|
||||
assert cvdiag_bootstrap.current_tier() == "default"
|
||||
assert cvdiag_bootstrap.is_enabled() is False
|
||||
|
||||
# Unresolved env is ALSO treated as production (fail-closed → degraded).
|
||||
cvdiag_bootstrap.reset_for_test()
|
||||
cvdiag_bootstrap.setup({"CVDIAG_DEBUG": "1"})
|
||||
assert cvdiag_bootstrap.current_tier() == "default"
|
||||
assert cvdiag_bootstrap.is_enabled() is False
|
||||
|
||||
# A non-production env with DEBUG is allowed (instrumentation enabled).
|
||||
cvdiag_bootstrap.reset_for_test()
|
||||
cvdiag_bootstrap.setup({"CVDIAG_DEBUG": "1", "SHOWCASE_ENV": "staging"})
|
||||
assert cvdiag_bootstrap.current_tier() == "debug"
|
||||
assert cvdiag_bootstrap.is_enabled() is True
|
||||
|
||||
|
||||
def test_basicconfig_captures_agents_logger_output(capsys):
|
||||
"""``setup()`` installs a handler so ``agents.*`` loggers actually emit.
|
||||
|
||||
This is the silent-drop regression guard: before bootstrap configures the
|
||||
root logger, an ``agents._header_forwarding`` ``logger.info`` is dropped;
|
||||
after ``setup()`` it reaches the stream.
|
||||
"""
|
||||
from _shared import cvdiag_bootstrap
|
||||
|
||||
cvdiag_bootstrap.reset_for_test()
|
||||
cvdiag_bootstrap.setup({"SHOWCASE_ENV": "staging"})
|
||||
fwd_logger = logging.getLogger("agents._header_forwarding")
|
||||
fwd_logger.info("CVDIAG component=test boundary=probe.start status=ok")
|
||||
captured = capsys.readouterr()
|
||||
combined = captured.out + captured.err
|
||||
assert "CVDIAG component=test boundary=probe.start" in combined
|
||||
@@ -0,0 +1,164 @@
|
||||
"""test_cvdiag_writer_bootstrap.py — functional regression suite for the three
|
||||
M5 CR R1 fixes in the Python ``_shared`` CVDIAG core:
|
||||
|
||||
FIX-1 cvdiag_pb_writer drain loop never-propagate: a non-JSON-serializable
|
||||
envelope must NOT kill the flush daemon (mirrors TS pb-writer
|
||||
writeBatch — one bad row degrades, the batch/daemon survives).
|
||||
FIX-2 cvdiag_bootstrap.setup() degrade-not-crash: a fail-closed DEBUG
|
||||
misconfig must DISABLE instrumentation, not raise out of import.
|
||||
FIX-3 cvdiag_bootstrap.setup() idempotency: a repeated call is a no-op and
|
||||
never orphans a second daemon thread / PB writer.
|
||||
|
||||
Imports the package as ``_shared.*`` to mirror the runtime layout; conftest.py
|
||||
puts ``showcase/integrations`` on sys.path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
|
||||
from _shared import cvdiag_bootstrap
|
||||
from _shared.cvdiag_pb_writer import CvdiagPbWriter
|
||||
|
||||
|
||||
class _Unserializable:
|
||||
"""An object json.dumps cannot encode (raises TypeError in _post)."""
|
||||
|
||||
|
||||
def _drain_thread_alive(writer: CvdiagPbWriter) -> bool:
|
||||
worker = writer._worker
|
||||
return worker is not None and worker.is_alive()
|
||||
|
||||
|
||||
# ── FIX-1: drain loop never-propagate ────────────────────────────────────────
|
||||
|
||||
|
||||
def test_drain_survives_non_json_serializable_envelope(monkeypatch):
|
||||
"""A non-JSON-serializable record must not kill the daemon; a later valid
|
||||
record still flushes.
|
||||
|
||||
RED (pre-fix): ``_post`` lets ``TypeError`` from ``json.dumps`` escape the
|
||||
``except (URLError, OSError, ValueError)`` clause; the exception unwinds
|
||||
``_run`` and the daemon thread dies — the later valid envelope is never
|
||||
POSTed.
|
||||
"""
|
||||
writer = CvdiagPbWriter(pb_url="http://pb.invalid", flush_window_s=0.05)
|
||||
|
||||
posted: list[dict] = []
|
||||
|
||||
# Stub the HTTP layer: record what reaches the wire after json.dumps.
|
||||
def _fake_post(envelope):
|
||||
# Re-run the real serialization seam so a bad envelope still throws
|
||||
# inside the drain loop, but a good one is "delivered" without network.
|
||||
import json as _json
|
||||
|
||||
_json.dumps(envelope) # raises TypeError on the bad envelope
|
||||
posted.append(envelope)
|
||||
|
||||
monkeypatch.setattr(writer, "_post", _fake_post)
|
||||
|
||||
# Bad record first — pre-fix this kills the drain thread.
|
||||
writer.enqueue({"bad": _Unserializable()})
|
||||
# Give the worker a couple of flush windows to process the bad record.
|
||||
time.sleep(0.2)
|
||||
|
||||
# Then a perfectly valid record.
|
||||
writer.enqueue({"ok": True, "n": 1})
|
||||
time.sleep(0.2)
|
||||
|
||||
assert _drain_thread_alive(writer), "drain daemon must survive a bad record"
|
||||
assert {"ok": True, "n": 1} in posted, "valid record must still flush"
|
||||
|
||||
|
||||
def test_post_swallows_typeerror_on_unserializable(monkeypatch):
|
||||
"""``_post`` itself must not raise on a non-serializable envelope.
|
||||
|
||||
RED (pre-fix): the ``TypeError`` from ``json.dumps`` is uncaught and
|
||||
propagates out of ``_post`` (only URLError/OSError/ValueError are caught).
|
||||
"""
|
||||
writer = CvdiagPbWriter(pb_url="http://pb.invalid")
|
||||
# Should NOT raise — _post is the never-throw persistence seam.
|
||||
writer._post({"bad": _Unserializable()})
|
||||
|
||||
|
||||
# ── FIX-2: bootstrap degrade-not-crash ───────────────────────────────────────
|
||||
|
||||
|
||||
def test_setup_degrades_on_failclosed_debug_misconfig():
|
||||
"""A fail-closed DEBUG misconfig DISABLES instrumentation instead of raising.
|
||||
|
||||
RED (pre-fix): ``setup({"CVDIAG_DEBUG": "1"})`` (unresolved env → treated as
|
||||
production) raises ``RuntimeError`` out of setup(), which at import time
|
||||
would abort the whole backend module import.
|
||||
"""
|
||||
cvdiag_bootstrap.reset_for_test()
|
||||
# Must NOT raise.
|
||||
cvdiag_bootstrap.setup({"CVDIAG_DEBUG": "1"})
|
||||
# Fail-closed intent preserved: instrumentation is OFF (tier not debug).
|
||||
assert cvdiag_bootstrap.current_tier() == "default"
|
||||
assert cvdiag_bootstrap.is_enabled() is False
|
||||
|
||||
# Explicit production env with DEBUG also degrades, never raises.
|
||||
cvdiag_bootstrap.reset_for_test()
|
||||
cvdiag_bootstrap.setup({"CVDIAG_DEBUG": "1", "SHOWCASE_ENV": "production"})
|
||||
assert cvdiag_bootstrap.current_tier() == "default"
|
||||
assert cvdiag_bootstrap.is_enabled() is False
|
||||
|
||||
|
||||
def test_setup_allows_debug_in_nonproduction():
|
||||
"""A non-production DEBUG request still enables debug tier (intent intact)."""
|
||||
cvdiag_bootstrap.reset_for_test()
|
||||
cvdiag_bootstrap.setup({"CVDIAG_DEBUG": "1", "SHOWCASE_ENV": "staging"})
|
||||
assert cvdiag_bootstrap.current_tier() == "debug"
|
||||
assert cvdiag_bootstrap.is_enabled() is True
|
||||
|
||||
|
||||
# ── FIX-3: bootstrap idempotency ─────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_setup_is_idempotent_no_orphan_daemon():
|
||||
"""A second setup() is a no-op: no second PB writer / daemon thread.
|
||||
|
||||
RED (pre-fix): no ``_SETUP_DONE`` guard in setup(), so a second call rebuilds
|
||||
``_PB_WRITER`` (orphaning the first writer's queue) and a subsequent enqueue
|
||||
spins a second ``cvdiag-pb-writer`` daemon thread.
|
||||
"""
|
||||
cvdiag_bootstrap.reset_for_test()
|
||||
|
||||
# Count daemon threads BEFORE this test so the assertion is robust to
|
||||
# daemons left alive by earlier tests in the suite (short-lived best-effort
|
||||
# daemons are not joined on reset).
|
||||
def _pb_daemon_count() -> int:
|
||||
return sum(1 for t in threading.enumerate() if t.name == "cvdiag-pb-writer")
|
||||
|
||||
before = _pb_daemon_count()
|
||||
|
||||
cvdiag_bootstrap.setup(
|
||||
{"SHOWCASE_ENV": "staging", "CVDIAG_PB_URL": "http://pb.invalid"}
|
||||
)
|
||||
first_writer = cvdiag_bootstrap._PB_WRITER
|
||||
assert first_writer is not None
|
||||
first_writer.enqueue({"n": 1})
|
||||
time.sleep(0.05)
|
||||
after_first = _pb_daemon_count()
|
||||
assert after_first == before + 1, (
|
||||
"first setup()+enqueue must spin exactly one daemon"
|
||||
)
|
||||
|
||||
cvdiag_bootstrap.setup(
|
||||
{"SHOWCASE_ENV": "staging", "CVDIAG_PB_URL": "http://pb.invalid"}
|
||||
)
|
||||
second_writer = cvdiag_bootstrap._PB_WRITER
|
||||
|
||||
assert second_writer is first_writer, (
|
||||
"second setup() must not rebuild the PB writer"
|
||||
)
|
||||
|
||||
second_writer.enqueue({"n": 2})
|
||||
time.sleep(0.05)
|
||||
after_second = _pb_daemon_count()
|
||||
# The second setup()+enqueue must NOT spin an additional daemon.
|
||||
assert after_second == after_first, (
|
||||
f"second setup() orphaned a daemon: {before}->{after_first}->{after_second}"
|
||||
)
|
||||
@@ -0,0 +1,248 @@
|
||||
"""test_cvdiag_writer_live_pb.py — live-PocketBase auth proof for the Python
|
||||
CVDIAG PB writer.
|
||||
|
||||
Boots an ACTUAL PocketBase 0.22 server using the EXACT production migrations
|
||||
(showcase/pocketbase/pb_migrations), which create ``cvdiag_events`` + the
|
||||
``cvdiag_api_keys`` auth collection + the seeded role-keyed records (writer /
|
||||
purge / migration). It then drives the real ``CvdiagPbWriter`` against that
|
||||
server and asserts the CREATE-only ACL is satisfied **as the writer role**.
|
||||
|
||||
WHY WRITER-ROLE (not superuser): the PB superuser bypasses ALL collection
|
||||
rules, so a writer that never authenticates would still "succeed" if probed as
|
||||
a superuser — that is exactly how the original defect (the inert
|
||||
``X-Cvdiag-Writer-Key`` header) escaped review. The createRule
|
||||
|
||||
@request.auth.collectionName = "cvdiag_api_keys" && @request.auth.role = "writer"
|
||||
|
||||
is only satisfiable by authenticating as the seeded writer record and sending
|
||||
``Authorization: Bearer <token>``. We therefore drive the writer with the
|
||||
writer record's PASSWORD as ``CVDIAG_WRITER_KEY`` and verify the row lands by
|
||||
querying ``cvdiag_events`` AS THE SUPERUSER (read-only, rule-bypassing) — the
|
||||
write path under test is never superuser.
|
||||
|
||||
Requires a ``pocketbase`` binary: set ``POCKETBASE_BIN`` to its path, put it on
|
||||
PATH, or drop it at ``/tmp/pb022/pocketbase``. The suite SKIPS (does not fail)
|
||||
when no binary is available so CI without the binary stays green.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
|
||||
from _shared.cvdiag_pb_writer import CvdiagPbWriter
|
||||
|
||||
# These mirror the seed records created by the cvdiag_api_keys migration
|
||||
# (showcase/pocketbase/pb_migrations/1779990200_create_cvdiag_events.js).
|
||||
WRITER_EMAIL = "cvdiag-writer@keys.local"
|
||||
WRITER_PASS = "cvdiagwriterpass123"
|
||||
|
||||
# A superuser used ONLY to read rows back (rule-bypassing read); never the
|
||||
# write path under test.
|
||||
ADMIN_EMAIL = "cvdiag-py-acl@test.local"
|
||||
ADMIN_PASS = "cvdiagpyaclpass123"
|
||||
|
||||
# showcase/integrations/_shared/tests/ → repo root → showcase/pocketbase
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
_MIGRATIONS_DIR = _REPO_ROOT / "showcase" / "pocketbase" / "pb_migrations"
|
||||
|
||||
|
||||
def _resolve_pb_binary() -> Optional[str]:
|
||||
explicit = os.environ.get("POCKETBASE_BIN")
|
||||
if explicit and Path(explicit).is_file():
|
||||
return explicit
|
||||
on_path = shutil.which("pocketbase")
|
||||
if on_path:
|
||||
return on_path
|
||||
if Path("/tmp/pb022/pocketbase").is_file():
|
||||
return "/tmp/pb022/pocketbase"
|
||||
return None
|
||||
|
||||
|
||||
_PB_BIN = _resolve_pb_binary()
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
_PB_BIN is None,
|
||||
reason="no pocketbase binary (set POCKETBASE_BIN, PATH, or /tmp/pb022/pocketbase)",
|
||||
)
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return s.getsockname()[1]
|
||||
|
||||
|
||||
def _wait_for_health(base: str, timeout_s: float = 15.0) -> None:
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(f"{base}/api/health", timeout=1.0) as r:
|
||||
if r.status == 200:
|
||||
return
|
||||
except (urllib.error.URLError, OSError):
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
raise RuntimeError("PocketBase did not become healthy in time")
|
||||
|
||||
|
||||
def _admin_token(base: str) -> str:
|
||||
body = json.dumps({"identity": ADMIN_EMAIL, "password": ADMIN_PASS}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"{base}/api/admins/auth-with-password",
|
||||
data=body,
|
||||
method="POST",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=5.0) as r:
|
||||
return json.loads(r.read())["token"]
|
||||
|
||||
|
||||
def _count_events(base: str, test_id: str) -> int:
|
||||
"""Count cvdiag_events rows for one test_id, reading AS SUPERUSER."""
|
||||
tok = _admin_token(base)
|
||||
flt = urllib.parse.quote(f'test_id="{test_id}"')
|
||||
req = urllib.request.Request(
|
||||
f"{base}/api/collections/cvdiag_events/records?filter={flt}",
|
||||
headers={"Authorization": tok},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=5.0) as r:
|
||||
return json.loads(r.read())["totalItems"]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def live_pb():
|
||||
"""Boot a real PB with the production migrations; yield its base URL."""
|
||||
# skipif(_PB_BIN is None) guarantees a binary at runtime; narrow to a
|
||||
# concrete str so subprocess calls don't take a `str | None` arg.
|
||||
pb_bin = _PB_BIN
|
||||
assert pb_bin is not None
|
||||
data_dir = tempfile.mkdtemp(prefix="pb-cvdiag-py-")
|
||||
try:
|
||||
mig = subprocess.run(
|
||||
[
|
||||
pb_bin,
|
||||
"migrate",
|
||||
"up",
|
||||
f"--dir={data_dir}",
|
||||
f"--migrationsDir={_MIGRATIONS_DIR}",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if mig.returncode != 0:
|
||||
raise RuntimeError(f"pb migrate up failed: {mig.stderr or mig.stdout}")
|
||||
admin = subprocess.run(
|
||||
[pb_bin, "admin", "create", ADMIN_EMAIL, ADMIN_PASS, f"--dir={data_dir}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if admin.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"pb admin create failed: {admin.stderr or admin.stdout}"
|
||||
)
|
||||
port = _free_port()
|
||||
base = f"http://127.0.0.1:{port}"
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
pb_bin,
|
||||
"serve",
|
||||
f"--http=127.0.0.1:{port}",
|
||||
f"--dir={data_dir}",
|
||||
f"--migrationsDir={_MIGRATIONS_DIR}",
|
||||
],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
)
|
||||
try:
|
||||
_wait_for_health(base)
|
||||
yield base
|
||||
finally:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
finally:
|
||||
shutil.rmtree(data_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def _flush_one(writer: CvdiagPbWriter, envelope: dict) -> None:
|
||||
"""Enqueue one envelope and let the daemon drain it (flush window 0.05s)."""
|
||||
writer.enqueue(envelope)
|
||||
# Give the worker a few flush windows to drain + POST + (re-)auth.
|
||||
time.sleep(1.0)
|
||||
|
||||
|
||||
def _event(test_id: str) -> dict:
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"test_id": test_id,
|
||||
"trace_id": test_id,
|
||||
"span_id": "0000000000000001",
|
||||
"parent_span_id": None,
|
||||
"layer": "backend",
|
||||
"boundary": "backend.handle",
|
||||
"slug": "langgraph-python",
|
||||
"demo": "chat",
|
||||
"ts": "2026-06-18T00:00:00.000Z",
|
||||
"mono_ns": 1,
|
||||
"duration_ms": None,
|
||||
"outcome": "ok",
|
||||
"edge_headers": {},
|
||||
"metadata": {"src": "live-pb-test"},
|
||||
}
|
||||
|
||||
|
||||
def test_writer_authenticates_as_writer_role_and_persists(live_pb):
|
||||
"""GREEN: the writer auth-with-passwords as the writer role → CREATE 201 →
|
||||
the row is present in cvdiag_events (read back as superuser).
|
||||
|
||||
This is the fix for the inert ``X-Cvdiag-Writer-Key`` header: without
|
||||
auth-with-password → Bearer the CREATE 403s under the writer createRule.
|
||||
"""
|
||||
test_id = "0190a0c0-0000-7000-8000-00000000a001"
|
||||
# Pre-condition: no row yet.
|
||||
assert _count_events(live_pb, test_id) == 0
|
||||
|
||||
writer = CvdiagPbWriter(
|
||||
pb_url=live_pb,
|
||||
writer_key=WRITER_PASS, # CVDIAG_WRITER_KEY == the writer record password
|
||||
flush_window_s=0.05,
|
||||
)
|
||||
_flush_one(writer, _event(test_id))
|
||||
|
||||
# The CREATE went through the CREATE-only writer rule → row present.
|
||||
assert _count_events(live_pb, test_id) == 1
|
||||
|
||||
|
||||
def test_writer_with_wrong_password_degrades_to_noop(live_pb):
|
||||
"""A WRONG password must degrade to a no-op (auth fails, CREATE 401/403)
|
||||
WITHOUT crashing the daemon — never-throw preserved.
|
||||
"""
|
||||
test_id = "0190a0c0-0000-7000-8000-00000000a002"
|
||||
assert _count_events(live_pb, test_id) == 0
|
||||
|
||||
writer = CvdiagPbWriter(
|
||||
pb_url=live_pb,
|
||||
writer_key="totally-wrong-password",
|
||||
flush_window_s=0.05,
|
||||
)
|
||||
_flush_one(writer, _event(test_id))
|
||||
|
||||
# No row landed (auth failed) AND the daemon survived (no crash).
|
||||
assert _count_events(live_pb, test_id) == 0
|
||||
worker = writer._worker
|
||||
assert worker is not None and worker.is_alive(), (
|
||||
"daemon must survive an auth failure (never-throw)"
|
||||
)
|
||||
Reference in New Issue
Block a user