chore: import upstream snapshot with attribution
Test / Code Quality (push) Has been cancelled
Test / Test (macos-latest, Python 3.10) (push) Has been cancelled
Test / Test (macos-latest, Python 3.11) (push) Has been cancelled
Test / Test (macos-latest, Python 3.12) (push) Has been cancelled
Test / Test (macos-latest, Python 3.13) (push) Has been cancelled
Test / Test (macos-latest, Python 3.14) (push) Has been cancelled
Test / Test (ubuntu-latest, Python 3.10) (push) Has been cancelled
Test / Test (ubuntu-latest, Python 3.11) (push) Has been cancelled
Test / Test (ubuntu-latest, Python 3.12) (push) Has been cancelled
Test / Test (ubuntu-latest, Python 3.13) (push) Has been cancelled
Test / Test (ubuntu-latest, Python 3.14) (push) Has been cancelled
Test / Test (windows-latest, Python 3.10) (push) Has been cancelled
Test / Test (windows-latest, Python 3.11) (push) Has been cancelled
Test / Test (windows-latest, Python 3.12) (push) Has been cancelled
Test / Test (windows-latest, Python 3.13) (push) Has been cancelled
Test / Test (windows-latest, Python 3.14) (push) Has been cancelled
CodeQL / Analyze (push) Has been cancelled
dependency-audit / pip-audit (push) Has been cancelled
Test / Code Quality (push) Has been cancelled
Test / Test (macos-latest, Python 3.10) (push) Has been cancelled
Test / Test (macos-latest, Python 3.11) (push) Has been cancelled
Test / Test (macos-latest, Python 3.12) (push) Has been cancelled
Test / Test (macos-latest, Python 3.13) (push) Has been cancelled
Test / Test (macos-latest, Python 3.14) (push) Has been cancelled
Test / Test (ubuntu-latest, Python 3.10) (push) Has been cancelled
Test / Test (ubuntu-latest, Python 3.11) (push) Has been cancelled
Test / Test (ubuntu-latest, Python 3.12) (push) Has been cancelled
Test / Test (ubuntu-latest, Python 3.13) (push) Has been cancelled
Test / Test (ubuntu-latest, Python 3.14) (push) Has been cancelled
Test / Test (windows-latest, Python 3.10) (push) Has been cancelled
Test / Test (windows-latest, Python 3.11) (push) Has been cancelled
Test / Test (windows-latest, Python 3.12) (push) Has been cancelled
Test / Test (windows-latest, Python 3.13) (push) Has been cancelled
Test / Test (windows-latest, Python 3.14) (push) Has been cancelled
CodeQL / Analyze (push) Has been cancelled
dependency-audit / pip-audit (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
# Repair allowlist — cassettes the Tier 8 phase-1 audit flagged for phase-2 repair.
|
||||
#
|
||||
# Entries are cassette basenames (one per line). Blank lines and comment
|
||||
# lines (starting with ``#``) are ignored.
|
||||
#
|
||||
# Cassettes listed here are SKIPPED by ``tests/scripts/check_cassettes_clean.py``
|
||||
# unless the tool is invoked with ``--strict``. This lets the guard be wired
|
||||
# into CI as a non-blocking step today while phase-2 cassette re-recording
|
||||
# catches up; the ``--strict`` flag will be flipped on once all entries below
|
||||
# are repaired and this file is empty / deleted.
|
||||
#
|
||||
# Two groupings (a cassette flagged for *both* reasons appears only once):
|
||||
#
|
||||
# 1. Spec-explicit entries — cassettes the audit named for repair on grounds
|
||||
# other than ``/ogw/`` avatar URLs alone (display-name leaks, upload-token
|
||||
# leaks, Drive-token leaks, etc.).
|
||||
#
|
||||
# 2. ``/ogw/`` avatar URL group — cleared in T8.B6 (this PR). The 61-entry
|
||||
# block that used to live here was bulk re-scrubbed through
|
||||
# ``scripts/rescrub-cassettes.py`` (the script collapses every avatar URL
|
||||
# to ``SCRUBBED_AVATAR_URL`` and re-derives chunk byte-counts via
|
||||
# ``cassette_patterns.recompute_chunk_prefix`` so every affected cassette
|
||||
# is now self-consistent under shape-lint). Adding a new cassette to this
|
||||
# file should now be a rare, deliberate event tied to an open audit
|
||||
# ticket — NOT a phase-1 catch-up obligation.
|
||||
#
|
||||
# When phase-2 lands the corresponding scrubber, re-record the cassette and
|
||||
# remove its entry from this list. Do NOT delete the file outright — the
|
||||
# guard tolerates an empty file and an empty file makes the eventual
|
||||
# ``--strict`` flip a no-op.
|
||||
|
||||
# --- illustrative documentation fixtures (not real recordings) -------------
|
||||
# ``example_batchexecute_pattern.yaml`` + ``example_scrubbed_cookies.yaml``
|
||||
# were relocated to ``tests/cassettes/examples/`` in T8.E2 per the cassette
|
||||
# naming convention in ``tests/cassettes/README.md``. The default guard scan
|
||||
# (``tests/scripts/check_cassettes_clean.py``) globs ``tests/cassettes/*.yaml``
|
||||
# non-recursively, so files under ``examples/`` are no longer scanned — and
|
||||
# the allowlist entries that previously suppressed their (false-positive)
|
||||
# trailing-quote line-boundary "leak" became dead code. Removed in T8.E2.
|
||||
|
||||
# --- spec-explicit entries (none remaining) -------------------------------
|
||||
# All spec-explicit phase-2 cassettes have been repaired or removed:
|
||||
# - ``artifacts_revise_slide.yaml`` — re-recorded in T8.B1 against the live
|
||||
# REVISE_SLIDE RPC; f.req is real urlencoded JSON now.
|
||||
# - ``chat_ask.yaml`` + ``chat_ask_with_references.yaml`` — re-recorded in
|
||||
# T8.B2 against the current 9-param streaming-chat builder.
|
||||
# - ``sharing_get_status.yaml`` + ``sharing_set_public.yaml`` — re-scrubbed
|
||||
# in T8.B3 for escaped display names + avatar URLs.
|
||||
# - ``sources_add_file.yaml`` — re-scrubbed in T8.B4 for upload tokens.
|
||||
# - ``sources_add_drive.yaml`` + ``sources_check_freshness_drive.yaml`` —
|
||||
# re-scrubbed in T8.B5 for Drive tokens.
|
||||
# - The 61 ``/ogw/`` avatar URL entries — bulk re-scrubbed in T8.B6 (this PR)
|
||||
# via scripts/rescrub-cassettes.py.
|
||||
# - ``example_httpbin_*.yaml`` — deleted in T8.B7 (illustrative fixtures).
|
||||
@@ -0,0 +1,409 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Pure-Python cassette guard — replacement for ``tests/check_cassettes_clean.sh``.
|
||||
|
||||
Walks ``tests/cassettes/*.yaml`` (or any explicit paths passed on the command
|
||||
line) and reports any cassette that contains sensitive data. Uses the
|
||||
canonical pattern registry in ``tests/cassette_patterns.py``.
|
||||
|
||||
Key differences vs. the legacy bash script:
|
||||
|
||||
* Cross-platform — pure stdlib, runs on Linux / macOS / Windows.
|
||||
* Explicit placeholder allowlist (``SCRUB_PLACEHOLDERS``) instead of the bash
|
||||
"starts with S" heuristic — closes the cookie-value leak gap (a real token
|
||||
whose first byte is ``S`` no longer slips through).
|
||||
* ``--strict`` flag disables the repair allowlist for CI gating once
|
||||
cleanup is done.
|
||||
* Reports ``file:line`` for every leak so a developer can jump straight to
|
||||
the offending interaction.
|
||||
|
||||
Usage::
|
||||
|
||||
uv run python tests/scripts/check_cassettes_clean.py
|
||||
uv run python tests/scripts/check_cassettes_clean.py path/to/cassette.yaml
|
||||
uv run python tests/scripts/check_cassettes_clean.py --strict
|
||||
|
||||
Exit codes:
|
||||
0 — every scanned cassette is clean
|
||||
1 — one or more leaks detected (printed to stdout)
|
||||
|
||||
Implementation note: the tool reads each cassette line-by-line (no PyYAML
|
||||
parse) so that:
|
||||
|
||||
* It runs in O(stream) memory even on multi-megabyte cassettes.
|
||||
* Reported line numbers map directly to the file on disk.
|
||||
* It can also scan partial / malformed YAML that a real recorder might emit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Allow `python tests/scripts/check_cassettes_clean.py` from anywhere by
|
||||
# putting the repo root on sys.path so ``tests.cassette_patterns`` resolves.
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(_REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_REPO_ROOT))
|
||||
|
||||
from tests.cassette_patterns import ( # noqa: E402
|
||||
find_cookie_leaks,
|
||||
find_credential_leaks,
|
||||
is_clean,
|
||||
)
|
||||
|
||||
DEFAULT_CASSETTE_DIR = _REPO_ROOT / "tests" / "cassettes"
|
||||
# Extensions scanned in ``--secrets-only`` mode. Golden RPC fixtures are
|
||||
# ``.json`` and the captured-page fixture is ``.html``; the default cassette
|
||||
# scan only globs ``.yaml``, so a Google API key smuggled into a golden HTML
|
||||
# fixture (the GET_INTERACTIVE_HTML.json shape) would slip past the cassette
|
||||
# guard entirely. ``--secrets-only`` widens both the file types AND the
|
||||
# directories scanned.
|
||||
_SECRETS_ONLY_EXTENSIONS: tuple[str, ...] = (".yaml", ".yml", ".json", ".html", ".txt")
|
||||
DEFAULT_ALLOWLIST = _REPO_ROOT / "tests" / "scripts" / "cassette_repair_allowlist.txt"
|
||||
|
||||
|
||||
def _load_allowlist(path: Path) -> set[str]:
|
||||
"""Read the repair allowlist as a set of cassette basenames.
|
||||
|
||||
Blank lines and ``#`` comments are ignored. Entries are interpreted as
|
||||
cassette basenames (e.g. ``chat_ask.yaml``) — paths or globs are not
|
||||
supported, by design, to keep the file boring and auditable.
|
||||
"""
|
||||
if not path.exists():
|
||||
return set()
|
||||
return {
|
||||
line.strip()
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip() and not line.strip().startswith("#")
|
||||
}
|
||||
|
||||
|
||||
# Subdirectories of ``tests/cassettes/`` that hold ILLUSTRATIVE fixtures
|
||||
# (not real recordings) and are filtered out of the recursive scan. These
|
||||
# files use placeholder cookie / token values with intentional formatting
|
||||
# quirks (truncated YAML strings, hand-edited content) that would trip the
|
||||
# leak detector even though they contain no actual secrets — see the
|
||||
# README in ``tests/cassettes/examples/`` for the design intent.
|
||||
#
|
||||
# The ``tests/integration/conftest.py`` cassette-availability check already
|
||||
# excludes ``example_*`` cassettes from the "real recordings present"
|
||||
# count via a similar filter; this constant carries the same exclusion
|
||||
# semantic onto the guard tool.
|
||||
_EXAMPLE_SUBDIRS: frozenset[str] = frozenset({"examples"})
|
||||
|
||||
|
||||
def _iter_cassettes(
|
||||
paths: list[str],
|
||||
recursive: bool = False,
|
||||
extensions: tuple[str, ...] = (".yaml",),
|
||||
skip_examples: bool = True,
|
||||
) -> list[Path]:
|
||||
"""Resolve CLI arguments into a concrete list of cassette files.
|
||||
|
||||
* If no paths are given, scan ``tests/cassettes/*.yaml`` (non-recursive)
|
||||
OR ``tests/cassettes/**/*.yaml`` when ``recursive=True``.
|
||||
* If a directory is given, scan ``*.yaml`` inside it (recursively when
|
||||
``recursive=True``).
|
||||
* If a file is given, scan it directly.
|
||||
* Non-existent paths are silently skipped — matches the bash original's
|
||||
"scan what exists" behaviour and keeps the tool friendly to pre-commit
|
||||
hooks that may pass deleted-but-still-staged paths.
|
||||
* When ``skip_examples`` is set (the default), files under an
|
||||
``examples/`` directory (any depth) and ``example_*`` files are excluded
|
||||
from directory scans — they are illustrative fixtures with placeholder
|
||||
cookies and intentional YAML quirks, not real recordings (see
|
||||
:data:`_EXAMPLE_SUBDIRS`). Their placeholder content trips the full
|
||||
:func:`is_clean` heuristics, so the default cassette scan filters them.
|
||||
|
||||
``skip_examples`` is set to ``False`` for ``--secrets-only`` scans: that
|
||||
mode only matches high-severity credential shapes (which never occur in
|
||||
placeholder fixtures), so there is no false-positive reason to skip
|
||||
``examples/`` — and skipping them WOULD be a blind spot for credential
|
||||
hunting over a directory like ``tests/fixtures/`` (coderabbit review on
|
||||
#1266). Explicitly-named file paths are always scanned regardless.
|
||||
|
||||
The ``recursive`` flag is what P1-5 adds: CI now scans subdirectories of
|
||||
``tests/cassettes/`` (e.g. ``gzip_coverage/``) so a recorder cannot
|
||||
smuggle a leak into a nested folder. The default stays non-recursive so
|
||||
existing developer workflows (running the guard on a single file or the
|
||||
top-level directory) are unchanged.
|
||||
"""
|
||||
glob_patterns = [(f"**/*{ext}" if recursive else f"*{ext}") for ext in extensions]
|
||||
|
||||
def _globdir(d: Path) -> list[Path]:
|
||||
found: list[Path] = []
|
||||
for pat in glob_patterns:
|
||||
found.extend(d.glob(pat))
|
||||
return sorted(set(found))
|
||||
|
||||
def _is_example_path(p: Path) -> bool:
|
||||
# An ``example_`` file at the top level OR any file under an
|
||||
# ``examples/`` directory anywhere is treated as illustrative and
|
||||
# excluded from directory scans (only when ``skip_examples``).
|
||||
if p.name.startswith("example_"):
|
||||
return True
|
||||
return any(part in _EXAMPLE_SUBDIRS for part in p.parts)
|
||||
|
||||
def _keep(p: Path) -> bool:
|
||||
return not (skip_examples and _is_example_path(p))
|
||||
|
||||
if not paths:
|
||||
if not DEFAULT_CASSETTE_DIR.exists():
|
||||
return []
|
||||
return [p for p in _globdir(DEFAULT_CASSETTE_DIR) if _keep(p)]
|
||||
|
||||
resolved: list[Path] = []
|
||||
for raw in paths:
|
||||
candidate = Path(raw)
|
||||
if candidate.is_dir():
|
||||
resolved.extend(p for p in _globdir(candidate) if _keep(p))
|
||||
elif candidate.is_file():
|
||||
# Explicit file paths are always scanned, even if under
|
||||
# ``examples/`` — the operator asked for them by name.
|
||||
resolved.append(candidate)
|
||||
return resolved
|
||||
|
||||
|
||||
def _scan_cookie_headers_yaml(path: Path) -> list[tuple[int, str]]:
|
||||
"""YAML-aware, name-agnostic cookie-value leak scan over a cassette.
|
||||
|
||||
The line-by-line :func:`is_clean` pass below cannot see a cookie pair that a
|
||||
folded YAML scalar split onto a continuation line (the session-cookie anchor
|
||||
that scopes the name-agnostic check lives on a different physical line). This
|
||||
pass parses the cassette with PyYAML, recovers each request ``Cookie:`` and
|
||||
response ``Set-Cookie:`` header as its LOGICAL (joined) value, and runs the
|
||||
name-agnostic :func:`tests.cassette_patterns.find_cookie_leaks` on it — so an
|
||||
off-allowlist cookie (``_ga`` / ``_gcl_au`` / ``AEC`` …) whose real value
|
||||
survives in a folded scalar is caught regardless of line wrapping.
|
||||
|
||||
Line number ``0`` is reported (the leak is logical, not tied to one physical
|
||||
line of a folded scalar). A YAML parse failure is non-fatal — the
|
||||
line-by-line pass still runs — so a malformed cassette degrades to the
|
||||
streaming scan rather than crashing the guard.
|
||||
"""
|
||||
leaks: list[tuple[int, str]] = []
|
||||
try:
|
||||
import yaml
|
||||
|
||||
try:
|
||||
from yaml import CSafeLoader as _Loader
|
||||
except ImportError: # pragma: no cover - libyaml optional
|
||||
from yaml import SafeLoader as _Loader # type: ignore[assignment]
|
||||
|
||||
raw = path.read_text(encoding="utf-8", errors="replace")
|
||||
data = yaml.load(raw, Loader=_Loader)
|
||||
except Exception: # noqa: BLE001 - degrade to line-scan on any parse/read error
|
||||
return leaks
|
||||
if not isinstance(data, dict):
|
||||
return leaks
|
||||
|
||||
def _values(header_block: object, key: str) -> list[str]:
|
||||
if not isinstance(header_block, dict):
|
||||
return []
|
||||
for hk, hv in header_block.items():
|
||||
if isinstance(hk, str) and hk.lower() == key.lower():
|
||||
if isinstance(hv, list):
|
||||
return [v for v in hv if isinstance(v, str)]
|
||||
if isinstance(hv, str):
|
||||
return [hv]
|
||||
return []
|
||||
|
||||
for interaction in data.get("interactions") or []:
|
||||
if not isinstance(interaction, dict):
|
||||
continue
|
||||
request = interaction.get("request") or {}
|
||||
response = interaction.get("response") or {}
|
||||
for value in _values(request.get("headers"), "Cookie"):
|
||||
for desc in find_cookie_leaks(value):
|
||||
leaks.append((0, desc))
|
||||
for value in _values(response.get("headers"), "Set-Cookie"):
|
||||
for desc in find_cookie_leaks(value, set_cookie=True):
|
||||
leaks.append((0, desc))
|
||||
return leaks
|
||||
|
||||
|
||||
def _scan_file(path: Path, secrets_only: bool = False) -> list[tuple[int, str]]:
|
||||
"""Return ``(line_number, leak_description)`` for each leak.
|
||||
|
||||
Reads the cassette line-by-line (no PyYAML parse) so the tool runs in
|
||||
streaming memory even on multi-megabyte cassettes, ``file:line`` numbers
|
||||
map directly to the on-disk file, and the guard can scan partial/malformed
|
||||
YAML a recorder might emit before VCR finalises the file.
|
||||
|
||||
Each leak description is the human-readable string emitted by
|
||||
:func:`tests.cassette_patterns.is_clean` (e.g. ``"Leak (cookie header):
|
||||
cookie 'SID' value 'abc' is not a known scrub placeholder"``).
|
||||
|
||||
When ``secrets_only`` is set the per-line check uses
|
||||
:func:`tests.cassette_patterns.find_credential_leaks` instead — only the
|
||||
high-severity credential shapes (auth tokens + Google API keys), which never
|
||||
match legitimate placeholder fixture content. That is what makes scanning
|
||||
fixture directories full of ``"Scrubbed ..."`` placeholders viable.
|
||||
|
||||
For full (non-``secrets_only``) scans a SECOND, YAML-aware cookie pass
|
||||
(:func:`_scan_cookie_headers_yaml`) runs name-agnostic cookie-value
|
||||
detection on each cassette's joined ``Cookie:`` / ``Set-Cookie:`` header
|
||||
values — catching off-allowlist cookies (``_ga`` …) that a folded YAML
|
||||
scalar split across lines the streaming pass cannot stitch back together.
|
||||
"""
|
||||
leaks: list[tuple[int, str]] = []
|
||||
try:
|
||||
# ``errors="replace"`` guarantees a corrupted cassette never crashes
|
||||
# the guard mid-scan; we still produce useful output.
|
||||
with path.open("r", encoding="utf-8", errors="replace") as fh:
|
||||
for line_no, line in enumerate(fh, start=1):
|
||||
if secrets_only:
|
||||
line_leaks = find_credential_leaks(line)
|
||||
else:
|
||||
ok, line_leaks = is_clean(line)
|
||||
if ok:
|
||||
continue
|
||||
for description in line_leaks:
|
||||
leaks.append((line_no, description))
|
||||
except OSError as exc:
|
||||
print(f"{path}: could not read ({exc})", file=sys.stderr)
|
||||
|
||||
if not secrets_only:
|
||||
# De-duplicate against the line-pass: the YAML pass catches folded-scalar
|
||||
# cookie leaks the streaming pass cannot stitch together; an on-allowlist
|
||||
# leak found by both is reported once.
|
||||
seen = {desc for _, desc in leaks}
|
||||
for line_no, desc in _scan_cookie_headers_yaml(path):
|
||||
if desc not in seen:
|
||||
seen.add(desc)
|
||||
leaks.append((line_no, desc))
|
||||
return leaks
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Scan VCR cassettes for sensitive-data leaks. "
|
||||
"Returns exit 1 on any leak; otherwise exit 0."
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"paths",
|
||||
nargs="*",
|
||||
help=(
|
||||
"Cassette file(s) or directory. If omitted, scans "
|
||||
"tests/cassettes/*.yaml from the repo root."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strict",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Ignore the repair allowlist AND fail with exit code 1 if any "
|
||||
"allowlist entry is present in the working tree. Use this in "
|
||||
"CI: the allowlist is a one-way ratchet for cleanup, and "
|
||||
"``--strict`` flips it from 'best-effort suppressor' to "
|
||||
"'must be empty'."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--recursive",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Scan ``tests/cassettes/**/*.yaml`` (recurse into subdirectories) "
|
||||
"instead of the default top-level-only ``tests/cassettes/*.yaml``. "
|
||||
"Required in CI so a recorder cannot smuggle a leak into a nested "
|
||||
"folder like ``tests/cassettes/gzip_coverage/``."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--secrets-only",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Scan only for high-severity credential shapes (Google auth tokens "
|
||||
"and Google API keys) instead of the full cookie/PII heuristics. "
|
||||
"Also widens the scanned file types to .json/.html/.txt (not just "
|
||||
".yaml). Use this to scan fixture directories such as "
|
||||
"tests/fixtures/ that contain intentional placeholder content "
|
||||
"(e.g. 'Scrubbed Note Title') which would false-positive under the "
|
||||
"full is_clean() heuristics."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--allowlist",
|
||||
type=Path,
|
||||
default=DEFAULT_ALLOWLIST,
|
||||
help=(
|
||||
"Path to the repair allowlist file "
|
||||
"(default: tests/scripts/cassette_repair_allowlist.txt)."
|
||||
),
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
extensions = _SECRETS_ONLY_EXTENSIONS if args.secrets_only else (".yaml",)
|
||||
# ``--secrets-only`` matches only credential shapes (no false positives on
|
||||
# placeholder content), so it must NOT skip ``examples/`` — doing so would
|
||||
# be a blind spot for credential hunting over fixture dirs (#1266). Default
|
||||
# (full-heuristic) scans keep skipping examples to avoid placeholder noise.
|
||||
cassettes = _iter_cassettes(
|
||||
args.paths,
|
||||
recursive=args.recursive,
|
||||
extensions=extensions,
|
||||
skip_examples=not args.secrets_only,
|
||||
)
|
||||
if not cassettes:
|
||||
# Fresh checkout with no recorded cassettes — that's a valid clean
|
||||
# state, matching the bash original's behaviour.
|
||||
print("OK: no cassettes to scan")
|
||||
return 0
|
||||
|
||||
if args.strict:
|
||||
# Strict mode: ignore the allowlist for scan-skip purposes AND
|
||||
# require the allowlist to be empty (or non-existent). Any entry
|
||||
# present in the file is treated as a CI failure so the allowlist
|
||||
# cannot quietly grow past the cleanup phase.
|
||||
allowlist: set[str] = set()
|
||||
present_in_allowlist = _load_allowlist(args.allowlist)
|
||||
if present_in_allowlist:
|
||||
print(
|
||||
f"ERROR (--strict): {args.allowlist} contains "
|
||||
f"{len(present_in_allowlist)} entries; --strict requires the "
|
||||
"allowlist to be empty. Either drop the entries or run "
|
||||
"without --strict for the cleanup-in-progress workflow."
|
||||
)
|
||||
for entry in sorted(present_in_allowlist):
|
||||
print(f" - {entry}")
|
||||
return 1
|
||||
else:
|
||||
allowlist = _load_allowlist(args.allowlist)
|
||||
|
||||
scanned = 0
|
||||
skipped = 0
|
||||
leaked_files: list[Path] = []
|
||||
total_leaks = 0
|
||||
|
||||
for cassette in cassettes:
|
||||
if not args.strict and cassette.name in allowlist:
|
||||
skipped += 1
|
||||
continue
|
||||
scanned += 1
|
||||
leaks = _scan_file(cassette, secrets_only=args.secrets_only)
|
||||
if leaks:
|
||||
leaked_files.append(cassette)
|
||||
total_leaks += len(leaks)
|
||||
try:
|
||||
rel = cassette.relative_to(_REPO_ROOT)
|
||||
except ValueError:
|
||||
rel = cassette
|
||||
for line_no, description in leaks:
|
||||
print(f"{rel}:{line_no}: {description}")
|
||||
|
||||
# Summary always prints, even on the happy path, so the operator gets a
|
||||
# one-line confirmation that the guard actually ran.
|
||||
print(
|
||||
f"\nSummary: {scanned} cassettes scanned"
|
||||
+ (f", {skipped} allow-listed" if skipped else "")
|
||||
+ f", {total_leaks} leaks found in {len(leaked_files)} files."
|
||||
)
|
||||
|
||||
return 1 if leaked_files else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,245 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Per-method RPC coverage gate.
|
||||
|
||||
Walks every member of :class:`notebooklm.rpc.types.RPCMethod` and asserts that
|
||||
each one has **both**:
|
||||
|
||||
1. **A test reference** — at least one file under ``tests/`` (excluding the
|
||||
coverage gate itself, its allowlist record, and helper data files that
|
||||
merely enumerate enum members) mentions the enum member by its qualified
|
||||
name (e.g. ``RPCMethod.LIST_NOTEBOOKS``) OR by its raw RPC id string value
|
||||
(e.g. ``"wXbhsf"``).
|
||||
2. **A cassette covering the RPC id** — at least one cassette YAML under
|
||||
``tests/cassettes/`` contains the raw RPC id string in its body. This is a
|
||||
pure-text grep — the cassette format is YAML but we never parse it; any
|
||||
occurrence of the id within the file counts, because ``batchexecute`` URLs
|
||||
and request bodies include the id verbatim.
|
||||
|
||||
Either failure prints a single line of the form::
|
||||
|
||||
MISSING: RPCMethod.<NAME> (id=<rpc_id>): <which check failed; suggestion>
|
||||
|
||||
and the script exits 1. Exit 0 on full coverage.
|
||||
|
||||
Pre-existing-gap allowlist
|
||||
--------------------------
|
||||
|
||||
When this gate first landed, some methods (notably newer or write-rarely
|
||||
exercised ones) already lacked one or both forms of coverage. Forcing
|
||||
contributors to backfill cassettes for unrelated methods before they could
|
||||
ship anything new would have stalled the arc, so the gate accepts a
|
||||
:data:`PREEXISTING_GAPS` set listing the (RPCMethod-name) entries grandfathered
|
||||
in at landing time.
|
||||
|
||||
The allowlist is intended as a **one-way ratchet**:
|
||||
|
||||
* It **must not grow** when a new ``RPCMethod`` member is added — new methods
|
||||
must ship with at least one test reference and at least one cassette.
|
||||
* It **must shrink** when a maintainer backfills coverage for a grandfathered
|
||||
method; stale entries in :data:`PREEXISTING_GAPS` fail the gate.
|
||||
|
||||
The script is intentionally a static check (pure text grep on the cassette
|
||||
files and on the contents of ``tests/``); it never runs pytest or imports
|
||||
anything that needs a network. That keeps it deterministic, fast, and safe
|
||||
to run as a CI gate on every PR.
|
||||
|
||||
Usage::
|
||||
|
||||
uv run python tests/scripts/check_method_coverage.py
|
||||
|
||||
Exit codes:
|
||||
0 — every method (modulo :data:`PREEXISTING_GAPS`) is covered
|
||||
1 — one or more methods are missing required coverage
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Allow running the script from any CWD by putting the repo root on sys.path
|
||||
# so ``from notebooklm.rpc.types import RPCMethod`` resolves the same way the
|
||||
# test suite resolves it.
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
_SRC = _REPO_ROOT / "src"
|
||||
for _path in (_SRC, _REPO_ROOT):
|
||||
if str(_path) not in sys.path:
|
||||
sys.path.insert(0, str(_path))
|
||||
|
||||
from notebooklm.rpc.types import RPCMethod # noqa: E402
|
||||
|
||||
TESTS_DIR = _REPO_ROOT / "tests"
|
||||
CASSETTES_DIR = TESTS_DIR / "cassettes"
|
||||
|
||||
# This script's own path — exclude it from the "test reference" search so the
|
||||
# enum-walk loop doesn't trivially satisfy the test-reference check.
|
||||
_SELF = Path(__file__).resolve()
|
||||
|
||||
# Files that mention every enum member structurally (the enum definition
|
||||
# itself, or helper scripts that iterate over ``RPCMethod`` by reflection) and
|
||||
# would therefore satisfy the test-reference check for everything. Excluding
|
||||
# them keeps the gate meaningful — we want a *real* test asserting behaviour,
|
||||
# not a structural enumeration.
|
||||
_TEST_REFERENCE_EXCLUDES: frozenset[Path] = frozenset(
|
||||
{
|
||||
_SELF,
|
||||
# Add other reflective enumerators here as they appear.
|
||||
}
|
||||
)
|
||||
|
||||
# Pre-existing gaps grandfathered in when this gate landed; new methods
|
||||
# must NOT be added here. See module docstring for the one-way-ratchet
|
||||
# policy. Each entry is the ``RPCMethod.<NAME>`` member name (without the
|
||||
# ``RPCMethod.`` prefix).
|
||||
# Intentionally empty: every RPCMethod has full coverage (a test reference +
|
||||
# a cassette). The source-label RPCs' cassettes were recorded in this change,
|
||||
# so the temporary grandfather entries were removed. Record a cassette for any
|
||||
# new method rather than re-adding it here.
|
||||
PREEXISTING_GAPS: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
def _iter_test_files() -> list[Path]:
|
||||
"""Return every file under ``tests/`` to grep for enum references.
|
||||
|
||||
We walk the directory tree once and snapshot it as a sorted list so
|
||||
repeated callers (and the per-method test-reference check) see a
|
||||
deterministic set. Cassettes live under ``tests/cassettes/`` but we
|
||||
intentionally exclude them — the cassette presence check covers those.
|
||||
"""
|
||||
files: list[Path] = []
|
||||
for path in TESTS_DIR.rglob("*"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
# Skip cassettes — they're covered by the separate cassette check.
|
||||
if CASSETTES_DIR in path.parents or path == CASSETTES_DIR:
|
||||
continue
|
||||
# Skip compiled bytecode under ``__pycache__``: ``.pyc`` files inline
|
||||
# source-level string constants like ``"wXbhsf"`` as raw UTF-8 bytes,
|
||||
# which would let a stale bytecode file silently satisfy the test-
|
||||
# reference check after the source was deleted. Cheap to skip, avoids
|
||||
# the spurious-match class entirely.
|
||||
if "__pycache__" in path.parts:
|
||||
continue
|
||||
if path.resolve() in _TEST_REFERENCE_EXCLUDES:
|
||||
continue
|
||||
files.append(path)
|
||||
files.sort()
|
||||
return files
|
||||
|
||||
|
||||
def _iter_cassette_files() -> list[Path]:
|
||||
"""Return every ``*.yaml`` cassette under ``tests/cassettes/`` (sorted).
|
||||
|
||||
Sorted output keeps the gate deterministic when reporting failures and
|
||||
matches the style of the sister script ``check_cassettes_clean.py``.
|
||||
Returns an empty list cleanly when the directory is missing so the gate
|
||||
can run on fresh checkouts without exploding.
|
||||
"""
|
||||
if not CASSETTES_DIR.exists():
|
||||
return []
|
||||
return sorted(CASSETTES_DIR.glob("*.yaml"))
|
||||
|
||||
|
||||
def _file_contains(path: Path, needles: tuple[str, ...]) -> bool:
|
||||
"""Return True iff ``path`` contains any of ``needles`` as raw text.
|
||||
|
||||
Reads the file as bytes to avoid encoding surprises on large cassette
|
||||
files (some contain mojibake-survivable HTTP payloads) and matches the
|
||||
UTF-8 bytes of each needle. ``OSError`` on read is treated as "no
|
||||
match" rather than crashing the gate — a corrupted cassette would
|
||||
already trip ``check_cassettes_clean.py``.
|
||||
"""
|
||||
try:
|
||||
data = path.read_bytes()
|
||||
except OSError:
|
||||
return False
|
||||
return any(needle.encode("utf-8") in data for needle in needles)
|
||||
|
||||
|
||||
def _has_test_reference(method: RPCMethod, test_files: list[Path]) -> bool:
|
||||
"""Return True iff any ``tests/`` file references the method by name or id.
|
||||
|
||||
Accepts either the qualified enum reference (``RPCMethod.LIST_NOTEBOOKS``)
|
||||
or the raw RPC id string (``"wXbhsf"``) — the latter catches tests that
|
||||
assert on encoded request payloads without importing the enum.
|
||||
"""
|
||||
needles = (f"RPCMethod.{method.name}", method.value)
|
||||
return any(_file_contains(p, needles) for p in test_files)
|
||||
|
||||
|
||||
def _has_cassette_coverage(method: RPCMethod, cassette_files: list[Path]) -> bool:
|
||||
"""Return True iff any cassette body contains the raw RPC id string."""
|
||||
needles = (method.value,)
|
||||
return any(_file_contains(p, needles) for p in cassette_files)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Run the gate and return the process exit code.
|
||||
|
||||
No CLI flags today — the gate is a pure pass/fail static check. We sort
|
||||
enum members by name before iterating so the failure output is stable
|
||||
across runs/machines. (The sister script ``check_cassettes_clean.py``
|
||||
accepts ``argv`` because it has ``--strict``/``--allowlist`` flags; this
|
||||
one has nothing to parse so we keep the signature flag-free rather than
|
||||
carrying an unused ``argv`` parameter.)
|
||||
"""
|
||||
test_files = _iter_test_files()
|
||||
cassette_files = _iter_cassette_files()
|
||||
|
||||
misses: list[str] = []
|
||||
unused_allowlist: list[str] = []
|
||||
|
||||
for method in sorted(RPCMethod, key=lambda m: m.name):
|
||||
has_test = _has_test_reference(method, test_files)
|
||||
has_cassette = _has_cassette_coverage(method, cassette_files)
|
||||
|
||||
if method.name in PREEXISTING_GAPS:
|
||||
# Track entries that no longer need allowlisting so maintainers
|
||||
# are nudged to shrink the ratchet.
|
||||
if has_test and has_cassette:
|
||||
unused_allowlist.append(method.name)
|
||||
continue
|
||||
|
||||
if has_test and has_cassette:
|
||||
continue
|
||||
|
||||
problems: list[str] = []
|
||||
if not has_test:
|
||||
problems.append(
|
||||
"missing test reference (add a test under tests/ that imports "
|
||||
f"RPCMethod.{method.name} or asserts on the raw id '{method.value}')"
|
||||
)
|
||||
if not has_cassette:
|
||||
problems.append(
|
||||
"missing cassette coverage (record a cassette under "
|
||||
f"tests/cassettes/ whose body contains the RPC id '{method.value}')"
|
||||
)
|
||||
misses.append(
|
||||
f"MISSING: RPCMethod.{method.name} (id={method.value}): {'; '.join(problems)}"
|
||||
)
|
||||
|
||||
for line in misses:
|
||||
print(line)
|
||||
|
||||
if unused_allowlist:
|
||||
print(
|
||||
"STALE: PREEXISTING_GAPS entries now have full coverage and "
|
||||
"must be removed: " + ", ".join(sorted(unused_allowlist)),
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
total = len(RPCMethod) # ``EnumMeta`` defines ``__len__`` directly.
|
||||
grandfathered = len(PREEXISTING_GAPS)
|
||||
checked = total - grandfathered
|
||||
print(
|
||||
f"\nSummary: {total} RPCMethod members, "
|
||||
f"{grandfathered} grandfathered (PREEXISTING_GAPS), "
|
||||
f"{checked} actively enforced, "
|
||||
f"{len(misses)} missing coverage."
|
||||
)
|
||||
|
||||
return 1 if misses or unused_allowlist else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,189 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compress the artifacts polling cassette after recording.
|
||||
|
||||
The live ``artifacts_poll_rename_wait.yaml`` cassette is recorded by
|
||||
``tests/integration/test_polling_vcr.py::TestPollingReplay::test_poll_rename_wait``
|
||||
running with ``NOTEBOOKLM_VCR_RECORD=1``. The raw recording can include
|
||||
50-100+ ``LIST_ARTIFACTS`` interactions because the live API takes 30-60 s
|
||||
to finish a flashcard generation and ``wait_for_completion`` polls at
|
||||
exponential intervals (1 s → 2 s → 4 s → ... → 5 s cap) until status
|
||||
flips to ``COMPLETED``.
|
||||
|
||||
We do not need every poll response on disk — the replay test only needs
|
||||
enough interactions to exercise the polling loop. This script keeps:
|
||||
|
||||
* The first ``LIST_ARTIFACTS`` interaction — consumed by the explicit
|
||||
``poll_status`` call in the test.
|
||||
* The next ``KEEP_PROCESSING`` ``LIST_ARTIFACTS`` interactions — exercise
|
||||
the ``wait_for_completion`` backoff path while the artifact is still
|
||||
``in_progress``.
|
||||
* The first ``LIST_ARTIFACTS`` interaction that reports
|
||||
``status == COMPLETED`` — terminates the wait loop.
|
||||
* The single ``CREATE_ARTIFACT`` and ``RENAME_ARTIFACT`` interactions
|
||||
that bookend the chain.
|
||||
|
||||
Run from the repo root::
|
||||
|
||||
uv run python tests/scripts/compress_polling_cassette.py
|
||||
|
||||
The script is idempotent — running it on an already-compressed cassette
|
||||
is a no-op because no PROCESSING interactions exist beyond the kept
|
||||
prefix. It writes the result back to the same file path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
CASSETTE_PATH = REPO_ROOT / "tests" / "cassettes" / "artifacts_poll_rename_wait.yaml"
|
||||
|
||||
# RPC IDs (mirror src/notebooklm/rpc/types.py — duplicated here so this script
|
||||
# can run without importing the notebooklm package).
|
||||
RPCID_CREATE_ARTIFACT = "R7cb6c"
|
||||
RPCID_LIST_ARTIFACTS = "gArtLc"
|
||||
RPCID_RENAME_ARTIFACT = "rc3d8d"
|
||||
|
||||
# How many in-progress LIST_ARTIFACTS responses to retain after the first
|
||||
# (explicit poll_status) one. 3 in-progress + 1 completed gives the
|
||||
# ``wait_for_completion`` loop four iterations to walk, comfortably above
|
||||
# the ``MIN_POLLING_INTERACTIONS = 3`` floor enforced by the test.
|
||||
KEEP_PROCESSING = 3
|
||||
|
||||
|
||||
def _rpcid(interaction: dict) -> str | None:
|
||||
qs = parse_qs(urlparse(interaction["request"]["uri"]).query)
|
||||
rpcids = qs.get("rpcids", [])
|
||||
return rpcids[0] if rpcids else None
|
||||
|
||||
|
||||
def _is_completed(interaction: dict, task_id: str) -> bool:
|
||||
"""True if this LIST_ARTIFACTS response reports the task as COMPLETED.
|
||||
|
||||
The artifact array shape is roughly::
|
||||
|
||||
[task_id, title, type, source_ids, status, ...]
|
||||
|
||||
where ``status == 3`` is ``COMPLETED`` (see ``ArtifactStatus`` in
|
||||
``src/notebooklm/rpc/types.py``). We don't fully decode the WRB
|
||||
envelope here — a simple substring check on ``,3,`` immediately after
|
||||
the task block is robust enough for this compression step.
|
||||
"""
|
||||
body = interaction["response"]["body"]["string"]
|
||||
if task_id not in body:
|
||||
return False
|
||||
idx = body.find(task_id)
|
||||
chunk = body[idx : idx + 400]
|
||||
# Status code 3 appears after the source_ids triple. A bare ``,3,`` in
|
||||
# the chunk is the COMPLETED marker; PROCESSING is ``,1,``.
|
||||
return ",3," in chunk
|
||||
|
||||
|
||||
def _extract_task_id(cassette: dict) -> str:
|
||||
"""Pull the task_id back out of the recorded CREATE_ARTIFACT response."""
|
||||
for interaction in cassette["interactions"]:
|
||||
if _rpcid(interaction) != RPCID_CREATE_ARTIFACT:
|
||||
continue
|
||||
body = interaction["response"]["body"]["string"]
|
||||
# The CREATE_ARTIFACT response carries the task_id as the first
|
||||
# quoted string in the inner JSON. We scan from the wrb.fr envelope
|
||||
# for the first ``\"...\"`` token. The inner JSON is itself a
|
||||
# JSON-encoded string, so quote characters are backslash-escaped:
|
||||
# ``[["wrb.fr","R7cb6c","[[\"<task_id>\", ...`` (two leading ``[``)
|
||||
# or ``[\"<task_id>\"`` depending on the response shape — both
|
||||
# start the same way once we look past the leading ``[`` runs.
|
||||
envelope = '"' + RPCID_CREATE_ARTIFACT + '","'
|
||||
env_idx = body.find(envelope)
|
||||
if env_idx == -1:
|
||||
continue
|
||||
# First escaped quote after the envelope opens the inner JSON
|
||||
# string. The task_id is whatever sits between that ``\"`` and the
|
||||
# next ``\"``.
|
||||
first_quote = body.find('\\"', env_idx)
|
||||
if first_quote == -1:
|
||||
continue
|
||||
start = first_quote + 2 # skip the ``\"`` escape
|
||||
end = body.find('\\"', start)
|
||||
if end == -1:
|
||||
continue
|
||||
return body[start:end]
|
||||
raise RuntimeError(
|
||||
"Could not locate task_id in CREATE_ARTIFACT response — has the "
|
||||
"response shape drifted? Re-record and inspect the raw cassette."
|
||||
)
|
||||
|
||||
|
||||
def compress(cassette_path: Path = CASSETTE_PATH) -> tuple[int, int]:
|
||||
"""Compress ``cassette_path`` in place. Returns ``(before, after)`` counts."""
|
||||
with cassette_path.open(encoding="utf-8") as fh:
|
||||
cassette = yaml.safe_load(fh)
|
||||
|
||||
interactions = cassette["interactions"]
|
||||
before = len(interactions)
|
||||
|
||||
task_id = _extract_task_id(cassette)
|
||||
|
||||
create_idx: int | None = None
|
||||
rename_idx: int | None = None
|
||||
list_processing: list[int] = []
|
||||
list_completed: int | None = None
|
||||
|
||||
for i, interaction in enumerate(interactions):
|
||||
rpc = _rpcid(interaction)
|
||||
if rpc == RPCID_CREATE_ARTIFACT and create_idx is None:
|
||||
create_idx = i
|
||||
elif rpc == RPCID_RENAME_ARTIFACT and rename_idx is None:
|
||||
rename_idx = i
|
||||
elif rpc == RPCID_LIST_ARTIFACTS:
|
||||
if list_completed is None and _is_completed(interaction, task_id):
|
||||
list_completed = i
|
||||
elif list_completed is None:
|
||||
list_processing.append(i)
|
||||
|
||||
if create_idx is None:
|
||||
raise RuntimeError("No CREATE_ARTIFACT interaction found in cassette")
|
||||
if rename_idx is None:
|
||||
raise RuntimeError("No RENAME_ARTIFACT interaction found in cassette")
|
||||
if list_completed is None:
|
||||
raise RuntimeError("No COMPLETED LIST_ARTIFACTS interaction found in cassette")
|
||||
|
||||
# Keep the first (1 + KEEP_PROCESSING) processing polls plus the completed one.
|
||||
# The first one is consumed by the explicit poll_status call in the test;
|
||||
# the rest drive wait_for_completion's iteration loop.
|
||||
keep_processing = list_processing[: 1 + KEEP_PROCESSING]
|
||||
|
||||
keep_indices = sorted({create_idx, *keep_processing, list_completed, rename_idx})
|
||||
cassette["interactions"] = [interactions[i] for i in keep_indices]
|
||||
|
||||
with cassette_path.open("w", encoding="utf-8") as fh:
|
||||
yaml.safe_dump(
|
||||
cassette,
|
||||
fh,
|
||||
default_flow_style=False,
|
||||
allow_unicode=True,
|
||||
sort_keys=False,
|
||||
)
|
||||
|
||||
return before, len(cassette["interactions"])
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not CASSETTE_PATH.exists():
|
||||
print(f"ERROR: cassette not found at {CASSETTE_PATH}", file=sys.stderr)
|
||||
print(
|
||||
"Record it first with: NOTEBOOKLM_VCR_RECORD=1 uv run pytest "
|
||||
"tests/integration/test_polling_vcr.py",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
before, after = compress()
|
||||
print(f"Compressed {CASSETTE_PATH.name}: {before} -> {after} interactions")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Re-encode a recorded cassette so its response body is gzip-compressed
|
||||
and the response advertises ``Content-Encoding: gzip``.
|
||||
|
||||
Why this exists
|
||||
---------------
|
||||
``decode_compressed_response=True`` in :mod:`tests.vcr_config` strips
|
||||
``Content-Encoding`` from the headers on record (after VCR transparently
|
||||
gunzips the body), so 89/89 cassettes in ``tests/cassettes/`` carry NO
|
||||
encoding header and a plaintext body. That hid issue #769 — every RPC
|
||||
failed in production with ``Error -3 while decompressing data: incorrect
|
||||
header check`` because :func:`notebooklm._streaming_post.stream_post_with_size_cap`
|
||||
rebuilt the response with the upstream ``Content-Encoding: gzip`` header
|
||||
attached to *already-decoded* body bytes, triggering a second gzip-decode
|
||||
inside :class:`httpx.Response.__init__`. Cassettes that don't carry the
|
||||
header on replay simply skip that branch and pass.
|
||||
|
||||
This script ports a cassette to the production wire shape: gzip the
|
||||
response body, restore the ``Content-Encoding: gzip`` header, and strip
|
||||
the now-meaningless ``Transfer-Encoding`` and ``Content-Length`` headers
|
||||
(the originals described the chunked / decoded wire form, not the gzipped
|
||||
buffer we hand to vcrpy on replay).
|
||||
|
||||
Run from the repo root::
|
||||
|
||||
uv run python tests/scripts/inject_gzip_into_cassette.py \\
|
||||
tests/cassettes/artifacts_revise_slide.yaml \\
|
||||
tests/cassettes/gzip_coverage/artifacts_revise_slide_gzipped.yaml
|
||||
|
||||
Idempotent: re-running on a cassette that already advertises
|
||||
``Content-Encoding: gzip`` rewrites it from scratch using the gzipped
|
||||
body that vcrpy has already deserialized to ``bytes``, so a second pass
|
||||
produces a byte-identical file.
|
||||
|
||||
Library use:
|
||||
Other tooling can ``from tests.scripts.inject_gzip_into_cassette
|
||||
import inject_gzip_into_cassette`` and pass a cassette dict in
|
||||
memory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
# Use the SAFE loader: cassettes are arbitrary YAML on disk and a malicious
|
||||
# ``!!python/object/apply`` tag would execute code under the full Loader.
|
||||
# ``!!binary`` (the encoding we rely on for gzipped bodies) is part of the
|
||||
# core YAML schema so it works under ``CSafeLoader`` / ``SafeLoader``.
|
||||
try:
|
||||
from yaml import CSafeDumper as Dumper
|
||||
from yaml import CSafeLoader as Loader
|
||||
except ImportError: # pragma: no cover — libyaml is bundled with PyYAML wheels
|
||||
from yaml import SafeDumper as Dumper # type: ignore[assignment]
|
||||
from yaml import SafeLoader as Loader
|
||||
|
||||
|
||||
# Headers stripped from the response when we rewrite the body. The recorded
|
||||
# Transfer-Encoding / Content-Length values described the chunked / decoded
|
||||
# wire form; once we re-gzip the body, neither matches reality and leaving
|
||||
# them in confuses httpx on replay. Content-Encoding is included so the
|
||||
# rewrite always re-adds the canonical-cased header regardless of how the
|
||||
# cassette was originally recorded.
|
||||
_STRIP_RESPONSE_HEADERS = frozenset(
|
||||
{
|
||||
"transfer-encoding",
|
||||
"content-length",
|
||||
"content-encoding",
|
||||
}
|
||||
)
|
||||
|
||||
_CONTENT_ENCODING_HEADER = "Content-Encoding"
|
||||
|
||||
|
||||
def _has_content_encoding(headers: dict[str, list[str]]) -> bool:
|
||||
"""Case-insensitive check for ``Content-Encoding`` in a cassette
|
||||
header dict."""
|
||||
return any(k.lower() == "content-encoding" for k in headers)
|
||||
|
||||
|
||||
def _coerce_body_to_bytes(body_string: Any) -> bytes:
|
||||
"""Normalize the recorded ``response.body.string`` to ``bytes``.
|
||||
|
||||
PyYAML deserializes ``!!binary`` blobs as :class:`bytes` and plain
|
||||
block scalars as :class:`str`. The cassette format permits either,
|
||||
so accept both and return raw bytes.
|
||||
"""
|
||||
if isinstance(body_string, bytes):
|
||||
return body_string
|
||||
if isinstance(body_string, str):
|
||||
return body_string.encode("utf-8")
|
||||
raise TypeError(
|
||||
f"Unexpected response.body.string type {type(body_string).__name__}; expected bytes or str."
|
||||
)
|
||||
|
||||
|
||||
def _inject_into_response(response: dict[str, Any]) -> bool:
|
||||
"""Mutate ``response`` in place so the body is gzip-compressed and
|
||||
``Content-Encoding: gzip`` is set.
|
||||
|
||||
Returns ``True`` when the body was (re)written; ``False`` only when
|
||||
the response has no ``body.string`` to process (missing key or
|
||||
``None``). A response that already advertises ``Content-Encoding``
|
||||
is gunzipped first and then re-gzipped from the decoded form so the
|
||||
rewrite is idempotent — still a rewrite, still ``True``.
|
||||
"""
|
||||
body = response.get("body") or {}
|
||||
if "string" not in body:
|
||||
return False
|
||||
|
||||
raw = body["string"]
|
||||
if raw is None:
|
||||
return False
|
||||
|
||||
body_bytes = _coerce_body_to_bytes(raw)
|
||||
|
||||
headers = response.setdefault("headers", {})
|
||||
already_encoded = _has_content_encoding(headers)
|
||||
|
||||
if already_encoded:
|
||||
# The body on disk is already gzipped — gunzip first so the
|
||||
# rewrite produces a byte-identical file on a second pass.
|
||||
try:
|
||||
body_bytes = gzip.decompress(body_bytes)
|
||||
except OSError as exc: # pragma: no cover — defensive
|
||||
raise ValueError(
|
||||
"Response already advertises Content-Encoding but body is not valid gzip."
|
||||
) from exc
|
||||
|
||||
# ``gzip.compress`` writes one byte that varies across Python releases
|
||||
# (the OS field at offset 9): 3.10 picks an OS-dependent constant
|
||||
# while 3.11+ pins it to ``0xff`` (unknown). Pin ``mtime=0`` and
|
||||
# overwrite the OS field so the same source cassette produces a
|
||||
# byte-identical artifact under every Python in the CI matrix
|
||||
# (``3.10`` – ``3.14``). ``0xff`` is the documented "unknown" value
|
||||
# from RFC 1952 §2.3.1 and round-trips through every gzip decoder.
|
||||
gzipped = bytearray(gzip.compress(body_bytes, mtime=0))
|
||||
gzipped[9] = 0xFF
|
||||
body["string"] = bytes(gzipped)
|
||||
|
||||
# Drop any case-variant of the headers in _STRIP_RESPONSE_HEADERS
|
||||
# (including Content-Encoding) so the rewrite re-adds the canonical
|
||||
# header name below.
|
||||
for key in list(headers):
|
||||
if key.lower() in _STRIP_RESPONSE_HEADERS:
|
||||
del headers[key]
|
||||
headers[_CONTENT_ENCODING_HEADER] = ["gzip"]
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def inject_gzip_into_cassette(cassette: dict[str, Any]) -> int:
|
||||
"""Rewrite every batchexecute-style response in ``cassette`` to be
|
||||
gzip-encoded.
|
||||
|
||||
Only responses whose request is a POST to a path containing
|
||||
``batchexecute`` are touched — those are the ones that flow through
|
||||
:func:`notebooklm._streaming_post.stream_post_with_size_cap` and
|
||||
exercise the rebuild step that bit #769. SPA-shell GET responses are
|
||||
left alone so the cassette diff stays focused.
|
||||
|
||||
Returns the number of responses that were rewritten.
|
||||
"""
|
||||
rewritten = 0
|
||||
for interaction in cassette.get("interactions", []) or []:
|
||||
request = interaction.get("request") or {}
|
||||
if request.get("method", "").upper() != "POST":
|
||||
continue
|
||||
uri = request.get("uri", "") or ""
|
||||
if "batchexecute" not in uri:
|
||||
continue
|
||||
response = interaction.get("response") or {}
|
||||
if _inject_into_response(response):
|
||||
rewritten += 1
|
||||
return rewritten
|
||||
|
||||
|
||||
def _load_cassette(path: Path) -> dict[str, Any]:
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
return yaml.load(fh, Loader=Loader)
|
||||
|
||||
|
||||
def _dump_cassette(cassette: dict[str, Any], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as fh:
|
||||
yaml.dump(cassette, fh, Dumper=Dumper)
|
||||
|
||||
|
||||
def _main(argv: list[str]) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("source", type=Path, help="cassette to read")
|
||||
parser.add_argument(
|
||||
"destination",
|
||||
type=Path,
|
||||
help="destination path (may equal source for in-place rewrite)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
cassette = _load_cassette(args.source)
|
||||
rewritten = inject_gzip_into_cassette(cassette)
|
||||
if rewritten == 0:
|
||||
print(
|
||||
f"warning: no batchexecute POST interactions found in {args.source}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
_dump_cassette(cassette, args.destination)
|
||||
print(f"rewrote {rewritten} response(s) in {args.destination}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
raise SystemExit(_main(sys.argv[1:]))
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Maintainer helper: record the MCP Studio mutating-op cassettes against live API.
|
||||
|
||||
``studio_delete`` and ``studio_rename`` are both cross-type (note∩artifact): they
|
||||
resolve ``item`` over the merged list, then route by resolved type. That issues
|
||||
``GET_NOTES_AND_MIND_MAPS`` (``cFji9``) + ``LIST_ARTIFACTS`` (``gArtLc``) several
|
||||
times — a merged-list resolve preflight plus, on the artifact path, a kind-aware
|
||||
``mind_maps`` probe — BEFORE the mutation RPC, a sequence no CLI cassette holds.
|
||||
This script drives the **MCP server** (real ``NotebookLMClient``, auth from your
|
||||
``~/.notebooklm`` profile) under VCR record mode so the recorded ``f.req`` shapes
|
||||
are exactly what the Studio tools emit.
|
||||
|
||||
It is SELF-CONTAINED and DESTRUCTIVE-SAFE: it creates a throwaway scratch notebook
|
||||
(one text source, one text note, one generated report), records the three cassettes
|
||||
against THAT notebook, then deletes the scratch notebook in a ``finally``. It never
|
||||
touches any pre-existing notebook.
|
||||
|
||||
Records (into ``tests/cassettes``):
|
||||
* ``mcp_studio_rename.yaml`` — studio_rename(report) → cFji9×4 + gArtLc×3 + rc3d8d
|
||||
* ``mcp_studio_delete_note.yaml`` — studio_delete(note) → cFji9×2 + gArtLc + AH0mwd
|
||||
* ``mcp_studio_delete_artifact.yaml`` — studio_delete(report) → cFji9×3 + gArtLc + V5N4be
|
||||
|
||||
Usage (maintainer, one Google account; consumes a little generation quota)::
|
||||
|
||||
uv run python tests/scripts/record_mcp_studio_cassettes.py
|
||||
|
||||
After recording, verify the cassettes are clean:
|
||||
|
||||
uv run python tests/scripts/check_cassettes_clean.py --strict --recursive
|
||||
|
||||
CI never runs this script. Replay uses scrubbed cassettes + mock auth.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Make the repo root importable so ``tests.vcr_config`` resolves under a plain
|
||||
# ``python`` run (pytest sets pythonpath=["."]; a direct run does not).
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
# Record mode + keepalive-poke disable MUST be set before importing the VCR config
|
||||
# (its record_mode is computed at import time) and before any client opens (so the
|
||||
# RotateCookies poke never records a stray interaction into a cassette).
|
||||
os.environ["NOTEBOOKLM_VCR_RECORD"] = "1"
|
||||
os.environ["NOTEBOOKLM_DISABLE_KEEPALIVE_POKE"] = "1"
|
||||
|
||||
from fastmcp import Client # noqa: E402
|
||||
from tests.vcr_config import notebooklm_vcr # noqa: E402
|
||||
|
||||
from notebooklm.mcp.server import create_server # noqa: E402
|
||||
|
||||
SCRATCH_TITLE = "zzz-mcp-studio-vcr-scratch"
|
||||
SOURCE_TEXT = (
|
||||
"The mitochondrion is the powerhouse of the cell. It generates most of the "
|
||||
"cell's supply of adenosine triphosphate (ATP), used as a source of chemical "
|
||||
"energy. This is scratch content for a throwaway VCR recording notebook."
|
||||
)
|
||||
|
||||
|
||||
def _sc(result: object) -> dict:
|
||||
"""Pull the structured_content dict off a FastMCP call_tool result."""
|
||||
sc = getattr(result, "structured_content", None)
|
||||
assert isinstance(sc, dict), f"expected structured_content dict, got {result!r}"
|
||||
return sc
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
server = create_server()
|
||||
async with Client(server) as mcp:
|
||||
nb_id: str | None = None
|
||||
try:
|
||||
# --- SETUP (outside any cassette → live, NOT recorded) ---------------
|
||||
nb_id = _sc(await mcp.call_tool("notebook_create", {"title": SCRATCH_TITLE}))[
|
||||
"notebook_id"
|
||||
]
|
||||
print(f"scratch notebook: {nb_id}")
|
||||
|
||||
await mcp.call_tool(
|
||||
"source_add",
|
||||
{
|
||||
"notebook": nb_id,
|
||||
"source_type": "text",
|
||||
"text": SOURCE_TEXT,
|
||||
"title": "Scratch source",
|
||||
},
|
||||
)
|
||||
await mcp.call_tool("source_wait", {"notebook": nb_id, "timeout": 180})
|
||||
|
||||
note_id = _sc(
|
||||
await mcp.call_tool(
|
||||
"note_save",
|
||||
{"notebook": nb_id, "title": "Scratch note", "content": "Delete me."},
|
||||
)
|
||||
)["note_id"]
|
||||
print(f"scratch note: {note_id}")
|
||||
|
||||
gen = _sc(
|
||||
await mcp.call_tool(
|
||||
"studio_generate", {"notebook": nb_id, "artifact_type": "report"}
|
||||
)
|
||||
)
|
||||
print(f"generating report: task_id={gen.get('task_id')}")
|
||||
|
||||
# The mutations resolve `item` over the MERGED notes+artifacts list, so
|
||||
# both the note and the report must be stably present there before we
|
||||
# record — otherwise resolve_studio_item misses and the op degrades to the
|
||||
# full-UUID carve-out (type "unknown"), which is NOT the cross-type route
|
||||
# these cassettes exist to pin. Settle until BOTH resolve to their expected
|
||||
# type via studio_list(item=…) on the SAME listing, and grab the report's
|
||||
# own listed id there (the generation task_id may differ from it).
|
||||
art_id: str | None = None
|
||||
for _ in range(40):
|
||||
items = _sc(await mcp.call_tool("studio_list", {"notebook": nb_id}))["items"]
|
||||
reports = [it for it in items if it.get("type") == "report"]
|
||||
has_note = any(it.get("id") == note_id and it.get("type") == "note" for it in items)
|
||||
if reports and has_note:
|
||||
art_id = str(reports[0]["id"])
|
||||
break
|
||||
await asyncio.sleep(3)
|
||||
if art_id is None:
|
||||
raise SystemExit("note+report never co-appeared in studio_list")
|
||||
print(f"report artifact: {art_id}")
|
||||
|
||||
# --- RECORD (each op in its own cassette) ---------------------------
|
||||
with notebooklm_vcr.use_cassette("mcp_studio_rename.yaml"):
|
||||
r = _sc(
|
||||
await mcp.call_tool(
|
||||
"studio_rename",
|
||||
{"notebook": nb_id, "item": art_id, "new_title": "Renamed by VCR"},
|
||||
)
|
||||
)
|
||||
print("rename ->", r)
|
||||
|
||||
with notebooklm_vcr.use_cassette("mcp_studio_delete_note.yaml"):
|
||||
r = _sc(
|
||||
await mcp.call_tool(
|
||||
"studio_delete", {"notebook": nb_id, "item": note_id, "confirm": True}
|
||||
)
|
||||
)
|
||||
print("delete note ->", r)
|
||||
|
||||
with notebooklm_vcr.use_cassette("mcp_studio_delete_artifact.yaml"):
|
||||
r = _sc(
|
||||
await mcp.call_tool(
|
||||
"studio_delete", {"notebook": nb_id, "item": art_id, "confirm": True}
|
||||
)
|
||||
)
|
||||
print("delete artifact ->", r)
|
||||
|
||||
print("\nRECORDED IDS (for the replay tests):")
|
||||
print(f" RENAME/DELETE_ARTIFACT nb={nb_id} art={art_id}")
|
||||
print(f" DELETE_NOTE nb={nb_id} note={note_id}")
|
||||
finally:
|
||||
if nb_id is not None:
|
||||
await mcp.call_tool("notebook_delete", {"notebook": nb_id, "confirm": True})
|
||||
print(f"cleaned up scratch notebook {nb_id}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Maintainer helper: record the REST-server artifact cassettes against live API.
|
||||
|
||||
The server adapter issues some RPCs with a *different request shape* than the CLI
|
||||
(e.g. the CLI resolves all sources first while the server passes ``source_ids``
|
||||
through verbatim), so the existing CLI cassettes don't match the server path.
|
||||
This script records server-shaped cassettes for the artifact lifecycle endpoints
|
||||
that ``tests/server/test_integration_real_client.py`` then replays.
|
||||
|
||||
It drives the **server** through FastAPI's ``TestClient`` with a real
|
||||
``NotebookLMClient`` (auth from your ``~/.notebooklm`` profile), under VCR record
|
||||
mode, so the recorded ``f.req`` shapes are exactly what the server emits.
|
||||
|
||||
Records (into ``tests/cassettes``):
|
||||
* ``server_generate_quiz.yaml`` — POST artifacts (quiz, all sources) → 202 + poll
|
||||
* ``server_download_mind_map.yaml`` — POST artifacts/download (mind-map) → bytes
|
||||
* ``server_add_file.yaml`` — POST sources/file (multipart upload) → 201
|
||||
|
||||
Usage (maintainer, one Google account with a populated generation notebook)::
|
||||
|
||||
NOTEBOOKLM_GENERATION_NOTEBOOK_ID=<uuid> \
|
||||
uv run python tests/scripts/record_server_cassettes.py
|
||||
|
||||
The notebook must own at least one source (for generation) and one completed
|
||||
mind-map (for the download leg). After recording, verify the cassettes are clean:
|
||||
|
||||
uv run python tests/scripts/check_cassettes_clean.py --strict --recursive
|
||||
|
||||
CI never runs this script. Replay uses scrubbed cassettes + mock auth.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Make the repo root importable so ``tests.vcr_config`` resolves when this script
|
||||
# is run directly (pytest sets pythonpath=["."]; a plain ``python`` run does not).
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
# Record mode + keepalive-poke disable MUST be set before importing the VCR
|
||||
# config (its record_mode is computed at import time) and before any client opens
|
||||
# (so the RotateCookies poke never records an extra interaction).
|
||||
os.environ["NOTEBOOKLM_VCR_RECORD"] = "1"
|
||||
os.environ["NOTEBOOKLM_DISABLE_KEEPALIVE_POKE"] = "1"
|
||||
|
||||
import io # noqa: E402
|
||||
from contextlib import asynccontextmanager # noqa: E402
|
||||
|
||||
from fastapi.testclient import TestClient # noqa: E402
|
||||
from tests.vcr_config import notebooklm_vcr # noqa: E402
|
||||
|
||||
from notebooklm.auth import AuthTokens # noqa: E402
|
||||
from notebooklm.client import NotebookLMClient # noqa: E402
|
||||
from notebooklm.server.app import create_app # noqa: E402
|
||||
|
||||
_TOKEN = "record-token" # nosec - local only; never leaves this process
|
||||
|
||||
#: Content for the recorded file-upload source. Enough text to be a valid source.
|
||||
_UPLOAD_BYTES = (
|
||||
b"NotebookLM REST server VCR upload fixture. This file exercises the multipart "
|
||||
b"upload path: spool to a server-owned temp file, then INIT_UPLOAD + PUT bytes "
|
||||
b"+ ADD_SOURCE through the real client.\n"
|
||||
)
|
||||
|
||||
|
||||
def _notebook_id() -> str:
|
||||
nb = os.environ.get("NOTEBOOKLM_GENERATION_NOTEBOOK_ID", "").strip()
|
||||
if not nb:
|
||||
sys.exit("Set NOTEBOOKLM_GENERATION_NOTEBOOK_ID=<uuid> (a notebook with sources).")
|
||||
return nb
|
||||
|
||||
|
||||
def _fresh_client() -> TestClient:
|
||||
"""A TestClient over a fresh real-client app (a TestClient is single-entry)."""
|
||||
|
||||
@asynccontextmanager
|
||||
async def factory():
|
||||
auth = await AuthTokens.from_storage() # real profile auth
|
||||
async with NotebookLMClient(auth) as client:
|
||||
yield client
|
||||
|
||||
os.environ["NOTEBOOKLM_SERVER_TOKEN"] = _TOKEN
|
||||
app = create_app(client_factory=factory)
|
||||
headers = {"Authorization": f"Bearer {_TOKEN}", "Host": "127.0.0.1"}
|
||||
return TestClient(app, headers=headers, raise_server_exceptions=False)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
nb = _notebook_id()
|
||||
|
||||
# Generate over ALL sources (no source_ids) — the server now defaults to all
|
||||
# sources like the CLI, so a bare generate no longer 502s "… unavailable".
|
||||
print("Recording server_generate_quiz.yaml (generate quiz, all sources + one poll)...")
|
||||
with notebooklm_vcr.use_cassette("server_generate_quiz.yaml"), _fresh_client() as c:
|
||||
gen = c.post(f"/v1/notebooks/{nb}/artifacts", json={"type": "quiz"})
|
||||
print(" generate ->", gen.status_code, str(gen.json())[:120])
|
||||
task_id = gen.json().get("task_id")
|
||||
if task_id:
|
||||
poll = c.get(f"/v1/notebooks/{nb}/artifacts/{task_id}")
|
||||
print(" poll ->", poll.status_code, str(poll.json())[:120])
|
||||
|
||||
print("Recording server_download_mind_map.yaml (download completed mind-map)...")
|
||||
with notebooklm_vcr.use_cassette("server_download_mind_map.yaml"), _fresh_client() as c:
|
||||
dl = c.post(f"/v1/notebooks/{nb}/artifacts/download", json={"type": "mind-map"})
|
||||
print(" download ->", dl.status_code, f"{len(dl.content)} bytes")
|
||||
|
||||
print("Recording server_add_file.yaml (multipart file upload)...")
|
||||
with notebooklm_vcr.use_cassette("server_add_file.yaml"), _fresh_client() as c:
|
||||
files = {"file": ("server-vcr-upload.txt", io.BytesIO(_UPLOAD_BYTES), "text/plain")}
|
||||
up = c.post(f"/v1/notebooks/{nb}/sources/file", files=files)
|
||||
print(" upload ->", up.status_code, str(up.json())[:120])
|
||||
|
||||
print(
|
||||
"Done. Now verify: uv run python tests/scripts/check_cassettes_clean.py --strict --recursive"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One-time maintainer setup for the VCR generation notebook.
|
||||
|
||||
This is a **manual, local-only** helper. It is NOT run by CI and must NOT be
|
||||
called from any test or workflow. Run it once per Google account that records
|
||||
VCR cassettes for mutation/generation endpoints:
|
||||
|
||||
uv run python tests/scripts/setup-generation-notebook.py
|
||||
|
||||
The script is idempotent: if a notebook titled
|
||||
"VCR Generation Notebook (Tier 8)" already exists on the authenticated
|
||||
account, its UUID is reported instead of creating a duplicate.
|
||||
|
||||
After running, export the printed UUID in your maintainer environment (e.g.
|
||||
`~/.zshrc` or a per-profile env file):
|
||||
|
||||
export NOTEBOOKLM_GENERATION_NOTEBOOK_ID=<printed-uuid>
|
||||
|
||||
The UUID is per-maintainer and MUST NOT be committed to the repository —
|
||||
notebook IDs are linkable to a Google account and leak account identity. See
|
||||
`docs/development.md` -> "Cassette recording" for the full env-var workflow.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
from notebooklm.client import NotebookLMClient
|
||||
|
||||
GENERATION_NOTEBOOK_TITLE = "VCR Generation Notebook (Tier 8)"
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
"""Ensure a generation notebook exists; print its UUID and export line."""
|
||||
async with await NotebookLMClient.from_storage() as client:
|
||||
print(f"Checking for existing notebook titled {GENERATION_NOTEBOOK_TITLE!r}...")
|
||||
notebooks = await client.notebooks.list()
|
||||
existing = next(
|
||||
(nb for nb in notebooks if nb.title == GENERATION_NOTEBOOK_TITLE),
|
||||
None,
|
||||
)
|
||||
|
||||
if existing is not None:
|
||||
notebook_id = existing.id
|
||||
print(f"Found existing notebook: {notebook_id}")
|
||||
else:
|
||||
print(f"Creating notebook {GENERATION_NOTEBOOK_TITLE!r}...")
|
||||
created = await client.notebooks.create(GENERATION_NOTEBOOK_TITLE)
|
||||
notebook_id = created.id
|
||||
print(f"Created notebook: {notebook_id}")
|
||||
|
||||
separator = "=" * 60
|
||||
print()
|
||||
print(separator)
|
||||
print("Generation notebook is ready.")
|
||||
print(f" Notebook ID: {notebook_id}")
|
||||
print(separator)
|
||||
print("Export this in your maintainer environment (NOT in the repo):")
|
||||
print()
|
||||
print(f" export NOTEBOOKLM_GENERATION_NOTEBOOK_ID={notebook_id}")
|
||||
print()
|
||||
print("See docs/development.md -> 'Cassette recording' for usage.")
|
||||
print(separator)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(asyncio.run(main()))
|
||||
except KeyboardInterrupt:
|
||||
print("Interrupted.", file=sys.stderr)
|
||||
sys.exit(130)
|
||||
except Exception as exc: # noqa: BLE001 — top-level CLI surface
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user