wehub-resource-sync
09e9f3545f
Test / Code Quality (push) Has been cancelled
Test / Test (macos-latest, Python 3.10) (push) Has been cancelled
Test / Test (macos-latest, Python 3.11) (push) Has been cancelled
Test / Test (macos-latest, Python 3.12) (push) Has been cancelled
Test / Test (macos-latest, Python 3.13) (push) Has been cancelled
Test / Test (macos-latest, Python 3.14) (push) Has been cancelled
Test / Test (ubuntu-latest, Python 3.10) (push) Has been cancelled
Test / Test (ubuntu-latest, Python 3.11) (push) Has been cancelled
Test / Test (ubuntu-latest, Python 3.12) (push) Has been cancelled
Test / Test (ubuntu-latest, Python 3.13) (push) Has been cancelled
Test / Test (ubuntu-latest, Python 3.14) (push) Has been cancelled
Test / Test (windows-latest, Python 3.10) (push) Has been cancelled
Test / Test (windows-latest, Python 3.11) (push) Has been cancelled
Test / Test (windows-latest, Python 3.12) (push) Has been cancelled
Test / Test (windows-latest, Python 3.13) (push) Has been cancelled
Test / Test (windows-latest, Python 3.14) (push) Has been cancelled
CodeQL / Analyze (push) Has been cancelled
dependency-audit / pip-audit (push) Has been cancelled
381 lines
18 KiB
YAML
381 lines
18 KiB
YAML
name: Nightly E2E Tests
|
|
|
|
on:
|
|
schedule:
|
|
# Main branch: 6 AM UTC (10 PM PST / 1 AM EST)
|
|
- cron: '0 6 * * *'
|
|
workflow_dispatch:
|
|
inputs:
|
|
test_filter:
|
|
description: 'Specific test to run (e.g., tests/e2e/test_downloads.py::TestDownloadReport)'
|
|
required: false
|
|
default: ''
|
|
custom_branch:
|
|
description: 'Override branch to test (leave empty to test the branch you triggered from)'
|
|
required: false
|
|
default: ''
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
# Determine which branch to test
|
|
resolve-branch:
|
|
runs-on: ubuntu-latest
|
|
if: github.repository == 'teng-lin/notebooklm-py'
|
|
outputs:
|
|
branch: ${{ steps.resolve.outputs.branch }}
|
|
is_standard: ${{ steps.resolve.outputs.is_standard }}
|
|
sha: ${{ steps.target.outputs.sha }}
|
|
short_sha: ${{ steps.target.outputs.short_sha }}
|
|
steps:
|
|
- name: Resolve target branch
|
|
id: resolve
|
|
# Route every workflow_dispatch input + GitHub-context value through
|
|
# the env block so crafted values (e.g. ``"; rm -rf / ; #``) reach
|
|
# the shell as literal strings, not as inlined script. ``GITHUB_OUTPUT``
|
|
# is set automatically by the runner.
|
|
env:
|
|
EVENT_NAME: ${{ github.event_name }}
|
|
CUSTOM_BRANCH: ${{ inputs.custom_branch }}
|
|
REF_NAME: ${{ github.ref_name }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [ "$EVENT_NAME" = "schedule" ]; then
|
|
# Scheduled runs test main only. Release branches are manual.
|
|
TARGET="main"
|
|
else
|
|
# Manual: use custom_branch if set, otherwise use triggering branch
|
|
if [ -n "$CUSTOM_BRANCH" ]; then
|
|
TARGET="$CUSTOM_BRANCH"
|
|
else
|
|
TARGET="$REF_NAME"
|
|
fi
|
|
fi
|
|
echo "branch=$TARGET" >> "$GITHUB_OUTPUT"
|
|
# Check if it's main or a release branch
|
|
case "$TARGET" in
|
|
main|release/*)
|
|
echo "is_standard=true" >> "$GITHUB_OUTPUT"
|
|
;;
|
|
*)
|
|
echo "is_standard=false" >> "$GITHUB_OUTPUT"
|
|
;;
|
|
esac
|
|
echo "Resolved branch: $TARGET"
|
|
|
|
- uses: actions/checkout@v7
|
|
if: steps.resolve.outputs.is_standard == 'true'
|
|
with:
|
|
ref: ${{ steps.resolve.outputs.branch }}
|
|
fetch-depth: 1
|
|
persist-credentials: false
|
|
|
|
- name: Resolve target commit
|
|
id: target
|
|
if: steps.resolve.outputs.is_standard == 'true'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
SHA=$(git rev-parse HEAD)
|
|
echo "sha=$SHA" >> "$GITHUB_OUTPUT"
|
|
echo "short_sha=$(git rev-parse --short=12 HEAD)" >> "$GITHUB_OUTPUT"
|
|
echo "Resolved commit: $SHA"
|
|
|
|
# Run E2E tests on the resolved branch
|
|
e2e:
|
|
name: E2E Tests (${{ needs.resolve-branch.outputs.branch }}@${{ needs.resolve-branch.outputs.short_sha }}/${{ matrix.os }})
|
|
needs: resolve-branch
|
|
if: needs.resolve-branch.outputs.is_standard == 'true'
|
|
runs-on: ${{ matrix.os }}
|
|
# Secret-bearing job. Two-layer gating:
|
|
#
|
|
# 1. ``needs.resolve-branch.outputs.is_standard`` — set by the upstream
|
|
# ``resolve-branch`` job. Only ``main`` and ``release/*`` branches
|
|
# (or scheduled cron triggers, which resolve to one of those) flip
|
|
# this to ``true``; any other branch keeps it ``false`` and the
|
|
# job-level ``if:`` above skips this whole job (no secret values
|
|
# land in the runner env). The step-level ``if:`` guards on the
|
|
# secret-bearing steps below are belt-and-suspenders so the gate
|
|
# stays visible if the job-level guard is ever loosened.
|
|
# 2. ``environment: protected-readonly`` (unconditional). The secrets
|
|
# this job consumes (``NOTEBOOKLM_AUTH_JSON`` and friends) live only
|
|
# in that environment, so the binding has to be unconditional —
|
|
# issue #1009 surfaced the scheduled cron failing when the previous
|
|
# conditional (``workflow_dispatch``-only) form fell back to
|
|
# repo-level secrets that no longer exist. Add a ``required
|
|
# reviewers`` rule on the environment if you want to block
|
|
# workflow_dispatch behind manual approval; scheduled runs would
|
|
# then queue too, so today this env carries no protection rules.
|
|
# See docs/development.md → "Workflow secret gates".
|
|
environment: protected-readonly
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
os: [ubuntu-latest, windows-latest]
|
|
env:
|
|
# Pin Playwright's browser install dir to a single workspace-relative
|
|
# path on every OS — see test.yml for the rationale (sidesteps the
|
|
# actions/cache@v6 Windows dual-path restore flake).
|
|
PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/.playwright-browsers
|
|
# Single source of truth for the coverage-floor sentinel path, referenced by
|
|
# both the E2E step (writes it) and the "Enforce coverage floors" step (gates
|
|
# on it). Harmless when unset-enforcement steps run — they never write it.
|
|
E2E_COVERAGE_FLOOR_SENTINEL: ${{ github.workspace }}/.e2e-coverage-floor-failed
|
|
|
|
concurrency:
|
|
group: nightly-${{ needs.resolve-branch.outputs.branch }}-${{ matrix.os }}
|
|
cancel-in-progress: true
|
|
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
ref: ${{ needs.resolve-branch.outputs.sha }}
|
|
fetch-depth: 1
|
|
persist-credentials: false
|
|
|
|
- name: Stamp E2E target
|
|
env:
|
|
EXPECTED_SHA: ${{ needs.resolve-branch.outputs.sha }}
|
|
TARGET_BRANCH: ${{ needs.resolve-branch.outputs.branch }}
|
|
MATRIX_OS: ${{ matrix.os }}
|
|
shell: bash
|
|
run: |
|
|
ACTUAL_SHA=$(git rev-parse HEAD)
|
|
if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then
|
|
echo "::error::Checked out $ACTUAL_SHA, expected $EXPECTED_SHA"
|
|
exit 1
|
|
fi
|
|
{
|
|
echo "### E2E target"
|
|
echo "- Branch: \`$TARGET_BRANCH\`"
|
|
echo "- Commit: \`$ACTUAL_SHA\`"
|
|
echo "- OS: \`$MATRIX_OS\`"
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: "3.12"
|
|
cache: 'pip'
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # @ v7.0.0
|
|
|
|
- name: Install dependencies
|
|
# `uv sync --frozen` uses `uv.lock` for deterministic dep resolution
|
|
# (same as the contributor workflow in docs/installation.md). Extras
|
|
# cover the full E2E surface: `browser` = playwright, `dev` = pytest +
|
|
# plugins, `markdown` = markdownify (used by source-conversion tests).
|
|
# Retried to ride out transient registry/network blips: a real lockfile
|
|
# problem fails all 3 attempts identically, so this only absorbs flakes
|
|
# — it never masks a genuine resolution error.
|
|
shell: bash
|
|
run: |
|
|
# --extra impersonate so the ubuntu curl_cffi smoke step below can run.
|
|
# --extra mcp so the MCP/CLI live e2e suite (tests/e2e/test_mcp*.py,
|
|
# test_cli_live.py) actually RUNS instead of importorskip-ping fastmcp.
|
|
# --extra server so the REST live e2e suite (tests/e2e/test_server_live.py)
|
|
# RUNS instead of importorskip-ping fastapi.
|
|
attempt=1; max=3
|
|
until uv sync --frozen --extra browser --extra dev --extra markdown --extra impersonate --extra mcp --extra server; do
|
|
if [ "$attempt" -ge "$max" ]; then
|
|
echo "::error::uv sync failed after $max attempts" >&2
|
|
exit 1
|
|
fi
|
|
echo "::warning::uv sync attempt $attempt failed; retrying in $((attempt * 15))s" >&2
|
|
sleep $((attempt * 15))
|
|
attempt=$((attempt + 1))
|
|
done
|
|
|
|
- name: Get Playwright version
|
|
id: playwright-version
|
|
shell: bash
|
|
# `uv pip show` / bare `pip show` after activation can resolve outside
|
|
# `.venv` when uv has not seeded pip into the project venv, yielding an
|
|
# empty version that collapses the browser cache key to
|
|
# `playwright-${{ matrix.os }}--ws`. `importlib.metadata` reads
|
|
# `.venv`'s actual installed dist-info via the active interpreter, so
|
|
# the version is always correct.
|
|
run: |
|
|
VERSION=$(uv run python -c 'import importlib.metadata as m; print(m.version("playwright"))')
|
|
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
|
|
|
- name: Cache Playwright browsers
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: ${{ github.workspace }}/.playwright-browsers
|
|
# ``-ws`` suffix invalidates archives keyed to the prior dual-path
|
|
# stanza so first restores after this change land on the new path.
|
|
key: playwright-${{ matrix.os }}-${{ steps.playwright-version.outputs.version }}-ws
|
|
|
|
- name: Install Playwright browsers
|
|
run: uv run playwright install chromium
|
|
|
|
- name: Install Playwright system dependencies (Linux)
|
|
if: runner.os == 'Linux'
|
|
run: uv run playwright install-deps chromium
|
|
|
|
# Fail-fast preflight. Without this, an empty NOTEBOOKLM_AUTH_JSON
|
|
# (env-binding misconfig, missing repo-level fallback, etc.) lets the
|
|
# E2E suite proceed; pytest then skips every auth-requiring test
|
|
# silently and the job lands green with 0 tests run (issue #1009).
|
|
- name: Verify auth secret is present
|
|
if: needs.resolve-branch.outputs.is_standard == 'true'
|
|
env:
|
|
NOTEBOOKLM_AUTH_JSON: ${{ secrets.NOTEBOOKLM_AUTH_JSON }}
|
|
NOTEBOOKLM_READ_ONLY_NOTEBOOK_ID: ${{ secrets.NOTEBOOKLM_READ_ONLY_NOTEBOOK_ID }}
|
|
shell: bash
|
|
run: |
|
|
if [ -z "${NOTEBOOKLM_AUTH_JSON:-}" ]; then
|
|
echo "::error::NOTEBOOKLM_AUTH_JSON resolved to empty. Env binding or secret config is broken — see docs/development.md → Workflow secret gates."
|
|
exit 1
|
|
fi
|
|
if [ -z "${NOTEBOOKLM_READ_ONLY_NOTEBOOK_ID:-}" ]; then
|
|
echo "::error::NOTEBOOKLM_READ_ONLY_NOTEBOOK_ID resolved to empty."
|
|
exit 1
|
|
fi
|
|
|
|
- name: Run E2E tests
|
|
id: e2e
|
|
# Hard gate: only standard branches (main/release/*, or scheduled cron
|
|
# runs which resolve to one of those) expose credentials. Any other
|
|
# branch — including ad-hoc workflow_dispatch on a feature branch —
|
|
# gets ``is_standard == 'false'`` from resolve-branch and skips here
|
|
# outright. The job-level ``environment:`` adds the maintainer approval
|
|
# layer for workflow_dispatch runs that DO resolve as standard.
|
|
if: needs.resolve-branch.outputs.is_standard == 'true'
|
|
# Don't fail the job here — the retry step below gets a 10-min cool-down
|
|
# shot at any failures, and its exit code is what marks the job red.
|
|
continue-on-error: true
|
|
env:
|
|
NOTEBOOKLM_AUTH_JSON: ${{ secrets.NOTEBOOKLM_AUTH_JSON }}
|
|
NOTEBOOKLM_READ_ONLY_NOTEBOOK_ID: ${{ secrets.NOTEBOOKLM_READ_ONLY_NOTEBOOK_ID }}
|
|
NOTEBOOKLM_GENERATION_NOTEBOOK_ID: ${{ secrets.NOTEBOOKLM_GENERATION_NOTEBOOK_ID }}
|
|
# Enforce the live coverage floors on the nightly only (fresh quota): if a
|
|
# monitored surface (live chat / live generation) produced zero real coverage
|
|
# because every marked test was rate-limited, record it to the sentinel that
|
|
# the "Enforce coverage floors" step below gates on. Written (not raised) so
|
|
# a pure-skip run stays exit 0 and doesn't trip the continue-on-error retry.
|
|
# The release job (verify-package) leaves this unset → skip-only (#1819).
|
|
# (E2E_COVERAGE_FLOOR_SENTINEL is defined once at job level.)
|
|
E2E_ENFORCE_COVERAGE_FLOOR: "1"
|
|
# Route every workflow_dispatch input + upstream needs.* value through
|
|
# the env block so crafted values (e.g. ``"; echo PWN``) reach pytest
|
|
# as literal strings, not as inlined shell. ``TARGET_BRANCH`` is
|
|
# derived from ``inputs.custom_branch`` upstream; ``TEST_FILTER`` is
|
|
# the direct workflow_dispatch input (empty for scheduled runs).
|
|
TEST_FILTER: ${{ inputs.test_filter }}
|
|
TARGET_BRANCH: ${{ needs.resolve-branch.outputs.branch }}
|
|
MATRIX_OS: ${{ matrix.os }}
|
|
shell: bash
|
|
run: |
|
|
echo "Testing branch: $TARGET_BRANCH"
|
|
# Start from a clean coverage-floor sentinel so a stale file from a prior
|
|
# run on a reused/self-hosted runner can't false-red the enforcement step.
|
|
rm -f "$E2E_COVERAGE_FLOOR_SENTINEL"
|
|
# --reruns 1 catches sub-minute network blips; longer chat-throttle
|
|
# recovery is handled by the cool-down retry step below.
|
|
if [ -n "$TEST_FILTER" ]; then
|
|
# Run specific test(s) when filter provided.
|
|
# ``--`` separator forces $TEST_FILTER to be parsed as positional
|
|
# file/path args, never as pytest options (e.g. a crafted ``--co``
|
|
# or ``-k "expr"`` value cannot inject pytest flags).
|
|
# The coverage floor is a whole-surface guarantee — meaningless for a
|
|
# single filtered test, and a marked-but-throttled pick would false-red
|
|
# the run — so disable enforcement for filtered dispatches (codex).
|
|
unset E2E_ENFORCE_COVERAGE_FLOOR
|
|
uv run pytest -s -v --tb=short --reruns 1 --reruns-delay 30 -- "$TEST_FILTER"
|
|
elif [ "$MATRIX_OS" = "ubuntu-latest" ]; then
|
|
# Linux: run all E2E tests except variants
|
|
uv run pytest tests/e2e -m "not variants" -s -v --tb=short --reruns 1 --reruns-delay 30
|
|
else
|
|
# Windows: run only read-only tests (safer, avoids write operations)
|
|
uv run pytest tests/e2e -m "readonly and not variants" -s -v --tb=short --reruns 1 --reruns-delay 30
|
|
fi
|
|
|
|
- name: Retry failed E2E tests after 10-min cool-down
|
|
# Tests still throttled after the cool-down become skips via the
|
|
# _install_chat_rate_limit_skip fixture in tests/e2e/conftest.py.
|
|
# Same ``is_standard`` hard gate as the primary E2E step above — the
|
|
# retry must never expose secrets on a non-standard branch even if a
|
|
# previous step somehow ran (e.g. re-run with edited workflow inputs).
|
|
if: |
|
|
needs.resolve-branch.outputs.is_standard == 'true'
|
|
&& steps.e2e.outcome == 'failure'
|
|
env:
|
|
NOTEBOOKLM_AUTH_JSON: ${{ secrets.NOTEBOOKLM_AUTH_JSON }}
|
|
NOTEBOOKLM_READ_ONLY_NOTEBOOK_ID: ${{ secrets.NOTEBOOKLM_READ_ONLY_NOTEBOOK_ID }}
|
|
NOTEBOOKLM_GENERATION_NOTEBOOK_ID: ${{ secrets.NOTEBOOKLM_GENERATION_NOTEBOOK_ID }}
|
|
# Enforce here too: a marked test that failed on the main run and turns into
|
|
# a rate-limit skip after the cool-down would otherwise leave no PASS event
|
|
# for its surface. Appends PASS/SKIP events to the same job-level sentinel
|
|
# (the main step's rm -f already cleared any stale one). (codex/coderabbit)
|
|
E2E_ENFORCE_COVERAGE_FLOOR: "1"
|
|
shell: bash
|
|
run: |
|
|
# Missing lastfailed after a failed e2e step means pytest crashed
|
|
# before writing the cache (import error, OOM, etc.) — that's a real
|
|
# failure, not a recoverable rate-limit, so fail the job rather than
|
|
# silently going green.
|
|
if [ ! -f .pytest_cache/v/cache/lastfailed ]; then
|
|
echo "::error::No lastfailed cache from previous step; pytest likely crashed before running tests."
|
|
exit 1
|
|
fi
|
|
echo "Initial E2E run failed. Sleeping 600s before retrying failed tests."
|
|
sleep 600
|
|
# Pin ``tests/e2e`` so pyproject's ``addopts = --ignore=tests/e2e``
|
|
# doesn't drop every lastfailed entry; otherwise pytest's default
|
|
# ``--last-failed-no-failures=all`` would expand to the full non-e2e
|
|
# suite under NOTEBOOKLM_AUTH_JSON. ``=none`` makes a missing cache
|
|
# fail fast instead.
|
|
uv run pytest tests/e2e --last-failed --last-failed-no-failures=none -s -v --tb=short
|
|
|
|
- name: curl_cffi transport smoke (live, minimal)
|
|
# Minimum live coverage of the opt-in curl_cffi transport: the handful of
|
|
# @pytest.mark.impersonate_smoke e2e tests (one per transport-critical op —
|
|
# read / ask / upload / download) run with NOTEBOOKLM_TRANSPORT=curl_cffi.
|
|
# ubuntu-only (the write path exercises the upload-fingerprint case that no
|
|
# offline test can reach; Windows just needs the read/ask path, already
|
|
# covered by the httpx legs + the test.yml hermetic matrix). Same is_standard
|
|
# secret gate as the steps above.
|
|
if: |
|
|
needs.resolve-branch.outputs.is_standard == 'true'
|
|
&& matrix.os == 'ubuntu-latest'
|
|
env:
|
|
NOTEBOOKLM_AUTH_JSON: ${{ secrets.NOTEBOOKLM_AUTH_JSON }}
|
|
NOTEBOOKLM_READ_ONLY_NOTEBOOK_ID: ${{ secrets.NOTEBOOKLM_READ_ONLY_NOTEBOOK_ID }}
|
|
NOTEBOOKLM_GENERATION_NOTEBOOK_ID: ${{ secrets.NOTEBOOKLM_GENERATION_NOTEBOOK_ID }}
|
|
NOTEBOOKLM_TRANSPORT: curl_cffi
|
|
shell: bash
|
|
run: uv run pytest tests/e2e -m impersonate_smoke -s -v --tb=short --reruns 1 --reruns-delay 30
|
|
|
|
- name: Enforce coverage floors
|
|
# Gate the nightly on the live coverage floors independently of the
|
|
# continue-on-error main run + --last-failed retry (which can't re-attempt
|
|
# throttled generation — those are skips, not failures). The main e2e step
|
|
# records a breach to the sentinel; fail here if any surface produced zero
|
|
# real coverage (every marked live-chat / live-generation test rate-limited).
|
|
# ``always()`` so a breach recorded before an unrelated later failure still
|
|
# surfaces. The release job never sets the sentinel, so this is nightly-only.
|
|
if: ${{ always() && needs.resolve-branch.outputs.is_standard == 'true' }}
|
|
shell: bash
|
|
run: |
|
|
sentinel="$E2E_COVERAGE_FLOOR_SENTINEL"
|
|
if [ ! -f "$sentinel" ]; then
|
|
echo "Coverage floors satisfied (no monitored surface emitted a skip)."
|
|
exit 0
|
|
fi
|
|
# Reconcile PASS/SKIP events accumulated across the main run + retry: a
|
|
# surface breaches only if it was SKIP-ped somewhere and PASSed nowhere.
|
|
passed="$(awk -F'\t' '$1=="PASS"{print $2}' "$sentinel" | sort -u)"
|
|
skipped="$(awk -F'\t' '$1=="SKIP"{print $2}' "$sentinel" | sort -u)"
|
|
breach="$(comm -23 <(printf '%s\n' "$skipped" | sed '/^$/d') \
|
|
<(printf '%s\n' "$passed" | sed '/^$/d'))"
|
|
if [ -n "$breach" ]; then
|
|
echo "::error::E2E coverage floor breached — surface(s) rate-limited with zero real coverage:"
|
|
printf ' %s\n' "$breach"
|
|
exit 1
|
|
fi
|
|
echo "Coverage floors satisfied (every skipped surface also had a passing test)."
|