Files
kage/scripts/ensure_cloudflare_pages.py
Duc-Tam Nguyen e6afa91e09 Add the clone engine, CLI, tests, CI, and docs
kage renders every page in headless Chrome, snapshots the final
DOM, strips all JavaScript, and localises CSS, images, and fonts
so a site can be browsed offline as a plain folder of files.

The engine is split into small packages:

  urlx      deterministic URL to local-path mapping and scope rules
  sanitize  remove scripts, on* handlers, and javascript: URLs
  asset     rewrite HTML and CSS references, download assets
  browser   headless Chrome pool over the DevTools protocol
  robots    robots.txt matcher
  clone     the orchestrator: a polite resumable breadth-first crawl

The cli package wires a cobra and fang command surface with two
commands, clone and serve. Every pure package has table tests; the
browser and clone packages add Chrome-driven end-to-end tests that
skip when no browser is present or under -short.

CI runs gofmt, vet, build, race tests, golangci-lint, govulncheck,
and a tidy check on Linux and macOS. A goreleaser config fans one
tag out to archives, deb/rpm/apk, a Chromium-bundled GHCR image,
and the package managers. A tago docs site builds to Pages and
Cloudflare.
2026-06-14 18:22:25 +07:00

145 lines
4.8 KiB
Python
Executable File

#!/usr/bin/env python3
"""Idempotently configure a Cloudflare Pages project, custom domain, and DNS."""
from __future__ import annotations
import argparse
import json
import os
import urllib.error
import urllib.parse
import urllib.request
API = "https://api.cloudflare.com/client/v4"
def request(method: str, url: str, token: str, payload: dict[str, object] | None = None) -> dict[str, object]:
data = None if payload is None else json.dumps(payload).encode()
req = urllib.request.Request(
url,
data=data,
method=method,
headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
)
try:
with urllib.request.urlopen(req, timeout=30) as resp:
body = resp.read().decode()
except urllib.error.HTTPError as exc:
body = exc.read().decode()
try:
parsed = json.loads(body) if body else {}
except json.JSONDecodeError:
parsed = {"raw": body}
raise SystemExit(f"{method} {url} failed: HTTP {exc.code} {parsed}") from exc
parsed = json.loads(body) if body else {}
if not parsed.get("success", True):
raise SystemExit(f"{method} {url} failed: {parsed}")
return parsed
def ensure_project(account_id: str, token: str, project: str, branch: str) -> str:
url = f"{API}/accounts/{account_id}/pages/projects/{project}"
try:
result = request("GET", url, token)
print(f"project exists: {project}")
return result["result"]["subdomain"]
except SystemExit as exc:
if "HTTP 404" not in str(exc):
raise
result = request(
"POST",
f"{API}/accounts/{account_id}/pages/projects",
token,
{"name": project, "production_branch": branch},
)
print(f"created project: {project}")
return result["result"]["subdomain"]
def ensure_custom_domain(account_id: str, token: str, project: str, domain: str) -> None:
base = f"{API}/accounts/{account_id}/pages/projects/{project}/domains"
try:
request("GET", f"{base}/{domain}", token)
print(f"domain exists: {domain}")
return
except SystemExit as exc:
if "HTTP 404" not in str(exc):
raise
request("POST", base, token, {"name": domain})
print(f"attached domain: {domain}")
def find_zone(token: str, zone_name: str) -> str:
query = urllib.parse.urlencode({"name": zone_name})
result = request("GET", f"{API}/zones?{query}", token)
zones = result.get("result") or []
if not zones:
raise SystemExit(f"zone not found: {zone_name}")
return zones[0]["id"]
def ensure_dns(token: str, zone_name: str, domain: str, target: str) -> None:
zone_id = find_zone(token, zone_name)
base = f"{API}/zones/{zone_id}/dns_records"
query = urllib.parse.urlencode({"name": domain})
result = request("GET", f"{base}?{query}", token)
payload = {"type": "CNAME", "name": domain, "content": target, "ttl": 1, "proxied": True}
matches = result.get("result") or []
if matches:
record_id = matches[0]["id"]
record = matches[0]
if (
record.get("type") == payload["type"]
and record.get("content") == payload["content"]
and record.get("proxied") is True
):
print(f"DNS exists: {domain} -> {target}")
return
request("PUT", f"{base}/{record_id}", token, payload)
print(f"updated DNS: {domain} -> {target}")
return
request("POST", base, token, payload)
print(f"created DNS: {domain} -> {target}")
def default_zone(domain: str) -> str:
parts = domain.split(".")
if len(parts) < 2:
raise SystemExit(f"cannot infer zone from domain: {domain}")
return ".".join(parts[-2:])
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--project", required=True)
parser.add_argument("--domain")
parser.add_argument("--zone")
parser.add_argument("--branch", default="main")
parser.add_argument("--dns-target")
args = parser.parse_args()
account_id = os.environ["CLOUDFLARE_ACCOUNT_ID"]
token = os.environ["CLOUDFLARE_API_TOKEN"]
# The pages.dev subdomain is not always {project}.pages.dev: when the
# name is taken globally, Cloudflare assigns a suffixed one. A CNAME at
# the guessed name points at a stranger's project and the custom domain
# never validates, so always ask the API for the real subdomain.
subdomain = ensure_project(account_id, token, args.project, args.branch)
if args.domain:
target = args.dns_target or subdomain
zone = args.zone or default_zone(args.domain)
ensure_custom_domain(account_id, token, args.project, args.domain)
ensure_dns(token, zone, args.domain, target)
return 0
if __name__ == "__main__":
raise SystemExit(main())