chore: import upstream snapshot with attribution

apps/aggregator/.env.example

@@ -0,0 +1,8 @@
+# Local-dev secrets for the aggregator Worker. Copy to `.env` (which is
+# gitignored) and customise. `wrangler dev` reads this file and binds the
+# values into `env`. Production secrets are managed via
+# `wrangler secret put <NAME>` and never appear in the repo.
+
+# Bearer token for the /_admin/* routes. Any non-empty string works locally;
+# pick something you'd recognise in `wrangler tail` logs.
+ADMIN_TOKEN=dev-only-not-for-production

apps/aggregator/migrations/0001_init.sql

@@ -0,0 +1,348 @@
+-- EmDash plugin registry aggregator: initial schema.
+--
+-- Lands every table that the v1 read API + ingest pipeline + label hydration
+-- + mirror tracking needs, at once on purpose: features that read these
+-- tables don't need to add new ones, so this is the only DDL we expect to
+-- ship while NSIDs remain experimental.
+
+------------------------------------------------------------------------------
+-- Records: package profiles + releases
+------------------------------------------------------------------------------
+
+CREATE TABLE packages (
+	did TEXT NOT NULL,
+	slug TEXT NOT NULL,
+	type TEXT NOT NULL,                         -- 'emdash-plugin'
+	name TEXT,
+	description TEXT,
+	license TEXT NOT NULL,
+	authors TEXT NOT NULL,                      -- JSON array
+	security TEXT NOT NULL,                     -- JSON array
+	keywords TEXT,                              -- JSON array
+	sections TEXT,                              -- JSON map
+	last_updated TEXT,
+	-- Denormalised from latest release for query convenience. Updated on every
+	-- new release insert; readers never compute "latest" by sorting.
+	latest_version TEXT,
+	capabilities TEXT,                          -- JSON array
+	-- Raw signed record bytes for verification + envelope passthrough. Clients
+	-- re-verify the MST signature against the publisher's DID document at
+	-- install time.
+	record_blob BLOB NOT NULL,
+	signature_metadata TEXT,                    -- JSON: head CID, signing key id
+	verified_at TEXT NOT NULL,
+	PRIMARY KEY (did, slug)
+);
+
+CREATE TABLE releases (
+	did TEXT NOT NULL,
+	package TEXT NOT NULL,                      -- matches the parent profile's rkey/slug (record.package field)
+	version TEXT NOT NULL,                      -- canonical (un-percent-encoded) semver from record.version
+	rkey TEXT NOT NULL,                         -- exact rkey of the form `<package>:<encoded-version>`
+	-- Pre-computed semver-precedence-ordered string for ORDER BY. Application
+	-- code writes this; SQLite cannot compute semver order natively. Format
+	-- packs zero-padded major.minor.patch with prerelease tags compared per
+	-- semver precedence rules.
+	version_sort TEXT NOT NULL,
+	artifacts TEXT NOT NULL,                    -- JSON
+	requires TEXT,                              -- JSON
+	suggests TEXT,                              -- JSON
+	-- com.emdashcms.experimental.package.releaseExtension contents:
+	-- { declaredAccess }. The capabilities-shaped projection lives in
+	-- packages.capabilities for query convenience.
+	emdash_extension TEXT NOT NULL,
+	repo_url TEXT,
+	cts TEXT NOT NULL,                          -- creation timestamp from the record
+	record_blob BLOB NOT NULL,
+	signature_metadata TEXT,
+	verified_at TEXT NOT NULL,
+	tombstoned_at TEXT,                         -- soft delete (publisher deleted record)
+	PRIMARY KEY (did, package, version),
+	-- ON DELETE CASCADE because Jetstream events for a publisher can arrive
+	-- in arbitrary order under network reorder. A publisher who deletes their
+	-- profile (and all releases) emits the events in author-order, but the
+	-- profile-delete might land at the consumer before the release-deletes.
+	-- Without cascade, the consumer would have to either skip the profile
+	-- delete (leaving stale rows) or sequence retries, neither of which is
+	-- worth the complexity. Releases are version-immutable from a publishing
+	-- perspective, but a publisher is still entitled to remove their entire
+	-- package; cascade mirrors that intent.
+	FOREIGN KEY (did, package) REFERENCES packages(did, slug) ON DELETE CASCADE
+);
+
+CREATE INDEX idx_releases_latest ON releases(did, package, version_sort DESC) WHERE tombstoned_at IS NULL;
+CREATE INDEX idx_releases_cts ON releases(cts);
+
+-- Audit trail for rejected duplicate-version attempts. FAIR PR #77 makes
+-- versions immutable: a second record at the same (did, package, version) is
+-- rejected at the SQL layer and logged here for forensics.
+--
+-- The UNIQUE constraint dedupes attempts by content (CID), not by raw bytes.
+-- CAR bytes include the publisher's commit + MST proof which churns whenever
+-- the publisher writes any other record in the same repo, so byte-equality
+-- would misclassify benign retries as new attempts and bloat the table.
+-- The CID is content-addressed and stable for an unchanged record.
+--
+-- The consumer's INSERT uses `ON CONFLICT … DO UPDATE SET rejected_at,
+-- attempted_record_blob = excluded.{rejected_at, attempted_record_blob}` so
+-- the row tracks the latest attempt timestamp + the latest envelope bytes
+-- (newer proofs supersede older ones in the forensics column).
+CREATE TABLE release_duplicate_attempts (
+	did TEXT NOT NULL,
+	package TEXT NOT NULL,
+	version TEXT NOT NULL,
+	-- CID of the verified record (stable for content; changes only when the
+	-- record itself changes). Used as the dedup key.
+	attempted_cid TEXT NOT NULL,
+	rejected_at TEXT NOT NULL,
+	reason TEXT NOT NULL,
+	-- Raw CAR bytes from the most recent attempt. Kept for forensics so
+	-- operators can inspect what was actually attempted even if the
+	-- publisher has since deleted the offending record.
+	attempted_record_blob BLOB NOT NULL,
+	UNIQUE (did, package, version, attempted_cid)
+);
+
+-- The UNIQUE constraint creates an implicit index on
+-- (did, package, version, attempted_record_blob); a separate index on the
+-- (did, package, version) prefix is redundant for both lookups (the implicit
+-- index handles prefix seeks) and inserts (one fewer index to maintain).
+
+------------------------------------------------------------------------------
+-- Publishers: identity-level publisher profiles + verification claims
+------------------------------------------------------------------------------
+
+-- One row per publisher DID (rkey is always literal `self`). Optional: a DID
+-- may publish packages without ever publishing a publisher.profile, in which
+-- case the row is absent and clients fall back to the handle. This table is
+-- the canonical source for "who is publishing these packages?" — distinct from
+-- `packages.authors`, which is per-package and remains authoritative for that
+-- package.
+CREATE TABLE publishers (
+	did TEXT PRIMARY KEY,
+	display_name TEXT NOT NULL,                 -- bound by verification records — see publisher_verifications
+	description TEXT,
+	url TEXT,
+	contact TEXT,                               -- JSON array of { kind, url?, email? }
+	updated_at TEXT,
+	record_blob BLOB NOT NULL,
+	signature_metadata TEXT,                    -- JSON: head CID, signing key id
+	verified_at TEXT NOT NULL
+);
+
+-- Verification claims: issuer DID vouches for subject DID as a trusted
+-- publisher. The rkey is a TID, so an issuer can issue multiple claims (e.g.
+-- delegated + official) and we store each as its own row. Validity is bound to
+-- the subject's handle + publisher.profile.displayName at issuance time:
+-- clients re-resolve those at read time and treat the claim as not in force if
+-- either has changed. Ingest stores the facts; the validity check is a
+-- query-time concern.
+CREATE TABLE publisher_verifications (
+	issuer_did TEXT NOT NULL,                   -- DID of the repo that wrote the record
+	rkey TEXT NOT NULL,                         -- TID
+	subject_did TEXT NOT NULL,
+	subject_handle TEXT NOT NULL,               -- bound at issuance; query-time validity check compares against current
+	subject_display_name TEXT NOT NULL,         -- bound at issuance; query-time validity check compares against current
+	created_at TEXT NOT NULL,
+	expires_at TEXT,
+	record_blob BLOB NOT NULL,
+	signature_metadata TEXT,
+	verified_at TEXT NOT NULL,
+	tombstoned_at TEXT,
+	PRIMARY KEY (issuer_did, rkey)
+);
+
+-- Hot path: "show me all unexpired, non-tombstoned verifications for subject X".
+-- Partial index keeps the index small by excluding tombstoned rows.
+CREATE INDEX idx_publisher_verifications_subject ON publisher_verifications(subject_did)
+	WHERE tombstoned_at IS NULL;
+
+-- For periodic expiry sweeps.
+CREATE INDEX idx_publisher_verifications_expires ON publisher_verifications(expires_at)
+	WHERE expires_at IS NOT NULL AND tombstoned_at IS NULL;
+
+------------------------------------------------------------------------------
+-- Mirror tracking (populated when the artifact mirror lands)
+------------------------------------------------------------------------------
+
+CREATE TABLE mirrored_artifacts (
+	did TEXT NOT NULL,
+	slug TEXT NOT NULL,
+	version TEXT NOT NULL,
+	artifact_id TEXT NOT NULL,                  -- 'package', 'icon', etc.
+	r2_key TEXT NOT NULL,
+	bytes INTEGER NOT NULL,
+	content_type TEXT NOT NULL,
+	mirrored_at TEXT NOT NULL,
+	PRIMARY KEY (did, slug, version, artifact_id)
+);
+
+------------------------------------------------------------------------------
+-- Labels (populated when the labeller integration lands)
+------------------------------------------------------------------------------
+
+-- Append-only label history. Every label received is written here, including
+-- negations. Current state is derived from latest cts per (src, uri, val) and
+-- projected into label_state below for hot-path lookups.
+CREATE TABLE labels (
+	src TEXT NOT NULL,                          -- labeller DID
+	uri TEXT NOT NULL,                          -- AT URI of subject
+	cid TEXT,                                   -- optional version-specific CID
+	val TEXT NOT NULL,                          -- e.g. 'security:yanked', '!takedown'
+	neg INTEGER NOT NULL DEFAULT 0,
+	cts TEXT NOT NULL,
+	exp TEXT,                                   -- optional expiry (RFC 3339)
+	sig BLOB NOT NULL,                          -- raw signature for client re-verification
+	ver INTEGER NOT NULL DEFAULT 1,
+	trusted INTEGER NOT NULL DEFAULT 0,
+	received_at TEXT NOT NULL,
+	PRIMARY KEY (src, uri, val, cts)
+);
+
+CREATE INDEX idx_labels_subject ON labels(uri);
+CREATE INDEX idx_labels_latest ON labels(src, uri, val, cts DESC);
+
+-- Latest-state projection: one row per (src, uri, val) holding the most recent
+-- cts seen, including the neg flag and exp timestamp. Updated on every label
+-- write within the same transaction. Query-time filters apply
+-- `neg = 0 AND (exp IS NULL OR exp > now())` to determine whether a label is
+-- currently in force.
+--
+-- Why retain rows for negated/expired labels rather than deleting them: an
+-- out-of-order delivery (a positive label arriving after its negation) could
+-- otherwise reinsert a row we'd already retracted. Keeping the row with its
+-- `cts` lets the upsert reject the older positive.
+CREATE TABLE label_state (
+	src TEXT NOT NULL,
+	uri TEXT NOT NULL,
+	val TEXT NOT NULL,
+	cid TEXT,
+	neg INTEGER NOT NULL DEFAULT 0,
+	cts TEXT NOT NULL,
+	exp TEXT,
+	trusted INTEGER NOT NULL DEFAULT 0,
+	PRIMARY KEY (src, uri, val)
+);
+
+-- Hot path for hard filters (yanked, takedown, etc.) from trusted issuers.
+-- Partial index keeps the index small by storing only currently-active rows.
+CREATE INDEX idx_label_state_enforce ON label_state(uri, val, trusted)
+	WHERE neg = 0 AND trusted = 1;
+
+-- Trusted/known labellers (operator config, edited via deployment).
+CREATE TABLE labellers (
+	did TEXT PRIMARY KEY,
+	endpoint TEXT NOT NULL,                     -- subscribeLabels URL
+	signing_key TEXT NOT NULL,                  -- cached #atproto_label key
+	signing_key_id TEXT NOT NULL,
+	trusted INTEGER NOT NULL DEFAULT 0,
+	added_at TEXT NOT NULL,
+	last_resolved_at TEXT NOT NULL,
+	notes TEXT
+);
+
+------------------------------------------------------------------------------
+-- Search: FTS5 over packages
+------------------------------------------------------------------------------
+
+CREATE VIRTUAL TABLE packages_fts USING fts5(
+	name,
+	description,
+	keywords,
+	authors,
+	sections,
+	content='packages',
+	content_rowid='rowid',
+	tokenize='porter unicode61 remove_diacritics 2'
+);
+
+CREATE TRIGGER packages_ai AFTER INSERT ON packages BEGIN
+	INSERT INTO packages_fts(rowid, name, description, keywords, authors, sections)
+	VALUES (new.rowid, new.name, new.description, new.keywords, new.authors, new.sections);
+END;
+
+CREATE TRIGGER packages_au AFTER UPDATE ON packages BEGIN
+	INSERT INTO packages_fts(packages_fts, rowid, name, description, keywords, authors, sections)
+	VALUES ('delete', old.rowid, old.name, old.description, old.keywords, old.authors, old.sections);
+	INSERT INTO packages_fts(rowid, name, description, keywords, authors, sections)
+	VALUES (new.rowid, new.name, new.description, new.keywords, new.authors, new.sections);
+END;
+
+CREATE TRIGGER packages_ad AFTER DELETE ON packages BEGIN
+	INSERT INTO packages_fts(packages_fts, rowid, name, description, keywords, authors, sections)
+	VALUES ('delete', old.rowid, old.name, old.description, old.keywords, old.authors, old.sections);
+END;
+
+------------------------------------------------------------------------------
+-- Ingest cursor state
+------------------------------------------------------------------------------
+
+-- Cursor state for ingest sources (Jetstream microsecond timestamp,
+-- subscribeLabels seq cursors per labeller, etc.).
+CREATE TABLE ingest_state (
+	source TEXT PRIMARY KEY,                    -- 'jetstream', 'labeller:did:web:labels.example.com', etc.
+	cursor TEXT NOT NULL,
+	updated_at TEXT NOT NULL
+);
+
+-- Known publisher DIDs we've seen via Jetstream or Constellation. Reconciliation
+-- iterates this table; cold-start backfill seeds it from Constellation.
+--
+-- Doubles as the DID-document resolution cache: `pds`, `signing_key`,
+-- `signing_key_id` are populated by the records consumer on first verification
+-- and refreshed when `pds_resolved_at` is older than the consumer's TTL
+-- (currently 24h, applied at query time as
+-- `pds_resolved_at > datetime('now', '-1 day')`). Backfill may insert a row
+-- with these fields null; the consumer's first event for that DID forces a
+-- resolution and UPDATE.
+CREATE TABLE known_publishers (
+	did TEXT PRIMARY KEY,
+	pds TEXT,                                   -- cached PDS endpoint from DID document
+	signing_key TEXT,                           -- cached #atproto signing key (multibase)
+	signing_key_id TEXT,                        -- e.g. 'did:plc:xxx#atproto'
+	pds_resolved_at TEXT,                       -- last successful DID-doc resolution
+	first_seen_at TEXT NOT NULL,
+	last_seen_at TEXT NOT NULL
+);
+
+------------------------------------------------------------------------------
+-- Verification-failure forensics
+------------------------------------------------------------------------------
+
+-- Records that failed PDS-verified ingest (signature, MST proof, AT-URI,
+-- lexicon, content-mismatch). Written instead of retrying, because these
+-- failures indicate malicious or broken upstream — retrying would just burn
+-- PDS round trips. Operators query this table to investigate suspected attacks
+-- or upstream regressions; it is NOT used as a retry queue.
+--
+-- Distinct from the configured Cloudflare DLQ (`emdash-aggregator-records-dlq`,
+-- see wrangler.jsonc), which receives messages after `max_retries` exhausted —
+-- that is for transient failures (PDS down, profile-not-yet-arrived). Two
+-- distinct failure modes, two distinct destinations.
+--
+-- `payload` holds the unverified record bytes from the Jetstream event so an
+-- operator can inspect what was attempted without going back to the source PDS.
+CREATE TABLE dead_letters (
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	did TEXT NOT NULL,
+	collection TEXT NOT NULL,
+	rkey TEXT NOT NULL,
+	-- Reason code; matches the `DeadLetterReason` union in records-consumer.ts.
+	-- Current values: 'RECORD_NOT_FOUND', 'RESPONSE_TOO_LARGE', 'INVALID_PROOF',
+	-- 'PDS_HTTP_ERROR', 'LEXICON_VALIDATION_FAILED', 'RKEY_MISMATCH',
+	-- 'CONTACT_VALIDATION_FAILED', 'INVALID_VERSION', 'UNKNOWN_COLLECTION',
+	-- 'UNEXPECTED_ERROR'.
+	reason TEXT NOT NULL,
+	-- Free-form context (which field, expected vs got, library error message, etc.).
+	detail TEXT,
+	-- UTF-8 encoded JSON bytes of `RecordsJob.jetstreamRecord` when present, or a
+	-- fallback envelope `{operation, cid}` for delete events that don't carry one.
+	-- Stored as BLOB so future formats (CBOR, raw record bytes) can land here
+	-- without a schema change; today operators must `CAST(payload AS TEXT)` to
+	-- read.
+	payload BLOB NOT NULL,
+	received_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+CREATE INDEX idx_dead_letters_did ON dead_letters(did);
+CREATE INDEX idx_dead_letters_received ON dead_letters(received_at);

apps/aggregator/migrations/0002_indexed_at.sql

@@ -0,0 +1,35 @@
+-- Add `indexed_at` to the four content tables that the read API exposes via
+-- the `packageView` / `releaseView` lexicon shapes.
+--
+-- Why a separate column from `verified_at`: `verified_at` tracks "when the
+-- aggregator last verified this record", and is bumped on every upsert
+-- (including no-op re-verifications). The read API needs "when the aggregator
+-- first observed this record" so the lexicon's `indexedAt` field is stable
+-- across re-ingest. They diverge whenever a record is re-fetched (e.g.
+-- backfill catching up on a record we already had via Jetstream).
+--
+-- Defaulting to `verified_at` for any rows that pre-date this migration: the
+-- closest approximation we can make for already-indexed content. Going
+-- forward, the consumer's INSERTs set `indexed_at` to `now()` on first write
+-- and COALESCE the existing value on conflict (see records-consumer.ts).
+--
+-- NOT NULL after backfill from `verified_at`. SQLite's `ALTER TABLE` doesn't
+-- support DEFAULT-with-NOT-NULL in one step, so we add the column nullable,
+-- backfill, then move the constraint via the new-table dance — except SQLite
+-- also can't add NOT NULL via ALTER, so we keep the column nullable at the
+-- schema level and have the writer (consumer) treat it as required. The
+-- read-API code defends against NULL by falling back to verified_at if
+-- indexed_at is somehow missing on a row, which won't happen for new writes
+-- but covers the historical-data corner case without a table rebuild.
+
+ALTER TABLE packages ADD COLUMN indexed_at TEXT;
+UPDATE packages SET indexed_at = verified_at WHERE indexed_at IS NULL;
+
+ALTER TABLE releases ADD COLUMN indexed_at TEXT;
+UPDATE releases SET indexed_at = verified_at WHERE indexed_at IS NULL;
+
+ALTER TABLE publishers ADD COLUMN indexed_at TEXT;
+UPDATE publishers SET indexed_at = verified_at WHERE indexed_at IS NULL;
+
+ALTER TABLE publisher_verifications ADD COLUMN indexed_at TEXT;
+UPDATE publisher_verifications SET indexed_at = verified_at WHERE indexed_at IS NULL;

apps/aggregator/package.json

@@ -0,0 +1,46 @@
+{
+	"name": "@emdash-cms/aggregator",
+	"version": "0.0.0",
+	"private": true,
+	"description": "EmDash plugin registry aggregator. Indexes com.emdashcms.experimental.package.* records via Jetstream + PDS-verified ingest, exposes XRPC reads.",
+	"type": "module",
+	"scripts": {
+		"dev": "vite dev",
+		"build": "vite build",
+		"preview": "vite preview",
+		"deploy": "vite build && wrangler deploy",
+		"typecheck": "tsgo --noEmit",
+		"test": "vitest run",
+		"db:migrate:local": "wrangler d1 migrations apply emdash-aggregator --local",
+		"db:migrate": "wrangler d1 migrations apply emdash-aggregator --remote"
+	},
+	"dependencies": {
+		"@atcute/atproto": "catalog:",
+		"@atcute/car": "catalog:",
+		"@atcute/cbor": "catalog:",
+		"@atcute/cid": "catalog:",
+		"@atcute/client": "catalog:",
+		"@atcute/crypto": "catalog:",
+		"@atcute/firehose": "catalog:",
+		"@atcute/identity": "catalog:",
+		"@atcute/identity-resolver": "catalog:",
+		"@atcute/jetstream": "catalog:",
+		"@atcute/lexicons": "catalog:",
+		"@atcute/mst": "catalog:",
+		"@atcute/multibase": "catalog:",
+		"@atcute/repo": "catalog:",
+		"@atcute/xrpc-server": "catalog:",
+		"@atcute/xrpc-server-cloudflare": "catalog:",
+		"@emdash-cms/registry-lexicons": "workspace:*"
+	},
+	"devDependencies": {
+		"@cloudflare/vite-plugin": "catalog:",
+		"@cloudflare/vitest-pool-workers": "catalog:",
+		"@emdash-cms/atproto-test-utils": "workspace:*",
+		"@types/node": "catalog:",
+		"typescript": "catalog:",
+		"vite": "catalog:",
+		"vitest": "catalog:",
+		"wrangler": "catalog:"
+	}
+}

apps/aggregator/src/backfill-consumer.ts

@@ -0,0 +1,144 @@
+/**
+ * Backfill queue consumer. Pulls one `BackfillJob` at a time and walks
+ * `com.atproto.repo.listRecords` for that (DID, collection) pair, batching
+ * results onto the records queue for the standard verify-and-write path.
+ *
+ * Why a separate queue from records: per-pair work (PDS resolution +
+ * paginated listRecords + sendBatch onto the records queue) is bounded but
+ * non-trivial — running it inside the records-queue consumer would burn the
+ * sub-request budget for jobs that should just be writing to D1. Keeping
+ * the queues separate also lets the operator throttle backfill work
+ * independently of live ingest.
+ *
+ * Error policy:
+ *   - Per-pair `processBackfillJob` throw → `message.retry()`. Cloudflare
+ *     Queues backs off and retries; after `max_retries` (3, configured in
+ *     wrangler.jsonc) the message lands in `emdash-aggregator-backfill-dlq`.
+ *   - Unexpected throws inside the batch loop are caught per-message so one
+ *     bad job can't poison the rest of the batch.
+ *
+ * DLQ drain (`drainBackfillDeadLetterBatch`): logs each dead-lettered
+ * pair at error level (so operators tailing `wrangler tail` see it loud)
+ * and acks so the DLQ doesn't accumulate unbounded. No D1 forensics row —
+ * the recovery action for a backfill pair that exhausted retries is
+ * "re-run backfill for the affected DID", which only needs the
+ * (did, collection) pair already on the log line.
+ */
+
+import {
+	AtprotoWebDidDocumentResolver,
+	CompositeDidDocumentResolver,
+	PlcDidDocumentResolver,
+} from "@atcute/identity-resolver";
+
+import { processBackfillJob, type ProcessBackfillJobDeps } from "./backfill.js";
+import { createD1DidDocCache, DidResolver } from "./did-resolver.js";
+import type { BackfillJob } from "./env.js";
+import type { MessageBatchLike } from "./records-consumer.js";
+import { boundFetch } from "./utils.js";
+
+/**
+ * Process one batch of backfill jobs. Mirrors `records-consumer.processBatch`'s
+ * shape: per-message try/catch, ack on success, retry on throw.
+ *
+ * `depsOverride` is the test seam — production calls without it and gets
+ * the standard composite resolver wired against `env.DB`.
+ */
+export async function processBackfillBatch(
+	batch: MessageBatchLike<BackfillJob>,
+	env: Env,
+	depsOverride?: ProcessBackfillJobDeps,
+): Promise<void> {
+	const deps = depsOverride ?? createProductionDeps(env);
+	for (const message of batch.messages) {
+		const job = message.body;
+		try {
+			const result = await processBackfillJob(job, deps);
+			console.log("[aggregator] backfill job complete", {
+				did: result.did,
+				collection: result.collection,
+				enqueued: result.enqueued,
+			});
+			message.ack();
+		} catch (err) {
+			// Resolution failures, listRecords 5xx, timeouts, and pagination
+			// runaway all land here. Retry — Cloudflare Queues backoff handles
+			// transient PDS failures; permanently broken DIDs land in the DLQ
+			// after max_retries.
+			console.error("[aggregator] backfill job failed, retrying", {
+				did: job.did,
+				collection: job.collection,
+				error: formatErrorChain(err),
+			});
+			message.retry();
+		}
+	}
+}
+
+/**
+ * Drain the backfill DLQ. Mirror of `records-consumer.drainDeadLetterBatch`
+ * but without the D1 forensics row — the recovery action for a backfill
+ * pair that exhausted retries is "re-run backfill for the affected DID",
+ * which only needs the (did, collection) pair from the log line.
+ *
+ * Logs at error level so operators tailing `wrangler tail` see the message
+ * loud, acks so the DLQ doesn't accumulate unbounded.
+ */
+export function drainBackfillDeadLetterBatch(
+	batch: MessageBatchLike<BackfillJob>,
+	_env: Env,
+): void {
+	for (const message of batch.messages) {
+		console.error("[aggregator] backfill DLQ drain: pair exhausted retries", {
+			did: message.body.did,
+			collection: message.body.collection,
+		});
+		message.ack();
+	}
+}
+
+/** Walk an Error's `cause` chain into a single human-readable string for
+ * log output. Library code (`@atcute/identity-resolver`,
+ * `@atcute/util-fetch`) wraps low-level failures in higher-level errors
+ * via `{ cause }`; logging only `err.message` hides the actual root
+ * cause. The chain is joined with `; cause: ` so a multi-level wrap
+ * reads top-down. Bounded at 5 levels so a circular chain can't loop
+ * forever. */
+function formatErrorChain(err: unknown): string {
+	const parts: string[] = [];
+	let current: unknown = err;
+	for (let i = 0; i < 5 && current; i++) {
+		if (current instanceof Error) {
+			parts.push(`${current.name}: ${current.message}`);
+			current = (current as { cause?: unknown }).cause;
+		} else {
+			// JSON.stringify (rather than String) so a plain object cause
+			// (some libraries throw `{ code, ... }` shapes) prints its
+			// fields instead of `[object Object]`.
+			parts.push(typeof current === "string" ? current : JSON.stringify(current));
+			break;
+		}
+	}
+	return parts.join("; cause: ");
+}
+
+function createProductionDeps(env: Env): ProcessBackfillJobDeps {
+	const composite = new CompositeDidDocumentResolver({
+		methods: {
+			plc: new PlcDidDocumentResolver({ fetch: boundFetch }),
+			web: new AtprotoWebDidDocumentResolver({ fetch: boundFetch }),
+		},
+	});
+	return {
+		resolver: new DidResolver({
+			cache: createD1DidDocCache(env.DB),
+			resolver: composite,
+		}),
+		queue: env.RECORDS_QUEUE,
+		// listRecords + sendBatch use this fetch for the PDS calls. Same
+		// bound-wrapper requirement as the resolver constructors —
+		// workerd's `fetch` rejects calls made through a stored
+		// reference (`const fetchImpl = deps.fetch ?? fetch; fetchImpl(url)`).
+		fetch: boundFetch,
+	};
+}

apps/aggregator/src/backfill.ts

@@ -0,0 +1,465 @@
+/**
+ * Cold-start discovery worker.
+ *
+ * Operator-triggered (via `POST /_admin/backfill`). Two trigger shapes:
+ *
+ *   - `{ "dids": [...] }` — explicit list, primarily for testing or recovery
+ *     of a known DID set.
+ *   - `{}` (empty body) — production cold-start. Calls
+ *     `com.atproto.sync.listReposByCollection` against the configured relay
+ *     for each NSID in `WANTED_COLLECTIONS`, paginates the full DID set,
+ *     dedupes, and feeds the union into the same backfill loop.
+ *
+ * Architecture: the POST handler synchronously discovers DIDs (or accepts an
+ * explicit list), then fans out one `BackfillJob = { did, collection }` per
+ * (DID × WANTED_COLLECTIONS) pair onto the dedicated `BACKFILL_QUEUE` via
+ * `sendBatch`. A separate consumer (`backfill-consumer.ts`) processes one
+ * pair at a time: resolve PDS, paginate `com.atproto.repo.listRecords`,
+ * batch-enqueue each returned record onto the existing Records Queue.
+ *
+ * Why a separate queue rather than running the per-DID loop inside the
+ * `ctx.waitUntil` of the POST handler: Cloudflare's hard 30-second
+ * wall-clock cap on `waitUntil` would limit a single backfill POST to
+ * ~15–25 DIDs before in-flight work was cancelled. The queue gives us
+ * automatic retry, concurrency, and per-pair invocation budgets that each
+ * fit comfortably under the sub-request ceiling.
+ *
+ * Live discovery (post-cold-start) is Jetstream's job, not this worker's;
+ * the consumer writes `known_publishers` opportunistically on any record
+ * event for an unseen DID. Backfill exists for the cold-start gap
+ * (publishers who published before the aggregator was listening) and for
+ * operator-triggered recovery after a known outage. There is deliberately
+ * no periodic scheduler — see plan §"Why no reconciliation cron".
+ */
+
+import {
+	parseCanonicalResourceUri,
+	type ParsedCanonicalResourceUri,
+} from "@atcute/lexicons/syntax";
+
+import { WANTED_COLLECTIONS } from "./constants.js";
+import type { DidResolver } from "./did-resolver.js";
+import type { BackfillJob, RecordsJob } from "./env.js";
+import { isPlainObject } from "./utils.js";
+
+const PAGE_SIZE = 100;
+/** Per-listRecords-page timeout. A hostile or hung publisher PDS that
+ * accepts the connection but stalls the body would otherwise block the
+ * fetch until workerd's overall sub-request budget exhausts — starving
+ * every later page in the same consumer invocation. Same shape as
+ * `pds-verify.ts`'s fetchCar timeout. */
+const LIST_RECORDS_TIMEOUT_MS = 15_000;
+/** Cap on listRecords pagination per (DID, collection) pair. A buggy or
+ * malicious PDS that echoes the same cursor would otherwise loop forever
+ * inside one consumer invocation. 1000 pages × 100 records = 100k records
+ * per pair, which is past anything we'd legitimately backfill in one shot. */
+const MAX_PAGES_PER_COLLECTION = 1000;
+/** Defensive cap on records per page. Real PDSes honour the `limit` query
+ * param; this guards against a hostile PDS returning an enormous array.
+ * Capped at the same width as Cloudflare Queues' sendBatch (100) so a
+ * compliant page maps 1:1 to one batch send; oversize pages are rejected
+ * rather than chunked, surfacing the PDS's spec violation as a partial
+ * failure the operator can investigate. */
+const MAX_RECORDS_PER_PAGE = PAGE_SIZE;
+/** Cloudflare Queues' hard cap on `sendBatch` size. Per-page enqueues are
+ * always ≤ this thanks to MAX_RECORDS_PER_PAGE; documented here so the
+ * relationship is visible at the call site. */
+export const QUEUE_SEND_BATCH_CAP = 100;
+// Static guard: bumping MAX_RECORDS_PER_PAGE above the queue's batch cap
+// would silently break sendBatch in production. Surface the violation at
+// module load rather than at the first batch send.
+if (MAX_RECORDS_PER_PAGE > QUEUE_SEND_BATCH_CAP) {
+	throw new Error(
+		`MAX_RECORDS_PER_PAGE (${MAX_RECORDS_PER_PAGE}) exceeds QUEUE_SEND_BATCH_CAP (${QUEUE_SEND_BATCH_CAP})`,
+	);
+}
+/** Producer-side records queue surface. The production binding
+ * `env.RECORDS_QUEUE` satisfies this; tests pass an in-memory implementation. */
+export interface RecordsQueueProducer {
+	sendBatch(messages: ReadonlyArray<{ body: RecordsJob }>): Promise<unknown>;
+}
+
+/** Producer-side backfill-jobs queue surface. The production binding
+ * `env.BACKFILL_QUEUE` satisfies this; tests pass an in-memory
+ * implementation. Same shape as `RecordsQueueProducer`, separated so the
+ * type system catches accidental cross-wiring. */
+export interface BackfillQueueProducer {
+	sendBatch(messages: ReadonlyArray<{ body: BackfillJob }>): Promise<unknown>;
+}
+
+/** Cap on pages walked per relay collection. Same shape as
+ * MAX_PAGES_PER_COLLECTION but for the discovery side. At 100 repos/page,
+ * 100 pages = 10k publishers — past anything we'd legitimately discover at
+ * Slice 1 scale. */
+const MAX_DISCOVERY_PAGES_PER_COLLECTION = 100;
+const DISCOVERY_PAGE_SIZE = 100;
+const DISCOVERY_TIMEOUT_MS = 15_000;
+/** Defensive cap on the union of discovered DIDs. A relay returning a
+ * runaway list (bug or hostile mirror) would otherwise let one POST fan
+ * out millions of jobs onto BACKFILL_QUEUE. At Slice 1 scale we expect a
+ * handful to a few hundred publishers. */
+export const MAX_DISCOVERED_DIDS = 1000;
+
+/**
+ * Discover all DIDs publishing any of `WANTED_COLLECTIONS` by querying the
+ * relay's `com.atproto.sync.listReposByCollection`. Returns the union of
+ * unique DIDs across all collections, in arbitrary order.
+ *
+ * Uses the same defenses as the per-pair listRecords loop: per-page
+ * timeout, max-page cap, cursor-equality check. Per-collection failures
+ * are logged and the loop continues — discovery via the relay is
+ * best-effort; a partial discovery list is better than none. Stops early
+ * once `MAX_DISCOVERED_DIDS` is reached so a runaway relay can't pump
+ * arbitrary fan-out into the queue.
+ */
+export async function discoverDids(
+	relayUrl: string,
+	opts: { fetch?: typeof fetch; timeoutMs?: number } = {},
+): Promise<string[]> {
+	const fetchImpl = opts.fetch ?? fetch;
+	const timeoutMs = opts.timeoutMs ?? DISCOVERY_TIMEOUT_MS;
+	const dids = new Set<string>();
+	for (const collection of WANTED_COLLECTIONS) {
+		try {
+			await discoverCollection(relayUrl, collection, fetchImpl, timeoutMs, dids);
+		} catch (err) {
+			console.error("[aggregator] backfill discovery failed for collection", {
+				collection,
+				error: err instanceof Error ? err.message : String(err),
+			});
+		}
+		if (dids.size >= MAX_DISCOVERED_DIDS) {
+			console.warn("[aggregator] backfill discovery hit DID cap, stopping early", {
+				cap: MAX_DISCOVERED_DIDS,
+				stoppedAfterCollection: collection,
+			});
+			break;
+		}
+	}
+	return [...dids];
+}
+
+async function discoverCollection(
+	relayUrl: string,
+	collection: string,
+	fetchImpl: typeof fetch,
+	timeoutMs: number,
+	dids: Set<string>,
+): Promise<void> {
+	let cursor: string | undefined;
+	let prevCursor: string | undefined;
+	let pages = 0;
+	do {
+		if (++pages > MAX_DISCOVERY_PAGES_PER_COLLECTION) {
+			throw new Error(`exceeded ${MAX_DISCOVERY_PAGES_PER_COLLECTION} discovery pages`);
+		}
+		if (cursor !== undefined && cursor === prevCursor) {
+			throw new Error("relay returned identical cursor twice");
+		}
+		prevCursor = cursor;
+
+		const url = new URL("/xrpc/com.atproto.sync.listReposByCollection", relayUrl);
+		url.searchParams.set("collection", collection);
+		url.searchParams.set("limit", String(DISCOVERY_PAGE_SIZE));
+		if (cursor) url.searchParams.set("cursor", cursor);
+
+		const controller = new AbortController();
+		const timer = setTimeout(() => controller.abort(), timeoutMs);
+		let response: Response;
+		try {
+			response = await fetchImpl(url.toString(), {
+				headers: { accept: "application/json" },
+				signal: controller.signal,
+			});
+		} catch (err) {
+			if (err instanceof Error && err.name === "AbortError") {
+				throw new Error(`listReposByCollection timed out after ${timeoutMs}ms`, { cause: err });
+			}
+			throw err;
+		} finally {
+			clearTimeout(timer);
+		}
+		if (!response.ok) {
+			throw new Error(`listReposByCollection returned ${response.status}`);
+		}
+		const body: unknown = await response.json();
+		if (!isPlainObject(body)) throw new Error("listReposByCollection returned non-object body");
+		const repos = body["repos"];
+		if (!Array.isArray(repos)) {
+			throw new Error("listReposByCollection response missing `repos` array");
+		}
+		for (const repo of repos) {
+			if (!isPlainObject(repo)) continue;
+			const did = repo["did"];
+			if (typeof did === "string") {
+				dids.add(did);
+				if (dids.size >= MAX_DISCOVERED_DIDS) return;
+			}
+		}
+		const nextCursor = body["cursor"];
+		cursor = typeof nextCursor === "string" && nextCursor.length > 0 ? nextCursor : undefined;
+	} while (cursor);
+}
+
+/**
+ * Fan out backfill work for a list of DIDs onto `BACKFILL_QUEUE`. Produces
+ * one `BackfillJob` per (DID × WANTED_COLLECTIONS) pair, sent in batches
+ * of up to `QUEUE_SEND_BATCH_CAP` to honour Cloudflare Queues' sendBatch
+ * limit. Returns the total number of jobs enqueued.
+ *
+ * Caller is responsible for capping `dids.length` (admin route enforces
+ * `MAX_BACKFILL_DIDS` for the explicit path; `discoverDids` enforces
+ * `MAX_DISCOVERED_DIDS` for the discovery path). This function trusts its
+ * input and just fans out — the per-call work is bounded by
+ * `dids.length * WANTED_COLLECTIONS.length` enqueues.
+ */
+export async function enqueueBackfillJobs(
+	dids: readonly string[],
+	queue: BackfillQueueProducer,
+): Promise<number> {
+	const messages: { body: BackfillJob }[] = [];
+	for (const did of dids) {
+		for (const collection of WANTED_COLLECTIONS) {
+			messages.push({ body: { did, collection } });
+		}
+	}
+	// Fan the sendBatch calls in parallel. Each is an independent outbound
+	// sub-request and the orchestrator runs inside the POST handler's 30s
+	// `waitUntil` budget — serial awaits on a `MAX_DISCOVERED_DIDS`-sized
+	// fan-out (1000 DIDs × 4 collections / 100 = 40 batches) would
+	// noticeably eat into the cold-start budget on top of discovery's own
+	// fetches.
+	const sends: Promise<unknown>[] = [];
+	for (let i = 0; i < messages.length; i += QUEUE_SEND_BATCH_CAP) {
+		sends.push(queue.sendBatch(messages.slice(i, i + QUEUE_SEND_BATCH_CAP)));
+	}
+	await Promise.all(sends);
+	return messages.length;
+}
+
+export interface ProcessBackfillJobDeps {
+	resolver: DidResolver;
+	queue: RecordsQueueProducer;
+	/** Inject for tests; defaults to `globalThis.fetch`. */
+	fetch?: typeof fetch;
+	/** Override for the per-listRecords-page timeout. Defaults to
+	 * `LIST_RECORDS_TIMEOUT_MS`. Tests use a small value to exercise the
+	 * abort path without burning the production budget. */
+	listRecordsTimeoutMs?: number;
+}
+
+export interface ProcessBackfillJobResult {
+	did: string;
+	collection: string;
+	enqueued: number;
+}
+
+/**
+ * Process one (DID, collection) pair: resolve the DID's PDS, paginate
+ * `com.atproto.repo.listRecords` for the collection, and batch-enqueue
+ * each returned record onto the records queue.
+ *
+ * Throws on any failure (resolution, listRecords status, pagination
+ * runaway). The queue consumer translates a thrown result into
+ * `message.retry()`; messages that exhaust max_retries land in the
+ * backfill DLQ for the operator to inspect.
+ *
+ * 404 from the PDS on the first page is treated as "publisher does not
+ * host this collection" and returns 0 enqueues without throwing — same
+ * shape as the previous serial-loop semantics.
+ */
+export async function processBackfillJob(
+	job: BackfillJob,
+	deps: ProcessBackfillJobDeps,
+): Promise<ProcessBackfillJobResult> {
+	const resolved = await deps.resolver.resolve(job.did);
+	const fetchImpl = deps.fetch ?? fetch;
+	const timeoutMs = deps.listRecordsTimeoutMs ?? LIST_RECORDS_TIMEOUT_MS;
+	const enqueued = await paginateAndEnqueue({
+		did: job.did,
+		pds: resolved.pds,
+		collection: job.collection,
+		queue: deps.queue,
+		fetchImpl,
+		timeoutMs,
+	});
+	return { did: job.did, collection: job.collection, enqueued };
+}
+
+interface PaginateOpts {
+	did: string;
+	pds: string;
+	collection: string;
+	queue: RecordsQueueProducer;
+	fetchImpl: typeof fetch;
+	timeoutMs: number;
+}
+
+/**
+ * Walk one DID's records for a single collection, paginating through
+ * `listRecords` and enqueuing each result via `sendBatch` (one batch per
+ * page). Returns the total records enqueued.
+ *
+ * 404 from the PDS on the FIRST page means the repo doesn't host this
+ * collection — silently treated as zero records, not an error. A 404
+ * mid-pagination is a partial-failure signal (the PDS is misrouting one
+ * page while the rest of the repo is fine) and throws.
+ *
+ * Pagination is capped at MAX_PAGES_PER_COLLECTION + cursor-equality check
+ * to defend against a PDS that echoes the same cursor forever.
+ *
+ * `MAX_RECORDS_PER_PAGE` matches Cloudflare Queues' `sendBatch` cap (100),
+ * so a compliant page maps 1:1 to one batch send. A PDS that ignores the
+ * `?limit=` query and returns more records than that throws — we'd rather
+ * surface the spec violation than silently chunk and hide the upstream bug.
+ */
+async function paginateAndEnqueue(opts: PaginateOpts): Promise<number> {
+	let cursor: string | undefined;
+	let prevCursor: string | undefined;
+	let pages = 0;
+	let totalEnqueued = 0;
+	do {
+		if (++pages > MAX_PAGES_PER_COLLECTION) {
+			throw new Error(`exceeded ${MAX_PAGES_PER_COLLECTION} pages`);
+		}
+		if (cursor !== undefined && cursor === prevCursor) {
+			throw new Error("PDS returned identical cursor twice");
+		}
+		prevCursor = cursor;
+
+		const url = new URL("/xrpc/com.atproto.repo.listRecords", opts.pds);
+		url.searchParams.set("repo", opts.did);
+		url.searchParams.set("collection", opts.collection);
+		url.searchParams.set("limit", String(PAGE_SIZE));
+		if (cursor) url.searchParams.set("cursor", cursor);
+
+		const controller = new AbortController();
+		const timer = setTimeout(() => controller.abort(), opts.timeoutMs);
+		let response: Response;
+		try {
+			response = await opts.fetchImpl(url.toString(), {
+				headers: { accept: "application/json" },
+				signal: controller.signal,
+			});
+		} catch (err) {
+			if (err instanceof Error && err.name === "AbortError") {
+				throw new Error(`listRecords timed out after ${opts.timeoutMs}ms`, { cause: err });
+			}
+			throw err;
+		} finally {
+			clearTimeout(timer);
+		}
+		if (response.status === 404) {
+			if (cursor === undefined) {
+				// First-page 404: publisher has no records of this collection.
+				return totalEnqueued;
+			}
+			// Mid-pagination 404 is a partial failure; surface it.
+			throw new Error(`listRecords returned 404 mid-pagination at cursor=${cursor}`);
+		}
+		if (!response.ok) {
+			throw new Error(`listRecords returned ${response.status}`);
+		}
+
+		const body: unknown = await response.json();
+		const records = extractListRecordsBody(body);
+		if (records.length > MAX_RECORDS_PER_PAGE) {
+			throw new Error(
+				`PDS returned ${records.length} records, exceeding per-page cap of ${MAX_RECORDS_PER_PAGE}`,
+			);
+		}
+		cursor = extractCursor(body);
+
+		const messages: { body: RecordsJob }[] = [];
+		for (const record of records) {
+			let parsed: ParsedCanonicalResourceUri;
+			try {
+				parsed = parseCanonicalResourceUri(record.uri);
+			} catch (err) {
+				if (err instanceof SyntaxError) continue;
+				throw err;
+			}
+			// Defence vs. a buggy/malicious PDS that returns records under
+			// a different DID (or a different collection) than the one we
+			// asked for. Such jobs would never verify (signature would be
+			// from a different key) and would just churn dead-letters; drop
+			// at the source. `parseCanonicalResourceUri` already validated
+			// the rkey grammar and the collection NSID for us, so we only
+			// need the cross-checks here.
+			if (parsed.repo !== opts.did) continue;
+			if (parsed.collection !== opts.collection) continue;
+			messages.push({
+				body: {
+					did: opts.did,
+					collection: opts.collection,
+					rkey: parsed.rkey,
+					operation: "create",
+					cid: record.cid,
+				},
+			});
+		}
+		if (messages.length > 0) {
+			// Page size is capped at QUEUE_SEND_BATCH_CAP at module load via
+			// the static assertion above, so this sendBatch never exceeds
+			// Cloudflare's 100-message limit by construction.
+			await opts.queue.sendBatch(messages);
+			totalEnqueued += messages.length;
+		}
+	} while (cursor);
+	return totalEnqueued;
+}
+
+interface ListRecordEntry {
+	uri: string;
+	cid: string;
+	value: unknown;
+}
+
+/**
+ * Parse a `com.atproto.repo.listRecords` response body. Throws on any
+ * structural mismatch — a PDS that 200s with the wrong shape is upstream
+ * breakage, not "no records", and silently treating it as the latter
+ * causes operator-invisible partial backfills. The thrown error
+ * propagates out of `processBackfillJob` and the queue consumer retries
+ * (then DLQs) per the standard policy.
+ */
+function extractListRecordsBody(body: unknown): ListRecordEntry[] {
+	if (!isPlainObject(body)) {
+		throw new Error("listRecords response was not a JSON object");
+	}
+	const records = body["records"];
+	if (!Array.isArray(records)) {
+		throw new Error("listRecords response missing `records` array");
+	}
+	const out: ListRecordEntry[] = [];
+	for (const r of records) {
+		if (!isPlainObject(r)) {
+			throw new Error("listRecords record entry was not a JSON object");
+		}
+		const uri = r["uri"];
+		const cid = r["cid"];
+		if (typeof uri !== "string" || typeof cid !== "string") {
+			throw new Error("listRecords record entry missing string `uri` or `cid`");
+		}
+		out.push({ uri, cid, value: r["value"] });
+	}
+	return out;
+}
+
+/**
+ * Pull the optional `cursor` out of a `listRecords` response. `undefined`
+ * (no key) is the spec-compliant signal for "end of pagination"; any other
+ * non-string value (number, object, etc.) is a PDS bug and throws so the
+ * pagination loop doesn't silently terminate on the wrong page.
+ */
+function extractCursor(body: unknown): string | undefined {
+	if (!isPlainObject(body)) {
+		throw new Error("listRecords response was not a JSON object");
+	}
+	const cursor = body["cursor"];
+	if (cursor === undefined) return undefined;
+	if (typeof cursor !== "string") {
+		throw new Error(`listRecords cursor was not a string (got ${typeof cursor})`);
+	}
+	return cursor;
+}

apps/aggregator/src/constants.ts

@@ -0,0 +1,28 @@
+/**
+ * Protocol-level constants. These are part of the aggregator's contract with
+ * the EmDash plugin lexicons, not per-deployment configuration — they don't
+ * vary across staging, prod, or self-hosted instances. Per-environment
+ * tunables (Jetstream URL, Constellation URL) live in wrangler.jsonc `vars`.
+ */
+
+/**
+ * NSIDs the aggregator subscribes to via Jetstream and verifies via PDS
+ * fetches. Will migrate to FAIR-namespaced equivalents once those NSIDs
+ * stabilise.
+ *
+ * Two record families:
+ *   - `package.*` — per-package metadata (profile + immutable releases) backing
+ *     the discovery / install path.
+ *   - `publisher.*` — identity-level metadata about the publishing entity
+ *     (`publisher.profile`, rkey `self`) and verification claims about it
+ *     (`publisher.verification`, rkey TID). Verifications are bound to the
+ *     subject's handle + publisher.profile.displayName at issuance time;
+ *     the consumer stores facts as observed and clients re-check validity at
+ *     read time.
+ */
+export const WANTED_COLLECTIONS = [
+	"com.emdashcms.experimental.package.profile",
+	"com.emdashcms.experimental.package.release",
+	"com.emdashcms.experimental.publisher.profile",
+	"com.emdashcms.experimental.publisher.verification",
+] as const;

apps/aggregator/src/did-resolver.ts

@@ -0,0 +1,229 @@
+/**
+ * DID document resolver with a TTL'd cache backed by `known_publishers`.
+ *
+ * The records consumer calls `resolve(did)` once per verification job to learn
+ * the publisher's PDS endpoint and `#atproto` signing key. The signing key is
+ * returned as a `PublicKey` instance (from `@atcute/crypto`) ready to hand to
+ * `verifyRecord` in `@atcute/repo`.
+ *
+ * Pure constructor injection — no D1 imports in the class itself, so tests
+ * pass an in-memory cache and a stub resolver. `createD1DidDocCache(db)` is
+ * the production binding to `known_publishers`.
+ */
+
+import {
+	getPublicKeyFromDidController,
+	P256PublicKey,
+	Secp256k1PublicKey,
+	type PublicKey,
+} from "@atcute/crypto";
+import { type DidDocument, getAtprotoVerificationMaterial, getPdsEndpoint } from "@atcute/identity";
+import { type Did, isDid } from "@atcute/lexicons/syntax";
+
+const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
+
+/** Cache entry shape; the multibase signing key is stored raw so the
+ * `PublicKey` instance is reconstructed on each `resolve()`. WebCrypto
+ * `importKey` is fast enough that an in-memory `PublicKey` cache isn't worth
+ * the complexity. */
+export interface CachedDidDoc {
+	pds: string;
+	signingKey: string; // multibase
+	signingKeyId: string; // e.g. 'did:plc:xxx#atproto'
+	resolvedAt: Date;
+}
+
+export interface DidDocCache {
+	read(did: string): Promise<CachedDidDoc | null>;
+	upsert(did: string, doc: Omit<CachedDidDoc, "resolvedAt">, now: Date): Promise<void>;
+	/** Force the cached row to look stale without disturbing other timestamps
+	 * the cache tracks (e.g. `last_seen_at` in the D1 binding). Used by
+	 * `DidResolver.invalidate()` after a signature failure suggests a key
+	 * rotation. The implementation chooses what "stale" means; the
+	 * Map-backed test cache rewrites `resolvedAt` to epoch, the D1 binding
+	 * sets `pds_resolved_at` only. */
+	expire(did: string): Promise<void>;
+}
+
+export interface DidDocumentResolverLike {
+	resolve(did: Did): Promise<DidDocument>;
+}
+
+export interface DidResolverOptions {
+	cache: DidDocCache;
+	resolver: DidDocumentResolverLike;
+	/** Default 24 hours. Cache entries older than this are re-resolved. */
+	ttlMs?: number;
+	/** Injected for deterministic tests. Defaults to `() => new Date()`. */
+	now?: () => Date;
+}
+
+export interface ResolvedDidDoc {
+	pds: string;
+	publicKey: PublicKey;
+	signingKeyId: string;
+}
+
+export class DidResolver {
+	private readonly cache: DidDocCache;
+	private readonly resolver: DidDocumentResolverLike;
+	private readonly ttlMs: number;
+	private readonly now: () => Date;
+
+	constructor(opts: DidResolverOptions) {
+		this.cache = opts.cache;
+		this.resolver = opts.resolver;
+		this.ttlMs = opts.ttlMs ?? DEFAULT_TTL_MS;
+		this.now = opts.now ?? (() => new Date());
+	}
+
+	async resolve(did: string): Promise<ResolvedDidDoc> {
+		const did_ = asDid(did);
+		const now = this.now();
+		const cached = await this.cache.read(did_);
+		if (cached && now.getTime() - cached.resolvedAt.getTime() < this.ttlMs) {
+			return materialise(cached);
+		}
+		const doc = await this.resolver.resolve(did_);
+		const fresh = extractCacheable(doc);
+		await this.cache.upsert(did_, fresh, now);
+		return materialise({ ...fresh, resolvedAt: now });
+	}
+
+	/** Force a re-resolution next time. Used by the verification path on
+	 * signature failure (the cached signing key may be stale after a
+	 * publisher key rotation). Delegates to the cache's `expire` so other
+	 * timestamps the cache tracks (e.g. `last_seen_at`) aren't disturbed —
+	 * we shouldn't pretend the publisher hasn't been seen since 1970 just
+	 * because we want to drop the cached crypto. */
+	async invalidate(did: string): Promise<void> {
+		await this.cache.expire(asDid(did));
+	}
+}
+
+function asDid(did: string): Did {
+	if (!isDid(did)) {
+		throw new Error(`invalid DID: ${did}`);
+	}
+	return did;
+}
+
+function extractCacheable(doc: DidDocument): Omit<CachedDidDoc, "resolvedAt"> {
+	const pds = getPdsEndpoint(doc);
+	if (!pds) {
+		throw new Error(`DID document has no atproto PDS service entry: ${doc.id}`);
+	}
+	const material = getAtprotoVerificationMaterial(doc);
+	if (!material) {
+		throw new Error(`DID document has no #atproto verification method: ${doc.id}`);
+	}
+	return {
+		pds,
+		signingKey: material.publicKeyMultibase,
+		// Verification method ids are returned by `getAtprotoVerificationMaterial`
+		// only as part of the wider doc; reconstruct the canonical id from the
+		// DID + the well-known fragment.
+		signingKeyId: `${doc.id}#atproto`,
+	};
+}
+
+async function materialise(cached: CachedDidDoc): Promise<ResolvedDidDoc> {
+	// `getPublicKeyFromDidController` only inspects `publicKeyMultibase`; the
+	// `type` field is ignored by the parser (the multibase prefix carries the
+	// curve). Pass a placeholder type — using the actual cached value would
+	// require persisting it in `known_publishers` for no benefit.
+	const found = getPublicKeyFromDidController({
+		type: "Multikey",
+		publicKeyMultibase: cached.signingKey,
+	});
+	let publicKey: PublicKey;
+	if (found.type === "p256") {
+		publicKey = await P256PublicKey.importRaw(found.publicKeyBytes);
+	} else if (found.type === "secp256k1") {
+		publicKey = await Secp256k1PublicKey.importRaw(found.publicKeyBytes);
+	} else {
+		// Exhaustiveness check — `FoundPublicKey` is a discriminated union of
+		// p256 and secp256k1 only. A new variant in a future @atcute/crypto
+		// release should be handled explicitly.
+		const _exhaustive: never = found;
+		throw new Error(`unsupported atproto signing key type`);
+	}
+	return {
+		pds: cached.pds,
+		publicKey,
+		signingKeyId: cached.signingKeyId,
+	};
+}
+
+/**
+ * D1-backed cache binding `known_publishers`. Used in production; tests pass
+ * an in-memory `Map`-backed `DidDocCache` instead.
+ *
+ * `first_seen_at` is set on the first insert and preserved on update. Tests
+ * confirm this — the consumer needs the discovery timestamp to be sticky for
+ * reconciliation reporting later.
+ */
+export function createD1DidDocCache(db: D1Database): DidDocCache {
+	return {
+		async read(did: string): Promise<CachedDidDoc | null> {
+			const row = await db
+				.prepare(
+					`SELECT pds, signing_key, signing_key_id, pds_resolved_at
+					 FROM known_publishers
+					 WHERE did = ?`,
+				)
+				.bind(did)
+				.first<{
+					pds: string | null;
+					signing_key: string | null;
+					signing_key_id: string | null;
+					pds_resolved_at: string | null;
+				}>();
+			if (
+				!row ||
+				row.pds === null ||
+				row.signing_key === null ||
+				row.signing_key_id === null ||
+				row.pds_resolved_at === null
+			) {
+				return null;
+			}
+			return {
+				pds: row.pds,
+				signingKey: row.signing_key,
+				signingKeyId: row.signing_key_id,
+				resolvedAt: new Date(row.pds_resolved_at),
+			};
+		},
+		async upsert(did, doc, now): Promise<void> {
+			const nowIso = now.toISOString();
+			await db
+				.prepare(
+					`INSERT INTO known_publishers
+					   (did, pds, signing_key, signing_key_id, pds_resolved_at, first_seen_at, last_seen_at)
+					 VALUES (?, ?, ?, ?, ?, ?, ?)
+					 ON CONFLICT(did) DO UPDATE SET
+					   pds = excluded.pds,
+					   signing_key = excluded.signing_key,
+					   signing_key_id = excluded.signing_key_id,
+					   pds_resolved_at = excluded.pds_resolved_at,
+					   last_seen_at = excluded.last_seen_at`,
+				)
+				.bind(did, doc.pds, doc.signingKey, doc.signingKeyId, nowIso, nowIso, nowIso)
+				.run();
+		},
+		async expire(did): Promise<void> {
+			// Touches only `pds_resolved_at`; first_seen_at / last_seen_at /
+			// the cached crypto are intentionally untouched. Setting to epoch
+			// is unambiguous "older than any plausible TTL". No-op when the
+			// row doesn't exist.
+			await db
+				.prepare(
+					`UPDATE known_publishers SET pds_resolved_at = '1970-01-01T00:00:00.000Z'
+					 WHERE did = ?`,
+				)
+				.bind(did)
+				.run();
+		},
+	};
+}

apps/aggregator/src/env.ts

@@ -0,0 +1,37 @@
+/**
+ * Project-specific message types passed across binding boundaries.
+ *
+ * The Worker's `Env` type itself is generated by `wrangler types` into
+ * `worker-configuration.d.ts` (committed alongside this file), exposed
+ * globally as `Env` and via `Cloudflare.Env`. Re-running `wrangler types`
+ * after editing `wrangler.jsonc` keeps it in sync.
+ */
+
+export interface RecordsJob {
+	did: string;
+	collection: string;
+	rkey: string;
+	operation: "create" | "update" | "delete";
+	cid: string;
+	/**
+	 * The Jetstream-supplied (unverified) record bytes. Compared against the
+	 * verified PDS copy after fetch as a Jetstream-correctness signal; the
+	 * verified copy always wins.
+	 */
+	jetstreamRecord?: unknown;
+}
+
+/**
+ * One unit of cold-start backfill work: walk every record under `collection`
+ * for `did` via that DID's PDS, then enqueue each record onto the records
+ * queue for the standard verify-and-write path.
+ *
+ * Granularity is (DID, collection) rather than per-DID because the per-DID
+ * fan-out can exceed Cloudflare's 30s `ctx.waitUntil` wall-clock cap. One
+ * collection's worth of pagination fits comfortably under that ceiling and
+ * gets queue-level retry + concurrency for free.
+ */
+export interface BackfillJob {
+	did: string;
+	collection: string;
+}

apps/aggregator/src/index.ts

@@ -0,0 +1,369 @@
+/**
+ * EmDash plugin registry aggregator: Worker entrypoint.
+ *
+ * Reading order for someone learning this code:
+ *   1. `constants.ts` — protocol-level constants (NSIDs, etc.).
+ *   2. `env.ts` — project-specific message types passed across binding
+ *      boundaries. The Worker `Env` is generated by `wrangler types` into
+ *      `worker-configuration.d.ts`.
+ *   3. `records-do.ts` — Jetstream connection.
+ *   4. `records-consumer.ts` — PDS-verified ingest.
+ *   5. `routes/*.ts` — XRPC read endpoints.
+ *
+ * The fetch/queue/scheduled handlers below are wired but no-op for now;
+ * the smoke test in `test/smoke.test.ts` proves migrations apply and the
+ * Worker boots.
+ */
+
+import { isDid } from "@atcute/lexicons/syntax";
+
+import { drainBackfillDeadLetterBatch, processBackfillBatch } from "./backfill-consumer.js";
+import { discoverDids, enqueueBackfillJobs } from "./backfill.js";
+import type { BackfillJob, RecordsJob } from "./env.js";
+import { drainDeadLetterBatch, processBatch } from "./records-consumer.js";
+import { RECORDS_DO_NAME } from "./records-do.js";
+import { handleXrpc } from "./routes/xrpc/router.js";
+import { isPlainObject } from "./utils.js";
+
+const RECORDS_QUEUE_NAME = "emdash-aggregator-records";
+const RECORDS_DLQ_NAME = "emdash-aggregator-records-dlq";
+const BACKFILL_QUEUE_NAME = "emdash-aggregator-backfill";
+const BACKFILL_DLQ_NAME = "emdash-aggregator-backfill-dlq";
+
+export { RecordsJetstreamDO } from "./records-do.js";
+
+/**
+ * Operational admin routes. Both gated by the `ADMIN_TOKEN` secret declared
+ * in `wrangler.jsonc`'s `secrets.required` and validated via constant-time
+ * compare against the `Authorization: Bearer <token>` header. Recommended
+ * deploy hook:
+ *
+ *   wrangler deploy && curl -X POST -H "Authorization: Bearer $ADMIN_TOKEN" \
+ *     https://api.emdashcms.com/_admin/start
+ *
+ * `/_admin/start` spins up the Records DO (idempotent on an already-running DO).
+ * `/_admin/backfill` discovers publishers (or accepts an explicit DID list)
+ * and fans (DID, collection) jobs onto `BACKFILL_QUEUE`; the consumer in
+ * `backfill-consumer.ts` does the per-pair listRecords + records-queue
+ * fan-out work asynchronously, getting queue retry + concurrency for free
+ * and sidestepping the 30s `waitUntil` cap on the POST handler.
+ *
+ * Auth-gating both routes: backfill specifically introduces caller-chosen
+ * URLs into the worker's outbound fetches (via `did:web` resolution + the
+ * publisher's PDS endpoint), so the unauth posture would expose an SSRF-shaped
+ * surface. Same gate on `/_admin/start` for symmetry — anyone with the token
+ * can do operational things.
+ */
+const BOOTSTRAP_PATH = "/_admin/start";
+const BACKFILL_PATH = "/_admin/backfill";
+const STATUS_PATH = "/_admin/status";
+
+/**
+ * Cap on the explicit DID list a single POST may submit. Lower than the
+ * old serial-loop cap (was 1000) because the queue-fan-out path amplifies
+ * a leaked-token attack: each submitted DID becomes
+ * `WANTED_COLLECTIONS.length` queue messages, each consuming a consumer
+ * invocation with its own outbound PDS fetches. 100 × 4 = 400 jobs from
+ * the explicit path is a meaningful operator-recovery batch but a
+ * tractable blast radius for the explicit path.
+ *
+ * The discovery path (empty body) has a separate ceiling at
+ * `MAX_DISCOVERED_DIDS = 1000` and is therefore actually the larger
+ * worst-case fan-out source — by design, since legitimate-publisher
+ * enumeration is the primary use case and we want the explicit list to
+ * be the tighter "operator types it themselves" bucket. Both paths share
+ * the same per-pair caps in the consumer.
+ */
+const MAX_BACKFILL_DIDS = 100;
+
+const tokenEncoder = new TextEncoder();
+
+/**
+ * Constant-time string equality via workerd's audited
+ * `crypto.subtle.timingSafeEqual`. The primitive returns `false` immediately
+ * for length-mismatched buffers, so the *prefix*-comparison is constant-time
+ * but a length difference is still observable via timing — acceptable here
+ * because the protected secret (`ADMIN_TOKEN`) has a fixed configured length
+ * known only to the operator, and any realistic length-via-timing attack
+ * would require so many requests that other defences (rate-limiting,
+ * Cloudflare Bot Management, log review) catch it first.
+ */
+function timingSafeEqual(a: string, b: string): boolean {
+	const aBuf = tokenEncoder.encode(a);
+	const bBuf = tokenEncoder.encode(b);
+	if (aBuf.byteLength !== bBuf.byteLength) return false;
+	return crypto.subtle.timingSafeEqual(aBuf, bBuf);
+}
+
+/**
+ * Validate the request's `Authorization: Bearer <token>` header against
+ * `env.ADMIN_TOKEN`. Returns null on success, or a 401 Response to return
+ * directly. Empty/missing token in env fails closed.
+ */
+function requireAdminAuth(request: Request, env: Env): Response | null {
+	const expected = env.ADMIN_TOKEN;
+	// `trim()` defends against a whitespace-only secret slipping past
+	// `secrets.required`'s presence check — `wrangler secret put ADMIN_TOKEN`
+	// followed by an accidental Enter would otherwise produce a working
+	// endpoint with a trivially guessable token.
+	if (!expected || expected.trim().length === 0) {
+		// Misconfigured production or unset dev — closed by default.
+		return new Response("admin endpoints not configured", { status: 503 });
+	}
+	const auth = request.headers.get("authorization");
+	const SCHEME_PREFIX = "bearer ";
+	// RFC 6750 §2.1: the auth scheme is case-insensitive. `curl -H
+	// "authorization: bearer ..."` and SDKs that don't canonicalise the
+	// scheme would otherwise fail with a confusing 401 even though the
+	// token is correct.
+	if (
+		!auth ||
+		auth.length < SCHEME_PREFIX.length ||
+		auth.slice(0, SCHEME_PREFIX.length).toLowerCase() !== SCHEME_PREFIX
+	) {
+		return new Response("unauthorized", {
+			status: 401,
+			headers: { "www-authenticate": "Bearer" },
+		});
+	}
+	const token = auth.slice(SCHEME_PREFIX.length);
+	if (!timingSafeEqual(token, expected)) {
+		return new Response("unauthorized", {
+			status: 401,
+			headers: { "www-authenticate": "Bearer" },
+		});
+	}
+	return null;
+}
+
+type BackfillRequest = { mode: "explicit"; dids: string[] } | { mode: "discover" };
+
+function parseBackfillBody(body: unknown): BackfillRequest | { error: string } {
+	if (!isPlainObject(body)) {
+		return { error: "request body must be a JSON object" };
+	}
+	const rawDids = body["dids"];
+	// Empty body OR `{ "dids": null }` OR `{ "dids": undefined }` ⇒ discovery
+	// mode. Production cold-start uses this path; the explicit list is the
+	// testing / recovery seam.
+	if (rawDids === undefined || rawDids === null) {
+		return { mode: "discover" };
+	}
+	if (!Array.isArray(rawDids)) {
+		return {
+			error: "`dids` must be an array of DID strings, or omitted to discover via the relay",
+		};
+	}
+	if (rawDids.length === 0) {
+		return {
+			error: "`dids` must not be empty (omit the field to discover via the relay)",
+		};
+	}
+	if (rawDids.length > MAX_BACKFILL_DIDS) {
+		return {
+			error: `\`dids\` must contain at most ${MAX_BACKFILL_DIDS} entries (got ${rawDids.length})`,
+		};
+	}
+	const seen = new Set<string>();
+	for (const did of rawDids) {
+		if (!isDid(did)) {
+			return { error: `invalid DID in list: ${JSON.stringify(did)}` };
+		}
+		seen.add(did);
+	}
+	// Set iteration order matches insertion; dedup preserves first-seen order
+	// so the operator sees jobs run in the order they submitted.
+	return { mode: "explicit", dids: [...seen] };
+}
+
+export default {
+	async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
+		const url = new URL(request.url);
+		if (url.pathname === BOOTSTRAP_PATH) {
+			if (request.method !== "POST") {
+				return new Response("method not allowed", {
+					status: 405,
+					headers: { allow: "POST" },
+				});
+			}
+			const denied = requireAdminAuth(request, env);
+			if (denied) return denied;
+			const id = env.RECORDS_DO.idFromName(RECORDS_DO_NAME);
+			const stub = env.RECORDS_DO.get(id);
+			// Fire-and-forget so the response shape doesn't depend on the
+			// DO's status output. Caller gets the same 204 whether the DO
+			// was already running, just woke up, or is mid-startup.
+			ctx.waitUntil(stub.fetch("https://do.internal/bootstrap"));
+			return new Response(null, { status: 204 });
+		}
+		if (url.pathname === STATUS_PATH) {
+			if (request.method !== "GET") {
+				return new Response("method not allowed", {
+					status: 405,
+					headers: { allow: "GET" },
+				});
+			}
+			const denied = requireAdminAuth(request, env);
+			if (denied) return denied;
+			// Proxy the DO's status JSON straight through. The DO returns
+			// `{cursor, consecutiveFailures}` — `consecutiveFailures: 0`
+			// means the latest connection attempt produced events; non-zero
+			// means Jetstream is unreachable, the wantedCollections filter
+			// is mismatched, or queue backpressure is biting.
+			const id = env.RECORDS_DO.idFromName(RECORDS_DO_NAME);
+			const stub = env.RECORDS_DO.get(id);
+			return stub.fetch("https://do.internal/status");
+		}
+		if (url.pathname === BACKFILL_PATH) {
+			if (request.method !== "POST") {
+				return new Response("method not allowed", {
+					status: 405,
+					headers: { allow: "POST" },
+				});
+			}
+			const denied = requireAdminAuth(request, env);
+			if (denied) return denied;
+			// Empty / no body is the production discovery path. JSON-parse
+			// failures with no content (Content-Length: 0) come back as
+			// SyntaxError; we treat that as "discover" rather than 400ing
+			// so `curl -X POST ... /_admin/backfill` (no body) does the
+			// expected thing.
+			let body: unknown = {};
+			const text = await request.text();
+			if (text.length > 0) {
+				try {
+					body = JSON.parse(text);
+				} catch {
+					return new Response("request body must be valid JSON", { status: 400 });
+				}
+			}
+			const parsed = parseBackfillBody(body);
+			if ("error" in parsed) {
+				return new Response(parsed.error, { status: 400 });
+			}
+			// Fire-and-forget via waitUntil so the route returns 202 quickly.
+			// `runBackfill` only does discovery + queue fan-out (both fast);
+			// per-pair work runs in the BACKFILL_QUEUE consumer with its own
+			// invocation budget. Operator-facing observability:
+			//   - this handler's logs ([aggregator] backfill discovery/enqueue)
+			//   - per-pair logs from `backfill-consumer.ts`
+			//   - DLQ inspection (`emdash-aggregator-backfill-dlq`) for
+			//     pairs that exhausted retries
+			ctx.waitUntil(runBackfill(parsed, env));
+			return new Response(null, { status: 202 });
+		}
+		// XRPC read API: aggregator endpoints + cached sync.getRecord
+		// passthrough. Returns null if pathname doesn't start with /xrpc/, so
+		// non-matching paths fall through to the catch-all below.
+		const xrpc = await handleXrpc(env, request);
+		if (xrpc) return xrpc;
+		return new Response("emdash-aggregator: not found", {
+			status: 404,
+			headers: { "content-type": "text/plain" },
+		});
+	},
+
+	async queue(batch: MessageBatch, env: Env, _ctx: ExecutionContext): Promise<void> {
+		// Workerd routes every consumer here; dispatch by queue name to the
+		// matching typed handler. The parameter is unparameterised
+		// (`MessageBatch` defaults to `MessageBatch<unknown>`) because the
+		// binding is shared across queues — narrowing happens per-case via
+		// the queue name, which is a runtime tag the compiler can't see.
+		switch (batch.queue) {
+			case RECORDS_QUEUE_NAME:
+				// eslint-disable-next-line typescript/no-unsafe-type-assertion -- narrowed by queue name
+				await processBatch(batch as MessageBatch<RecordsJob>, env);
+				return;
+			case RECORDS_DLQ_NAME:
+				// eslint-disable-next-line typescript/no-unsafe-type-assertion -- narrowed by queue name
+				await drainDeadLetterBatch(batch as MessageBatch<RecordsJob>, env);
+				return;
+			case BACKFILL_QUEUE_NAME:
+				// eslint-disable-next-line typescript/no-unsafe-type-assertion -- narrowed by queue name
+				await processBackfillBatch(batch as MessageBatch<BackfillJob>, env);
+				return;
+			case BACKFILL_DLQ_NAME:
+				// eslint-disable-next-line typescript/no-unsafe-type-assertion -- narrowed by queue name
+				drainBackfillDeadLetterBatch(batch as MessageBatch<BackfillJob>, env);
+				return;
+			default:
+				console.error("[aggregator] unknown queue, acking batch", { queue: batch.queue });
+				for (const m of batch.messages) m.ack();
+		}
+	},
+
+	async scheduled(_event: ScheduledEvent, env: Env, ctx: ExecutionContext): Promise<void> {
+		// DO liveness. The records DO is meant to hold an outbound WebSocket
+		// continuously, but during a Jetstream outage it spends time in
+		// backoff sleeps — that's when CF can evict it. Hitting the DO from
+		// the cron wakes it back up; constructor-time `ingestor.run()`
+		// resumes from the persisted cursor. Reconciliation work will share
+		// this trigger when it lands.
+		const id = env.RECORDS_DO.idFromName(RECORDS_DO_NAME);
+		const stub = env.RECORDS_DO.get(id);
+		ctx.waitUntil(stub.fetch("https://do.internal/liveness"));
+	},
+};
+
+type BackfillRequestParsed = { mode: "explicit"; dids: string[] } | { mode: "discover" };
+
+/**
+ * Backfill orchestrator. Runs inside the POST handler's `ctx.waitUntil`.
+ * Two paths:
+ *
+ *   - "discover" — call `com.atproto.sync.listReposByCollection` against
+ *     `env.RELAY_URL` for each NSID in WANTED_COLLECTIONS, union the DIDs,
+ *     then fan them out as backfill jobs.
+ *   - "explicit" — operator-supplied DID list, used for testing or for
+ *     recovery of a known DID set.
+ *
+ * In both cases the actual per-(DID, collection) work runs in the
+ * `BACKFILL_QUEUE` consumer (see `backfill-consumer.ts`). This orchestrator
+ * just discovers + enqueues — both fast operations that fit well within
+ * Cloudflare's 30s `waitUntil` ceiling regardless of fan-out size, even
+ * after a `MAX_DISCOVERED_DIDS`-sized union.
+ *
+ * Per-pair progress is logged from the consumer; this function only logs
+ * the discovery + fan-out result so an operator watching `wrangler tail`
+ * sees how many jobs were enqueued.
+ */
+async function runBackfill(req: BackfillRequestParsed, env: Env): Promise<void> {
+	try {
+		let dids: readonly string[];
+		if (req.mode === "discover") {
+			console.log("[aggregator] backfill discovery starting", {
+				relay: env.RELAY_URL,
+			});
+			dids = await discoverDids(env.RELAY_URL);
+			console.log("[aggregator] backfill discovery complete", {
+				relay: env.RELAY_URL,
+				didCount: dids.length,
+			});
+		} else {
+			dids = req.dids;
+			console.log("[aggregator] backfill enqueue starting", {
+				mode: "explicit",
+				didCount: dids.length,
+			});
+		}
+
+		if (dids.length === 0) {
+			console.warn("[aggregator] backfill produced zero DIDs, nothing to enqueue", {
+				mode: req.mode,
+			});
+			return;
+		}
+
+		const enqueued = await enqueueBackfillJobs(dids, env.BACKFILL_QUEUE);
+		console.log("[aggregator] backfill enqueue complete", {
+			mode: req.mode,
+			didCount: dids.length,
+			jobsEnqueued: enqueued,
+		});
+	} catch (err) {
+		console.error("[aggregator] backfill aborted", {
+			mode: req.mode,
+			error: err instanceof Error ? (err.stack ?? err.message) : String(err),
+		});
+	}
+}

apps/aggregator/src/jetstream-client.ts

@@ -0,0 +1,198 @@
+/**
+ * Jetstream client abstraction.
+ *
+ * Production wraps `@atcute/jetstream`'s `JetstreamSubscription`. Tests bind
+ * `MockJetstream` from `@emdash-cms/atproto-test-utils`. The ingestor only
+ * depends on this interface, so the same code path runs in both worlds.
+ *
+ * The shape mirrors the subset of `JetstreamSubscription` we actually use:
+ *   - async-iterable of commit events (we don't process identity/account
+ *     events today),
+ *   - a `cursor` getter exposing the time_us of the most recent event the
+ *     iterator has yielded — used to persist the cursor for reconnection,
+ *   - an explicit close.
+ *
+ * Open question we may revisit: real Jetstream emits identity + account
+ * events alongside commits. The ingestor narrows to commits today; if we
+ * grow to care about identity events for handle changes, widen the event
+ * type here and update the consumer.
+ */
+
+import { JetstreamSubscription } from "@atcute/jetstream";
+
+export interface JetstreamCommitEvent {
+	did: `did:${string}:${string}`;
+	time_us: number;
+	kind: "commit";
+	commit:
+		| {
+				rev: string;
+				collection: string;
+				rkey: string;
+				operation: "create" | "update";
+				cid: string;
+				record: Record<string, unknown>;
+		  }
+		| {
+				rev: string;
+				collection: string;
+				rkey: string;
+				operation: "delete";
+		  };
+}
+
+export interface JetstreamSubscribeOptions {
+	wantedCollections: readonly string[];
+	cursor?: number;
+}
+
+export interface JetstreamSubscriptionHandle extends AsyncIterable<JetstreamCommitEvent> {
+	readonly cursor: number;
+	close(): void;
+}
+
+export interface JetstreamClient {
+	subscribe(opts: JetstreamSubscribeOptions): JetstreamSubscriptionHandle;
+}
+
+/**
+ * Production client backed by `@atcute/jetstream`. Filters non-commit events
+ * before yielding, so the ingestor doesn't have to switch on `kind` every
+ * iteration.
+ */
+export class RealJetstreamClient implements JetstreamClient {
+	constructor(private readonly url: string) {}
+
+	subscribe(opts: JetstreamSubscribeOptions): JetstreamSubscriptionHandle {
+		const sub = new JetstreamSubscription({
+			url: this.url,
+			wantedCollections: [...opts.wantedCollections],
+			...(opts.cursor !== undefined ? { cursor: opts.cursor } : {}),
+		});
+		return wrapAtcuteSubscription(sub);
+	}
+}
+
+/**
+ * Minimum shape `wrapAtcuteSubscription` needs from its input: a `cursor`
+ * getter and an async iterable of events with a `kind` discriminator. Both
+ * `JetstreamSubscription` (production) and a stub-with-never-resolving-next
+ * (the C2 regression test) satisfy this without casts.
+ */
+export interface RawJetstreamSubscription<E extends { kind: string }> extends AsyncIterable<E> {
+	readonly cursor: number;
+}
+
+/**
+ * Exported so tests can drive the wrapper against a stub subscription with a
+ * never-resolving `next()` and verify that `close()` actually cancels the
+ * pending await. Production callers should construct via `RealJetstreamClient`.
+ */
+export function wrapAtcuteSubscription<E extends { kind: string }>(
+	sub: RawJetstreamSubscription<E>,
+): JetstreamSubscriptionHandle {
+	// Hoist the inner iterator so `close()` can reach it from outside the
+	// iterator factory.
+	let inner: AsyncIterator<E> | null = null;
+	// Shutdown signal raced against `inner.next()`. We can't rely on
+	// `inner.return()` to unblock a pending `next()` — `@mary-ext/event-iterator`
+	// drops its resolver on `return()` without invoking it (lib/index.ts:55-67),
+	// so a quiescent stream's pending `next()` Promise leaks. Racing against
+	// `closedSignal` lets the consumer wake regardless of the inner iterator's
+	// behaviour. The orphaned `it.next()` Promise is one of:
+	//   - resolved later when an event arrives (harmless, garbage-collected).
+	//   - leaked forever if Jetstream stays quiescent (no value held; only
+	//     the Promise object is GC-rooted by the inner iterator's #resolve).
+	let resolveClosed: (() => void) | null = null;
+	const closedSignal = new Promise<void>((resolve) => {
+		resolveClosed = resolve;
+	});
+	const fireClosed = () => {
+		if (resolveClosed) {
+			const r = resolveClosed;
+			resolveClosed = null;
+			r();
+		}
+	};
+
+	return {
+		get cursor() {
+			return sub.cursor;
+		},
+		close: () => {
+			fireClosed();
+			// `.catch` swallows rejections from the inner iterator's cleanup
+			// (an EventIterator's `return()` shouldn't reject, but a future
+			// implementation could). Without this, a rejection here would
+			// surface as an unhandled-promise warning in workerd.
+			inner?.return?.()?.catch(() => {});
+		},
+		[Symbol.asyncIterator](): AsyncIterator<JetstreamCommitEvent> {
+			inner ??= sub[Symbol.asyncIterator]();
+			const it = inner;
+			return {
+				async next(): Promise<IteratorResult<JetstreamCommitEvent>> {
+					for (;;) {
+						const result = await Promise.race([
+							it.next(),
+							closedSignal.then((): IteratorResult<E> => ({ value: undefined, done: true })),
+						]);
+						if (result.done) return { value: undefined, done: true };
+						const event = result.value;
+						if (isCommitEvent(event)) {
+							return { value: event, done: false };
+						}
+						// Skip identity/account events; loop until next commit.
+					}
+				},
+				async return(): Promise<IteratorResult<JetstreamCommitEvent>> {
+					fireClosed();
+					await it.return?.();
+					return { value: undefined, done: true };
+				},
+			};
+		},
+	};
+}
+
+/**
+ * Discriminator + structural predicate that narrows to `JetstreamCommitEvent`.
+ *
+ * The runtime check verifies BOTH `kind === "commit"` AND that `commit` is
+ * present and shaped enough for the ingestor's downstream access (it reads
+ * `event.commit.collection`, `event.commit.rkey`, `event.commit.operation`,
+ * `event.commit.cid`). Without the structural check, a producer emitting
+ * `{kind: "commit"}` with no `commit` field would crash the ingestor on
+ * access; the cursor wouldn't advance; Jetstream would replay the same
+ * malformed event forever.
+ */
+/** Wider parameter type than the bare `{ kind: string }` constraint so the
+ * predicate can inspect `commit` without an unsafe cast. Any producer
+ * conforming to `RawJetstreamSubscription<E>` where `E extends { kind: string }`
+ * is assignable here because `commit` is optional. */
+type MaybeCommitEvent = {
+	kind: string;
+	commit?: {
+		collection?: unknown;
+		rkey?: unknown;
+		operation?: unknown;
+		cid?: unknown;
+	};
+};
+
+const KNOWN_OPERATIONS = new Set(["create", "update", "delete"]);
+
+function isCommitEvent(event: MaybeCommitEvent): event is JetstreamCommitEvent {
+	if (event.kind !== "commit" || event.commit === undefined) return false;
+	const c = event.commit;
+	if (typeof c.collection !== "string" || typeof c.rkey !== "string") return false;
+	// Restrict to the operations the downstream RecordsJob + applyDelete
+	// dispatcher know about. An unknown operation slipping through would
+	// produce a job the consumer can't process and would land in
+	// dead_letters as UNEXPECTED_ERROR — better to drop it at the source.
+	if (typeof c.operation !== "string" || !KNOWN_OPERATIONS.has(c.operation)) return false;
+	// `cid` is required for create/update (the ingestor reads it into the
+	// RecordsJob); delete events legitimately have no cid.
+	if (c.operation !== "delete" && typeof c.cid !== "string") return false;
+	return true;
+}

apps/aggregator/src/jetstream-ingestor.ts

@@ -0,0 +1,276 @@
+/**
+ * Jetstream ingestor. Subscribes to a Jetstream client, converts commit
+ * events into `RecordsJob` messages, enqueues them, and persists the
+ * cursor so reconnects resume cleanly.
+ *
+ * Owns:
+ *   - Connection lifecycle (connect → consume → disconnect → backoff →
+ *     reconnect, indefinitely until `stop()` is called).
+ *   - Cursor persistence after each successful enqueue. Persisting AFTER
+ *     enqueue means a crash can replay the most recent event; downstream
+ *     ingest is idempotent (DO NOTHING on duplicate releases) so this is
+ *     safe and strictly better than the alternative of skipping events.
+ *   - Exponential backoff with jitter, capped, reset on each successful
+ *     event.
+ *
+ * Pure constructor injection — no DO/D1/Queue infrastructure imports — so
+ * unit tests instantiate it directly with `MockJetstream` + an in-memory
+ * queue + a `Map`-backed storage.
+ */
+
+import { WANTED_COLLECTIONS } from "./constants.js";
+import type { RecordsJob } from "./env.js";
+import type {
+	JetstreamClient,
+	JetstreamCommitEvent,
+	JetstreamSubscriptionHandle,
+} from "./jetstream-client.js";
+
+const CURSOR_STORAGE_KEY = "jetstream:cursor";
+
+/**
+ * Subset of `Queue.send` we use. Return type is loose because workerd's
+ * `Queue.send` resolves to `QueueSendResponse` while a hand-rolled
+ * in-memory test queue resolves to `void` — neither piece of metadata
+ * matters to the ingestor.
+ */
+export interface JobQueue {
+	send(job: RecordsJob): Promise<unknown>;
+}
+
+/** Subset of DurableObjectStorage we use. Tests pass a Map-backed shim. */
+export interface IngestorStorage {
+	get(key: string): Promise<number | undefined>;
+	put(key: string, value: number): Promise<void>;
+}
+
+export interface IngestorBackoffConfig {
+	/** Initial delay after the first disconnect (ms). Default 1s. */
+	initialDelayMs?: number;
+	/** Cap (ms). Default 60s. */
+	maxDelayMs?: number;
+	/** Multiplier per retry. Default 2. */
+	multiplier?: number;
+	/** ±jitter as a fraction of the delay. Default 0.2 (±20%). 0 disables. */
+	jitter?: number;
+}
+
+export interface IngestorLogger {
+	info?(msg: string, ctx?: Record<string, unknown>): void;
+	warn?(msg: string, ctx?: Record<string, unknown>): void;
+	error?(msg: string, ctx?: Record<string, unknown>): void;
+}
+
+export interface JetstreamIngestorOptions {
+	client: JetstreamClient;
+	queue: JobQueue;
+	storage: IngestorStorage;
+	/** Defaults to the protocol-level WANTED_COLLECTIONS constant. */
+	wantedCollections?: readonly string[];
+	backoff?: IngestorBackoffConfig;
+	logger?: IngestorLogger;
+	/** Sleep impl, swap in tests to skip real backoff waits. */
+	sleep?: (ms: number) => Promise<void>;
+	/** Random source for jitter, swap in tests for determinism. */
+	random?: () => number;
+	/**
+	 * Called when no cursor is persisted in DO storage (fresh deploy,
+	 * regional failover, dev-state wipe). Should return a Jetstream
+	 * `time_us` (microseconds since epoch) to start from, or `null` to
+	 * fall back to the subscription library's default (effectively
+	 * "now"). The DO supplies `MAX(verified_at)` across the four
+	 * content tables — events older than the latest record we've
+	 * already verified are no-ops thanks to consumer idempotency, but
+	 * events newer than that are the gap we'd otherwise miss between
+	 * our last known data and the moment Jetstream reconnects.
+	 *
+	 * Optional so tests can omit it; production wiring always supplies
+	 * it via the DO.
+	 */
+	cursorFloor?: () => Promise<number | null>;
+}
+
+const DEFAULT_BACKOFF: Required<IngestorBackoffConfig> = {
+	initialDelayMs: 1_000,
+	maxDelayMs: 60_000,
+	multiplier: 2,
+	jitter: 0.2,
+};
+
+export class JetstreamIngestor {
+	private readonly client: JetstreamClient;
+	private readonly queue: JobQueue;
+	private readonly storage: IngestorStorage;
+	private readonly wantedCollections: readonly string[];
+	private readonly backoff: Required<IngestorBackoffConfig>;
+	private readonly logger: IngestorLogger;
+	private readonly sleep: (ms: number) => Promise<void>;
+	private readonly random: () => number;
+
+	private stopped = false;
+	private currentSub: JetstreamSubscriptionHandle | null = null;
+	private cursor: number | null = null;
+	/** Set on every successful enqueue. The reconnect loop resets the
+	 * backoff counter when this is true at the start of a new attempt, so a
+	 * subscription that connects, consumes events, and then drops doesn't
+	 * spiral into ever-larger backoffs. */
+	private madeProgress = false;
+
+	private readonly cursorFloor: (() => Promise<number | null>) | null;
+
+	constructor(opts: JetstreamIngestorOptions) {
+		this.client = opts.client;
+		this.queue = opts.queue;
+		this.storage = opts.storage;
+		this.wantedCollections = opts.wantedCollections ?? WANTED_COLLECTIONS;
+		this.backoff = { ...DEFAULT_BACKOFF, ...opts.backoff };
+		this.logger = opts.logger ?? {};
+		this.sleep = opts.sleep ?? defaultSleep;
+		this.random = opts.random ?? Math.random;
+		this.cursorFloor = opts.cursorFloor ?? null;
+	}
+
+	/** The cursor most recently enqueued + persisted. `null` until the first event. */
+	get currentCursor(): number | null {
+		return this.cursor;
+	}
+
+	/**
+	 * Run the connect-consume-reconnect loop until `stop()` is called.
+	 *
+	 * Resolves when `stop()` is called and the current subscription drains.
+	 * Does NOT reject for transient failures — connection drops, parse
+	 * errors, queue.send rejections all increment the backoff counter and
+	 * retry. The DO observes liveness via the `currentCursor` getter and
+	 * the failure counter exposed on the ingestor. We could instead bubble
+	 * queue failures and let the DO crash loud; current choice is to
+	 * absorb them because Cloudflare Queues have transient failures and
+	 * the cron liveness ping recovers the DO either way.
+	 */
+	async run(): Promise<void> {
+		this.cursor = (await this.storage.get(CURSOR_STORAGE_KEY)) ?? null;
+		if (this.cursor === null && this.cursorFloor !== null) {
+			// Storage was empty (fresh deploy / regional failover / dev wipe)
+			// but our content tables may have rows from a prior backfill.
+			// Derive a cursor from the latest verified record so we don't
+			// silently skip events between backfill-time and reconnect-time.
+			// Persist immediately so a crash before the first Jetstream event
+			// arrives doesn't re-derive on the next run (the floor function
+			// is allowed to be expensive).
+			try {
+				const floor = await this.cursorFloor();
+				if (floor !== null) {
+					this.cursor = floor;
+					await this.storage.put(CURSOR_STORAGE_KEY, floor);
+					this.logger.info?.("jetstream cursor seeded from D1 floor", { cursor: floor });
+				}
+			} catch (err) {
+				this.logger.warn?.("cursorFloor lookup failed, falling back to subscription default", {
+					error: err instanceof Error ? err.message : String(err),
+				});
+			}
+		}
+
+		while (!this.stopped) {
+			try {
+				await this.connectAndConsume();
+				// Subscription ended cleanly. If we consumed at least one
+				// event, the connection was healthy — reset the counter and
+				// reconnect with the floor delay. Otherwise treat as a soft
+				// failure and grow the backoff.
+				if (this.madeProgress) this._consecutiveFailures = 0;
+				else this._consecutiveFailures += 1;
+			} catch (err) {
+				if (this.madeProgress) this._consecutiveFailures = 0;
+				else this._consecutiveFailures += 1;
+				this.logger.warn?.("jetstream subscription failed", {
+					error: err instanceof Error ? err.message : String(err),
+					consecutiveFailures: this._consecutiveFailures,
+				});
+			}
+			if (this.stopped) break;
+			await this.sleep(this.computeBackoff(this._consecutiveFailures));
+		}
+	}
+
+	/** Number of consecutive failed/empty connection attempts. Exposed for
+	 * liveness probes; `0` means the most recent attempt produced events. */
+	get consecutiveFailures(): number {
+		return this._consecutiveFailures;
+	}
+	private _consecutiveFailures = 0;
+
+	stop(): void {
+		this.stopped = true;
+		this.currentSub?.close();
+	}
+
+	private async connectAndConsume(): Promise<void> {
+		// Tied to one connection attempt: set true when we actually enqueue
+		// an event, read by the run loop to decide whether to reset
+		// backoff. Resetting per-attempt (rather than per-loop-iteration at
+		// the top of run()) keeps the flag's lifetime crisp.
+		this.madeProgress = false;
+		const sub = this.client.subscribe({
+			wantedCollections: this.wantedCollections,
+			...(this.cursor !== null ? { cursor: this.cursor } : {}),
+		});
+		this.currentSub = sub;
+		try {
+			for await (const event of sub) {
+				if (this.stopped) break;
+				await this.handleEvent(event);
+			}
+		} finally {
+			sub.close();
+			if (this.currentSub === sub) this.currentSub = null;
+		}
+	}
+
+	private async handleEvent(event: JetstreamCommitEvent): Promise<void> {
+		// Defence in depth: Jetstream filters server-side, but a future
+		// subscription change or a malicious relay could deliver something
+		// off-list. Trust nothing.
+		if (!this.wantedCollections.includes(event.commit.collection)) return;
+
+		const job: RecordsJob = {
+			did: event.did,
+			collection: event.commit.collection,
+			rkey: event.commit.rkey,
+			operation: event.commit.operation,
+			cid: event.commit.operation === "delete" ? "" : event.commit.cid,
+			...(event.commit.operation !== "delete" ? { jetstreamRecord: event.commit.record } : {}),
+		};
+
+		await this.queue.send(job);
+		// Persist cursor only after the queue has accepted the message.
+		// A crash between enqueue and persist replays the latest event on
+		// recovery; the consumer's idempotency rules (DO NOTHING on
+		// duplicate releases, upsert on profiles) absorb the duplicate.
+		this.cursor = event.time_us;
+		await this.storage.put(CURSOR_STORAGE_KEY, event.time_us);
+		this.madeProgress = true;
+	}
+
+	private computeBackoff(failures: number): number {
+		// Defensive: `failures` is always >= 1 when called from the run loop
+		// (the increment happens before computeBackoff), but a future caller
+		// passing 0 would give `initialDelayMs / multiplier`, which is below
+		// the floor. Clamp explicitly.
+		const exp = Math.min(
+			Math.max(
+				this.backoff.initialDelayMs,
+				this.backoff.initialDelayMs * this.backoff.multiplier ** (failures - 1),
+			),
+			this.backoff.maxDelayMs,
+		);
+		if (this.backoff.jitter <= 0) return exp;
+		const range = exp * this.backoff.jitter;
+		const offset = (this.random() * 2 - 1) * range;
+		return Math.max(0, Math.round(exp + offset));
+	}
+}
+
+function defaultSleep(ms: number): Promise<void> {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}

apps/aggregator/src/pds-verify.ts

@@ -0,0 +1,233 @@
+/**
+ * Fetch + verify a single record from a publisher's PDS.
+ *
+ * Two-stage pipeline:
+ *
+ *   1. Fetch CAR bytes via `com.atproto.sync.getRecord` against the publisher's
+ *      PDS endpoint (resolved upstream by `DidResolver`).
+ *   2. Hand the bytes to `@atcute/repo`'s `verifyRecord`, which does MST
+ *      inclusion proof + commit signature verification in one call against the
+ *      publisher's `#atproto` signing key.
+ *
+ * Failures are reported via a structured `PdsVerificationError` carrying a
+ * `reason` code. The consumer decides retry vs. forensics-and-ack based on the
+ * code (network/5xx → retry; 404, response too large, invalid proof →
+ * forensics + ack). Doing the classification here keeps the consumer's catch
+ * block readable and lets future call sites (backfill, reconciliation) reuse
+ * the same semantics.
+ */
+
+import type { PublicKey } from "@atcute/crypto";
+import { type AtprotoDid, isDid } from "@atcute/lexicons/syntax";
+import { verifyRecord } from "@atcute/repo";
+
+const DEFAULT_TIMEOUT_MS = 15_000;
+/** 5 MB ceiling. Records and their proofs are tiny (sub-KB typical); this is
+ * a defence against a hostile or broken PDS streaming an unbounded body. */
+const DEFAULT_MAX_RESPONSE_BYTES = 5 * 1024 * 1024;
+
+export type VerificationFailureReason =
+	| "PDS_NETWORK_ERROR"
+	| "PDS_HTTP_ERROR"
+	| "RECORD_NOT_FOUND"
+	| "RESPONSE_TOO_LARGE"
+	| "INVALID_PROOF";
+
+export class PdsVerificationError extends Error {
+	override readonly name = "PdsVerificationError";
+	constructor(
+		readonly reason: VerificationFailureReason,
+		message: string,
+		readonly status?: number,
+		override readonly cause?: unknown,
+	) {
+		super(message);
+	}
+}
+
+export interface FetchAndVerifyOptions {
+	pds: string;
+	did: string;
+	collection: string;
+	rkey: string;
+	publicKey: PublicKey;
+	/** Default 15s. Aborts the fetch if the PDS is slow. */
+	timeoutMs?: number;
+	/** Default 5 MB. Rejects with `RESPONSE_TOO_LARGE` if exceeded. */
+	maxResponseBytes?: number;
+	/** Inject for tests; defaults to `globalThis.fetch`. */
+	fetch?: typeof fetch;
+}
+
+export interface VerifiedPdsRecord {
+	cid: string;
+	record: unknown;
+	/** Raw CAR bytes the PDS served. Stored verbatim in `*.record_blob` so the
+	 * read API can passthrough the signed envelope to clients without re-fetching. */
+	carBytes: Uint8Array;
+}
+
+export async function fetchAndVerifyRecord(
+	opts: FetchAndVerifyOptions,
+): Promise<VerifiedPdsRecord> {
+	const fetchImpl = opts.fetch ?? fetch;
+	const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+	const maxResponseBytes = opts.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES;
+
+	if (!isAtprotoDid(opts.did)) {
+		// Caller is expected to have validated this upstream (the resolver
+		// rejects non-DID strings before reaching here), but the verifier's
+		// type contract is narrower than `string` so guard explicitly.
+		throw new PdsVerificationError(
+			"INVALID_PROOF",
+			`unsupported DID method (expected did:plc or did:web): ${opts.did}`,
+		);
+	}
+	const url = buildGetRecordUrl(opts.pds, opts.did, opts.collection, opts.rkey);
+	const carBytes = await fetchCar(fetchImpl, url, timeoutMs, maxResponseBytes);
+
+	try {
+		const result = await verifyRecord({
+			did: opts.did,
+			collection: opts.collection,
+			rkey: opts.rkey,
+			publicKey: opts.publicKey,
+			carBytes,
+		});
+		return { cid: result.cid, record: result.record, carBytes };
+	} catch (err) {
+		// `verifyRecord` rejects on signature failure, MST proof failure,
+		// malformed CAR, or rkey/collection mismatch. All four are "drop and
+		// log" outcomes — distinguishing them isn't load-bearing here, the
+		// detail goes into the dead_letters detail column for forensics.
+		throw new PdsVerificationError(
+			"INVALID_PROOF",
+			`verifyRecord failed: ${err instanceof Error ? err.message : String(err)}`,
+			undefined,
+			err,
+		);
+	}
+}
+
+function buildGetRecordUrl(pds: string, did: string, collection: string, rkey: string): string {
+	const url = new URL("/xrpc/com.atproto.sync.getRecord", pds);
+	url.searchParams.set("did", did);
+	url.searchParams.set("collection", collection);
+	url.searchParams.set("rkey", rkey);
+	return url.toString();
+}
+
+async function fetchCar(
+	fetchImpl: typeof fetch,
+	url: string,
+	timeoutMs: number,
+	maxBytes: number,
+): Promise<Uint8Array> {
+	const controller = new AbortController();
+	const timer = setTimeout(() => controller.abort(), timeoutMs);
+	let response: Response;
+	try {
+		response = await fetchImpl(url, {
+			signal: controller.signal,
+			headers: { accept: "application/vnd.ipld.car" },
+		});
+	} catch (err) {
+		// Whether the fetch threw because we aborted (timeout) or because the
+		// network failed at the OS layer, the right caller behaviour is the
+		// same: retry. Lump them under PDS_NETWORK_ERROR.
+		throw new PdsVerificationError(
+			"PDS_NETWORK_ERROR",
+			err instanceof Error && err.name === "AbortError"
+				? `PDS fetch aborted after ${timeoutMs}ms`
+				: `PDS fetch failed: ${err instanceof Error ? err.message : String(err)}`,
+			undefined,
+			err,
+		);
+	} finally {
+		clearTimeout(timer);
+	}
+
+	if (response.status === 404) {
+		// Distinct from a generic 4xx. The publisher may have deleted the
+		// record between Jetstream emitting and us fetching, which is the
+		// common cause; other 4xx (auth, bad request) suggest programming
+		// errors. Both end up dead-lettered by the consumer (the audit trail
+		// is useful even for legitimate races so operators can spot
+		// systematic Jetstream-vs-PDS skew); the distinct reason code keeps
+		// them queryable separately.
+		throw new PdsVerificationError("RECORD_NOT_FOUND", `PDS returned 404 for ${url}`, 404);
+	}
+	if (!response.ok) {
+		throw new PdsVerificationError(
+			"PDS_HTTP_ERROR",
+			`PDS returned ${response.status} for ${url}`,
+			response.status,
+		);
+	}
+
+	// Buffer the body up to the size limit. Don't trust Content-Length; a
+	// hostile or buggy PDS could under-report and stream more bytes than
+	// advertised.
+	const reader = response.body?.getReader();
+	if (!reader) {
+		throw new PdsVerificationError("INVALID_PROOF", "PDS response body is null");
+	}
+	const chunks: Uint8Array[] = [];
+	let total = 0;
+	try {
+		// biome-ignore lint/correctness/noConstantCondition: drains the stream
+		while (true) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			total += value.byteLength;
+			if (total > maxBytes) {
+				throw new PdsVerificationError(
+					"RESPONSE_TOO_LARGE",
+					`PDS response exceeded ${maxBytes} bytes`,
+				);
+			}
+			chunks.push(value);
+		}
+	} catch (err) {
+		// Cancel the stream so the underlying socket isn't left dangling.
+		await reader.cancel().catch(() => {
+			/* swallow — we already have a primary error to surface */
+		});
+		throw err;
+	} finally {
+		reader.releaseLock();
+	}
+
+	const out = new Uint8Array(total);
+	let offset = 0;
+	for (const chunk of chunks) {
+		out.set(chunk, offset);
+		offset += chunk.byteLength;
+	}
+	return out;
+}
+
+/**
+ * Map a `PdsVerificationError.reason` to "should the consumer retry?". Network
+ * blips and 5xx are transient; everything else is permanent (forensics + ack).
+ *
+ * Exposed so the consumer can write `if (isTransient(err.reason, err.status))
+ * message.retry()` without re-encoding the policy in the catch block.
+ */
+function isAtprotoDid(value: string): value is AtprotoDid {
+	// Library `isDid` enforces the full grammar (length, terminator chars);
+	// the prefix check then narrows to the atproto-supported method subset
+	// (`did:plc:` or `did:web:`). A bare prefix check would accept things
+	// like `did:plc:` (empty body), so the library call carries weight here.
+	if (!isDid(value)) return false;
+	return value.startsWith("did:plc:") || value.startsWith("did:web:");
+}
+
+export function isTransient(
+	reason: VerificationFailureReason,
+	status: number | undefined,
+): boolean {
+	if (reason === "PDS_NETWORK_ERROR") return true;
+	if (reason === "PDS_HTTP_ERROR" && status !== undefined && status >= 500) return true;
+	return false;
+}

apps/aggregator/src/records-do.ts

@@ -0,0 +1,110 @@
+/**
+ * Records Jetstream DO: holds a long-lived outbound WebSocket to Jetstream,
+ * filters our experimental package collections, and enqueues verification
+ * jobs onto the Records Queue.
+ *
+ * Why a DO at all: outbound WebSockets stay open across requests, but a
+ * Worker isolate doesn't. A single DO instance keeps the connection alive
+ * continuously. The Hibernation API doesn't apply here — it's server-side
+ * only, and our connection is outbound.
+ *
+ * The DO is thin by design — all the loop / cursor / backoff logic lives in
+ * `JetstreamIngestor`. The DO just wires real bindings into the ingestor;
+ * its `fetch` handler returns the current ingestor status for the
+ * `/_admin/start` bootstrap path and any later admin/status surface.
+ */
+
+import { DurableObject } from "cloudflare:workers";
+
+import { RealJetstreamClient } from "./jetstream-client.js";
+import { JetstreamIngestor, type IngestorStorage } from "./jetstream-ingestor.js";
+
+/** SQL `time_us` floor across the four content tables. Microseconds since
+ * epoch (Jetstream's cursor unit). Returns null when no rows exist
+ * (truly fresh install) so the subscription falls back to its own
+ * "now" default — there's nothing to gap-fill for. */
+async function deriveJetstreamCursorFloor(db: D1Database): Promise<number | null> {
+	const row = await db
+		.prepare(
+			`SELECT MAX(t) AS latest FROM (
+				SELECT MAX(verified_at) AS t FROM packages
+				UNION ALL
+				SELECT MAX(verified_at) AS t FROM releases
+				UNION ALL
+				SELECT MAX(verified_at) AS t FROM publishers
+				UNION ALL
+				SELECT MAX(verified_at) AS t FROM publisher_verifications
+			)`,
+		)
+		.first<{ latest: string | null }>();
+	if (!row?.latest) return null;
+	const ms = new Date(row.latest).getTime();
+	if (!Number.isFinite(ms)) return null;
+	return ms * 1000;
+}
+
+/** Singleton DO ID. There's exactly one ingestor per deployment. */
+export const RECORDS_DO_NAME = "main";
+
+export class RecordsJetstreamDO extends DurableObject {
+	private readonly ingestor: JetstreamIngestor;
+	/** Held so the run loop isn't garbage-collected. */
+	private readonly runPromise: Promise<void>;
+
+	constructor(state: DurableObjectState, env: Env) {
+		super(state, env);
+		this.ingestor = new JetstreamIngestor({
+			client: new RealJetstreamClient(env.JETSTREAM_URL),
+			queue: env.RECORDS_QUEUE,
+			storage: wrapDoStorage(state.storage),
+			cursorFloor: () => deriveJetstreamCursorFloor(env.DB),
+		});
+		// Fire-and-forget. The run loop absorbs every error path internally
+		// today (transient queue failures, connection drops, parse errors
+		// all retry with backoff). The catch is here defensively — if a
+		// future change introduces a non-recoverable rejection, we want it
+		// in the logs rather than as an unhandled promise.
+		this.runPromise = this.ingestor.run().catch((err) => {
+			console.error("[aggregator] jetstream ingestor crashed", err);
+		});
+	}
+
+	/**
+	 * Status surface for the `/_admin/start` bootstrap and the 5-minute cron
+	 * liveness pump. Idempotent — calling it on an already-running DO just
+	 * reports the current cursor and consecutive-failure count. `0` means
+	 * the most recent connection attempt produced at least one event; a
+	 * non-zero value indicates the latest reconnect cycle hasn't yet
+	 * delivered an event (Jetstream unreachable, wantedCollections
+	 * mismatch, or queue backpressure).
+	 *
+	 * The bootstrap route in the worker doesn't proxy this body — it
+	 * fires-and-forgets the DO fetch and returns 204 — so this surface is
+	 * effectively internal to the DO + cron pump.
+	 */
+	override async fetch(_request: Request): Promise<Response> {
+		return Response.json({
+			cursor: this.ingestor.currentCursor,
+			consecutiveFailures: this.ingestor.consecutiveFailures,
+		});
+	}
+}
+
+/**
+ * Adapt the workerd `DurableObjectStorage` (Promise-based key/value with
+ * unknown values) to the narrow `IngestorStorage` shape (string→number).
+ * Keeping the adaptation here means the ingestor stays free of workerd
+ * imports and the DO is the only place that needs to know about storage's
+ * type-erasure.
+ */
+function wrapDoStorage(storage: DurableObjectStorage): IngestorStorage {
+	return {
+		async get(key: string): Promise<number | undefined> {
+			const value = await storage.get<number>(key);
+			return typeof value === "number" ? value : undefined;
+		},
+		async put(key: string, value: number): Promise<void> {
+			await storage.put(key, value);
+		},
+	};
+}

apps/aggregator/src/routes/xrpc/cursor.ts

@@ -0,0 +1,126 @@
+/**
+ * Cursor encoding for paginated XRPC responses.
+ *
+ * Cursors are opaque base64url strings to clients; internally they encode
+ * a small JSON object specific to the endpoint's ORDER BY clause. We use
+ * different shapes for different endpoints (list cursors carry sort
+ * keys; offset cursors carry an integer).
+ *
+ * Decode discipline: an *absent* cursor (`undefined`) means "first page".
+ * A *provided* cursor that fails to decode throws `InvalidCursorError`,
+ * which handlers translate into a 400 InvalidRequest. Earlier drafts
+ * silently restarted on bad cursors to defend against DOS via forged
+ * cursors, but that policy lets a legitimate client (with a corrupted
+ * cursor from URL-encoding mishaps, mid-rollout schema changes, etc.)
+ * loop forever re-fetching page 1 thinking they're paginating. The DOS
+ * argument doesn't apply because the workload is bounded by `limit`
+ * regardless. Throw loud, fail fast.
+ *
+ * Offset cursors additionally cap `offset` to `MAX_OFFSET` so a forged
+ * cursor with a wildly large offset can't trigger an arbitrarily deep
+ * SQL scan. Real clients never paginate that deep.
+ */
+
+import { isPlainObject } from "../../utils.js";
+
+/** Throw when a *provided* cursor fails to decode. Handler catches and
+ * converts to 400 InvalidRequest. */
+export class InvalidCursorError extends Error {
+	override readonly name = "InvalidCursorError";
+}
+
+interface ListCursor {
+	versionSort: string;
+	version: string;
+}
+
+export function encodeListCursor(cursor: ListCursor): string {
+	return base64UrlEncode(JSON.stringify(cursor));
+}
+
+export function decodeListCursor(raw: string | undefined): ListCursor | null {
+	if (raw === undefined) return null;
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(base64UrlDecode(raw));
+	} catch {
+		throw new InvalidCursorError("cursor is not a valid base64url-encoded JSON object");
+	}
+	if (!isPlainObject(parsed)) {
+		throw new InvalidCursorError("cursor must decode to a JSON object");
+	}
+	const versionSort = parsed["versionSort"];
+	const version = parsed["version"];
+	if (typeof versionSort !== "string" || typeof version !== "string") {
+		throw new InvalidCursorError("cursor must carry string `versionSort` and `version` fields");
+	}
+	return { versionSort, version };
+}
+
+interface OffsetCursor {
+	offset: number;
+}
+
+/** Cap on offset values accepted from a client cursor. Real pagination
+ * never reaches this depth — at the lexicon's `limit: 100` cap that's
+ * 100 pages of 100 results = 10k items; defaulting to `limit: 25` makes
+ * it 400 pages. Past that, a forged cursor is the more likely
+ * explanation than a legitimate client. */
+const MAX_OFFSET = 10_000;
+
+export function encodeOffsetCursor(cursor: OffsetCursor): string {
+	return base64UrlEncode(JSON.stringify(cursor));
+}
+
+export function decodeOffsetCursor(raw: string | undefined): OffsetCursor | null {
+	if (raw === undefined) return null;
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(base64UrlDecode(raw));
+	} catch {
+		throw new InvalidCursorError("cursor is not a valid base64url-encoded JSON object");
+	}
+	if (!isPlainObject(parsed)) {
+		throw new InvalidCursorError("cursor must decode to a JSON object");
+	}
+	const offset = parsed["offset"];
+	if (typeof offset !== "number" || !Number.isInteger(offset) || offset < 0) {
+		throw new InvalidCursorError("cursor `offset` must be a non-negative integer");
+	}
+	if (offset > MAX_OFFSET) {
+		throw new InvalidCursorError(`cursor offset exceeds maximum (${MAX_OFFSET})`);
+	}
+	return { offset };
+}
+
+const BASE64URL_PLUS = /\+/g;
+const BASE64URL_SLASH = /\//g;
+const BASE64URL_TRAILING_EQUALS = /=+$/;
+const BASE64URL_DASH = /-/g;
+const BASE64URL_UNDERSCORE = /_/g;
+
+function base64UrlEncode(value: string): string {
+	// btoa needs latin-1; encode UTF-8 first via TextEncoder.
+	const bytes = new TextEncoder().encode(value);
+	let str = "";
+	for (let i = 0; i < bytes.length; i++) {
+		const byte = bytes[i];
+		if (byte === undefined) continue;
+		str += String.fromCharCode(byte);
+	}
+	return btoa(str)
+		.replace(BASE64URL_PLUS, "-")
+		.replace(BASE64URL_SLASH, "_")
+		.replace(BASE64URL_TRAILING_EQUALS, "");
+}
+
+function base64UrlDecode(value: string): string {
+	const padded = value
+		.replace(BASE64URL_DASH, "+")
+		.replace(BASE64URL_UNDERSCORE, "/")
+		.padEnd(value.length + ((4 - (value.length % 4)) % 4), "=");
+	const binary = atob(padded);
+	const bytes = new Uint8Array(binary.length);
+	for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+	return new TextDecoder().decode(bytes);
+}

apps/aggregator/src/routes/xrpc/getLatestRelease.ts

@@ -0,0 +1,67 @@
+/**
+ * `com.emdashcms.experimental.aggregator.getLatestRelease` — single-release
+ * lookup that returns the highest-precedence non-tombstoned release for a
+ * (did, package).
+ *
+ * The aggregator's writer maintains `packages.latest_version` denormalised
+ * from each `releases` insert (see `refreshPackageLatestStmt` in
+ * records-consumer.ts), so the fast path is a single JOIN. If that misses
+ * — typically because `latest_version` points at a release that was
+ * tombstoned after the pointer was written, and the refresh hasn't
+ * propagated yet (or failed transactionally) — we fall back to the
+ * authoritative ORDER BY query.
+ *
+ * Without the fallback, a tombstoned-but-still-pointed-at release would
+ * make this endpoint return `NotFound` even though the package
+ * demonstrably has other live releases (visible via `listReleases`). The
+ * denormalisation is an optimisation, not a correctness gate.
+ */
+
+import { json, XRPCError } from "@atcute/xrpc-server";
+import { type AggregatorGetLatestRelease } from "@emdash-cms/registry-lexicons";
+
+import { type ReleaseRow, releaseColumns, releaseView } from "./views.js";
+
+export async function getLatestRelease(
+	env: Env,
+	params: AggregatorGetLatestRelease.$params,
+): Promise<Response> {
+	const session = env.DB.withSession("first-primary");
+
+	// Fast path: pull the latest_version pointer + matching release in one
+	// query. The tombstoned_at filter is the integrity gate — if the
+	// pointer is stale (release tombstoned, refresh pending), this misses
+	// and we fall through.
+	const fast = await session
+		.prepare(
+			`SELECT ${releaseColumns("r.")}
+			 FROM packages p
+			 JOIN releases r ON r.did = p.did AND r.package = p.slug AND r.version = p.latest_version
+			 WHERE p.did = ? AND p.slug = ? AND r.tombstoned_at IS NULL`,
+		)
+		.bind(params.did, params.package)
+		.first<ReleaseRow>();
+	if (fast) return json(releaseView(fast));
+
+	// Slow-path fallback: the authoritative ORDER BY. Costs an extra D1
+	// round-trip on the rare miss but guarantees we don't 404 on a package
+	// that has live releases. Tiebreakers (version, rkey) keep the result
+	// deterministic if version_sort ties, matching listReleases' ordering.
+	const slow = await session
+		.prepare(
+			`SELECT ${releaseColumns()}
+			 FROM releases
+			 WHERE did = ? AND package = ? AND tombstoned_at IS NULL
+			 ORDER BY version_sort DESC, version DESC, rkey DESC
+			 LIMIT 1`,
+		)
+		.bind(params.did, params.package)
+		.first<ReleaseRow>();
+	if (slow) return json(releaseView(slow));
+
+	throw new XRPCError({
+		status: 404,
+		error: "NotFound",
+		message: `No eligible release for (${params.did}, ${params.package}).`,
+	});
+}

apps/aggregator/src/routes/xrpc/getPackage.ts

@@ -0,0 +1,38 @@
+/**
+ * `com.emdashcms.experimental.aggregator.getPackage` — single package by
+ * (did, slug). Returns the lexicon's `packageView` envelope (decoded
+ * record + cid + indexedAt + empty mirrors/labels).
+ *
+ * Throws `XRPCError("NotFound")` when no row matches. Tombstone is not a
+ * separate state on `packages` (deletes hard-delete the row), so a NotFound
+ * covers both cases — the lexicon's documented "Tombstoned" error name is
+ * reserved for if/when we move to soft-delete on packages.
+ */
+
+import { json, XRPCError } from "@atcute/xrpc-server";
+import { type AggregatorDefs, type AggregatorGetPackage } from "@emdash-cms/registry-lexicons";
+
+import { type PackageRow, packageColumns, packageView } from "./views.js";
+
+export async function getPackage(
+	env: Env,
+	params: AggregatorGetPackage.$params,
+): Promise<Response> {
+	// `first-primary` because the same row could become subject to a takedown
+	// label between two reads; once the labeller (Slice 2) writes, the next
+	// read everywhere should reflect it. Per plan §XRPC endpoints.
+	const session = env.DB.withSession("first-primary");
+	const row = await session
+		.prepare(`SELECT ${packageColumns()} FROM packages WHERE did = ? AND slug = ?`)
+		.bind(params.did, params.slug)
+		.first<PackageRow>();
+	if (!row) {
+		throw new XRPCError({
+			status: 404,
+			error: "NotFound",
+			message: `No package indexed under (${params.did}, ${params.slug}).`,
+		});
+	}
+	const view: AggregatorDefs.PackageView = packageView(row);
+	return json(view);
+}

apps/aggregator/src/routes/xrpc/listReleases.ts

@@ -0,0 +1,112 @@
+/**
+ * `com.emdashcms.experimental.aggregator.listReleases` — releases for a
+ * (did, package), descending semver. Cursor pagination over
+ * `(version_sort, version)` so tied semver-precedence cases (shouldn't
+ * happen in practice but defensive) still page deterministically.
+ *
+ * Returns `NotFound` when the parent package isn't indexed, even if a
+ * (orphaned) release row exists — the lexicon's contract is "list
+ * releases of a known package", not "list any release rows for this
+ * (did, package)".
+ */
+
+import { InvalidRequestError, json, XRPCError } from "@atcute/xrpc-server";
+import { type AggregatorDefs, type AggregatorListReleases } from "@emdash-cms/registry-lexicons";
+
+import { decodeListCursor, encodeListCursor, InvalidCursorError } from "./cursor.js";
+import { type ReleaseRow, releaseColumns, releaseView } from "./views.js";
+
+const DEFAULT_LIMIT = 25;
+const MAX_LIMIT = 100;
+
+export async function listReleases(
+	env: Env,
+	params: AggregatorListReleases.$params,
+): Promise<Response> {
+	const limit = clampLimit(params.limit);
+	const session = env.DB.withSession("first-primary");
+
+	// Confirm parent package exists. One extra D1 read per request — could be
+	// folded into a JOIN, but the explicit existence check keeps the NotFound
+	// signal cheap and unambiguous (the empty-list response shape would
+	// otherwise mean "package exists, no releases" or "package doesn't exist"
+	// indistinguishably).
+	const parentExists = await session
+		.prepare(`SELECT 1 AS hit FROM packages WHERE did = ? AND slug = ?`)
+		.bind(params.did, params.package)
+		.first<{ hit: number }>();
+	if (!parentExists) {
+		throw new XRPCError({
+			status: 404,
+			error: "NotFound",
+			message: `No package indexed under (${params.did}, ${params.package}).`,
+		});
+	}
+
+	// Cursor encodes the LAST seen (version_sort, version) on the previous
+	// page so the next page picks up below it in DESC order. `WHERE`
+	// half-tuple inequality so SQLite's index on (did, package, version_sort
+	// DESC) stays useful. A *provided* cursor that fails to decode 400s
+	// (would otherwise loop the client through page 1 forever).
+	let cursor: ReturnType<typeof decodeListCursor>;
+	try {
+		cursor = decodeListCursor(params.cursor);
+	} catch (err) {
+		if (err instanceof InvalidCursorError) {
+			throw new InvalidRequestError({ error: "InvalidRequest", message: err.message });
+		}
+		throw err;
+	}
+	const rows = await session
+		.prepare(
+			`SELECT ${releaseColumns()}, version_sort
+			 FROM releases
+			 WHERE did = ? AND package = ? AND tombstoned_at IS NULL
+			 ${cursor ? "AND (version_sort < ? OR (version_sort = ? AND version < ?))" : ""}
+			 ORDER BY version_sort DESC, version DESC
+			 LIMIT ?`,
+		)
+		.bind(
+			...(cursor
+				? [
+						params.did,
+						params.package,
+						cursor.versionSort,
+						cursor.versionSort,
+						cursor.version,
+						limit + 1,
+					]
+				: [params.did, params.package, limit + 1]),
+		)
+		.all<ReleaseRow & { version_sort: string }>();
+
+	const items = rows.results ?? [];
+	// Read limit+1 to detect a next page without a trailing COUNT query.
+	const hasMore = items.length > limit;
+	const page = hasMore ? items.slice(0, limit) : items;
+	const last = page.at(-1);
+
+	const response: {
+		releases: AggregatorDefs.ReleaseView[];
+		cursor?: string;
+	} = {
+		releases: page.map(releaseView),
+	};
+	if (hasMore && last) {
+		// Cursor encodes the internal `version_sort` format. If the
+		// `computeVersionSort` encoding ever changes, in-flight cursors
+		// will be cursor-incompatible across the deploy — clients will
+		// 400 (per the strict-cursor policy) and fall back to fetching
+		// page 1. Acceptable for the experimental NSID; revisit if/when
+		// we stabilise.
+		response.cursor = encodeListCursor({ versionSort: last.version_sort, version: last.version });
+	}
+	return json(response);
+}
+
+function clampLimit(raw: number | undefined): number {
+	if (raw === undefined) return DEFAULT_LIMIT;
+	if (raw < 1) return 1;
+	if (raw > MAX_LIMIT) return MAX_LIMIT;
+	return raw;
+}

apps/aggregator/src/routes/xrpc/resolvePackage.ts

@@ -0,0 +1,85 @@
+/**
+ * `com.emdashcms.experimental.aggregator.resolvePackage` — handle/slug →
+ * package view. Convenience wrapper that:
+ *
+ *   1. Resolves the handle to a DID via the publisher's DID document.
+ *   2. Looks up the package by (resolved DID, slug).
+ *
+ * Throws `HandleNotFound` when handle resolution fails (publisher's
+ * `_atproto` TXT record / `.well-known/atproto-did` file missing or
+ * mismatched). Throws `NotFound` when handle resolves but no package is
+ * indexed under (did, slug).
+ *
+ * Resolution path: bidirectional handle resolution per atproto spec
+ * (`@atcute/identity-resolver`'s `HandleResolver`). We deliberately don't
+ * cache the handle→DID mapping at the aggregator: handles are mutable and
+ * resolvable on every request keeps the contract honest. If/when this
+ * becomes a hot path, add a short-TTL cache here keyed by handle.
+ */
+
+import {
+	CompositeHandleResolver,
+	DohJsonHandleResolver,
+	WellKnownHandleResolver,
+} from "@atcute/identity-resolver";
+import { json, XRPCError } from "@atcute/xrpc-server";
+import { type AggregatorResolvePackage } from "@emdash-cms/registry-lexicons";
+
+import { boundFetch } from "../../utils.js";
+import { type PackageRow, packageColumns, packageView } from "./views.js";
+
+/** Cache the resolver per worker isolate. Construction is allocation-only
+ * (no I/O), but reusing a single instance avoids per-request setup. */
+let cachedResolver: CompositeHandleResolver | null = null;
+function getHandleResolver(): CompositeHandleResolver {
+	if (!cachedResolver) {
+		cachedResolver = new CompositeHandleResolver({
+			strategy: "race",
+			methods: {
+				dns: new DohJsonHandleResolver({
+					dohUrl: "https://mozilla.cloudflare-dns.com/dns-query",
+					fetch: boundFetch,
+				}),
+				http: new WellKnownHandleResolver({ fetch: boundFetch }),
+			},
+		});
+	}
+	return cachedResolver;
+}
+
+export async function resolvePackage(
+	env: Env,
+	params: AggregatorResolvePackage.$params,
+): Promise<Response> {
+	let did: string;
+	try {
+		// Lexicon validates `handle` format upstream so `params.handle` is
+		// already typed `${string}.${string}`, which structurally satisfies
+		// the resolver's `Handle` parameter — no cast needed.
+		did = await getHandleResolver().resolve(params.handle);
+	} catch (err) {
+		throw new XRPCError({
+			status: 404,
+			error: "HandleNotFound",
+			message: `Could not resolve handle '${params.handle}': ${err instanceof Error ? err.message : String(err)}`,
+		});
+	}
+
+	const session = env.DB.withSession("first-primary");
+	const row = await session
+		.prepare(`SELECT ${packageColumns()} FROM packages WHERE did = ? AND slug = ?`)
+		.bind(did, params.slug)
+		.first<PackageRow>();
+	if (!row) {
+		throw new XRPCError({
+			status: 404,
+			error: "NotFound",
+			message: `No package indexed under resolved (${did}, ${params.slug}).`,
+		});
+	}
+	const view = packageView(row);
+	// Surface the handle we resolved — the lexicon's view has an optional
+	// `handle` field for exactly this case (best-effort current handle).
+	view.handle = params.handle;
+	return json(view);
+}

apps/aggregator/src/routes/xrpc/router.ts

@@ -0,0 +1,152 @@
+/**
+ * XRPC dispatcher for the aggregator's read API.
+ *
+ * Aggregator endpoints (`com.emdashcms.experimental.aggregator.*`) flow
+ * through `@atcute/xrpc-server`'s typed router — handlers receive
+ * lexicon-validated `params` and return `JSONResponse<…>` typed against
+ * the lexicon's output schema. The router handles 400 (bad params),
+ * 404 (no handler), and 500 (unexpected throw) automatically; handlers
+ * throw `XRPCError` for typed application errors (`NotFound`, etc.).
+ *
+ * `com.atproto.sync.getRecord` is intercepted *before* the router because
+ * we don't have a generated lexicon binding for atproto's own NSIDs and
+ * the response is `application/vnd.ipld.car` not JSON. See
+ * `sync-get-record.ts`.
+ *
+ * Caching headers:
+ *   - All aggregator endpoints: `private, no-store` (label-state can change
+ *     at any time and Cloudflare's Cache API is colo-local — see plan
+ *     §Caching).
+ *   - `sync.getRecord`: `public, max-age=300` set on the response itself
+ *     (immutable bytes).
+ */
+
+import { XRPCRouter } from "@atcute/xrpc-server";
+import {
+	AggregatorGetLatestRelease,
+	AggregatorGetPackage,
+	AggregatorListReleases,
+	AggregatorResolvePackage,
+	AggregatorSearchPackages,
+} from "@emdash-cms/registry-lexicons";
+
+import { getLatestRelease } from "./getLatestRelease.js";
+import { getPackage } from "./getPackage.js";
+import { listReleases } from "./listReleases.js";
+import { resolvePackage } from "./resolvePackage.js";
+import { searchPackages } from "./searchPackages.js";
+import { syncGetRecord } from "./sync-get-record.js";
+
+const NO_STORE = "private, no-store";
+const SYNC_GET_RECORD_PATH = "/xrpc/com.atproto.sync.getRecord";
+
+/**
+ * CORS for the aggregator's XRPC surface.
+ *
+ * The aggregator is a public read-only service: admin UIs running on
+ * arbitrary EmDash sites call it directly from the browser. The atproto
+ * spec doesn't standardize CORS for XRPC services, but browser clients
+ * need `Access-Control-Allow-Origin` to access the JSON responses.
+ *
+ * `*` is correct here because nothing in our responses depends on the
+ * caller's origin or credentials -- there are no cookies, no auth, no
+ * per-origin policy. We allow `atproto-accept-labelers` and
+ * `content-type` as request headers (the only two clients send), echo
+ * back the labellers header for symmetry with atproto's labeller-aware
+ * clients, and cap preflight cache at 24h.
+ */
+const CORS_HEADERS: Record<string, string> = {
+	"access-control-allow-origin": "*",
+	"access-control-allow-methods": "GET, POST, OPTIONS",
+	"access-control-allow-headers": "content-type, atproto-accept-labelers",
+	"access-control-expose-headers": "atproto-accept-labelers, content-language",
+	"access-control-max-age": "86400",
+};
+
+function applyCorsHeaders(headers: Headers): void {
+	for (const [name, value] of Object.entries(CORS_HEADERS)) {
+		headers.set(name, value);
+	}
+}
+
+/**
+ * Dispatch any `/xrpc/*` request. Returns null when the path isn't an
+ * XRPC route (caller falls through to other route matching).
+ */
+export async function handleXrpc(env: Env, request: Request): Promise<Response | null> {
+	const url = new URL(request.url);
+	if (!url.pathname.startsWith("/xrpc/")) return null;
+
+	// CORS preflight. Browsers send OPTIONS before any cross-origin XRPC
+	// call; we answer with the same allow-list as the actual response
+	// so the real request goes through.
+	if (request.method === "OPTIONS") {
+		const headers = new Headers();
+		applyCorsHeaders(headers);
+		return new Response(null, { status: 204, headers });
+	}
+
+	if (url.pathname === SYNC_GET_RECORD_PATH) {
+		const response = await syncGetRecord(env, request);
+		const headers = new Headers(response.headers);
+		applyCorsHeaders(headers);
+		return new Response(response.body, {
+			status: response.status,
+			statusText: response.statusText,
+			headers,
+		});
+	}
+
+	const router = getRouter(env);
+	const response = await router.fetch(request);
+	// Override Cache-Control unconditionally on aggregator endpoints — the
+	// takedown story requires `no-store` regardless of which endpoint
+	// responded, and it's deliberately not per-handler-overridable (a
+	// future endpoint that wants public caching has to be intercepted
+	// before the router, like sync.getRecord, where the cache contract
+	// can be reasoned about end-to-end). Cloning so we don't mutate a
+	// frozen Response from `json()`.
+	const headers = new Headers(response.headers);
+	headers.set("cache-control", NO_STORE);
+	applyCorsHeaders(headers);
+	return new Response(response.body, {
+		status: response.status,
+		statusText: response.statusText,
+		headers,
+	});
+}
+
+/** Cache the router per worker isolate. Construction registers handler
+ * closures that capture `env`; env is stable across requests within an
+ * isolate so single-instance is fine. */
+let cachedRouter: XRPCRouter | null = null;
+let cachedEnvRef: Env | null = null;
+function getRouter(env: Env): XRPCRouter {
+	// If somehow re-invoked with a different env reference (shouldn't happen
+	// in workerd but cheap to guard), rebuild — better than serving stale
+	// closures pointing at a swapped-out env.
+	if (cachedRouter && cachedEnvRef === env) return cachedRouter;
+	cachedRouter = createRouter(env);
+	cachedEnvRef = env;
+	return cachedRouter;
+}
+
+function createRouter(env: Env): XRPCRouter {
+	const router = new XRPCRouter();
+	router.addQuery(AggregatorGetPackage.mainSchema, {
+		handler: ({ params }) => getPackage(env, params),
+	});
+	router.addQuery(AggregatorListReleases.mainSchema, {
+		handler: ({ params }) => listReleases(env, params),
+	});
+	router.addQuery(AggregatorGetLatestRelease.mainSchema, {
+		handler: ({ params }) => getLatestRelease(env, params),
+	});
+	router.addQuery(AggregatorSearchPackages.mainSchema, {
+		handler: ({ params }) => searchPackages(env, params),
+	});
+	router.addQuery(AggregatorResolvePackage.mainSchema, {
+		handler: ({ params }) => resolvePackage(env, params),
+	});
+	return router;
+}

apps/aggregator/src/routes/xrpc/searchPackages.ts

@@ -0,0 +1,186 @@
+/**
+ * `com.emdashcms.experimental.aggregator.searchPackages` — FTS5 search over
+ * `packages_fts` with optional capability filter.
+ *
+ * Pagination: offset-based (cursor encodes `{offset}`), since BM25 ranking
+ * isn't stable across queries — a cursor encoding `(rank, slug)` would
+ * misbehave if the corpus changed between calls. Offset is the simplest
+ * stable pagination contract for ranked search; the trade is that deep
+ * pagination scans more rows. At Slice 1 scale (hundreds of packages) it's
+ * a non-issue.
+ *
+ * Takedown filter joins `label_state` even though that table is empty in
+ * Slice 1 — keeps the contract honest for Slice 2 (when the labeller starts
+ * writing), and the optimiser short-circuits the NOT EXISTS subquery
+ * cheaply when no rows match. See plan §Search.
+ *
+ * `q` is passed directly to FTS5 MATCH. Special characters in user input
+ * are escaped via `quoteFtsQuery` so a stray `"`/`*`/`(` doesn't blow up
+ * the FTS parser. Empty query returns all packages (paginated, ordered by
+ * last_updated DESC) — the lexicon's documented behaviour.
+ */
+
+import { InvalidRequestError, json } from "@atcute/xrpc-server";
+import {
+	type AggregatorDefs,
+	NSID,
+	type AggregatorSearchPackages,
+} from "@emdash-cms/registry-lexicons";
+
+import { decodeOffsetCursor, encodeOffsetCursor, InvalidCursorError } from "./cursor.js";
+import { type PackageRow, packageColumns, packageView } from "./views.js";
+
+const DEFAULT_LIMIT = 25;
+const MAX_LIMIT = 100;
+
+export async function searchPackages(
+	env: Env,
+	params: AggregatorSearchPackages.$params,
+): Promise<Response> {
+	const limit = clampLimit(params.limit);
+	let offset: number;
+	try {
+		offset = decodeOffsetCursor(params.cursor)?.offset ?? 0;
+	} catch (err) {
+		if (err instanceof InvalidCursorError) {
+			throw new InvalidRequestError({ error: "InvalidRequest", message: err.message });
+		}
+		throw err;
+	}
+	const session = env.DB.withSession("first-primary");
+
+	const hasQuery = typeof params.q === "string" && params.q.trim().length > 0;
+	const hasCapability = typeof params.capability === "string" && params.capability.length > 0;
+
+	let rows: PackageRow[];
+	if (hasQuery) {
+		const ftsQuery = quoteFtsQuery(params.q!);
+		const result = await session
+			.prepare(buildFtsSearchSql(hasCapability))
+			.bind(
+				...buildFtsBindings(
+					ftsQuery,
+					hasCapability ? params.capability : undefined,
+					limit + 1,
+					offset,
+				),
+			)
+			.all<PackageRow>();
+		rows = result.results ?? [];
+	} else {
+		// No query → ordered list of all packages, label-filtered. last_updated
+		// DESC keeps the "what's new" view sensible for an empty search box.
+		const result = await session
+			.prepare(buildBrowseSql(hasCapability))
+			.bind(
+				...buildBrowseBindings(hasCapability ? params.capability : undefined, limit + 1, offset),
+			)
+			.all<PackageRow>();
+		rows = result.results ?? [];
+	}
+
+	const hasMore = rows.length > limit;
+	const page = hasMore ? rows.slice(0, limit) : rows;
+
+	const response: {
+		packages: AggregatorDefs.PackageView[];
+		cursor?: string;
+	} = {
+		packages: page.map(packageView),
+	};
+	if (hasMore) response.cursor = encodeOffsetCursor({ offset: offset + limit });
+	return json(response);
+}
+
+function buildFtsSearchSql(hasCapability: boolean): string {
+	return `
+		SELECT ${packageColumns("p.")}
+		FROM packages_fts
+		JOIN packages p ON p.rowid = packages_fts.rowid
+		WHERE packages_fts MATCH ?
+		${ENFORCEMENT_FILTER_SQL}
+		${hasCapability ? CAPABILITY_FILTER_SQL : ""}
+		ORDER BY bm25(packages_fts), p.last_updated DESC, p.did ASC, p.slug ASC
+		LIMIT ? OFFSET ?
+	`;
+}
+
+function buildFtsBindings(
+	ftsQuery: string,
+	capability: string | undefined,
+	limit: number,
+	offset: number,
+): unknown[] {
+	const out: unknown[] = [ftsQuery];
+	if (capability !== undefined) out.push(capability);
+	out.push(limit, offset);
+	return out;
+}
+
+function buildBrowseSql(hasCapability: boolean): string {
+	// Stable tiebreakers (did, slug) so offset pagination doesn't shuffle
+	// rows across pages when many packages share `last_updated` (or it's
+	// NULL — `last_updated` comes from the optional record.lastUpdated
+	// field). NULLS LAST keeps NULL `last_updated` rows out of the way
+	// of the freshness sort but still reachable via pagination.
+	return `
+		SELECT ${packageColumns("p.")}
+		FROM packages p
+		WHERE 1=1
+		${ENFORCEMENT_FILTER_SQL}
+		${hasCapability ? CAPABILITY_FILTER_SQL : ""}
+		ORDER BY p.last_updated IS NULL, p.last_updated DESC, p.did ASC, p.slug ASC
+		LIMIT ? OFFSET ?
+	`;
+}
+
+function buildBrowseBindings(
+	capability: string | undefined,
+	limit: number,
+	offset: number,
+): unknown[] {
+	const out: unknown[] = [];
+	if (capability !== undefined) out.push(capability);
+	out.push(limit, offset);
+	return out;
+}
+
+/** Hard-enforcement label filter. The Slice 2 labeller writes to
+ * `label_state`; until then the table is empty so this clause is a no-op
+ * (the NOT EXISTS short-circuits). The plan calls for shipping the filter
+ * now to lock the contract — adding it later would be a behaviour change
+ * for cached clients. */
+const ENFORCEMENT_FILTER_SQL = `
+	AND NOT EXISTS (
+		SELECT 1 FROM label_state ls
+		WHERE ls.uri = 'at://' || p.did || '/${NSID.packageProfile}/' || p.slug
+		  AND ls.val IN ('!takedown', 'security:yanked')
+		  AND ls.trusted = 1
+		  AND ls.neg = 0
+		  AND (ls.exp IS NULL OR ls.exp > strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+	)
+`;
+
+const CAPABILITY_FILTER_SQL = `
+	AND p.capabilities IS NOT NULL
+	AND EXISTS (SELECT 1 FROM json_each(p.capabilities) WHERE value = ?)
+`;
+
+/** Quote a user-supplied search string for FTS5 MATCH. FTS5 treats `"`,
+ * `*`, `(`, `)`, `.`, `:`, `^`, `+`, `-` as syntax. The simplest robust
+ * escape is to wrap the whole query as a single phrase string and double
+ * any embedded quotes. This loses prefix-search functionality
+ * (`"foo*"` is treated literally) but is safe and sufficient for v1; if
+ * advanced query syntax becomes a product feature we'll layer a parsed
+ * mode on top. */
+const FTS_QUOTE_RE = /"/g;
+function quoteFtsQuery(raw: string): string {
+	return `"${raw.replace(FTS_QUOTE_RE, '""')}"`;
+}
+
+function clampLimit(raw: number | undefined): number {
+	if (raw === undefined) return DEFAULT_LIMIT;
+	if (raw < 1) return 1;
+	if (raw > MAX_LIMIT) return MAX_LIMIT;
+	return raw;
+}

apps/aggregator/src/routes/xrpc/sync-get-record.ts

@@ -0,0 +1,153 @@
+/**
+ * `com.atproto.sync.getRecord` — passthrough of stored CAR bytes.
+ *
+ * Slingshot-style behaviour: the aggregator is also a cached record store
+ * for the four collections it indexes. Install-time clients fetch the
+ * signed CAR from us instead of round-tripping to per-publisher PDSes —
+ * faster install, resilient when a publisher's PDS is slow/down, lower
+ * load on publisher PDSes.
+ *
+ * Trust model holds because clients re-verify the signature against the
+ * publisher's signing key (resolved independently from the DID document).
+ * The aggregator can serve stale or withhold but cannot forge.
+ *
+ * Implemented as a manual route rather than via @atcute/xrpc-server because
+ * we don't have a generated lexicon binding for atproto's
+ * `com.atproto.sync.getRecord` (it's atproto's, not ours), and the response
+ * is `application/vnd.ipld.car` not JSON. Wiring it through the typed
+ * router would mean authoring a stub lexicon for the sole purpose of
+ * registration, which is more code than this module.
+ */
+
+import { isDid } from "@atcute/lexicons/syntax";
+import { NSID } from "@emdash-cms/registry-lexicons";
+
+const CAR_CONTENT_TYPE = "application/vnd.ipld.car";
+/** 5 minutes. The CAR bytes are content-addressed (CID-derivable) so a
+ * stale cache is detectable client-side; we trade absolute freshness for
+ * lower aggregator load on the install-time hot path. Tighter than the
+ * aggregator endpoints (`no-store`) because there's no label-dependent
+ * filtering on this passthrough. */
+const CACHE_CONTROL = "public, max-age=300";
+
+interface ParsedQuery {
+	did: string;
+	collection: string;
+	rkey: string;
+}
+
+export async function syncGetRecord(env: Env, request: Request): Promise<Response> {
+	if (request.method !== "GET" && request.method !== "HEAD") {
+		return jsonError(405, "MethodNotAllowed", "method not allowed", { allow: "GET, HEAD" });
+	}
+	const parseResult = parseQuery(request);
+	if ("error" in parseResult) return parseResult.error;
+	const { did, collection, rkey } = parseResult;
+
+	const carBytes = await fetchRecordBlob(env, did, collection, rkey);
+	if (!carBytes) {
+		return jsonError(
+			404,
+			"RecordNotFound",
+			`record not found for (${did}, ${collection}, ${rkey})`,
+		);
+	}
+	// Normalise to a Uint8Array view. D1 returns BLOBs as ArrayBuffer in
+	// production but as Uint8Array via miniflare in tests; wrapping in
+	// `new Uint8Array(buf)` produces the byte-stream shape workerd's
+	// Response constructor wants. (Without this normalisation, passing an
+	// `ArrayBuffer` directly works in production but trips miniflare's
+	// "ReadableStream did not return bytes" check.)
+	//
+	// `new Uint8Array(arrayBuffer)` *views* the buffer rather than copying.
+	// That's safe here because workerd buffers the entire body during
+	// Response construction (the body isn't streamed asynchronously back
+	// to the client over the lifetime of the underlying buffer).
+	const body = carBytes instanceof Uint8Array ? carBytes : new Uint8Array(carBytes);
+	return new Response(request.method === "HEAD" ? null : body, {
+		status: 200,
+		headers: {
+			"content-type": CAR_CONTENT_TYPE,
+			"content-length": String(body.byteLength),
+			"cache-control": CACHE_CONTROL,
+		},
+	});
+}
+
+function parseQuery(request: Request): ParsedQuery | { error: Response } {
+	const url = new URL(request.url);
+	const did = url.searchParams.get("did");
+	const collection = url.searchParams.get("collection");
+	const rkey = url.searchParams.get("rkey");
+	if (!did || !collection || !rkey) {
+		return {
+			error: jsonError(400, "InvalidRequest", "missing required param: did, collection, rkey"),
+		};
+	}
+	if (!isDid(did)) {
+		return { error: jsonError(400, "InvalidRequest", `invalid did: ${did}`) };
+	}
+	return { did, collection, rkey };
+}
+
+async function fetchRecordBlob(
+	env: Env,
+	did: string,
+	collection: string,
+	rkey: string,
+): Promise<ArrayBuffer | Uint8Array | null> {
+	// Cacheable read — `first-unconstrained` lets D1 hit the nearest
+	// replica. The CAR bytes are immutable per record version (the writer
+	// rejects content changes via CID-based dedup), so even a slightly stale
+	// replica returns the same bytes a primary would.
+	const session = env.DB.withSession("first-unconstrained");
+	switch (collection) {
+		case NSID.packageProfile:
+			return selectBlob(session, `SELECT record_blob FROM packages WHERE did = ? AND slug = ?`, [
+				did,
+				rkey,
+			]);
+		case NSID.packageRelease:
+			return selectBlob(
+				session,
+				`SELECT record_blob FROM releases WHERE did = ? AND rkey = ? AND tombstoned_at IS NULL`,
+				[did, rkey],
+			);
+		case NSID.publisherProfile:
+			// publisher.profile rkey is always 'self' per the writer's enforcement.
+			if (rkey !== "self") return null;
+			return selectBlob(session, `SELECT record_blob FROM publishers WHERE did = ?`, [did]);
+		case NSID.publisherVerification:
+			return selectBlob(
+				session,
+				`SELECT record_blob FROM publisher_verifications WHERE issuer_did = ? AND rkey = ? AND tombstoned_at IS NULL`,
+				[did, rkey],
+			);
+		default:
+			return null;
+	}
+}
+
+async function selectBlob(
+	session: D1DatabaseSession,
+	sql: string,
+	bindings: unknown[],
+): Promise<ArrayBuffer | Uint8Array | null> {
+	const row = await session
+		.prepare(sql)
+		.bind(...bindings)
+		.first<{ record_blob: ArrayBuffer | Uint8Array }>();
+	return row?.record_blob ?? null;
+}
+
+function jsonError(
+	status: number,
+	error: string,
+	message: string,
+	headers: Record<string, string> = {},
+): Response {
+	return new Response(JSON.stringify({ error, message }), {
+		status,
+		headers: { "content-type": "application/json", ...headers },
+	});
+}

apps/aggregator/src/routes/xrpc/views.ts

@@ -0,0 +1,247 @@
+/**
+ * Row → lexicon-view mappers for the Read API.
+ *
+ * The `packages` and `releases` tables are normalised projections of the
+ * signed records (with the raw CAR bytes also kept verbatim in
+ * `record_blob` for the `sync.getRecord` passthrough). The Read API needs
+ * to return a JSON shape that mirrors what the publisher signed — for
+ * display on the wire, with `cid` carried alongside so clients can re-verify
+ * against `sync.getRecord` if they want byte-identical bytes.
+ *
+ * These mappers are the single source of truth for that round-trip. Adding
+ * a new column to the schema means updating both the writer (in
+ * `records-consumer.ts`) and the relevant mapper here, in lock-step.
+ *
+ * Mirrors and labels: the lexicon allows them on both views; v1 always
+ * returns empty arrays. Mirror integration ships in Slice 3; label
+ * hydration ships in Slice 2. The contract is in place so adding either
+ * later doesn't change the response shape.
+ */
+
+import { type AggregatorDefs, NSID } from "@emdash-cms/registry-lexicons";
+
+import { isPlainObject, parseSignatureMetadataCid } from "../../utils.js";
+
+/** Subset of columns from `packages` we read for `packageView`. Selecting
+ * exactly these columns keeps the SQL query auditable and cheap. */
+export interface PackageRow {
+	did: string;
+	slug: string;
+	type: string;
+	name: string | null;
+	description: string | null;
+	license: string;
+	authors: string; // JSON array
+	security: string; // JSON array
+	keywords: string | null; // JSON array
+	sections: string | null; // JSON map
+	last_updated: string | null;
+	latest_version: string | null;
+	signature_metadata: string | null;
+	verified_at: string;
+	indexed_at: string | null;
+}
+
+/** Subset of columns from `releases` we read for `releaseView`. */
+export interface ReleaseRow {
+	did: string;
+	package: string;
+	version: string;
+	rkey: string;
+	artifacts: string; // JSON
+	requires: string | null; // JSON
+	suggests: string | null; // JSON
+	emdash_extension: string; // JSON of validated releaseExtension contents
+	repo_url: string | null;
+	signature_metadata: string | null;
+	verified_at: string;
+	indexed_at: string | null;
+}
+
+/** Column list backing `PackageRow`. Single source of truth so a column
+ * added to the schema needs writer + reader updates in one grep target. */
+const PACKAGE_VIEW_COLUMN_NAMES = [
+	"did",
+	"slug",
+	"type",
+	"name",
+	"description",
+	"license",
+	"authors",
+	"security",
+	"keywords",
+	"sections",
+	"last_updated",
+	"latest_version",
+	"signature_metadata",
+	"verified_at",
+	"indexed_at",
+] as const;
+
+/** Column list backing `ReleaseRow`. */
+const RELEASE_VIEW_COLUMN_NAMES = [
+	"did",
+	"package",
+	"version",
+	"rkey",
+	"artifacts",
+	"requires",
+	"suggests",
+	"emdash_extension",
+	"repo_url",
+	"signature_metadata",
+	"verified_at",
+	"indexed_at",
+] as const;
+
+/** SELECT-clause string for `PackageRow`. Pass an alias prefix (with the
+ * trailing dot) for use in JOINs: `packageColumns("p.")` →
+ * `"p.did, p.slug, ..."`. No prefix is unambiguous when only one table is
+ * in scope. */
+export function packageColumns(prefix = ""): string {
+	return PACKAGE_VIEW_COLUMN_NAMES.map((c) => `${prefix}${c}`).join(", ");
+}
+
+/** SELECT-clause string for `ReleaseRow`, optionally prefixed for JOINs. */
+export function releaseColumns(prefix = ""): string {
+	return RELEASE_VIEW_COLUMN_NAMES.map((c) => `${prefix}${c}`).join(", ");
+}
+
+/**
+ * Map a `packages` row to the lexicon's `packageView`. The synthesized
+ * `profile` field reconstructs the package.profile record JSON from the
+ * normalised columns — same field values the publisher signed. For
+ * byte-identical bytes, clients call `sync.getRecord` and re-verify.
+ *
+ * `indexedAt` falls back to `verified_at` for any historical row that
+ * predates migration 0002 (`indexed_at` is nullable at the schema level —
+ * see migration comment).
+ */
+export function packageView(row: PackageRow): AggregatorDefs.PackageView {
+	const uri = `at://${row.did}/${NSID.packageProfile}/${row.slug}` as const;
+	const cid = parseSignatureMetadataCid(row.signature_metadata) ?? "";
+	// `mirrors` is on releaseView, not packageView — packages aren't
+	// mirrored, releases are. Don't add it here even though they share the
+	// "envelope" idiom in plan-doc shorthand.
+	const view: AggregatorDefs.PackageView = {
+		uri,
+		cid,
+		// eslint-disable-next-line typescript/no-unsafe-type-assertion -- `did` is consumer-validated at write time
+		did: row.did as `did:${string}:${string}`,
+		slug: row.slug,
+		profile: synthesizePackageProfile(row, uri),
+		indexedAt: row.indexed_at ?? row.verified_at,
+		labels: [],
+	};
+	if (row.latest_version !== null) {
+		view.latestVersion = row.latest_version;
+	}
+	return view;
+}
+
+/**
+ * Map a `releases` row to the lexicon's `releaseView`. The synthesized
+ * `release` field reconstructs the package.release record JSON from the
+ * normalised columns. `mirrors: []` is intentional — the artifact mirror
+ * worker (Slice 3) is what populates real mirror URLs; until then the
+ * field is the empty contract.
+ */
+export function releaseView(row: ReleaseRow): AggregatorDefs.ReleaseView {
+	const uri = `at://${row.did}/${NSID.packageRelease}/${row.rkey}` as const;
+	const cid = parseSignatureMetadataCid(row.signature_metadata) ?? "";
+	return {
+		uri,
+		cid,
+		// eslint-disable-next-line typescript/no-unsafe-type-assertion -- `did` is consumer-validated at write time
+		did: row.did as `did:${string}:${string}`,
+		package: row.package,
+		version: row.version,
+		release: synthesizePackageRelease(row),
+		mirrors: [],
+		indexedAt: row.indexed_at ?? row.verified_at,
+		labels: [],
+	};
+}
+
+/** Reconstruct the `com.emdashcms.experimental.package.profile` record
+ * JSON from the row's columns. Field set matches what the consumer's
+ * `ingestPackageProfile` writer accepts; optional fields are omitted
+ * (rather than emitted as null) so the JSON shape matches what a
+ * publisher would have written.
+ *
+ * Returned as `Record<string, unknown>` rather than typed to the lexicon
+ * Main schema — the columns hold writer-validated JSON but TypeScript
+ * can't narrow `JSON.parse` output to the lexicon's structural types
+ * without re-validating, and the lexicon explicitly types `packageView.profile`
+ * as `unknown` for exactly this reason: the value is a passthrough on
+ * the wire and clients re-validate against the published lexicon. */
+function synthesizePackageProfile(row: PackageRow, uri: string): Record<string, unknown> {
+	const profile: Record<string, unknown> = {
+		$type: NSID.packageProfile,
+		id: uri,
+		type: row.type,
+		license: row.license,
+		authors: parseJsonArray(row.authors),
+		security: parseJsonArray(row.security),
+	};
+	if (row.name !== null) profile["name"] = row.name;
+	if (row.description !== null) profile["description"] = row.description;
+	if (row.keywords !== null) profile["keywords"] = parseJsonArray(row.keywords);
+	if (row.sections !== null) {
+		const sections = parseJsonObject(row.sections);
+		if (sections) profile["sections"] = sections;
+	}
+	if (row.last_updated !== null) profile["lastUpdated"] = row.last_updated;
+	// `slug` in the record is optional but, when present, must equal the
+	// rkey. We always have it as the PK of the row, so always emit.
+	profile["slug"] = row.slug;
+	return profile;
+}
+
+/** Reconstruct the `com.emdashcms.experimental.package.release` record
+ * JSON from the row's columns. The `extensions` map is rebuilt from the
+ * stored `emdash_extension` payload (which the writer validates against
+ * the `releaseExtension` lexicon at ingest time). Same passthrough-shape
+ * caveat as `synthesizePackageProfile`. */
+function synthesizePackageRelease(row: ReleaseRow): Record<string, unknown> {
+	const release: Record<string, unknown> = {
+		$type: NSID.packageRelease,
+		package: row.package,
+		version: row.version,
+		artifacts: parseJsonObject(row.artifacts) ?? {},
+	};
+	if (row.requires !== null) {
+		const requires = parseJsonObject(row.requires);
+		if (requires) release["requires"] = requires;
+	}
+	if (row.suggests !== null) {
+		const suggests = parseJsonObject(row.suggests);
+		if (suggests) release["suggests"] = suggests;
+	}
+	if (row.repo_url !== null) release["repo"] = row.repo_url;
+	const ext = parseJsonObject(row.emdash_extension);
+	if (ext) {
+		release["extensions"] = {
+			[NSID.packageReleaseExtension]: { ...ext, $type: NSID.packageReleaseExtension },
+		};
+	}
+	return release;
+}
+
+function parseJsonArray(json: string): unknown[] {
+	try {
+		const parsed: unknown = JSON.parse(json);
+		return Array.isArray(parsed) ? parsed : [];
+	} catch {
+		return [];
+	}
+}
+
+function parseJsonObject(json: string): Record<string, unknown> | null {
+	try {
+		const parsed: unknown = JSON.parse(json);
+		return isPlainObject(parsed) ? parsed : null;
+	} catch {
+		return null;
+	}
+}

apps/aggregator/src/utils.ts

@@ -0,0 +1,40 @@
+/**
+ * Shared utility helpers used across multiple modules.
+ */
+
+/**
+ * Type guard for narrowing `unknown` to `Record<string, unknown>` so
+ * subsequent `value["key"]` accesses are typesafe without an `as` cast.
+ * Excludes arrays (which are also `typeof === "object"`) so consumers
+ * checking for "JSON-shaped object" get what they expect.
+ */
+export function isPlainObject(value: unknown): value is Record<string, unknown> {
+	return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+/**
+ * `globalThis.fetch` pre-bound. Pass this whenever a library wants a
+ * `fetch` callable (resolver constructors, deps objects); the bare
+ * global trips workerd's "Illegal invocation" check when called via a
+ * stored reference.
+ */
+export const boundFetch: typeof fetch = globalThis.fetch.bind(globalThis);
+
+/**
+ * Pull the verified record's CID out of a JSON-stringified
+ * `signature_metadata` column value. Returns null on malformed input
+ * (missing column, non-JSON, missing `cid` key) — callers decide what
+ * to do; in writer-controlled data this never happens, but the
+ * fallback keeps read-side comparisons robust against future schema
+ * drift.
+ */
+export function parseSignatureMetadataCid(signatureMetadata: string | null): string | null {
+	if (signatureMetadata === null) return null;
+	try {
+		const parsed: unknown = JSON.parse(signatureMetadata);
+		if (isPlainObject(parsed) && typeof parsed["cid"] === "string") return parsed["cid"];
+	} catch {
+		// fall through
+	}
+	return null;
+}

apps/aggregator/test/did-resolver.test.ts

@@ -0,0 +1,367 @@
+/**
+ * DidResolver unit tests + the D1 binding's contract test.
+ *
+ * The class is exercised end-to-end with a real WebCrypto-backed signing key
+ * (generated once in `beforeAll`) and a Map-backed cache so cache behaviour is
+ * verified independently of D1 wiring. A separate suite runs the D1 binding
+ * against the test pool's in-memory D1 and re-runs the cache contract via
+ * the same scenarios — that way the contract is the same in tests and in
+ * production.
+ */
+
+import { P256PrivateKeyExportable } from "@atcute/crypto";
+import type { DidDocument } from "@atcute/identity";
+import type { Did } from "@atcute/lexicons/syntax";
+import { applyD1Migrations, env } from "cloudflare:test";
+import { beforeAll, beforeEach, describe, expect, it } from "vitest";
+
+import {
+	type CachedDidDoc,
+	createD1DidDocCache,
+	type DidDocCache,
+	type DidDocumentResolverLike,
+	DidResolver,
+} from "../src/did-resolver.js";
+
+interface TestEnv {
+	DB: D1Database;
+	TEST_MIGRATIONS: Parameters<typeof applyD1Migrations>[1];
+}
+
+const testEnv = env as unknown as TestEnv;
+
+const TEST_DID = "did:plc:test00000000000000000000";
+const TEST_PDS = "https://pds.test.example";
+
+let signingKeyMultibase: string;
+
+beforeAll(async () => {
+	const kp = await P256PrivateKeyExportable.createKeypair();
+	signingKeyMultibase = await kp.exportPublicKey("multikey");
+});
+
+class MapDidDocCache implements DidDocCache {
+	private readonly entries = new Map<string, CachedDidDoc>();
+	readonly reads: string[] = [];
+	readonly upserts: Array<{ did: string; doc: Omit<CachedDidDoc, "resolvedAt">; now: Date }> = [];
+	readonly expires: string[] = [];
+
+	read(did: string): Promise<CachedDidDoc | null> {
+		this.reads.push(did);
+		return Promise.resolve(this.entries.get(did) ?? null);
+	}
+	upsert(did: string, doc: Omit<CachedDidDoc, "resolvedAt">, now: Date): Promise<void> {
+		this.upserts.push({ did, doc, now });
+		this.entries.set(did, { ...doc, resolvedAt: now });
+		return Promise.resolve();
+	}
+	expire(did: string): Promise<void> {
+		this.expires.push(did);
+		const entry = this.entries.get(did);
+		if (entry) {
+			this.entries.set(did, { ...entry, resolvedAt: new Date(0) });
+		}
+		return Promise.resolve();
+	}
+}
+
+class StubResolver implements DidDocumentResolverLike {
+	readonly calls: Did[] = [];
+	private response: DidDocument;
+	private error: Error | null = null;
+
+	constructor(response: DidDocument) {
+		this.response = response;
+	}
+
+	resolve(did: Did): Promise<DidDocument> {
+		this.calls.push(did);
+		if (this.error) return Promise.reject(this.error);
+		return Promise.resolve(this.response);
+	}
+
+	setResponse(doc: DidDocument): void {
+		this.response = doc;
+	}
+	setError(err: Error): void {
+		this.error = err;
+	}
+}
+
+function buildDidDoc(overrides: Partial<DidDocument> = {}): DidDocument {
+	return {
+		id: TEST_DID as `did:${string}:${string}`,
+		verificationMethod: [
+			{
+				id: `${TEST_DID}#atproto`,
+				type: "Multikey",
+				controller: TEST_DID as `did:${string}:${string}`,
+				publicKeyMultibase: signingKeyMultibase,
+			},
+		],
+		service: [
+			{
+				id: "#atproto_pds",
+				type: "AtprotoPersonalDataServer",
+				serviceEndpoint: TEST_PDS,
+			},
+		],
+		...overrides,
+	};
+}
+
+describe("DidResolver", () => {
+	describe("with in-memory cache", () => {
+		it("resolves on cache miss, writes the row, returns a usable PublicKey", async () => {
+			const cache = new MapDidDocCache();
+			const resolver = new StubResolver(buildDidDoc());
+			const subject = new DidResolver({ cache, resolver, now: () => new Date(1000) });
+
+			const result = await subject.resolve(TEST_DID);
+
+			expect(result.pds).toBe(TEST_PDS);
+			expect(result.signingKeyId).toBe(`${TEST_DID}#atproto`);
+			expect(typeof result.publicKey.verify).toBe("function");
+			expect(resolver.calls).toEqual([TEST_DID]);
+			expect(cache.upserts).toHaveLength(1);
+			expect(cache.upserts[0]).toMatchObject({
+				did: TEST_DID,
+				doc: { pds: TEST_PDS, signingKey: signingKeyMultibase },
+			});
+		});
+
+		it("hits cache on second call within TTL — no resolver call", async () => {
+			const cache = new MapDidDocCache();
+			const resolver = new StubResolver(buildDidDoc());
+			const subject = new DidResolver({
+				cache,
+				resolver,
+				ttlMs: 60_000,
+				now: () => new Date(1000),
+			});
+
+			await subject.resolve(TEST_DID);
+			await subject.resolve(TEST_DID);
+
+			expect(resolver.calls).toHaveLength(1);
+			expect(cache.upserts).toHaveLength(1);
+		});
+
+		it("re-resolves when cached entry is past TTL", async () => {
+			const cache = new MapDidDocCache();
+			const resolver = new StubResolver(buildDidDoc());
+			let now = 1_000;
+			const subject = new DidResolver({
+				cache,
+				resolver,
+				ttlMs: 60_000,
+				now: () => new Date(now),
+			});
+
+			await subject.resolve(TEST_DID);
+			now = 1_000 + 60_001;
+			await subject.resolve(TEST_DID);
+
+			expect(resolver.calls).toHaveLength(2);
+			expect(cache.upserts).toHaveLength(2);
+		});
+
+		it("propagates resolver errors without writing to cache", async () => {
+			const cache = new MapDidDocCache();
+			const resolver = new StubResolver(buildDidDoc());
+			resolver.setError(new Error("plc unreachable"));
+			const subject = new DidResolver({ cache, resolver });
+
+			await expect(subject.resolve(TEST_DID)).rejects.toThrow("plc unreachable");
+			expect(cache.upserts).toHaveLength(0);
+		});
+
+		it("rejects DID documents with no PDS service entry", async () => {
+			const cache = new MapDidDocCache();
+			const resolver = new StubResolver(buildDidDoc({ service: [] }));
+			const subject = new DidResolver({ cache, resolver });
+
+			await expect(subject.resolve(TEST_DID)).rejects.toThrow(/no atproto PDS/i);
+		});
+
+		it("rejects DID documents with no #atproto verification method", async () => {
+			const cache = new MapDidDocCache();
+			const resolver = new StubResolver(buildDidDoc({ verificationMethod: [] }));
+			const subject = new DidResolver({ cache, resolver });
+
+			await expect(subject.resolve(TEST_DID)).rejects.toThrow(/#atproto verification method/i);
+		});
+
+		it("rejects malformed DIDs without calling the resolver or cache", async () => {
+			const cache = new MapDidDocCache();
+			const resolver = new StubResolver(buildDidDoc());
+			const subject = new DidResolver({ cache, resolver });
+
+			await expect(subject.resolve("not-a-did")).rejects.toThrow(/invalid DID/i);
+			expect(resolver.calls).toHaveLength(0);
+			expect(cache.reads).toHaveLength(0);
+		});
+
+		it("invalidate() forces re-resolution on the next call", async () => {
+			const cache = new MapDidDocCache();
+			const resolver = new StubResolver(buildDidDoc());
+			// Use a real-world `now` so invalidate's "epoch" sentinel falls
+			// well outside the TTL window.
+			const subject = new DidResolver({
+				cache,
+				resolver,
+				ttlMs: 60_000,
+				now: () => new Date("2026-05-09T12:00:00.000Z"),
+			});
+
+			await subject.resolve(TEST_DID);
+			await subject.invalidate(TEST_DID);
+			await subject.resolve(TEST_DID);
+
+			expect(resolver.calls).toHaveLength(2);
+		});
+
+		it("invalidate() on an unknown DID is a no-op", async () => {
+			const cache = new MapDidDocCache();
+			const resolver = new StubResolver(buildDidDoc());
+			const subject = new DidResolver({ cache, resolver });
+
+			await expect(subject.invalidate(TEST_DID)).resolves.toBeUndefined();
+			expect(cache.upserts).toHaveLength(0);
+		});
+	});
+
+	describe("createD1DidDocCache", () => {
+		beforeAll(async () => {
+			await applyD1Migrations(testEnv.DB, testEnv.TEST_MIGRATIONS);
+		});
+
+		beforeEach(async () => {
+			await testEnv.DB.exec("DELETE FROM known_publishers");
+		});
+
+		it("returns null when no row exists", async () => {
+			const cache = createD1DidDocCache(testEnv.DB);
+			expect(await cache.read(TEST_DID)).toBeNull();
+		});
+
+		it("returns null when row exists but cache fields are unpopulated", async () => {
+			// Backfill or future code may insert a known_publishers row before the
+			// consumer has resolved the DID doc; the read should treat that as
+			// a cache miss, not a stale cache hit.
+			const now = new Date().toISOString();
+			await testEnv.DB.prepare(
+				`INSERT INTO known_publishers (did, first_seen_at, last_seen_at)
+				 VALUES (?, ?, ?)`,
+			)
+				.bind(TEST_DID, now, now)
+				.run();
+
+			const cache = createD1DidDocCache(testEnv.DB);
+			expect(await cache.read(TEST_DID)).toBeNull();
+		});
+
+		it("upsert + read round-trips a cache entry", async () => {
+			const cache = createD1DidDocCache(testEnv.DB);
+			const now = new Date("2026-05-09T12:00:00.000Z");
+			await cache.upsert(
+				TEST_DID,
+				{
+					pds: TEST_PDS,
+					signingKey: signingKeyMultibase,
+					signingKeyId: `${TEST_DID}#atproto`,
+				},
+				now,
+			);
+
+			const out = await cache.read(TEST_DID);
+			expect(out).not.toBeNull();
+			expect(out?.pds).toBe(TEST_PDS);
+			expect(out?.signingKey).toBe(signingKeyMultibase);
+			expect(out?.signingKeyId).toBe(`${TEST_DID}#atproto`);
+			expect(out?.resolvedAt.toISOString()).toBe(now.toISOString());
+		});
+
+		it("upsert preserves first_seen_at across updates", async () => {
+			const cache = createD1DidDocCache(testEnv.DB);
+			const t1 = new Date("2026-05-09T12:00:00.000Z");
+			const t2 = new Date("2026-05-10T12:00:00.000Z");
+
+			await cache.upsert(
+				TEST_DID,
+				{
+					pds: TEST_PDS,
+					signingKey: signingKeyMultibase,
+					signingKeyId: `${TEST_DID}#atproto`,
+				},
+				t1,
+			);
+			await cache.upsert(
+				TEST_DID,
+				{
+					pds: TEST_PDS,
+					signingKey: signingKeyMultibase,
+					signingKeyId: `${TEST_DID}#atproto`,
+				},
+				t2,
+			);
+
+			const row = await testEnv.DB.prepare(
+				`SELECT first_seen_at, last_seen_at, pds_resolved_at FROM known_publishers WHERE did = ?`,
+			)
+				.bind(TEST_DID)
+				.first<{ first_seen_at: string; last_seen_at: string; pds_resolved_at: string }>();
+			expect(row?.first_seen_at).toBe(t1.toISOString());
+			expect(row?.last_seen_at).toBe(t2.toISOString());
+			expect(row?.pds_resolved_at).toBe(t2.toISOString());
+		});
+
+		it("end-to-end: resolver wired to D1 cache", async () => {
+			const cache = createD1DidDocCache(testEnv.DB);
+			const resolver = new StubResolver(buildDidDoc());
+			const subject = new DidResolver({ cache, resolver });
+
+			await subject.resolve(TEST_DID);
+			await subject.resolve(TEST_DID);
+
+			// One resolver call for the cache miss; the second resolve hits D1.
+			expect(resolver.calls).toHaveLength(1);
+		});
+
+		it("expire only touches pds_resolved_at; preserves first_seen_at and last_seen_at", async () => {
+			const cache = createD1DidDocCache(testEnv.DB);
+			const seenAt = new Date("2026-05-09T12:00:00.000Z");
+			await cache.upsert(
+				TEST_DID,
+				{
+					pds: TEST_PDS,
+					signingKey: signingKeyMultibase,
+					signingKeyId: `${TEST_DID}#atproto`,
+				},
+				seenAt,
+			);
+
+			await cache.expire(TEST_DID);
+
+			const row = await testEnv.DB.prepare(
+				`SELECT first_seen_at, last_seen_at, pds_resolved_at FROM known_publishers WHERE did = ?`,
+			)
+				.bind(TEST_DID)
+				.first<{ first_seen_at: string; last_seen_at: string; pds_resolved_at: string }>();
+			expect(row?.first_seen_at).toBe(seenAt.toISOString());
+			expect(row?.last_seen_at).toBe(seenAt.toISOString());
+			// pds_resolved_at gets pushed to epoch so the next resolve()
+			// sees the row as stale per the TTL check.
+			expect(row?.pds_resolved_at).toBe("1970-01-01T00:00:00.000Z");
+		});
+
+		it("expire on an unknown DID is a no-op (no row appears)", async () => {
+			const cache = createD1DidDocCache(testEnv.DB);
+			await cache.expire(TEST_DID);
+			const row = await testEnv.DB.prepare(`SELECT did FROM known_publishers WHERE did = ?`)
+				.bind(TEST_DID)
+				.first();
+			expect(row).toBeNull();
+		});
+	});
+});

apps/aggregator/test/jetstream-client.test.ts

@@ -0,0 +1,217 @@
+/**
+ * `wrapAtcuteSubscription` regression test.
+ *
+ * The wrapper's `close()` MUST cancel a pending `for await` even when the
+ * underlying subscription is quiescent (no events arriving). MockJetstream
+ * actively resolves pending awaiters on close, so it can't catch a
+ * misbehaving production wrapper — this test pairs the wrapper with a stub
+ * whose `next()` never resolves, and asserts the for-await terminates within
+ * a small grace window after `close()`.
+ *
+ * Without the fix, the wrapper's `close()` flipped a flag the iterator only
+ * checked AFTER `inner.next()` resolved, so a quiescent stream would hang
+ * `stop()` indefinitely. This test failing means we've regressed there.
+ */
+
+import { describe, expect, it } from "vitest";
+
+import { wrapAtcuteSubscription, type RawJetstreamSubscription } from "../src/jetstream-client.js";
+
+interface QuiescentEvent {
+	kind: string;
+}
+
+/**
+ * Subscription stub whose `next()` returns a Promise that NEVER resolves,
+ * even after `return()` is called. This mirrors `@mary-ext/event-iterator`'s
+ * actual behaviour: `EventIterator.return()` drops its resolver reference
+ * without invoking it (`lib/index.ts:55-67`), so a pending `next()` Promise
+ * is orphaned. If the wrapper relies on `inner.return()` resolving the
+ * pending await (the C1 mistake), this stub catches it — the for-await
+ * never wakes from `inner.return()` alone, only the closed-signal race in
+ * the wrapper can unblock it.
+ */
+function quiescentSubscription(): RawJetstreamSubscription<QuiescentEvent> {
+	let returned = false;
+	const innerIter: AsyncIterator<QuiescentEvent> = {
+		next() {
+			if (returned) return Promise.resolve({ value: undefined, done: true });
+			// Return a Promise that never settles. Mirrors EventIterator's
+			// behaviour: it stashes the resolver in a private field and
+			// drops it on return() without ever calling it.
+			return new Promise<IteratorResult<QuiescentEvent>>(() => {});
+		},
+		async return() {
+			returned = true;
+			return { value: undefined, done: true };
+		},
+	};
+	return {
+		cursor: 0,
+		[Symbol.asyncIterator]: () => innerIter,
+	};
+}
+
+describe("wrapAtcuteSubscription", () => {
+	it("close() unblocks a for-await waiting on a quiescent subscription", async () => {
+		const sub = quiescentSubscription();
+		const handle = wrapAtcuteSubscription(sub);
+
+		// Start consuming on a background promise. The for-await will block
+		// on the first iter.next() because the underlying inner iterator
+		// only resolves when return() is called.
+		const consumed: QuiescentEvent[] = [];
+		const consumePromise = (async () => {
+			for await (const event of handle) {
+				consumed.push(event);
+			}
+		})();
+
+		// Yield a few microtasks so the consumer reaches the awaiting state.
+		await Promise.resolve();
+		await Promise.resolve();
+		await new Promise<void>((r) => setTimeout(r, 0));
+
+		// The consumer should be wedged, not done. Use Promise.race to
+		// detect — if it's wedged, the timeout wins.
+		const beforeClose = await Promise.race([
+			consumePromise.then(() => "done" as const),
+			new Promise<"pending">((r) => setTimeout(r, 10, "pending")),
+		]);
+		expect(beforeClose).toBe("pending");
+
+		// close() must cancel the pending await.
+		handle.close();
+
+		// Now the consumer should resolve quickly.
+		await expect(
+			Promise.race([
+				consumePromise.then(() => "done" as const),
+				new Promise<"timeout">((r) => setTimeout(r, 100, "timeout")),
+			]),
+		).resolves.toBe("done");
+
+		expect(consumed).toEqual([]);
+	});
+
+	it("filters non-commit events", async () => {
+		// `isCommitEvent` requires the full commit shape (collection + rkey +
+		// operation) — a `{kind: "commit"}` envelope without a structurally
+		// valid `commit` object is correctly rejected as "malformed", so the
+		// stub must mirror what production producers emit.
+		const events: Array<{
+			kind: string;
+			commit?: { collection: string; rkey: string; operation: string; cid?: string };
+		}> = [
+			{ kind: "identity" },
+			{
+				kind: "commit",
+				commit: { collection: "x", rkey: "r1", operation: "create", cid: "bafyc1" },
+			},
+			{ kind: "account" },
+			{
+				kind: "commit",
+				commit: { collection: "y", rkey: "r2", operation: "create", cid: "bafyc2" },
+			},
+		];
+		let i = 0;
+		const sub: RawJetstreamSubscription<(typeof events)[number]> = {
+			cursor: 0,
+			[Symbol.asyncIterator]: () => ({
+				async next() {
+					if (i >= events.length) return { value: undefined, done: true };
+					const value = events[i++];
+					return { value: value as (typeof events)[number], done: false };
+				},
+			}),
+		};
+		const handle = wrapAtcuteSubscription(sub);
+		const out: unknown[] = [];
+		for await (const event of handle) out.push(event);
+		expect(out).toHaveLength(2);
+		expect(out.every((e) => (e as { kind: string }).kind === "commit")).toBe(true);
+	});
+
+	it("rejects commits with missing cid on non-delete operations", async () => {
+		// `create`/`update` events without a `cid` would produce a RecordsJob
+		// with `cid: undefined`, breaking the consumer's verification step.
+		// Predicate must drop them at the source.
+		const events = [
+			{ kind: "commit", commit: { collection: "x", rkey: "r1", operation: "create" } },
+			{
+				kind: "commit",
+				commit: { collection: "x", rkey: "r2", operation: "update" },
+			},
+		];
+		let i = 0;
+		const sub: RawJetstreamSubscription<(typeof events)[number]> = {
+			cursor: 0,
+			[Symbol.asyncIterator]: () => ({
+				async next() {
+					if (i >= events.length) return { value: undefined, done: true };
+					const value = events[i++];
+					return { value: value as (typeof events)[number], done: false };
+				},
+			}),
+		};
+		const handle = wrapAtcuteSubscription(sub);
+		const out: unknown[] = [];
+		for await (const event of handle) out.push(event);
+		expect(out).toHaveLength(0);
+	});
+
+	it("rejects commits whose operation isn't one of create/update/delete", async () => {
+		// A producer emitting an unknown operation would otherwise produce a
+		// RecordsJob the consumer can't handle, ending up as
+		// UNEXPECTED_ERROR in dead_letters. Better to drop at the source.
+		const events = [
+			{
+				kind: "commit",
+				commit: {
+					collection: "x",
+					rkey: "r1",
+					operation: "rebase", // not a real atproto op
+					cid: "bafyc1",
+				},
+			},
+		];
+		let i = 0;
+		const sub: RawJetstreamSubscription<(typeof events)[number]> = {
+			cursor: 0,
+			[Symbol.asyncIterator]: () => ({
+				async next() {
+					if (i >= events.length) return { value: undefined, done: true };
+					const value = events[i++];
+					return { value: value as (typeof events)[number], done: false };
+				},
+			}),
+		};
+		const handle = wrapAtcuteSubscription(sub);
+		const out: unknown[] = [];
+		for await (const event of handle) out.push(event);
+		expect(out).toHaveLength(0);
+	});
+
+	it("accepts delete commits without cid", async () => {
+		// Delete events legitimately have no cid; predicate must let them
+		// through.
+		const events = [
+			{ kind: "commit", commit: { collection: "x", rkey: "r1", operation: "delete" } },
+		];
+		let i = 0;
+		const sub: RawJetstreamSubscription<(typeof events)[number]> = {
+			cursor: 0,
+			[Symbol.asyncIterator]: () => ({
+				async next() {
+					if (i >= events.length) return { value: undefined, done: true };
+					const value = events[i++];
+					return { value: value as (typeof events)[number], done: false };
+				},
+			}),
+		};
+		const handle = wrapAtcuteSubscription(sub);
+		const out: unknown[] = [];
+		for await (const event of handle) out.push(event);
+		expect(out).toHaveLength(1);
+	});
+});

apps/aggregator/test/jetstream-ingestor.test.ts

@@ -0,0 +1,481 @@
+/**
+ * JetstreamIngestor unit tests.
+ *
+ * Drives the ingestor against MockJetstream, an in-memory queue, and a
+ * Map-backed storage. No DO/D1/Queue runtime needed; the only Cloudflare
+ * binding the ingestor depends on is "something with a send method", which
+ * the mock queue trivially satisfies.
+ *
+ * Tests cover the ingestor's whole contract: event-to-job conversion,
+ * cursor persistence, reconnect with backoff, stop semantics. Each one
+ * pins a behaviour the production DO needs to honour; if a future change
+ * regresses any of them the failure surfaces here, not in production.
+ */
+
+// Subpath imports avoid pulling in `@atproto/repo` (Node-crypto only) which
+// the package's main entry transitively re-exports via FakeRepo. workerd
+// can't load that, but we don't need it here — only MockJetstream + the
+// NSID constants.
+import { MockJetstream } from "@emdash-cms/atproto-test-utils/jetstream";
+import { PROFILE_NSID, RELEASE_NSID } from "@emdash-cms/atproto-test-utils/nsid";
+import { describe, expect, it } from "vitest";
+
+import type { RecordsJob } from "../src/env.js";
+import type {
+	JetstreamClient,
+	JetstreamSubscribeOptions,
+	JetstreamSubscriptionHandle,
+} from "../src/jetstream-client.js";
+import {
+	JetstreamIngestor,
+	type IngestorStorage,
+	type JobQueue,
+} from "../src/jetstream-ingestor.js";
+
+const TEST_DID = "did:plc:test00000000000000000000";
+
+class InMemoryQueue implements JobQueue {
+	readonly jobs: RecordsJob[] = [];
+	send(job: RecordsJob): Promise<void> {
+		this.jobs.push(job);
+		return Promise.resolve();
+	}
+}
+
+class MapStorage implements IngestorStorage {
+	private readonly map = new Map<string, number>();
+	get(key: string): Promise<number | undefined> {
+		return Promise.resolve(this.map.get(key));
+	}
+	put(key: string, value: number): Promise<void> {
+		this.map.set(key, value);
+		return Promise.resolve();
+	}
+}
+
+/**
+ * Adapter that turns a MockJetstream into the JetstreamClient interface
+ * the ingestor uses. MockJetstream's subscribe() already returns an
+ * AsyncIterable with a cursor + close, so this is a thin pass-through —
+ * its only job is shaping the type.
+ */
+class MockJetstreamClient implements JetstreamClient {
+	constructor(private readonly stream: MockJetstream) {}
+	subscribe(opts: JetstreamSubscribeOptions): JetstreamSubscriptionHandle {
+		return this.stream.subscribe({
+			wantedCollections: [...opts.wantedCollections],
+			...(opts.cursor !== undefined ? { cursor: opts.cursor } : {}),
+		});
+	}
+}
+
+interface Harness {
+	stream: MockJetstream;
+	queue: InMemoryQueue;
+	storage: MapStorage;
+	ingestor: JetstreamIngestor;
+	runPromise: Promise<void>;
+}
+
+function buildHarness(opts: { wantedCollections?: readonly string[] } = {}): Harness {
+	const stream = new MockJetstream();
+	const queue = new InMemoryQueue();
+	const storage = new MapStorage();
+	const ingestor = new JetstreamIngestor({
+		client: new MockJetstreamClient(stream),
+		queue,
+		storage,
+		wantedCollections: opts.wantedCollections ?? [PROFILE_NSID, RELEASE_NSID],
+		// Tight backoff so tests don't sit in real timers; jitter off for
+		// deterministic assertions.
+		backoff: { initialDelayMs: 1, maxDelayMs: 5, multiplier: 2, jitter: 0 },
+		sleep: () => Promise.resolve(),
+	});
+	const runPromise = ingestor.run();
+	return { stream, queue, storage, ingestor, runPromise };
+}
+
+/** Wait until the predicate returns true or the test times out. Polls
+ * the microtask queue rather than wall-clock; the ingestor reads events
+ * eagerly inside an async iterator, so a small loop is enough. */
+async function waitFor(predicate: () => boolean, label: string, attempts = 200): Promise<void> {
+	for (let i = 0; i < attempts; i++) {
+		if (predicate()) return;
+		await Promise.resolve();
+		await new Promise<void>((r) => setTimeout(r, 0));
+	}
+	throw new Error(`waitFor timed out: ${label}`);
+}
+
+describe("JetstreamIngestor", () => {
+	it("converts a commit create event into a RecordsJob and enqueues it", async () => {
+		const h = buildHarness();
+		const event = h.stream.emitCommit({
+			did: TEST_DID,
+			collection: PROFILE_NSID,
+			rkey: "p",
+			cid: "bafyrecord",
+			record: { slug: "p", license: "MIT" },
+		});
+
+		await waitFor(() => h.queue.jobs.length === 1, "first job enqueued");
+
+		expect(h.queue.jobs[0]).toEqual({
+			did: TEST_DID,
+			collection: PROFILE_NSID,
+			rkey: "p",
+			operation: "create",
+			cid: "bafyrecord",
+			jetstreamRecord: { slug: "p", license: "MIT" },
+		});
+		expect(h.ingestor.currentCursor).toBe(event.time_us);
+
+		h.ingestor.stop();
+		await h.runPromise;
+	});
+
+	it("persists cursor to storage after each successful enqueue", async () => {
+		const h = buildHarness();
+		const e1 = h.stream.emitCommit({
+			did: TEST_DID,
+			collection: PROFILE_NSID,
+			rkey: "a",
+		});
+		const e2 = h.stream.emitCommit({
+			did: TEST_DID,
+			collection: PROFILE_NSID,
+			rkey: "b",
+		});
+
+		await waitFor(() => h.queue.jobs.length === 2, "both jobs enqueued");
+
+		expect(await h.storage.get("jetstream:cursor")).toBe(e2.time_us);
+		expect(e2.time_us).toBeGreaterThan(e1.time_us);
+
+		h.ingestor.stop();
+		await h.runPromise;
+	});
+
+	it("resumes from the persisted cursor on a fresh ingestor", async () => {
+		const stream = new MockJetstream();
+		const queue = new InMemoryQueue();
+		const storage = new MapStorage();
+		const earlier = stream.emitCommit({
+			did: TEST_DID,
+			collection: PROFILE_NSID,
+			rkey: "earlier",
+		});
+		const later = stream.emitCommit({
+			did: TEST_DID,
+			collection: PROFILE_NSID,
+			rkey: "later",
+		});
+
+		// Pretend a previous run consumed the earlier event.
+		await storage.put("jetstream:cursor", earlier.time_us);
+
+		const ingestor = new JetstreamIngestor({
+			client: new MockJetstreamClient(stream),
+			queue,
+			storage,
+			wantedCollections: [PROFILE_NSID, RELEASE_NSID],
+			backoff: { initialDelayMs: 1, maxDelayMs: 5, multiplier: 2, jitter: 0 },
+			sleep: () => Promise.resolve(),
+		});
+		const runPromise = ingestor.run();
+
+		await waitFor(() => queue.jobs.length === 1, "later event enqueued");
+
+		expect(queue.jobs).toHaveLength(1);
+		expect(queue.jobs[0]?.rkey).toBe("later");
+		expect(ingestor.currentCursor).toBe(later.time_us);
+
+		ingestor.stop();
+		await runPromise;
+	});
+
+	it("seeds cursor from cursorFloor when storage is empty", async () => {
+		// DO storage is empty (fresh deploy / regional failover) but the
+		// cursorFloor callback supplies a time_us derived from D1 — typically
+		// MAX(verified_at) across content tables. The ingestor should adopt
+		// it AND persist it immediately so a crash before the first event
+		// doesn't re-derive on next run.
+		const stream = new MockJetstream();
+		const queue = new InMemoryQueue();
+		const storage = new MapStorage();
+		const floorTimeUs = 1_700_000_000_000_000; // arbitrary stable epoch us
+		let calls = 0;
+
+		const ingestor = new JetstreamIngestor({
+			client: new MockJetstreamClient(stream),
+			queue,
+			storage,
+			wantedCollections: [PROFILE_NSID],
+			backoff: { initialDelayMs: 1, maxDelayMs: 5, multiplier: 2, jitter: 0 },
+			sleep: () => Promise.resolve(),
+			cursorFloor: () => {
+				calls += 1;
+				return Promise.resolve(floorTimeUs);
+			},
+		});
+		const runPromise = ingestor.run();
+
+		// Persistence is the contract — the run loop should write the floor
+		// to storage before opening any subscription.
+		await waitFor(() => ingestor.currentCursor === floorTimeUs, "floor adopted by ingestor");
+		expect(calls).toBe(1);
+		expect(await storage.get("jetstream:cursor")).toBe(floorTimeUs);
+
+		ingestor.stop();
+		await runPromise;
+	});
+
+	it("ignores cursorFloor when storage already has a cursor (no override)", async () => {
+		// The persisted cursor wins — cursorFloor is for the empty-storage
+		// case only. A call to floor() on a warm DO would silently roll the
+		// cursor back to a stale value.
+		const stream = new MockJetstream();
+		const queue = new InMemoryQueue();
+		const storage = new MapStorage();
+		const persisted = 2_000_000_000_000_000;
+		await storage.put("jetstream:cursor", persisted);
+		let floorCalled = false;
+
+		const ingestor = new JetstreamIngestor({
+			client: new MockJetstreamClient(stream),
+			queue,
+			storage,
+			wantedCollections: [PROFILE_NSID],
+			backoff: { initialDelayMs: 1, maxDelayMs: 5, multiplier: 2, jitter: 0 },
+			sleep: () => Promise.resolve(),
+			cursorFloor: () => {
+				floorCalled = true;
+				return Promise.resolve(1_000_000_000_000_000);
+			},
+		});
+		const runPromise = ingestor.run();
+		// Give the run loop a tick to read storage.
+		await new Promise((resolve) => setTimeout(resolve, 5));
+
+		expect(floorCalled).toBe(false);
+		expect(ingestor.currentCursor).toBe(persisted);
+
+		ingestor.stop();
+		await runPromise;
+	});
+
+	it("falls back to subscription default when cursorFloor returns null", async () => {
+		// Truly fresh install — D1 has no rows so floor() returns null.
+		// Storage stays empty; the subscription's own default kicks in
+		// (effectively "now"). Ingestor's cursor stays null until the
+		// first event lands.
+		const stream = new MockJetstream();
+		const queue = new InMemoryQueue();
+		const storage = new MapStorage();
+
+		const ingestor = new JetstreamIngestor({
+			client: new MockJetstreamClient(stream),
+			queue,
+			storage,
+			wantedCollections: [PROFILE_NSID],
+			backoff: { initialDelayMs: 1, maxDelayMs: 5, multiplier: 2, jitter: 0 },
+			sleep: () => Promise.resolve(),
+			cursorFloor: () => Promise.resolve(null),
+		});
+		const runPromise = ingestor.run();
+		await new Promise((resolve) => setTimeout(resolve, 5));
+
+		expect(ingestor.currentCursor).toBeNull();
+		expect(await storage.get("jetstream:cursor")).toBeUndefined();
+
+		ingestor.stop();
+		await runPromise;
+	});
+
+	it("handles delete operations (no record body, empty cid)", async () => {
+		const h = buildHarness();
+		h.stream.emit({
+			did: TEST_DID,
+			time_us: Date.now() * 1000,
+			kind: "commit",
+			commit: {
+				rev: "rev-del",
+				collection: PROFILE_NSID,
+				rkey: "p",
+				operation: "delete",
+			},
+		});
+
+		await waitFor(() => h.queue.jobs.length === 1, "delete job enqueued");
+
+		expect(h.queue.jobs[0]).toEqual({
+			did: TEST_DID,
+			collection: PROFILE_NSID,
+			rkey: "p",
+			operation: "delete",
+			cid: "",
+		});
+		expect(h.queue.jobs[0]?.jetstreamRecord).toBeUndefined();
+
+		h.ingestor.stop();
+		await h.runPromise;
+	});
+
+	it("filters events outside wantedCollections (defence in depth)", async () => {
+		// Real Jetstream filters server-side, but a malicious or buggy relay
+		// could send something off-list. The ingestor must not enqueue it.
+		const h = buildHarness({ wantedCollections: [PROFILE_NSID] });
+		h.stream.emitCommit({
+			did: TEST_DID,
+			collection: RELEASE_NSID, // not in wantedCollections
+			rkey: "ignored:1.0.0",
+		});
+		h.stream.emitCommit({
+			did: TEST_DID,
+			collection: PROFILE_NSID,
+			rkey: "kept",
+		});
+
+		await waitFor(() => h.queue.jobs.length === 1, "filtered job enqueued");
+
+		expect(h.queue.jobs).toHaveLength(1);
+		expect(h.queue.jobs[0]?.rkey).toBe("kept");
+
+		h.ingestor.stop();
+		await h.runPromise;
+	});
+
+	it("stop() ends the run loop cleanly", async () => {
+		const h = buildHarness();
+		h.stream.emitCommit({ did: TEST_DID, collection: PROFILE_NSID, rkey: "p" });
+		await waitFor(() => h.queue.jobs.length === 1, "first job");
+
+		h.ingestor.stop();
+		await expect(h.runPromise).resolves.toBeUndefined();
+	});
+
+	it("consecutiveFailures stays 0 across disconnect-with-events cycles", async () => {
+		// Per the documented contract: 0 means the most recent connection
+		// attempt produced at least one event. A connect → consume → close
+		// cycle must NOT bump the counter to 1.
+		const stream = new MockJetstream();
+		const queue = new InMemoryQueue();
+		const storage = new MapStorage();
+		const ingestor = new JetstreamIngestor({
+			client: new MockJetstreamClient(stream),
+			queue,
+			storage,
+			wantedCollections: [PROFILE_NSID],
+			backoff: { initialDelayMs: 1, maxDelayMs: 5, multiplier: 2, jitter: 0 },
+			sleep: () => Promise.resolve(),
+		});
+		const runPromise = ingestor.run();
+
+		// Three full cycles of: connect → emit → close. After each, the
+		// counter should still be 0 because each attempt made progress.
+		for (let i = 0; i < 3; i++) {
+			stream.emitCommit({ did: TEST_DID, collection: PROFILE_NSID, rkey: `r${i}` });
+			await waitFor(() => queue.jobs.length === i + 1, `event ${i}`);
+			stream.closeAll();
+			// Yield enough microtasks for the run loop to process the close
+			// and complete its bookkeeping before we inspect.
+			await new Promise<void>((r) => setTimeout(r, 5));
+			expect(ingestor.consecutiveFailures).toBe(0);
+		}
+
+		ingestor.stop();
+		await runPromise;
+	});
+
+	it("resets backoff after a successful event, even across reconnects", async () => {
+		// Without a reset, a subscription that disconnects → reconnects →
+		// processes an event → disconnects again would back off as if the
+		// failures were continuous. The "made progress" flag should reset
+		// the counter so each disconnect that consumed events starts fresh
+		// from the initial delay.
+		const stream = new MockJetstream();
+		const queue = new InMemoryQueue();
+		const storage = new MapStorage();
+		const sleeps: number[] = [];
+		const ingestor = new JetstreamIngestor({
+			client: new MockJetstreamClient(stream),
+			queue,
+			storage,
+			wantedCollections: [PROFILE_NSID],
+			backoff: { initialDelayMs: 10, maxDelayMs: 1000, multiplier: 10, jitter: 0 },
+			sleep: (ms) => {
+				sleeps.push(ms);
+				return Promise.resolve();
+			},
+		});
+		const runPromise = ingestor.run();
+
+		// Cycle 1: emit event, close. After backoff this should still be
+		// the initial delay because we made progress.
+		stream.emitCommit({ did: TEST_DID, collection: PROFILE_NSID, rkey: "a" });
+		await waitFor(() => queue.jobs.length === 1, "first job");
+		stream.closeAll();
+		await waitFor(() => sleeps.length >= 1, "first backoff");
+
+		// Cycle 2: emit another event, close. Backoff should still be the
+		// initial delay (10ms), not 100ms (10×10), because progress in
+		// between resets the counter.
+		stream.emitCommit({ did: TEST_DID, collection: PROFILE_NSID, rkey: "b" });
+		await waitFor(() => queue.jobs.length === 2, "second job", 500);
+		stream.closeAll();
+		await waitFor(() => sleeps.length >= 2, "second backoff", 500);
+
+		expect(sleeps[0]).toBe(10);
+		expect(sleeps[1]).toBe(10);
+
+		ingestor.stop();
+		await runPromise;
+	});
+
+	it("computes exponential backoff with cap, no jitter for determinism", async () => {
+		// Direct unit on the backoff calc via stop()/restart of subscription.
+		// The straightforward way: drive the stream, close the subscription
+		// from underneath the ingestor (simulating a Jetstream disconnect),
+		// observe the sleep delays the ingestor passes to our injected
+		// `sleep`. Without driving real time we can't easily probe — instead
+		// verify the run loop survives a series of forced disconnects and
+		// keeps consuming after each.
+		const stream = new MockJetstream();
+		const queue = new InMemoryQueue();
+		const storage = new MapStorage();
+		const sleeps: number[] = [];
+		const ingestor = new JetstreamIngestor({
+			client: new MockJetstreamClient(stream),
+			queue,
+			storage,
+			wantedCollections: [PROFILE_NSID],
+			backoff: { initialDelayMs: 10, maxDelayMs: 80, multiplier: 2, jitter: 0 },
+			sleep: (ms) => {
+				sleeps.push(ms);
+				return Promise.resolve();
+			},
+		});
+		const runPromise = ingestor.run();
+
+		// Emit one event so the subscription has work, then close it from the
+		// MockJetstream side to simulate a server disconnect. The ingestor's
+		// loop sees the iterator end, sleeps with backoff, reconnects.
+		stream.emitCommit({ did: TEST_DID, collection: PROFILE_NSID, rkey: "a" });
+		await waitFor(() => queue.jobs.length === 1, "first job");
+		stream.closeAll();
+
+		// After the disconnect, post a second event so the new subscription
+		// has something to consume. Wait until it lands.
+		stream.emitCommit({ did: TEST_DID, collection: PROFILE_NSID, rkey: "b" });
+		await waitFor(() => queue.jobs.length === 2, "post-reconnect job", 500);
+
+		// At least one backoff sleep happened between the disconnect and
+		// the next successful subscription.
+		expect(sleeps.length).toBeGreaterThanOrEqual(1);
+		expect(sleeps[0]).toBeGreaterThanOrEqual(10);
+		expect(sleeps[0]).toBeLessThanOrEqual(80);
+
+		ingestor.stop();
+		await runPromise;
+	});
+});

apps/aggregator/test/pds-verify.test.ts

@@ -0,0 +1,174 @@
+/**
+ * pds-verify unit tests.
+ *
+ * Cover the HTTP / error-shaping logic with a stub `fetch`. The actual
+ * verification handoff to `@atcute/repo`'s `verifyRecord` is NOT exercised
+ * end-to-end anywhere in this suite — building a valid signed CAR by hand
+ * would re-implement what `@atcute/repo` already tests internally, and the
+ * consumer-level test path stubs verification via `ConsumerDeps.verify`
+ * (the FakePublisher / MockPds fixture from `@emdash-cms/atproto-test-utils`
+ * can't load inside `@cloudflare/vitest-pool-workers` due to a transitive
+ * `@atproto/lex-data` incompatibility; see records-consumer test header).
+ *
+ * What we DO test here is the surface every reason code can be reached
+ * through, plus the `isTransient` policy mapping the consumer relies on.
+ */
+
+import { P256PublicKey, P256PrivateKeyExportable } from "@atcute/crypto";
+import { beforeAll, describe, expect, it } from "vitest";
+
+import { fetchAndVerifyRecord, isTransient, PdsVerificationError } from "../src/pds-verify.js";
+
+const TEST_DID = "did:plc:test00000000000000000000";
+const TEST_PDS = "https://pds.test.example";
+
+let publicKey: P256PublicKey;
+
+beforeAll(async () => {
+	const kp = await P256PrivateKeyExportable.createKeypair();
+	const raw = await kp.exportPublicKey("raw");
+	publicKey = await P256PublicKey.importRaw(raw);
+});
+
+async function captureRejection<T>(promise: Promise<T>): Promise<PdsVerificationError> {
+	try {
+		await promise;
+	} catch (err) {
+		if (err instanceof PdsVerificationError) return err;
+		throw err;
+	}
+	throw new Error("expected promise to reject with PdsVerificationError");
+}
+
+function buildOpts(overrides: {
+	fetch: typeof fetch;
+	timeoutMs?: number;
+	maxResponseBytes?: number;
+}) {
+	return {
+		pds: TEST_PDS,
+		did: TEST_DID,
+		collection: "com.emdashcms.experimental.package.profile",
+		rkey: "demo",
+		publicKey,
+		...overrides,
+	};
+}
+
+describe("fetchAndVerifyRecord — HTTP path", () => {
+	it("builds the canonical sync.getRecord URL with did/collection/rkey", async () => {
+		let observedUrl: string | undefined;
+		const fetchImpl: typeof fetch = async (input) => {
+			observedUrl =
+				typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
+			return new Response(new Uint8Array([0]), { status: 200 });
+		};
+		await fetchAndVerifyRecord(buildOpts({ fetch: fetchImpl })).catch(() => {
+			/* verifyRecord rejects on the dummy bytes — we only care about the URL */
+		});
+		expect(observedUrl).toBe(
+			`${TEST_PDS}/xrpc/com.atproto.sync.getRecord?did=${encodeURIComponent(TEST_DID)}&collection=com.emdashcms.experimental.package.profile&rkey=demo`,
+		);
+	});
+
+	it("maps a network error to PDS_NETWORK_ERROR", async () => {
+		const fetchImpl: typeof fetch = () => Promise.reject(new TypeError("connection refused"));
+		await expect(fetchAndVerifyRecord(buildOpts({ fetch: fetchImpl }))).rejects.toMatchObject({
+			name: "PdsVerificationError",
+			reason: "PDS_NETWORK_ERROR",
+		});
+	});
+
+	it("maps an aborted fetch (timeout) to PDS_NETWORK_ERROR with the timeout in the message", async () => {
+		const fetchImpl: typeof fetch = (_input, init) => {
+			return new Promise((_resolve, reject) => {
+				init?.signal?.addEventListener("abort", () => {
+					const err = new DOMException("aborted", "AbortError");
+					reject(err);
+				});
+			});
+		};
+		const err = await captureRejection(
+			fetchAndVerifyRecord(buildOpts({ fetch: fetchImpl, timeoutMs: 25 })),
+		);
+		expect(err.reason).toBe("PDS_NETWORK_ERROR");
+		expect(err.message).toMatch(/aborted after 25ms/);
+	});
+
+	it("maps a 404 to RECORD_NOT_FOUND with status", async () => {
+		const fetchImpl: typeof fetch = () => Promise.resolve(new Response("", { status: 404 }));
+		const err = await captureRejection(fetchAndVerifyRecord(buildOpts({ fetch: fetchImpl })));
+		expect(err.reason).toBe("RECORD_NOT_FOUND");
+		expect(err.status).toBe(404);
+	});
+
+	it("maps a 500 to PDS_HTTP_ERROR with status", async () => {
+		const fetchImpl: typeof fetch = () => Promise.resolve(new Response("", { status: 503 }));
+		const err = await captureRejection(fetchAndVerifyRecord(buildOpts({ fetch: fetchImpl })));
+		expect(err.reason).toBe("PDS_HTTP_ERROR");
+		expect(err.status).toBe(503);
+	});
+
+	it("maps a non-404 4xx to PDS_HTTP_ERROR with status", async () => {
+		const fetchImpl: typeof fetch = () => Promise.resolve(new Response("", { status: 401 }));
+		const err = await captureRejection(fetchAndVerifyRecord(buildOpts({ fetch: fetchImpl })));
+		expect(err.reason).toBe("PDS_HTTP_ERROR");
+		expect(err.status).toBe(401);
+	});
+
+	it("rejects responses larger than maxResponseBytes with RESPONSE_TOO_LARGE", async () => {
+		const big = new Uint8Array(64);
+		big.fill(0xff);
+		const fetchImpl: typeof fetch = () => Promise.resolve(new Response(big, { status: 200 }));
+		const err = await captureRejection(
+			fetchAndVerifyRecord(buildOpts({ fetch: fetchImpl, maxResponseBytes: 16 })),
+		);
+		expect(err.reason).toBe("RESPONSE_TOO_LARGE");
+	});
+
+	it("rejects a null body with INVALID_PROOF", async () => {
+		const fetchImpl: typeof fetch = () => {
+			// Construct a Response with a null body. The Response constructor
+			// allows null for HEAD-style responses; we never get null in
+			// practice but the guard is defensive.
+			return Promise.resolve(new Response(null, { status: 200 }));
+		};
+		const err = await captureRejection(fetchAndVerifyRecord(buildOpts({ fetch: fetchImpl })));
+		expect(err.reason).toBe("INVALID_PROOF");
+	});
+
+	it("hands successful body bytes to verifyRecord (which rejects malformed input as INVALID_PROOF)", async () => {
+		// Random bytes are guaranteed not to parse as a valid CAR. The point
+		// of the test is that we got past HTTP and INTO verifyRecord — and
+		// that verifyRecord's rejection is wrapped as INVALID_PROOF.
+		const garbage = new Uint8Array([1, 2, 3, 4, 5]);
+		const fetchImpl: typeof fetch = () => Promise.resolve(new Response(garbage, { status: 200 }));
+		const err = await captureRejection(fetchAndVerifyRecord(buildOpts({ fetch: fetchImpl })));
+		expect(err.reason).toBe("INVALID_PROOF");
+		expect(err.cause).toBeDefined();
+	});
+});
+
+describe("isTransient policy", () => {
+	it("network errors retry", () => {
+		expect(isTransient("PDS_NETWORK_ERROR", undefined)).toBe(true);
+	});
+	it("HTTP 5xx retries", () => {
+		expect(isTransient("PDS_HTTP_ERROR", 500)).toBe(true);
+		expect(isTransient("PDS_HTTP_ERROR", 503)).toBe(true);
+	});
+	it("HTTP 4xx is permanent", () => {
+		expect(isTransient("PDS_HTTP_ERROR", 401)).toBe(false);
+		expect(isTransient("PDS_HTTP_ERROR", 400)).toBe(false);
+	});
+	it("missing status on PDS_HTTP_ERROR is treated as permanent", () => {
+		// Defensive: PDS_HTTP_ERROR is always raised with a status, but the
+		// policy must not blow up if a future code path drops it.
+		expect(isTransient("PDS_HTTP_ERROR", undefined)).toBe(false);
+	});
+	it("404, oversized response, and invalid proof are permanent", () => {
+		expect(isTransient("RECORD_NOT_FOUND", 404)).toBe(false);
+		expect(isTransient("RESPONSE_TOO_LARGE", undefined)).toBe(false);
+		expect(isTransient("INVALID_PROOF", undefined)).toBe(false);
+	});
+});

apps/aggregator/test/read-api.test.ts

@@ -0,0 +1,514 @@
+/**
+ * Read API integration tests.
+ *
+ * Each test seeds D1 directly with the columns the handlers read, then
+ * exercises the handler via `SELF.fetch` to a `/xrpc/...` URL — same path
+ * a real client would take. Asserts on the envelope shape (uri, cid, did,
+ * indexedAt, mirrors, labels) and on error mappings (404 NotFound, 400
+ * InvalidRequest).
+ *
+ * `mirrors: []` and `labels: []` are the v1 contract; Slice 2 (labels)
+ * and Slice 3 (mirrors) populate them, but the contract is locked now so
+ * cached clients don't see a shape change later.
+ */
+
+import { NSID } from "@emdash-cms/registry-lexicons";
+import { applyD1Migrations, env, SELF } from "cloudflare:test";
+import { beforeAll, beforeEach, describe, expect, it } from "vitest";
+
+interface TestEnv {
+	DB: D1Database;
+	TEST_MIGRATIONS: Parameters<typeof applyD1Migrations>[1];
+}
+const testEnv = env as unknown as TestEnv;
+
+const DID_A = "did:plc:read000000000000000000aa";
+const DID_B = "did:plc:read000000000000000000bb";
+const NOW = new Date("2026-05-10T12:00:00.000Z");
+
+beforeAll(async () => {
+	await applyD1Migrations(testEnv.DB, testEnv.TEST_MIGRATIONS);
+});
+
+beforeEach(async () => {
+	// Tables in dependency order: releases → packages (FK), then publishers
+	// + verifications.
+	await testEnv.DB.prepare("DELETE FROM releases").run();
+	await testEnv.DB.prepare("DELETE FROM packages").run();
+	await testEnv.DB.prepare("DELETE FROM publishers").run();
+	await testEnv.DB.prepare("DELETE FROM publisher_verifications").run();
+});
+
+interface SeedPackageOpts {
+	did?: string;
+	slug?: string;
+	type?: string;
+	name?: string | null;
+	description?: string | null;
+	license?: string;
+	keywords?: string[] | null;
+	latestVersion?: string | null;
+	cid?: string;
+	indexedAt?: string;
+	verifiedAt?: string;
+	carBytes?: Uint8Array;
+}
+
+async function seedPackage(opts: SeedPackageOpts = {}): Promise<void> {
+	const did = opts.did ?? DID_A;
+	const slug = opts.slug ?? "demo";
+	const indexedAt = opts.indexedAt ?? NOW.toISOString();
+	await testEnv.DB.prepare(
+		`INSERT INTO packages
+		   (did, slug, type, name, description, license, authors, security, keywords,
+		    sections, last_updated, latest_version, capabilities, record_blob,
+		    signature_metadata, verified_at, indexed_at)
+		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	)
+		.bind(
+			did,
+			slug,
+			opts.type ?? "emdash-plugin",
+			opts.name ?? "Demo Plugin",
+			opts.description ?? "A demo plugin",
+			opts.license ?? "MIT",
+			JSON.stringify([{ name: "Tester" }]),
+			JSON.stringify([{ email: "x@y.test" }]),
+			opts.keywords === null ? null : JSON.stringify(opts.keywords ?? ["demo"]),
+			null,
+			NOW.toISOString(),
+			opts.latestVersion ?? null,
+			null,
+			opts.carBytes ?? new Uint8Array([0xa1, 0xa2, 0xa3]),
+			JSON.stringify({ cid: opts.cid ?? "bafyseed" }),
+			opts.verifiedAt ?? NOW.toISOString(),
+			indexedAt,
+		)
+		.run();
+}
+
+interface SeedReleaseOpts {
+	did?: string;
+	package?: string;
+	version: string;
+	versionSort?: string;
+	tombstoned?: boolean;
+	cid?: string;
+	carBytes?: Uint8Array;
+}
+
+async function seedRelease(opts: SeedReleaseOpts): Promise<void> {
+	const did = opts.did ?? DID_A;
+	const pkg = opts.package ?? "demo";
+	const rkey = `${pkg}:${opts.version}`;
+	await testEnv.DB.prepare(
+		`INSERT INTO releases
+		   (did, package, version, rkey, version_sort, artifacts, requires, suggests,
+		    emdash_extension, repo_url, cts, record_blob, signature_metadata,
+		    verified_at, indexed_at, tombstoned_at)
+		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+	)
+		.bind(
+			did,
+			pkg,
+			opts.version,
+			rkey,
+			opts.versionSort ?? defaultVersionSort(opts.version),
+			JSON.stringify({ package: { url: "https://x.test/d.tgz", checksum: "bsha256-abc" } }),
+			null,
+			null,
+			JSON.stringify({ declaredAccess: {} }),
+			null,
+			NOW.toISOString(),
+			opts.carBytes ?? new Uint8Array([0xb1, 0xb2, 0xb3]),
+			JSON.stringify({ cid: opts.cid ?? `bafrel-${opts.version}` }),
+			NOW.toISOString(),
+			NOW.toISOString(),
+			opts.tombstoned ? NOW.toISOString() : null,
+		)
+		.run();
+}
+
+/** Naive 1.x.y zero-padded version_sort for the test fixtures. Real values
+ * come from the consumer's `computeVersionSort`; tests just need the
+ * relative ordering to be right. */
+function defaultVersionSort(version: string): string {
+	const [major = "0", minor = "0", patch = "0"] = version.split(".");
+	const pad = (s: string) => s.padStart(10, "0");
+	return `${pad(major)}.${pad(minor)}.${pad(patch)}.~`;
+}
+
+describe("getPackage", () => {
+	it("returns the packageView envelope for an indexed package", async () => {
+		await seedPackage({ slug: "demo", latestVersion: "1.0.0" });
+
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorGetPackage}?did=${DID_A}&slug=demo`,
+		);
+		expect(res.status).toBe(200);
+		const body = (await res.json()) as Record<string, unknown>;
+		expect(body).toMatchObject({
+			uri: `at://${DID_A}/${NSID.packageProfile}/demo`,
+			cid: "bafyseed",
+			did: DID_A,
+			slug: "demo",
+			latestVersion: "1.0.0",
+			indexedAt: NOW.toISOString(),
+			labels: [],
+		});
+		// `mirrors` is on releaseView only — assert it's NOT on packageView.
+		expect(body).not.toHaveProperty("mirrors");
+		const profile = body["profile"] as Record<string, unknown>;
+		expect(profile["$type"]).toBe(NSID.packageProfile);
+		expect(profile["id"]).toBe(`at://${DID_A}/${NSID.packageProfile}/demo`);
+		expect(profile["license"]).toBe("MIT");
+		expect(profile["slug"]).toBe("demo");
+	});
+
+	it("returns 404 NotFound when no row matches", async () => {
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorGetPackage}?did=${DID_A}&slug=missing`,
+		);
+		expect(res.status).toBe(404);
+		const body = (await res.json()) as { error: string };
+		expect(body.error).toBe("NotFound");
+	});
+
+	it("returns 400 InvalidRequest on missing required params", async () => {
+		const res = await SELF.fetch(`https://test/xrpc/${NSID.aggregatorGetPackage}?did=${DID_A}`);
+		expect(res.status).toBe(400);
+	});
+
+	it("sets Cache-Control: private, no-store on success", async () => {
+		await seedPackage({ slug: "demo" });
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorGetPackage}?did=${DID_A}&slug=demo`,
+		);
+		expect(res.headers.get("cache-control")).toBe("private, no-store");
+	});
+
+	it("omits latestVersion when no release has been written yet", async () => {
+		await seedPackage({ slug: "fresh", latestVersion: null });
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorGetPackage}?did=${DID_A}&slug=fresh`,
+		);
+		const body = (await res.json()) as Record<string, unknown>;
+		expect(body).not.toHaveProperty("latestVersion");
+	});
+});
+
+describe("listReleases", () => {
+	it("returns releases ordered by descending semver", async () => {
+		await seedPackage({ slug: "demo", latestVersion: "2.0.0" });
+		await seedRelease({ version: "1.0.0" });
+		await seedRelease({ version: "1.10.0" });
+		await seedRelease({ version: "2.0.0" });
+
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorListReleases}?did=${DID_A}&package=demo`,
+		);
+		expect(res.status).toBe(200);
+		const body = (await res.json()) as { releases: Array<{ version: string }>; cursor?: string };
+		expect(body.releases.map((r) => r.version)).toEqual(["2.0.0", "1.10.0", "1.0.0"]);
+		expect(body).not.toHaveProperty("cursor");
+	});
+
+	it("filters tombstoned releases", async () => {
+		await seedPackage({ slug: "demo" });
+		await seedRelease({ version: "1.0.0" });
+		await seedRelease({ version: "1.1.0", tombstoned: true });
+		await seedRelease({ version: "1.2.0" });
+
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorListReleases}?did=${DID_A}&package=demo`,
+		);
+		const body = (await res.json()) as { releases: Array<{ version: string }> };
+		expect(body.releases.map((r) => r.version)).toEqual(["1.2.0", "1.0.0"]);
+	});
+
+	it("paginates via cursor", async () => {
+		await seedPackage({ slug: "demo" });
+		for (let i = 1; i <= 5; i++) await seedRelease({ version: `1.${i}.0` });
+
+		const first = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorListReleases}?did=${DID_A}&package=demo&limit=2`,
+		);
+		const firstBody = (await first.json()) as {
+			releases: Array<{ version: string }>;
+			cursor: string;
+		};
+		expect(firstBody.releases.map((r) => r.version)).toEqual(["1.5.0", "1.4.0"]);
+		expect(firstBody.cursor).toBeTruthy();
+
+		const second = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorListReleases}?did=${DID_A}&package=demo&limit=2&cursor=${encodeURIComponent(firstBody.cursor)}`,
+		);
+		const secondBody = (await second.json()) as {
+			releases: Array<{ version: string }>;
+			cursor?: string;
+		};
+		expect(secondBody.releases.map((r) => r.version)).toEqual(["1.3.0", "1.2.0"]);
+		expect(secondBody.cursor).toBeTruthy();
+	});
+
+	it("returns 404 when the parent package is missing", async () => {
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorListReleases}?did=${DID_A}&package=missing`,
+		);
+		expect(res.status).toBe(404);
+	});
+
+	it("400s on a provided-but-malformed cursor", async () => {
+		await seedPackage({ slug: "demo" });
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorListReleases}?did=${DID_A}&package=demo&cursor=garbage!!!`,
+		);
+		expect(res.status).toBe(400);
+	});
+});
+
+describe("getLatestRelease", () => {
+	it("returns the release pointed to by packages.latest_version", async () => {
+		await seedPackage({ slug: "demo", latestVersion: "2.0.0" });
+		await seedRelease({ version: "1.0.0" });
+		await seedRelease({ version: "2.0.0" });
+
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorGetLatestRelease}?did=${DID_A}&package=demo`,
+		);
+		expect(res.status).toBe(200);
+		const body = (await res.json()) as Record<string, unknown>;
+		expect(body["version"]).toBe("2.0.0");
+		expect(body["uri"]).toBe(`at://${DID_A}/${NSID.packageRelease}/demo:2.0.0`);
+	});
+
+	it("returns 404 when ALL releases are tombstoned (or none exist)", async () => {
+		await seedPackage({ slug: "demo", latestVersion: "1.0.0" });
+		await seedRelease({ version: "1.0.0", tombstoned: true });
+
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorGetLatestRelease}?did=${DID_A}&package=demo`,
+		);
+		expect(res.status).toBe(404);
+	});
+
+	it("falls back to the next-best release when latest_version points at a tombstoned one", async () => {
+		// `latest_version` was set to 2.0.0, then 2.0.0 was tombstoned but
+		// `refreshPackageLatestStmt` hasn't run yet (or failed). The fast-path
+		// JOIN misses; the slow-path ORDER BY should still find 1.0.0 and
+		// return it instead of 404ing.
+		await seedPackage({ slug: "demo", latestVersion: "2.0.0" });
+		await seedRelease({ version: "1.0.0" });
+		await seedRelease({ version: "2.0.0", tombstoned: true });
+
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorGetLatestRelease}?did=${DID_A}&package=demo`,
+		);
+		expect(res.status).toBe(200);
+		const body = (await res.json()) as Record<string, unknown>;
+		expect(body["version"]).toBe("1.0.0");
+	});
+
+	it("returns 404 when no package row exists", async () => {
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorGetLatestRelease}?did=${DID_A}&package=missing`,
+		);
+		expect(res.status).toBe(404);
+	});
+});
+
+describe("searchPackages", () => {
+	it("returns FTS-matched packages", async () => {
+		await seedPackage({ slug: "gallery", name: "Gallery Plugin", description: "image gallery" });
+		await seedPackage({ slug: "form", name: "Form Plugin", description: "form builder" });
+
+		const res = await SELF.fetch(`https://test/xrpc/${NSID.aggregatorSearchPackages}?q=gallery`);
+		expect(res.status).toBe(200);
+		const body = (await res.json()) as { packages: Array<{ slug: string }> };
+		expect(body.packages.map((p) => p.slug)).toContain("gallery");
+		expect(body.packages.map((p) => p.slug)).not.toContain("form");
+	});
+
+	it("returns all packages ordered by last_updated DESC when q is empty", async () => {
+		await seedPackage({ slug: "alpha" });
+		await seedPackage({ slug: "beta" });
+		const res = await SELF.fetch(`https://test/xrpc/${NSID.aggregatorSearchPackages}`);
+		const body = (await res.json()) as { packages: Array<{ slug: string }> };
+		expect(body.packages.map((p) => p.slug).toSorted()).toEqual(["alpha", "beta"]);
+	});
+
+	it("paginates via offset cursor", async () => {
+		for (let i = 0; i < 5; i++) await seedPackage({ slug: `pkg${i}` });
+
+		const first = await SELF.fetch(`https://test/xrpc/${NSID.aggregatorSearchPackages}?limit=2`);
+		const firstBody = (await first.json()) as {
+			packages: Array<{ slug: string }>;
+			cursor: string;
+		};
+		expect(firstBody.packages).toHaveLength(2);
+		expect(firstBody.cursor).toBeTruthy();
+
+		const second = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorSearchPackages}?limit=2&cursor=${encodeURIComponent(firstBody.cursor)}`,
+		);
+		const secondBody = (await second.json()) as { packages: Array<{ slug: string }> };
+		expect(secondBody.packages).toHaveLength(2);
+		// Distinct from page 1 — seeded slugs don't overlap.
+		const overlap = firstBody.packages
+			.map((p) => p.slug)
+			.filter((s) => secondBody.packages.some((p) => p.slug === s));
+		expect(overlap).toEqual([]);
+	});
+
+	it("doesn't blow up on FTS-unsafe query chars (defensive quoting)", async () => {
+		await seedPackage({ slug: "demo", name: "Demo" });
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorSearchPackages}?q=${encodeURIComponent('demo "*"(')}`,
+		);
+		// Either matches nothing or matches normally — but doesn't 500.
+		expect(res.status).toBe(200);
+	});
+
+	it("treats FTS operators as literal tokens (the escape actually works)", async () => {
+		await seedPackage({ slug: "alpha", name: "Alpha" });
+		await seedPackage({ slug: "beta", name: "Beta" });
+		// `alpha OR beta` would match both packages if `OR` were interpreted
+		// as the FTS5 operator. With proper escaping the whole string is one
+		// literal phrase that can't possibly appear in either record's
+		// indexed text → zero matches. A buggy escape that stripped the
+		// quotes would return *both* packages.
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorSearchPackages}?q=${encodeURIComponent("alpha OR beta")}`,
+		);
+		expect(res.status).toBe(200);
+		const body = (await res.json()) as { packages: Array<{ slug: string }> };
+		expect(body.packages).toEqual([]);
+	});
+
+	it("400s on a provided-but-malformed cursor (no silent restart)", async () => {
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorSearchPackages}?cursor=not-a-valid-cursor`,
+		);
+		expect(res.status).toBe(400);
+	});
+
+	it("400s on a forged cursor with an out-of-range offset", async () => {
+		// Encode {offset: 1_000_000} → over MAX_OFFSET → 400.
+		const forged = btoa(JSON.stringify({ offset: 1_000_000 }))
+			.replace(/\+/g, "-")
+			.replace(/\//g, "_")
+			.replace(/=+$/, "");
+		const res = await SELF.fetch(
+			`https://test/xrpc/${NSID.aggregatorSearchPackages}?cursor=${forged}`,
+		);
+		expect(res.status).toBe(400);
+	});
+});
+
+describe("sync.getRecord", () => {
+	const PATH = "/xrpc/com.atproto.sync.getRecord";
+
+	it("returns CAR bytes for an indexed package profile", async () => {
+		await seedPackage({ slug: "demo", carBytes: new Uint8Array([0x11, 0x22, 0x33]) });
+		const res = await SELF.fetch(
+			`https://test${PATH}?did=${DID_A}&collection=${NSID.packageProfile}&rkey=demo`,
+		);
+		expect(res.status).toBe(200);
+		expect(res.headers.get("content-type")).toBe("application/vnd.ipld.car");
+		expect(res.headers.get("cache-control")).toBe("public, max-age=300");
+		const bytes = new Uint8Array(await res.arrayBuffer());
+		expect([...bytes]).toEqual([0x11, 0x22, 0x33]);
+	});
+
+	it("returns CAR bytes for an indexed release", async () => {
+		await seedPackage({ slug: "demo" });
+		await seedRelease({ version: "1.0.0", carBytes: new Uint8Array([0x44, 0x55]) });
+		const res = await SELF.fetch(
+			`https://test${PATH}?did=${DID_A}&collection=${NSID.packageRelease}&rkey=demo:1.0.0`,
+		);
+		expect(res.status).toBe(200);
+		const bytes = new Uint8Array(await res.arrayBuffer());
+		expect([...bytes]).toEqual([0x44, 0x55]);
+	});
+
+	it("returns 404 for a tombstoned release", async () => {
+		await seedPackage({ slug: "demo" });
+		await seedRelease({ version: "1.0.0", tombstoned: true });
+		const res = await SELF.fetch(
+			`https://test${PATH}?did=${DID_A}&collection=${NSID.packageRelease}&rkey=demo:1.0.0`,
+		);
+		expect(res.status).toBe(404);
+	});
+
+	it("returns 404 for an unknown rkey", async () => {
+		const res = await SELF.fetch(
+			`https://test${PATH}?did=${DID_A}&collection=${NSID.packageProfile}&rkey=does-not-exist`,
+		);
+		expect(res.status).toBe(404);
+	});
+
+	it("returns 400 InvalidRequest on missing query params", async () => {
+		const res = await SELF.fetch(`https://test${PATH}?did=${DID_A}`);
+		expect(res.status).toBe(400);
+	});
+
+	it("returns 400 on a malformed DID", async () => {
+		const res = await SELF.fetch(
+			`https://test${PATH}?did=not-a-did&collection=${NSID.packageProfile}&rkey=demo`,
+		);
+		expect(res.status).toBe(400);
+	});
+
+	it("returns HEAD with content-length but no body", async () => {
+		await seedPackage({ slug: "demo", carBytes: new Uint8Array([0x11, 0x22, 0x33]) });
+		const res = await SELF.fetch(
+			`https://test${PATH}?did=${DID_A}&collection=${NSID.packageProfile}&rkey=demo`,
+			{ method: "HEAD" },
+		);
+		expect(res.status).toBe(200);
+		expect(res.headers.get("content-length")).toBe("3");
+		const bytes = new Uint8Array(await res.arrayBuffer());
+		expect(bytes.byteLength).toBe(0);
+	});
+
+	it("rejects non-GET/HEAD methods with 405", async () => {
+		const res = await SELF.fetch(
+			`https://test${PATH}?did=${DID_A}&collection=${NSID.packageProfile}&rkey=demo`,
+			{ method: "POST" },
+		);
+		expect(res.status).toBe(405);
+		expect(res.headers.get("allow")).toBe("GET, HEAD");
+	});
+
+	it("only matches publisher.profile when rkey='self'", async () => {
+		// publisher.profile rkey is always 'self'; any other rkey returns 404
+		// even if the (did) row exists.
+		await testEnv.DB.prepare(
+			`INSERT INTO publishers
+			   (did, display_name, record_blob, verified_at, indexed_at)
+			 VALUES (?, ?, ?, ?, ?)`,
+		)
+			.bind(DID_B, "Pub", new Uint8Array([0x99]), NOW.toISOString(), NOW.toISOString())
+			.run();
+		const wrongRkey = await SELF.fetch(
+			`https://test${PATH}?did=${DID_B}&collection=${NSID.publisherProfile}&rkey=other`,
+		);
+		expect(wrongRkey.status).toBe(404);
+		const correctRkey = await SELF.fetch(
+			`https://test${PATH}?did=${DID_B}&collection=${NSID.publisherProfile}&rkey=self`,
+		);
+		expect(correctRkey.status).toBe(200);
+	});
+});
+
+describe("XRPC dispatcher", () => {
+	it("returns 404 on non-XRPC paths", async () => {
+		const res = await SELF.fetch("https://test/some/random/path");
+		expect(res.status).toBe(404);
+	});
+
+	it("returns 404 on unknown XRPC NSIDs", async () => {
+		const res = await SELF.fetch("https://test/xrpc/com.example.notARealEndpoint");
+		expect(res.status).toBe(404);
+	});
+});

apps/aggregator/test/smoke.test.ts

@@ -0,0 +1,113 @@
+/**
+ * Smoke test for the aggregator's test infrastructure.
+ *
+ * Proves the workers-pool wiring is healthy: migrations apply into the test
+ * D1 instance, the schema accepts a write, and the worker module loads. No
+ * business logic is exercised — the actual ingest and read paths are tested
+ * in their own files as later PRs land them.
+ *
+ * If this test fails after a migration change, fix the migration; the rest
+ * of the suite assumes this passes.
+ */
+
+import { applyD1Migrations, env } from "cloudflare:test";
+import { beforeAll, describe, expect, it } from "vitest";
+
+interface TestEnv {
+	DB: D1Database;
+	TEST_MIGRATIONS: Parameters<typeof applyD1Migrations>[1];
+}
+
+const testEnv = env as unknown as TestEnv;
+
+beforeAll(async () => {
+	await applyD1Migrations(testEnv.DB, testEnv.TEST_MIGRATIONS);
+});
+
+describe("aggregator scaffold smoke test", () => {
+	it("applies the initial migration and round-trips a packages row", async () => {
+		const now = new Date().toISOString();
+		await testEnv.DB.prepare(
+			`INSERT INTO packages (
+				did, slug, type, license, authors, security, record_blob, verified_at
+			) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+		)
+			.bind(
+				"did:plc:test",
+				"smoke",
+				"emdash-plugin",
+				"MIT",
+				JSON.stringify([{ name: "Tester" }]),
+				JSON.stringify([{ email: "x@y.test" }]),
+				new Uint8Array([0x00]),
+				now,
+			)
+			.run();
+
+		const row = await testEnv.DB.prepare(
+			"SELECT did, slug, license FROM packages WHERE did = ? AND slug = ?",
+		)
+			.bind("did:plc:test", "smoke")
+			.first<{ did: string; slug: string; license: string }>();
+
+		expect(row).not.toBeNull();
+		expect(row?.license).toBe("MIT");
+	});
+
+	it("populates packages_fts on insert via trigger", async () => {
+		const now = new Date().toISOString();
+		await testEnv.DB.prepare(
+			`INSERT INTO packages (
+				did, slug, type, name, description, license, authors, security, record_blob, verified_at
+			) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+		)
+			.bind(
+				"did:plc:fts",
+				"searchable",
+				"emdash-plugin",
+				"A Searchable Plugin",
+				"Does something searchable",
+				"MIT",
+				JSON.stringify([{ name: "Tester" }]),
+				JSON.stringify([{ email: "x@y.test" }]),
+				new Uint8Array([0x00]),
+				now,
+			)
+			.run();
+
+		const result = await testEnv.DB.prepare(
+			"SELECT p.slug FROM packages_fts JOIN packages p ON p.rowid = packages_fts.rowid WHERE packages_fts MATCH ?",
+		)
+			.bind("searchable")
+			.first<{ slug: string }>();
+
+		expect(result?.slug).toBe("searchable");
+	});
+
+	it("rejects a release whose did/package does not match an existing profile (FK)", async () => {
+		// Releases reference packages via composite FK; SQLite enforces this when
+		// FK checks are enabled. workers-pool's miniflare D1 has FK checks on by
+		// default per Cloudflare's runtime configuration.
+		const now = new Date().toISOString();
+		const insertOrphan = testEnv.DB.prepare(
+			`INSERT INTO releases (
+				did, package, version, rkey, version_sort, artifacts, emdash_extension, cts, record_blob, verified_at
+			) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+		)
+			.bind(
+				"did:plc:orphan",
+				"nonexistent",
+				"1.0.0",
+				"nonexistent:1.0.0",
+				"00000000001.00000000000.00000000000.zzz",
+				"{}",
+				"{}",
+				now,
+				new Uint8Array([0x00]),
+				now,
+			)
+			.run();
+
+		await expect(insertOrphan).rejects.toThrow();
+	});
+});

apps/aggregator/tsconfig.json

@@ -0,0 +1,13 @@
+{
+	"extends": "../../tsconfig.base.json",
+	"compilerOptions": {
+		// Runtime types come from `worker-configuration.d.ts` (generated by
+		// `wrangler types`) — referenced via `include` below. The vitest-pool
+		// types subpath provides the `cloudflare:test` ambient declarations
+		// for the smoke test rig.
+		"types": ["@cloudflare/vitest-pool-workers/types"],
+		"verbatimModuleSyntax": true,
+		"noEmit": true
+	},
+	"include": ["src/**/*", "test/**/*", "worker-configuration.d.ts"]
+}

apps/aggregator/vite.config.ts

@@ -0,0 +1,20 @@
+/**
+ * Aggregator dev/build via the Cloudflare Vite plugin.
+ *
+ * The plugin reads `wrangler.jsonc` for bindings (D1, Queues, DOs, vars) and
+ * runs the worker inside a workerd-backed Vite dev server. `vite dev` gives
+ * us HMR + proper module resolution, `vite build` produces the deployable
+ * bundle that `wrangler deploy` ships.
+ *
+ * Test config is separate: `vitest.config.ts` uses `@cloudflare/vitest-pool-workers`,
+ * which manages its own miniflare instance. The two pipelines don't share
+ * configuration but read the same `wrangler.jsonc` for binding shape, so the
+ * test environment matches dev/prod by construction.
+ */
+
+import { cloudflare } from "@cloudflare/vite-plugin";
+import { defineConfig } from "vite";
+
+export default defineConfig({
+	plugins: [cloudflare()],
+});

apps/aggregator/vitest.config.ts

@@ -0,0 +1,48 @@
+/**
+ * Aggregator test config.
+ *
+ * Uses `@cloudflare/vitest-pool-workers` (v0.16+) so tests run inside a real
+ * workerd isolate with real D1, real DOs, and real Queues. The
+ * `cloudflareTest` plugin reads `wrangler.jsonc` for binding shape, so the
+ * test environment matches dev/prod by construction.
+ *
+ * D1 migrations are read at config time and exposed to the worker isolate as
+ * the `TEST_MIGRATIONS` binding. Tests apply them in a `beforeAll` via
+ * `applyD1Migrations(env.DB, env.TEST_MIGRATIONS)` from `cloudflare:test`.
+ *
+ * External services (PDS, Jetstream, DID resolver, Constellation) become
+ * dependency-injected via env bindings populated from
+ * `@emdash-cms/atproto-test-utils`. The smoke test only exercises the schema;
+ * suites that drive the ingest pipeline use the mock infrastructure.
+ *
+ * Why workers-pool over plain vitest: aggregator behaviour depends on D1
+ * transaction semantics, DO storage durability, and Queue batching — all
+ * workerd-specific. Mocked node tests would pass while production fails.
+ */
+
+import { fileURLToPath } from "node:url";
+
+import { cloudflareTest, readD1Migrations } from "@cloudflare/vitest-pool-workers";
+import { defineConfig } from "vitest/config";
+
+const migrationsPath = fileURLToPath(new URL("./migrations", import.meta.url));
+const migrations = await readD1Migrations(migrationsPath);
+
+export default defineConfig({
+	plugins: [
+		cloudflareTest({
+			wrangler: { configPath: "./wrangler.jsonc" },
+			miniflare: {
+				bindings: {
+					TEST_MIGRATIONS: migrations,
+					// Stub admin auth token so tests can exercise the auth-gated
+					// admin routes without needing a real secret in the test
+					// environment. Production deploys pull from
+					// `wrangler secret put ADMIN_TOKEN`; the value below only
+					// applies inside the workers test pool.
+					ADMIN_TOKEN: "test-admin-token",
+				},
+			},
+		}),
+	],
+});

apps/aggregator/wrangler.jsonc

@@ -0,0 +1,148 @@
+{
+	"$schema": "node_modules/wrangler/config-schema.json",
+	"name": "emdash-aggregator",
+	"main": "./src/index.ts",
+	"compatibility_date": "2026-02-24",
+	"compatibility_flags": ["nodejs_compat"],
+	// Production routes are configured at deploy time. The CDN
+	// (`cdn.emdashcms.com`) is a separate Worker that we'll add later
+	// alongside R2. Local development (`wrangler dev`) doesn't need a route
+	// binding.
+	"d1_databases": [
+		{
+			"binding": "DB",
+			"database_name": "emdash-aggregator",
+			// `database_id` intentionally omitted — wrangler auto-provisions
+			// the D1 instance on first deploy. A placeholder ID would make
+			// wrangler think the binding is already configured and skip
+			// provisioning.
+			"migrations_dir": "./migrations",
+		},
+	],
+	"queues": {
+		"producers": [
+			{
+				"binding": "RECORDS_QUEUE",
+				"queue": "emdash-aggregator-records",
+			},
+			{
+				// Backfill orchestrator (POST /_admin/backfill) fans
+				// (DID, collection) pairs onto this queue. Per-pair work
+				// (resolve PDS → listRecords → sendBatch onto RECORDS_QUEUE)
+				// fits well within a single consumer invocation's waitUntil,
+				// solving the 30s wall-clock cap that would otherwise limit
+				// us to ~15–25 DIDs per backfill POST.
+				"binding": "BACKFILL_QUEUE",
+				"queue": "emdash-aggregator-backfill",
+			},
+		],
+		"consumers": [
+			{
+				"queue": "emdash-aggregator-records",
+				"max_batch_size": 25,
+				"max_batch_timeout": 5,
+				"max_retries": 5,
+				"dead_letter_queue": "emdash-aggregator-records-dlq",
+			},
+			{
+				// Drains the DLQ. Today the consumer logs each dead-lettered
+				// job to Workers logs and writes a `dead_letters` row, then
+				// acks. On D1 failure the handler calls `message.retry()`
+				// (configured `max_retries: 3` per below); after that, the
+				// DLQ has no DLQ-of-DLQ so workerd drops the message. Once
+				// the reconciliation pass lands it'll replace this with
+				// retry-from-listRecords.
+				"queue": "emdash-aggregator-records-dlq",
+				"max_batch_size": 25,
+				"max_batch_timeout": 30,
+				"max_retries": 3,
+			},
+			{
+				// Backfill (DID, collection) consumer. Each job: resolve
+				// PDS, paginate listRecords for one collection, batch
+				// results onto RECORDS_QUEUE. `max_batch_size: 1` so each
+				// pair gets its own consumer invocation — listRecords can
+				// chain ~15s per page × `MAX_PAGES_PER_COLLECTION`, so
+				// batching multiple pairs per invocation risks blowing
+				// the consumer wall-clock budget. Throughput is fine
+				// because backfill is operator-triggered, not steady-state.
+				"queue": "emdash-aggregator-backfill",
+				"max_batch_size": 1,
+				"max_batch_timeout": 5,
+				"max_retries": 3,
+				"dead_letter_queue": "emdash-aggregator-backfill-dlq",
+			},
+			{
+				// Drains the backfill DLQ. Logs each dead-lettered pair
+				// loud enough that operators tailing logs see it, then
+				// acks so the DLQ doesn't accumulate unbounded. We don't
+				// write a forensics row to D1 here (no `backfill_dead_letters`
+				// table) because the operator's recovery action is
+				// "re-trigger backfill for the affected DID" — they don't
+				// need the per-pair payload, just the (did, collection)
+				// pair, which is on the log line.
+				"queue": "emdash-aggregator-backfill-dlq",
+				"max_batch_size": 25,
+				"max_batch_timeout": 30,
+				"max_retries": 3,
+			},
+		],
+	},
+	"durable_objects": {
+		"bindings": [
+			{
+				"name": "RECORDS_DO",
+				"class_name": "RecordsJetstreamDO",
+			},
+		],
+	},
+	"migrations": [
+		{
+			"tag": "v1",
+			"new_sqlite_classes": ["RecordsJetstreamDO"],
+		},
+	],
+	"triggers": {
+		// Liveness ping for the records DO. The DO holds Jetstream open and
+		// stays alive while the WebSocket is up, but during a Jetstream
+		// outage it spends time in backoff sleeps with no active connection
+		// — that's when CF can evict it. The 5-minute cron wakes it back up
+		// quickly; constructor-time `ingestor.run()` resumes from the
+		// persisted cursor. Reconciliation work will share this trigger when
+		// it lands.
+		"crons": ["*/5 * * * *"],
+	},
+	"vars": {
+		"NODE_OPTIONS": "--max-old-space-size=6144",
+		// Jetstream endpoint. Override per environment for self-hosted relays
+		// or staging backends.
+		"JETSTREAM_URL": "wss://jetstream2.us-east.bsky.network/subscribe",
+		// Relay base URL implementing `com.atproto.sync.listReposByCollection`,
+		// used by `/_admin/backfill` to enumerate publishers of our NSIDs at
+		// cold-start time. Defaults to Bluesky's main relay (the canonical
+		// atproto relay; same trust orbit as our Jetstream endpoint).
+		// Override for self-hosted indexers (e.g. Microcosm's Lightrail)
+		// if/when we want a non-bsky.network discovery path.
+		"RELAY_URL": "https://bsky.network",
+	},
+	// Required secrets, declared in config so `wrangler types` generates the
+	// typed binding and `wrangler deploy` validates the secret is set on the
+	// target Worker before publishing. `wrangler dev` warns at startup when a
+	// required secret isn't in `.env`.
+	//
+	// Set in production with `wrangler secret put ADMIN_TOKEN`. Tests bind
+	// a stub via miniflare in vitest.config.ts. Local dev pulls from
+	// `.env` (gitignored; see `.env.example`).
+	//
+	// The `requireAdminAuth` runtime guard fails closed (503) if the binding
+	// is somehow empty at request time, so even a misconfigured deploy can't
+	// leave the admin routes unauth-passable. NOTE: this `secrets` block is
+	// not inherited by named environments — repeat it under each `env.<name>`
+	// you add or that env's deploys won't enforce the secret.
+	"secrets": {
+		"required": ["ADMIN_TOKEN"],
+	},
+	"observability": {
+		"enabled": true,
+	},
+}