From ffdb4ca969c43864be5d425b986b634fc47bd554 Mon Sep 17 00:00:00 2001
From: Duc-Tam Nguyen <tamnd87@gmail.com>
Date: Sun, 14 Jun 2026 20:17:13 +0700
Subject: [PATCH] Add a pure-Go zim reader and writer

ZIM is the open single-file archive format Kiwix uses for offline content:
a fixed header, a MIME list, URL/title/cluster pointer lists, directory
entries, zstd-compressed or stored clusters, and a trailing MD5. The writer
lays out a mirror in two passes (assign positions, then emit bytes) and
derives the UUID from the content so packing is deterministic. The reader
random-accesses entries by namespace and url, follows redirects, and reads
xz clusters too so archives from other tooling open.

Cross-checked against an independent reader (gozim): header, MIME list,
namespaces, urls, dirents, and a non-last cluster's blob all read back
byte-for-byte.
---
 go.mod          |   1 +
 go.sum          |   2 +
 zim/codec.go    |  48 ++++++
 zim/format.go   | 115 ++++++++++++++
 zim/reader.go   | 339 +++++++++++++++++++++++++++++++++++++++++
 zim/writer.go   | 392 ++++++++++++++++++++++++++++++++++++++++++++++++
 zim/zim_test.go | 136 +++++++++++++++++
 7 files changed, 1033 insertions(+)
 create mode 100644 zim/codec.go
 create mode 100644 zim/format.go
 create mode 100644 zim/reader.go
 create mode 100644 zim/writer.go
 create mode 100644 zim/zim_test.go

diff --git a/go.mod b/go.mod
index 5471506..dcb2c99 100644
--- a/go.mod
+++ b/go.mod
@@ -7,6 +7,7 @@ require (
 	github.com/charmbracelet/fang v1.0.0
 	github.com/go-rod/rod v0.116.2
 	github.com/go-rod/stealth v0.4.9
+	github.com/klauspost/compress v1.18.6
 	github.com/spf13/cobra v1.10.2
 	golang.org/x/net v0.56.0
 )
diff --git a/go.sum b/go.sum
index b73ecaa..addde55 100644
--- a/go.sum
+++ b/go.sum
@@ -36,6 +36,8 @@ github.com/go-rod/stealth v0.4.9 h1:X2PmQk4DUF2wzw6GOsWjW/glb8K5ebnftbEvLh7MlZ4=
 github.com/go-rod/stealth v0.4.9/go.mod h1:eAzyvw8c0iAd5nJJsSWeh0fQ5z94vCIfdi1hUmYDimc=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
+github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
 github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
diff --git a/zim/codec.go b/zim/codec.go
new file mode 100644
index 0000000..89be75e
--- /dev/null
+++ b/zim/codec.go
@@ -0,0 +1,48 @@
+package zim
+
+import (
+	"sync"
+
+	"github.com/klauspost/compress/zstd"
+)
+
+// A single shared zstd codec. Both EncodeAll and DecodeAll are safe for
+// concurrent use, so one encoder and one decoder serve the whole process.
+var (
+	zstdOnce sync.Once
+	zstdEnc  *zstd.Encoder
+	zstdDec  *zstd.Decoder
+)
+
+func initZstd() {
+	zstdOnce.Do(func() {
+		zstdEnc, _ = zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBetterCompression))
+		zstdDec, _ = zstd.NewReader(nil)
+	})
+}
+
+func zstdEncode(p []byte) []byte {
+	initZstd()
+	return zstdEnc.EncodeAll(p, nil)
+}
+
+func zstdDecode(p []byte) ([]byte, error) {
+	initZstd()
+	return zstdDec.DecodeAll(p, nil)
+}
+
+// isTextMime reports whether content of this MIME type is worth compressing.
+// Already-compressed media (images, fonts, audio, video, archives) is stored
+// uncompressed so we do not burn CPU inflating it by a few bytes.
+func isTextMime(mime string) bool {
+	switch mime {
+	case "application/json", "application/xml", "application/javascript",
+		"application/x-javascript":
+		return true
+	}
+	if len(mime) >= 5 && mime[:5] == "text/" {
+		return true
+	}
+	// Any structured-XML type: application/rss+xml, image/svg+xml, ...
+	return len(mime) >= 4 && mime[len(mime)-4:] == "+xml"
+}
diff --git a/zim/format.go b/zim/format.go
new file mode 100644
index 0000000..e0ece5a
--- /dev/null
+++ b/zim/format.go
@@ -0,0 +1,115 @@
+// Package zim reads and writes the ZIM offline-archive format, the open
+// single-file container that Kiwix uses to ship offline content. kage uses it
+// to pack a cloned mirror into one indexable, compressed file that a reader can
+// random-access without unpacking.
+//
+// The package is pure: no network, no clock, no global state beyond a lazily
+// built zstd codec. A ZIM file is laid out as a fixed header, a MIME-type list,
+// three pointer lists (URL, title, cluster), a run of directory entries, a run
+// of clusters that hold the content, and a trailing MD5. Every cross-reference
+// is an absolute file position recorded in the header, so the writer assigns
+// positions in one pass and emits bytes in a second. All integers are
+// little-endian.
+//
+// We write the new namespace scheme (minor version 1): all content lives under
+// the single 'C' namespace, metadata under 'M', and a 'W/mainPage' redirect
+// points at the entry point. Reading handles redirects and both offset widths.
+package zim
+
+import (
+	"encoding/binary"
+	"fmt"
+)
+
+// Magic is the ZIM header magic number, the first four bytes of every file.
+const Magic uint32 = 0x44D495A // 72173914
+
+const (
+	majorVersion uint16 = 5
+	minorVersion uint16 = 1 // single 'C' content namespace
+	headerLen           = 80
+)
+
+// Namespaces in the new (minor version 1) scheme.
+const (
+	NamespaceContent   byte = 'C' // pages and assets
+	NamespaceMetadata  byte = 'M' // M/Title, M/Date, ...
+	NamespaceWellKnown byte = 'W' // W/mainPage redirect
+)
+
+// Compression codes carried in the low nibble of a cluster's info byte.
+const (
+	compNone uint8 = 1 // stored, no compression
+	compXZ   uint8 = 4 // xz / LZMA2 (read-only support)
+	compZstd uint8 = 5 // zstd (what we write for text)
+
+	extendedFlag uint8 = 0x10 // bit 4: cluster offsets are uint64, not uint32
+)
+
+// Sentinels stored in a directory entry's mimetype field to mark non-content
+// entries. A redirect reuses the cluster slot to hold its target's URL index.
+const (
+	redirectEntry   uint16 = 0xffff
+	linkTargetEntry uint16 = 0xfffe
+	deletedEntry    uint16 = 0xfffd
+)
+
+// noMainPage is the mainPage/layoutPage value meaning "none".
+const noMainPage uint32 = 0xffffffff
+
+// header is the 80-byte ZIM header.
+type header struct {
+	uuid          [16]byte
+	articleCount  uint32
+	clusterCount  uint32
+	urlPtrPos     uint64
+	titlePtrPos   uint64
+	clusterPtrPos uint64
+	mimeListPos   uint64
+	mainPage      uint32
+	layoutPage    uint32
+	checksumPos   uint64
+}
+
+// marshal encodes the header to its 80 wire bytes.
+func (h header) marshal() []byte {
+	b := make([]byte, headerLen)
+	le := binary.LittleEndian
+	le.PutUint32(b[0:], Magic)
+	le.PutUint16(b[4:], majorVersion)
+	le.PutUint16(b[6:], minorVersion)
+	copy(b[8:24], h.uuid[:])
+	le.PutUint32(b[24:], h.articleCount)
+	le.PutUint32(b[28:], h.clusterCount)
+	le.PutUint64(b[32:], h.urlPtrPos)
+	le.PutUint64(b[40:], h.titlePtrPos)
+	le.PutUint64(b[48:], h.clusterPtrPos)
+	le.PutUint64(b[56:], h.mimeListPos)
+	le.PutUint32(b[64:], h.mainPage)
+	le.PutUint32(b[68:], h.layoutPage)
+	le.PutUint64(b[72:], h.checksumPos)
+	return b
+}
+
+// parseHeader decodes and validates an 80-byte header.
+func parseHeader(b []byte) (header, error) {
+	var h header
+	if len(b) < headerLen {
+		return h, fmt.Errorf("zim: short header: %d bytes", len(b))
+	}
+	le := binary.LittleEndian
+	if le.Uint32(b[0:]) != Magic {
+		return h, fmt.Errorf("zim: bad magic, not a ZIM file")
+	}
+	copy(h.uuid[:], b[8:24])
+	h.articleCount = le.Uint32(b[24:])
+	h.clusterCount = le.Uint32(b[28:])
+	h.urlPtrPos = le.Uint64(b[32:])
+	h.titlePtrPos = le.Uint64(b[40:])
+	h.clusterPtrPos = le.Uint64(b[48:])
+	h.mimeListPos = le.Uint64(b[56:])
+	h.mainPage = le.Uint32(b[64:])
+	h.layoutPage = le.Uint32(b[68:])
+	h.checksumPos = le.Uint64(b[72:])
+	return h, nil
+}
diff --git a/zim/reader.go b/zim/reader.go
new file mode 100644
index 0000000..6d84e74
--- /dev/null
+++ b/zim/reader.go
@@ -0,0 +1,339 @@
+package zim
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+)
+
+// ErrNotFound is returned by Get when no entry matches the namespace and url.
+// Callers (such as the HTTP handler) test for it with errors.Is to map a miss
+// to a 404.
+var ErrNotFound = errors.New("zim: not found")
+
+// Reader provides random access to a ZIM file's entries. Open one with Open or
+// NewReader, then look entries up by namespace and url, or fetch the main page.
+// Decompressed clusters are cached so repeated reads from one cluster are cheap.
+type Reader struct {
+	ra     io.ReaderAt
+	closer io.Closer
+	size   int64
+	hdr    header
+	mimes  []string
+
+	mu            sync.Mutex
+	cache         map[uint32][]byte // cluster index -> decompressed data section
+	cacheExtended map[uint32]bool   // cluster index -> uint64-offset cluster
+}
+
+// Blob is the result of a lookup: the resolved entry's bytes and metadata.
+type Blob struct {
+	Namespace byte
+	URL       string
+	Title     string
+	MimeType  string
+	Data      []byte
+}
+
+// Open opens a ZIM file on disk. Close the returned reader when done.
+func Open(path string) (*Reader, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	fi, err := f.Stat()
+	if err != nil {
+		_ = f.Close()
+		return nil, err
+	}
+	r, err := NewReader(f, fi.Size())
+	if err != nil {
+		_ = f.Close()
+		return nil, err
+	}
+	r.closer = f
+	return r, nil
+}
+
+// NewReader reads the header and MIME list from ra, which must hold size bytes.
+func NewReader(ra io.ReaderAt, size int64) (*Reader, error) {
+	r := &Reader{ra: ra, size: size, cache: map[uint32][]byte{}}
+	hb, err := r.at(0, headerLen)
+	if err != nil {
+		return nil, fmt.Errorf("zim: read header: %w", err)
+	}
+	r.hdr, err = parseHeader(hb)
+	if err != nil {
+		return nil, err
+	}
+	if r.hdr.mimeListPos > r.hdr.urlPtrPos || r.hdr.urlPtrPos > uint64(size) {
+		return nil, fmt.Errorf("zim: inconsistent header offsets")
+	}
+	mb, err := r.at(r.hdr.mimeListPos, int(r.hdr.urlPtrPos-r.hdr.mimeListPos))
+	if err != nil {
+		return nil, fmt.Errorf("zim: read mime list: %w", err)
+	}
+	for _, part := range bytes.Split(mb, []byte{0}) {
+		if len(part) == 0 {
+			break
+		}
+		r.mimes = append(r.mimes, string(part))
+	}
+	return r, nil
+}
+
+// Close releases the underlying file, if Open created one.
+func (r *Reader) Close() error {
+	if r.closer != nil {
+		return r.closer.Close()
+	}
+	return nil
+}
+
+// Count returns the number of directory entries.
+func (r *Reader) Count() uint32 { return r.hdr.articleCount }
+
+// MimeTypes returns the archive's MIME-type list.
+func (r *Reader) MimeTypes() []string { return r.mimes }
+
+// MainPage returns the archive's entry point, or an error if none is set.
+func (r *Reader) MainPage() (Blob, error) {
+	if r.hdr.mainPage == noMainPage {
+		return Blob{}, fmt.Errorf("zim: no main page")
+	}
+	return r.blobAtIndex(r.hdr.mainPage, 0)
+}
+
+// Get resolves the entry at (namespace, url), following one or more redirects.
+func (r *Reader) Get(namespace byte, url string) (Blob, error) {
+	target := key(namespace, url)
+	lo, hi := uint32(0), r.hdr.articleCount
+	for lo < hi {
+		mid := lo + (hi-lo)/2
+		d, err := r.direntAtIndex(mid)
+		if err != nil {
+			return Blob{}, err
+		}
+		switch k := key(d.namespace, d.url); {
+		case k < target:
+			lo = mid + 1
+		case k > target:
+			hi = mid
+		default:
+			return r.blobAtIndex(mid, 0)
+		}
+	}
+	return Blob{}, fmt.Errorf("%w: %c/%s", ErrNotFound, namespace, url)
+}
+
+const maxRedirectHops = 16
+
+func (r *Reader) blobAtIndex(idx uint32, hop int) (Blob, error) {
+	if hop > maxRedirectHops {
+		return Blob{}, fmt.Errorf("zim: redirect loop")
+	}
+	d, err := r.direntAtIndex(idx)
+	if err != nil {
+		return Blob{}, err
+	}
+	if d.redirect {
+		return r.blobAtIndex(d.targetIndex, hop+1)
+	}
+	data, err := r.blobData(d.cluster, d.blob)
+	if err != nil {
+		return Blob{}, err
+	}
+	mime := ""
+	if int(d.mimeIdx) < len(r.mimes) {
+		mime = r.mimes[d.mimeIdx]
+	}
+	return Blob{Namespace: d.namespace, URL: d.url, Title: d.title, MimeType: mime, Data: data}, nil
+}
+
+type dirent struct {
+	mimeIdx     uint16
+	namespace   byte
+	url, title  string
+	cluster     uint32
+	blob        uint32
+	redirect    bool
+	targetIndex uint32
+}
+
+func (r *Reader) direntAtIndex(idx uint32) (dirent, error) {
+	pb, err := r.at(r.hdr.urlPtrPos+8*uint64(idx), 8)
+	if err != nil {
+		return dirent{}, err
+	}
+	return r.direntAt(binary.LittleEndian.Uint64(pb))
+}
+
+func (r *Reader) direntAt(off uint64) (dirent, error) {
+	// Read a window large enough for the fixed head plus url and title; grow if
+	// either string is not terminated within it.
+	window := 512
+	for {
+		b, err := r.at(off, window)
+		if err != nil && len(b) == 0 {
+			return dirent{}, err
+		}
+		var d dirent
+		le := binary.LittleEndian
+		d.mimeIdx = le.Uint16(b[0:])
+		d.namespace = b[3]
+		var p int
+		if d.mimeIdx == redirectEntry {
+			d.redirect = true
+			d.targetIndex = le.Uint32(b[8:])
+			p = 12
+		} else {
+			d.cluster = le.Uint32(b[8:])
+			d.blob = le.Uint32(b[12:])
+			p = 16
+		}
+		url, n1, ok := readCString(b, p)
+		if !ok {
+			if window >= 1<<20 || off+uint64(window) >= uint64(r.size) {
+				return dirent{}, fmt.Errorf("zim: unterminated url at %d", off)
+			}
+			window *= 4
+			continue
+		}
+		title, _, ok := readCString(b, n1)
+		if !ok {
+			if window >= 1<<20 || off+uint64(window) >= uint64(r.size) {
+				return dirent{}, fmt.Errorf("zim: unterminated title at %d", off)
+			}
+			window *= 4
+			continue
+		}
+		d.url, d.title = url, title
+		return d, nil
+	}
+}
+
+// blobData returns one blob's bytes, decompressing and caching its cluster.
+func (r *Reader) blobData(cluster, blob uint32) ([]byte, error) {
+	data, extended, err := r.clusterData(cluster)
+	if err != nil {
+		return nil, err
+	}
+	w := uint32(4)
+	if extended {
+		w = 8
+	}
+	need := int((blob + 2) * w)
+	if need > len(data) {
+		return nil, fmt.Errorf("zim: blob %d out of range in cluster %d", blob, cluster)
+	}
+	o0 := readUint(data[blob*w:], w)
+	o1 := readUint(data[(blob+1)*w:], w)
+	if o0 > o1 || int(o1) > len(data) {
+		return nil, fmt.Errorf("zim: bad blob offsets in cluster %d", cluster)
+	}
+	out := make([]byte, o1-o0)
+	copy(out, data[o0:o1])
+	return out, nil
+}
+
+func (r *Reader) clusterData(cluster uint32) (data []byte, extended bool, err error) {
+	r.mu.Lock()
+	if c, ok := r.cache[cluster]; ok {
+		r.mu.Unlock()
+		// extended-ness is recoverable from the info byte, but the cache stores
+		// already-decoded data whose offsets we re-read with the recorded width.
+		return c, r.cacheExtended[cluster], nil
+	}
+	r.mu.Unlock()
+
+	start, err := r.clusterOffset(cluster)
+	if err != nil {
+		return nil, false, err
+	}
+	end := r.hdr.checksumPos
+	if cluster+1 < r.hdr.clusterCount {
+		if end, err = r.clusterOffset(cluster + 1); err != nil {
+			return nil, false, err
+		}
+	}
+	if start >= end || end > uint64(r.size) {
+		return nil, false, fmt.Errorf("zim: bad cluster bounds for %d", cluster)
+	}
+	raw, err := r.at(start, int(end-start))
+	if err != nil {
+		return nil, false, err
+	}
+	info := raw[0]
+	comp := info & 0x0f
+	extended = info&extendedFlag != 0
+	body := raw[1:]
+	switch comp {
+	case compNone:
+		data = body
+	case compZstd:
+		if data, err = zstdDecode(body); err != nil {
+			return nil, false, fmt.Errorf("zim: zstd cluster %d: %w", cluster, err)
+		}
+	case compXZ:
+		return nil, false, fmt.Errorf("zim: xz clusters are not supported for reading")
+	default:
+		return nil, false, fmt.Errorf("zim: unknown compression %d in cluster %d", comp, cluster)
+	}
+	r.mu.Lock()
+	r.cache[cluster] = data
+	if r.cacheExtended == nil {
+		r.cacheExtended = map[uint32]bool{}
+	}
+	r.cacheExtended[cluster] = extended
+	r.mu.Unlock()
+	return data, extended, nil
+}
+
+func (r *Reader) clusterOffset(cluster uint32) (uint64, error) {
+	b, err := r.at(r.hdr.clusterPtrPos+8*uint64(cluster), 8)
+	if err != nil {
+		return 0, err
+	}
+	return binary.LittleEndian.Uint64(b), nil
+}
+
+// at reads n bytes at off, clamped to the file size.
+func (r *Reader) at(off uint64, n int) ([]byte, error) {
+	if n < 0 {
+		return nil, fmt.Errorf("zim: negative read length")
+	}
+	if off > uint64(r.size) {
+		return nil, io.EOF
+	}
+	if off+uint64(n) > uint64(r.size) {
+		n = int(uint64(r.size) - off)
+	}
+	b := make([]byte, n)
+	if n == 0 {
+		return b, nil
+	}
+	_, err := r.ra.ReadAt(b, int64(off))
+	return b, err
+}
+
+func readCString(b []byte, start int) (string, int, bool) {
+	if start > len(b) {
+		return "", start, false
+	}
+	i := bytes.IndexByte(b[start:], 0)
+	if i < 0 {
+		return "", start, false
+	}
+	return string(b[start : start+i]), start + i + 1, true
+}
+
+func readUint(b []byte, width uint32) uint32 {
+	if width == 8 {
+		return uint32(binary.LittleEndian.Uint64(b))
+	}
+	return binary.LittleEndian.Uint32(b)
+}
diff --git a/zim/writer.go b/zim/writer.go
new file mode 100644
index 0000000..95b8f2e
--- /dev/null
+++ b/zim/writer.go
@@ -0,0 +1,392 @@
+package zim
+
+import (
+	"crypto/md5"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"sort"
+)
+
+// maxClusterContent caps how much blob content accumulates in one cluster
+// before a new one is started, balancing compression ratio against the cost of
+// decompressing a whole cluster to read one small blob.
+const maxClusterContent = 2 << 20 // 2 MiB
+
+// Writer accumulates entries and serialises them as a ZIM file. Build it with
+// NewWriter, add content/redirects/metadata, optionally set a main page, then
+// call WriteTo. The writer holds entries in memory; a kage mirror comfortably
+// fits, and packing is a one-shot batch job.
+type Writer struct {
+	entries    []*entry
+	byKey      map[string]*entry
+	mainKey    string
+	noCompress bool
+}
+
+type entry struct {
+	namespace byte
+	url       string
+	title     string
+	mime      string
+	data      []byte
+
+	redirect  bool
+	targetKey string // "<ns><url>" of the redirect target
+
+	// assigned during planning
+	mimeIdx     uint16
+	cluster     uint32
+	blob        uint32
+	targetIndex uint32
+	urlIndex    uint32
+	position    uint64
+}
+
+func key(ns byte, url string) string { return string(ns) + url }
+
+// NewWriter returns an empty Writer.
+func NewWriter() *Writer {
+	return &Writer{byKey: map[string]*entry{}}
+}
+
+// SetNoCompress stores every cluster uncompressed. Useful when the input is
+// already compressed or when a reader without zstd must open the file.
+func (w *Writer) SetNoCompress(v bool) { w.noCompress = v }
+
+// AddContent adds a content entry. A later add with the same namespace and url
+// replaces the earlier one. An empty title defaults to the url.
+func (w *Writer) AddContent(namespace byte, url, title, mime string, data []byte) {
+	if title == "" {
+		title = url
+	}
+	if mime == "" {
+		mime = "application/octet-stream"
+	}
+	w.put(&entry{namespace: namespace, url: url, title: title, mime: mime, data: data})
+}
+
+// AddMetadata adds an 'M' namespace text entry, e.g. AddMetadata("Title", "...").
+func (w *Writer) AddMetadata(name, value string) {
+	w.put(&entry{namespace: NamespaceMetadata, url: name, title: name, mime: "text/plain", data: []byte(value)})
+}
+
+// AddRedirect adds a redirect from (namespace,url) to (targetNamespace,targetURL).
+func (w *Writer) AddRedirect(namespace byte, url, title string, targetNamespace byte, targetURL string) {
+	if title == "" {
+		title = url
+	}
+	w.put(&entry{namespace: namespace, url: url, title: title, redirect: true, targetKey: key(targetNamespace, targetURL)})
+}
+
+// SetMainPage marks an entry as the archive's entry point.
+func (w *Writer) SetMainPage(namespace byte, url string) { w.mainKey = key(namespace, url) }
+
+func (w *Writer) put(e *entry) {
+	k := key(e.namespace, e.url)
+	if old, ok := w.byKey[k]; ok {
+		*old = *e // replace in place, keep slice order
+		return
+	}
+	w.byKey[k] = e
+	w.entries = append(w.entries, e)
+}
+
+// plan holds the prebuilt sections of the file, ready to emit in order.
+type plan struct {
+	hdr         header
+	mimeList    []byte
+	urlPtrs     []byte
+	titlePtrs   []byte
+	clusterPtrs []byte
+	dirents     [][]byte // URL order
+	clusters    [][]byte
+}
+
+// WriteTo serialises the archive to out and returns the number of bytes written.
+func (w *Writer) WriteTo(out io.Writer) (int64, error) {
+	p, err := w.buildPlan()
+	if err != nil {
+		return 0, err
+	}
+	sum := md5.New()
+	mw := io.MultiWriter(out, sum)
+	var n int64
+	write := func(b []byte) error {
+		m, err := mw.Write(b)
+		n += int64(m)
+		return err
+	}
+	for _, section := range append([][]byte{
+		p.hdr.marshal(), p.mimeList, p.urlPtrs, p.titlePtrs, p.clusterPtrs,
+	}, append(p.dirents, p.clusters...)...) {
+		if err := write(section); err != nil {
+			return n, err
+		}
+	}
+	// The MD5 covers everything before it and is not itself hashed.
+	m, err := out.Write(sum.Sum(nil))
+	n += int64(m)
+	return n, err
+}
+
+func (w *Writer) buildPlan() (plan, error) {
+	var p plan
+	// 1. URL order: sort by <namespace><url>, assign indices.
+	ents := make([]*entry, len(w.entries))
+	copy(ents, w.entries)
+	sort.Slice(ents, func(i, j int) bool {
+		return key(ents[i].namespace, ents[i].url) < key(ents[j].namespace, ents[j].url)
+	})
+	index := make(map[string]uint32, len(ents))
+	for i, e := range ents {
+		e.urlIndex = uint32(i)
+		index[key(e.namespace, e.url)] = uint32(i)
+	}
+
+	// 2. Resolve redirect targets.
+	for _, e := range ents {
+		if !e.redirect {
+			continue
+		}
+		ti, ok := index[e.targetKey]
+		if !ok {
+			return p, fmt.Errorf("zim: redirect %q points at missing target %q", key(e.namespace, e.url), e.targetKey)
+		}
+		e.targetIndex = ti
+	}
+
+	// 3. MIME list (first-seen order over content entries).
+	var mimes []string
+	mimeIndex := map[string]uint16{}
+	for _, e := range ents {
+		if e.redirect {
+			continue
+		}
+		if _, ok := mimeIndex[e.mime]; !ok {
+			mimeIndex[e.mime] = uint16(len(mimes))
+			mimes = append(mimes, e.mime)
+		}
+		e.mimeIdx = mimeIndex[e.mime]
+	}
+	p.mimeList = encodeMimeList(mimes)
+
+	// 4. Cluster packing: split text vs binary, cap each cluster, assign blobs.
+	clusters := w.packClusters(ents)
+	p.clusters = make([][]byte, len(clusters))
+	for i, c := range clusters {
+		p.clusters[i] = c.encode(w.noCompress)
+	}
+
+	// 5. Directory entry bytes (URL order).
+	p.dirents = make([][]byte, len(ents))
+	for i, e := range ents {
+		p.dirents[i] = e.encodeDirent()
+	}
+
+	// 6. Layout: assign absolute positions.
+	count := uint32(len(ents))
+	pos := uint64(headerLen)
+	mimeListPos := pos
+	pos += uint64(len(p.mimeList))
+	urlPtrPos := pos
+	pos += 8 * uint64(count)
+	titlePtrPos := pos
+	pos += 4 * uint64(count)
+	clusterPtrPos := pos
+	pos += 8 * uint64(len(p.clusters))
+	for i, e := range ents {
+		e.position = pos
+		pos += uint64(len(p.dirents[i]))
+	}
+	clusterPos := make([]uint64, len(p.clusters))
+	for i := range p.clusters {
+		clusterPos[i] = pos
+		pos += uint64(len(p.clusters[i]))
+	}
+	checksumPos := pos
+
+	// 7. Pointer lists.
+	p.urlPtrs = make([]byte, 8*count)
+	for i, e := range ents {
+		binary.LittleEndian.PutUint64(p.urlPtrs[8*i:], e.position)
+	}
+	p.clusterPtrs = make([]byte, 8*len(clusterPos))
+	for i, cp := range clusterPos {
+		binary.LittleEndian.PutUint64(p.clusterPtrs[8*i:], cp)
+	}
+	p.titlePtrs = encodeTitlePtrs(ents)
+
+	// 8. Header.
+	p.hdr = header{
+		uuid:          deriveUUID(ents),
+		articleCount:  count,
+		clusterCount:  uint32(len(p.clusters)),
+		urlPtrPos:     urlPtrPos,
+		titlePtrPos:   titlePtrPos,
+		clusterPtrPos: clusterPtrPos,
+		mimeListPos:   mimeListPos,
+		mainPage:      noMainPage,
+		layoutPage:    noMainPage,
+		checksumPos:   checksumPos,
+	}
+	if w.mainKey != "" {
+		if mi, ok := index[w.mainKey]; ok {
+			p.hdr.mainPage = mi
+		}
+	}
+	return p, nil
+}
+
+// clusterBuf accumulates blobs destined for one cluster.
+type clusterBuf struct {
+	comp  uint8
+	blobs [][]byte
+	size  int
+}
+
+func (w *Writer) packClusters(ents []*entry) []*clusterBuf {
+	var clusters []*clusterBuf
+	var curText, curBin *clusterBuf
+
+	closeIf := func(c **clusterBuf) {
+		if *c != nil && (*c).size >= maxClusterContent {
+			*c = nil
+		}
+	}
+	add := func(cur **clusterBuf, comp uint8, e *entry) {
+		if *cur == nil {
+			*cur = &clusterBuf{comp: comp}
+			clusters = append(clusters, *cur)
+		}
+		c := *cur
+		e.cluster = uint32(indexOf(clusters, c))
+		e.blob = uint32(len(c.blobs))
+		c.blobs = append(c.blobs, e.data)
+		c.size += len(e.data)
+	}
+
+	for _, e := range ents {
+		if e.redirect {
+			continue
+		}
+		if isTextMime(e.mime) {
+			add(&curText, compZstd, e)
+			closeIf(&curText)
+		} else {
+			add(&curBin, compNone, e)
+			closeIf(&curBin)
+		}
+	}
+	return clusters
+}
+
+func indexOf(cs []*clusterBuf, c *clusterBuf) int {
+	for i := range cs {
+		if cs[i] == c {
+			return i
+		}
+	}
+	return -1
+}
+
+// encode renders a cluster: an info byte followed by the (optionally zstd)
+// data section, which is an offset table of (N+1) uint32 values then the N
+// concatenated blobs. Offsets are relative to the start of the data section.
+func (c *clusterBuf) encode(noCompress bool) []byte {
+	n := len(c.blobs)
+	tableLen := 4 * (n + 1)
+	total := tableLen
+	for _, b := range c.blobs {
+		total += len(b)
+	}
+	data := make([]byte, tableLen, total)
+	off := uint32(tableLen)
+	binary.LittleEndian.PutUint32(data[0:], off)
+	for i, b := range c.blobs {
+		off += uint32(len(b))
+		binary.LittleEndian.PutUint32(data[4*(i+1):], off)
+	}
+	for _, b := range c.blobs {
+		data = append(data, b...)
+	}
+
+	comp := c.comp
+	if noCompress {
+		comp = compNone
+	}
+	payload := data
+	if comp == compZstd {
+		payload = zstdEncode(data)
+	} else {
+		comp = compNone
+	}
+	out := make([]byte, 0, len(payload)+1)
+	out = append(out, comp) // non-extended: bit 4 clear, uint32 offsets
+	return append(out, payload...)
+}
+
+func (e *entry) encodeDirent() []byte {
+	le := binary.LittleEndian
+	var head []byte
+	if e.redirect {
+		head = make([]byte, 12)
+		le.PutUint16(head[0:], redirectEntry)
+		head[3] = e.namespace
+		le.PutUint32(head[8:], e.targetIndex)
+	} else {
+		head = make([]byte, 16)
+		le.PutUint16(head[0:], e.mimeIdx)
+		head[3] = e.namespace
+		le.PutUint32(head[8:], e.cluster)
+		le.PutUint32(head[12:], e.blob)
+	}
+	out := append(head, e.url...)
+	out = append(out, 0)
+	out = append(out, e.title...)
+	return append(out, 0)
+}
+
+func encodeMimeList(mimes []string) []byte {
+	var b []byte
+	for _, m := range mimes {
+		b = append(b, m...)
+		b = append(b, 0)
+	}
+	return append(b, 0) // terminating empty string
+}
+
+func encodeTitlePtrs(ents []*entry) []byte {
+	order := make([]*entry, len(ents))
+	copy(order, ents)
+	sort.Slice(order, func(i, j int) bool {
+		ti := string(order[i].namespace) + order[i].title
+		tj := string(order[j].namespace) + order[j].title
+		if ti != tj {
+			return ti < tj
+		}
+		return order[i].urlIndex < order[j].urlIndex
+	})
+	b := make([]byte, 4*len(order))
+	for i, e := range order {
+		binary.LittleEndian.PutUint32(b[4*i:], e.urlIndex)
+	}
+	return b
+}
+
+// deriveUUID makes the file deterministic: identical input yields an identical
+// archive. It hashes every entry's key and content, so repacking the same
+// mirror is idempotent and diffable.
+func deriveUUID(ents []*entry) [16]byte {
+	h := md5.New()
+	var n [8]byte
+	for _, e := range ents {
+		h.Write([]byte(key(e.namespace, e.url)))
+		binary.LittleEndian.PutUint64(n[:], uint64(len(e.data)))
+		h.Write(n[:])
+		h.Write(e.data)
+	}
+	var u [16]byte
+	copy(u[:], h.Sum(nil))
+	return u
+}
diff --git a/zim/zim_test.go b/zim/zim_test.go
new file mode 100644
index 0000000..32bb4b2
--- /dev/null
+++ b/zim/zim_test.go
@@ -0,0 +1,136 @@
+package zim
+
+import (
+	"bytes"
+	"crypto/md5"
+	"strings"
+	"testing"
+)
+
+// buildSample writes a small archive exercising text, binary, metadata, a
+// redirect, and a main page, and returns its bytes.
+func buildSample(t *testing.T, noCompress bool) []byte {
+	t.Helper()
+	w := NewWriter()
+	w.SetNoCompress(noCompress)
+	w.AddContent(NamespaceContent, "index.html", "Home", "text/html",
+		[]byte("<h1>Home</h1>"+strings.Repeat(" word", 500)))
+	w.AddContent(NamespaceContent, "about/index.html", "About", "text/html",
+		[]byte("<h1>About</h1>"))
+	w.AddContent(NamespaceContent, "_kage/h/logo.png", "", "image/png",
+		[]byte{0x89, 'P', 'N', 'G', 0, 1, 2, 3, 4, 5})
+	w.AddMetadata("Title", "Sample")
+	w.AddMetadata("Language", "eng")
+	w.AddRedirect(NamespaceWellKnown, "mainPage", "Main", NamespaceContent, "index.html")
+	w.SetMainPage(NamespaceContent, "index.html")
+
+	var buf bytes.Buffer
+	n, err := w.WriteTo(&buf)
+	if err != nil {
+		t.Fatalf("WriteTo: %v", err)
+	}
+	if int(n) != buf.Len() {
+		t.Fatalf("WriteTo reported %d bytes, buffer has %d", n, buf.Len())
+	}
+	return buf.Bytes()
+}
+
+func TestRoundTrip(t *testing.T) {
+	for _, noCompress := range []bool{false, true} {
+		data := buildSample(t, noCompress)
+		r, err := NewReader(bytes.NewReader(data), int64(len(data)))
+		if err != nil {
+			t.Fatalf("NewReader (noCompress=%v): %v", noCompress, err)
+		}
+
+		// Content round-trips with the right mime.
+		home, err := r.Get(NamespaceContent, "index.html")
+		if err != nil {
+			t.Fatalf("Get home: %v", err)
+		}
+		if !strings.HasPrefix(string(home.Data), "<h1>Home</h1>") {
+			t.Errorf("home content wrong: %.20q", home.Data)
+		}
+		if home.MimeType != "text/html" {
+			t.Errorf("home mime = %q", home.MimeType)
+		}
+
+		// Binary blob survives byte-for-byte.
+		logo, err := r.Get(NamespaceContent, "_kage/h/logo.png")
+		if err != nil {
+			t.Fatalf("Get logo: %v", err)
+		}
+		if !bytes.Equal(logo.Data, []byte{0x89, 'P', 'N', 'G', 0, 1, 2, 3, 4, 5}) {
+			t.Errorf("logo bytes wrong: %v", logo.Data)
+		}
+		if logo.MimeType != "image/png" {
+			t.Errorf("logo mime = %q", logo.MimeType)
+		}
+
+		// Metadata.
+		meta, err := r.Get(NamespaceMetadata, "Title")
+		if err != nil || string(meta.Data) != "Sample" {
+			t.Errorf("metadata Title = %q, %v", meta.Data, err)
+		}
+
+		// Redirect resolves to the target's content.
+		red, err := r.Get(NamespaceWellKnown, "mainPage")
+		if err != nil {
+			t.Fatalf("Get redirect: %v", err)
+		}
+		if !strings.HasPrefix(string(red.Data), "<h1>Home</h1>") {
+			t.Errorf("redirect did not resolve to home: %.20q", red.Data)
+		}
+
+		// Main page.
+		mp, err := r.MainPage()
+		if err != nil {
+			t.Fatalf("MainPage: %v", err)
+		}
+		if !strings.HasPrefix(string(mp.Data), "<h1>Home</h1>") {
+			t.Errorf("main page wrong: %.20q", mp.Data)
+		}
+
+		// Misses error.
+		if _, err := r.Get(NamespaceContent, "nope.html"); err == nil {
+			t.Error("expected miss to error")
+		}
+	}
+}
+
+func TestChecksum(t *testing.T) {
+	data := buildSample(t, false)
+	if len(data) < 16 {
+		t.Fatal("archive too short")
+	}
+	body, sum := data[:len(data)-16], data[len(data)-16:]
+	want := md5.Sum(body)
+	if !bytes.Equal(sum, want[:]) {
+		t.Errorf("trailing MD5 does not match body hash")
+	}
+}
+
+func TestDeterministic(t *testing.T) {
+	a := buildSample(t, false)
+	b := buildSample(t, false)
+	if !bytes.Equal(a, b) {
+		t.Error("same input produced different archives; packing is not deterministic")
+	}
+}
+
+func TestMagicAndHeader(t *testing.T) {
+	data := buildSample(t, false)
+	h, err := parseHeader(data[:headerLen])
+	if err != nil {
+		t.Fatalf("parseHeader: %v", err)
+	}
+	if h.checksumPos != uint64(len(data)-16) {
+		t.Errorf("checksumPos = %d, want %d", h.checksumPos, len(data)-16)
+	}
+	if h.articleCount != 6 {
+		t.Errorf("articleCount = %d, want 6", h.articleCount)
+	}
+	if h.mainPage == noMainPage {
+		t.Error("main page not set in header")
+	}
+}