From 108b80f2fbe06abc9729fa6c1fc7351f90961266 Mon Sep 17 00:00:00 2001
From: Tam Nguyen Duc <1218621+tamnd@users.noreply.github.com>
Date: Mon, 15 Jun 2026 19:12:16 +0700
Subject: [PATCH] Add incremental packing that reuses unchanged compression
 (#22)

Compressing clusters with zstd is the slow part of packing a large
mirror. When a mirror is re-packed after a small change, most clusters
are byte-identical and there is no reason to compress them again.

pack --incremental keeps a content-addressed cache of compressed
clusters in a sidecar next to the output. On the next pack, a cluster
whose uncompressed bytes match the cache is served from it instead of
being recompressed; only new or changed clusters go through zstd. A
cache hit returns exactly what a fresh compression would, so the archive
stays deterministic and byte-identical to a cold pack.

The zim writer gains a settable cluster compressor so the cache can hook
in without changing the format. The cache only writes back the clusters
it touched this run, so clusters that left the mirror drop out and it
cannot grow without bound.
---
 CHANGELOG.md       |   8 +++
 cli/pack.go        |  59 +++++++++++++++++---
 pack/cache.go      | 134 +++++++++++++++++++++++++++++++++++++++++++++
 pack/cache_test.go | 122 +++++++++++++++++++++++++++++++++++++++++
 pack/zim.go        |  72 +++++++++++++++++++++---
 zim/codec.go       |   6 ++
 zim/writer.go      |  27 +++++++--
 7 files changed, 408 insertions(+), 20 deletions(-)
 create mode 100644 pack/cache.go
 create mode 100644 pack/cache_test.go

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2a5a5a1..2eec67b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,14 @@ All notable changes to kage are recorded here. The format follows
 
 ### Added
 
+- `kage pack --incremental` keeps a small cache sidecar next to the output and
+  reuses the compression of any cluster whose bytes have not changed since the
+  last pack. Compressing clusters with zstd is the dominant cost of packing a
+  large mirror, so re-packing after a small change (a `--refresh`, a handful of
+  edited pages) only compresses what actually changed instead of the whole
+  archive. A cached cluster is byte-for-byte what a fresh compression produces,
+  so the archive stays deterministic and valid. The pack reports how many
+  clusters it reused versus compressed.
 - Identical pages are now stored once. When a rendered page's bytes match a page
   already written, kage stores it as a hard link to the first copy instead of a
   second full file. This collapses the duplicate content a faceted site spawns
diff --git a/cli/pack.go b/cli/pack.go
index c40be60..a7ae674 100644
--- a/cli/pack.go
+++ b/cli/pack.go
@@ -28,8 +28,13 @@ type packFlags struct {
 	description string
 	language    string
 	date        string
+	incremental bool
 }
 
+// cacheSuffix names the cluster-cache sidecar kage writes next to a packed
+// artifact when --incremental is set.
+const cacheSuffix = ".kagecache"
+
 func newPackCmd() *cobra.Command {
 	f := &packFlags{}
 	cmd := &cobra.Command{
@@ -40,7 +45,9 @@ func newPackCmd() *cobra.Command {
 			"ZIM reader can browse. With --format binary it appends that archive to a copy\n" +
 			"of kage, producing a single executable that serves the site offline when run.\n" +
 			"Add --app to wrap that executable in a double-click desktop app (a .app bundle\n" +
-			"on macOS, an AppImage-style .AppDir on Linux) with the site's favicon as the icon.",
+			"on macOS, an AppImage-style .AppDir on Linux) with the site's favicon as the icon.\n" +
+			"Add --incremental to keep a cache sidecar so re-packing a mirror only compresses\n" +
+			"the clusters that changed, not the whole archive.",
 		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return runPack(args[0], f)
@@ -53,6 +60,7 @@ func newPackCmd() *cobra.Command {
 	fs.BoolVar(&f.app, "app", false, "wrap the viewer in a double-click desktop app (.app on macOS, .AppImage/.AppDir on Linux)")
 	fs.StringVar(&f.icon, "icon", "", "icon file for --app (default the site's favicon)")
 	fs.BoolVar(&f.noCompress, "no-compress", false, "store every cluster raw, no zstd")
+	fs.BoolVar(&f.incremental, "incremental", false, "reuse compression from a cache sidecar so re-packing a mirror only compresses what changed")
 	fs.StringVar(&f.title, "title", "", "archive title (default the main page's <title>)")
 	fs.StringVar(&f.description, "description", "", "archive description")
 	fs.StringVar(&f.language, "language", "eng", "archive language code")
@@ -80,19 +88,26 @@ func runPack(mirrorArg string, f *packFlags) error {
 
 	switch f.format {
 	case "zim":
-		out, size, err := pack.BuildZIM(dir, zopts)
+		out := f.out
+		if out == "" {
+			out = filepath.Base(dir) + ".zim"
+		}
+		zopts.Out = out
+		var st pack.PackStats
+		if f.incremental {
+			zopts.CachePath = out + cacheSuffix
+			zopts.Stats = &st
+		}
+		outPath, size, err := pack.BuildZIM(dir, zopts)
 		if err != nil {
 			return err
 		}
-		printPackResult(out, size)
-		fmt.Fprintf(os.Stderr, "  open %s\n", styleAccent.Render("kage open "+out))
+		printPackResult(outPath, size)
+		printCacheLine(f.incremental, st)
+		fmt.Fprintf(os.Stderr, "  open %s\n", styleAccent.Render("kage open "+outPath))
 		return nil
 
 	case "binary":
-		zbytes, err := pack.BuildZIMBytes(dir, zopts)
-		if err != nil {
-			return err
-		}
 		target := resolveTargetOS(f.base)
 		out := f.out
 		if out == "" {
@@ -103,11 +118,21 @@ func runPack(mirrorArg string, f *packFlags) error {
 		if target == "windows" && !strings.HasSuffix(strings.ToLower(out), ".exe") {
 			out += ".exe"
 		}
+		var st pack.PackStats
+		if f.incremental {
+			zopts.CachePath = out + cacheSuffix
+			zopts.Stats = &st
+		}
+		zbytes, err := pack.BuildZIMBytes(dir, zopts)
+		if err != nil {
+			return err
+		}
 		path, size, err := pack.BuildBinary(zbytes, pack.BinaryOptions{Out: out, Base: f.base})
 		if err != nil {
 			return err
 		}
 		printPackResult(path, size)
+		printCacheLine(f.incremental, st)
 		printRunHint(path, target)
 		return nil
 
@@ -142,10 +167,16 @@ func runPackApp(dir string, f *packFlags, zopts pack.ZIMOptions) error {
 	if err != nil {
 		return err
 	}
+	var st pack.PackStats
+	if f.incremental {
+		zopts.CachePath = prog + cacheSuffix
+		zopts.Stats = &st
+	}
 	zbytes, err := pack.BuildZIMBytes(dir, zopts)
 	if err != nil {
 		return err
 	}
+	printCacheLine(f.incremental, st)
 
 	switch target {
 	case "darwin":
@@ -345,6 +376,18 @@ func printPackResult(path string, size int64) {
 	fmt.Fprintf(os.Stderr, "  %s %s\n", styleAccent.Render("size"), humanBytes(size))
 }
 
+// printCacheLine reports how much compression the incremental cache reused. On
+// the first pack everything is compressed fresh; on a re-pack after a small
+// change most clusters are reused and only the rest are compressed.
+func printCacheLine(incremental bool, st pack.PackStats) {
+	if !incremental {
+		return
+	}
+	total := st.ClustersReused + st.ClustersCompressed
+	fmt.Fprintf(os.Stderr, "  %s %d reused, %d compressed of %d clusters\n",
+		styleDim.Render("cache"), st.ClustersReused, st.ClustersCompressed, total)
+}
+
 func printRunHint(path, target string) {
 	rel := path
 	if !strings.ContainsAny(path, "/\\") {
diff --git a/pack/cache.go b/pack/cache.go
new file mode 100644
index 0000000..e105b11
--- /dev/null
+++ b/pack/cache.go
@@ -0,0 +1,134 @@
+package pack
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
+	"io"
+	"os"
+	"sort"
+
+	"github.com/tamnd/kage/zim"
+)
+
+// cacheMagic tags the sidecar so a stale or foreign file is recognised and
+// ignored rather than misread. The trailing digit is the format version.
+var cacheMagic = [8]byte{'k', 'a', 'g', 'e', 'z', 'c', 'h', '1'}
+
+// clusterCache is a content-addressed store of compressed ZIM clusters, kept in
+// a sidecar next to the archive. Compressing clusters with zstd is the dominant
+// cost of packing a large mirror, so a re-pack reuses the compression of every
+// cluster whose uncompressed bytes are unchanged and only compresses the rest.
+//
+// A cache hit returns exactly what a fresh compression would have produced, so
+// the archive stays deterministic and valid; the cache only saves CPU. Clusters
+// are keyed by the SHA-256 of their uncompressed data section, which the zim
+// writer assembles before compression.
+type clusterCache struct {
+	prev map[[32]byte][]byte // clusters loaded from the previous pack
+	used map[[32]byte][]byte // clusters touched this pack, written back on save
+
+	reused     int // clusters served from the cache this pack
+	compressed int // clusters compressed fresh this pack
+}
+
+func newClusterCache() *clusterCache {
+	return &clusterCache{prev: map[[32]byte][]byte{}, used: map[[32]byte][]byte{}}
+}
+
+// loadClusterCache reads the sidecar at path. A missing, unreadable, or
+// truncated cache is not an error: packing starts cold and rebuilds it.
+func loadClusterCache(path string) *clusterCache {
+	c := newClusterCache()
+	f, err := os.Open(path)
+	if err != nil {
+		return c
+	}
+	defer func() { _ = f.Close() }()
+
+	r := bufio.NewReader(f)
+	var magic [8]byte
+	if _, err := io.ReadFull(r, magic[:]); err != nil || magic != cacheMagic {
+		return c
+	}
+	var count uint32
+	if err := binary.Read(r, binary.LittleEndian, &count); err != nil {
+		return c
+	}
+	for i := uint32(0); i < count; i++ {
+		var h [32]byte
+		if _, err := io.ReadFull(r, h[:]); err != nil {
+			return newClusterCache() // truncated: drop the partial load
+		}
+		var n uint32
+		if err := binary.Read(r, binary.LittleEndian, &n); err != nil {
+			return newClusterCache()
+		}
+		b := make([]byte, n)
+		if _, err := io.ReadFull(r, b); err != nil {
+			return newClusterCache()
+		}
+		c.prev[h] = b
+	}
+	return c
+}
+
+// Compress returns the stored bytes for an uncompressed cluster: the cached
+// compression when the cluster's bytes are unchanged, a fresh zstd compression
+// on a miss. It is the function handed to zim.Writer.SetCompress.
+func (c *clusterCache) Compress(data []byte) []byte {
+	h := sha256.Sum256(data)
+	if b, ok := c.used[h]; ok {
+		return b
+	}
+	if b, ok := c.prev[h]; ok {
+		c.used[h] = b
+		c.reused++
+		return b
+	}
+	b := zim.Compress(data)
+	c.used[h] = b
+	c.compressed++
+	return b
+}
+
+// save writes the clusters touched this pack to path, replacing the previous
+// sidecar. Only touched clusters are written, so clusters that left the mirror
+// drop out and the cache cannot grow without bound. Entries are sorted by hash
+// so the sidecar itself is reproducible. The write goes through a temp file and
+// a rename so a crash mid-write cannot corrupt an existing cache.
+func (c *clusterCache) save(path string) error {
+	hashes := make([][32]byte, 0, len(c.used))
+	for h := range c.used {
+		hashes = append(hashes, h)
+	}
+	sort.Slice(hashes, func(i, j int) bool {
+		return bytes.Compare(hashes[i][:], hashes[j][:]) < 0
+	})
+
+	tmp := path + ".tmp"
+	f, err := os.Create(tmp)
+	if err != nil {
+		return err
+	}
+	w := bufio.NewWriter(f)
+	_, _ = w.Write(cacheMagic[:])
+	_ = binary.Write(w, binary.LittleEndian, uint32(len(hashes)))
+	for _, h := range hashes {
+		b := c.used[h]
+		_, _ = w.Write(h[:])
+		_ = binary.Write(w, binary.LittleEndian, uint32(len(b)))
+		_, _ = w.Write(b)
+	}
+	if err := w.Flush(); err != nil {
+		_ = f.Close()
+		_ = os.Remove(tmp)
+		return err
+	}
+	if err := f.Close(); err != nil {
+		_ = os.Remove(tmp)
+		return err
+	}
+	return os.Rename(tmp, path)
+}
diff --git a/pack/cache_test.go b/pack/cache_test.go
new file mode 100644
index 0000000..bc88fba
--- /dev/null
+++ b/pack/cache_test.go
@@ -0,0 +1,122 @@
+package pack
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/tamnd/kage/zim"
+)
+
+// TestIncrementalPackReusesCompression checks the core promise of the cache: a
+// second pack of an unchanged mirror compresses nothing, reuses every cluster,
+// and produces a byte-identical archive.
+func TestIncrementalPackReusesCompression(t *testing.T) {
+	host := writeMirror(t)
+	dir := t.TempDir()
+	out := filepath.Join(dir, "example.zim")
+	cache := out + ".kagecache"
+
+	var first PackStats
+	a, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &first})
+	if err != nil {
+		t.Fatalf("first pack: %v", err)
+	}
+	if first.ClustersCompressed == 0 {
+		t.Fatal("first pack reported no clusters compressed")
+	}
+	if first.ClustersReused != 0 {
+		t.Errorf("first pack reused %d clusters, want 0", first.ClustersReused)
+	}
+	if _, err := os.Stat(cache); err != nil {
+		t.Fatalf("cache sidecar not written: %v", err)
+	}
+	bytesA, err := os.ReadFile(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var second PackStats
+	b, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &second})
+	if err != nil {
+		t.Fatalf("second pack: %v", err)
+	}
+	if second.ClustersCompressed != 0 {
+		t.Errorf("second pack compressed %d clusters, want 0 (all reused)", second.ClustersCompressed)
+	}
+	if second.ClustersReused != first.ClustersCompressed {
+		t.Errorf("second pack reused %d clusters, want %d", second.ClustersReused, first.ClustersCompressed)
+	}
+	bytesB, err := os.ReadFile(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(bytesA, bytesB) {
+		t.Error("a cached re-pack produced a different archive than the cold pack")
+	}
+}
+
+// TestIncrementalPackRecompressesChange checks that editing one page forces a
+// fresh compression of the cluster that holds it while the rest are reused, and
+// that the archive still opens and serves the new content.
+func TestIncrementalPackRecompressesChange(t *testing.T) {
+	host := writeMirror(t)
+	dir := t.TempDir()
+	out := filepath.Join(dir, "example.zim")
+	cache := out + ".kagecache"
+
+	if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache}); err != nil {
+		t.Fatalf("first pack: %v", err)
+	}
+
+	// Change one page. Its text cluster must be recompressed.
+	edited := filepath.Join(host, "about", "index.html")
+	if err := os.WriteFile(edited, []byte("<!doctype html><title>About</title><h1>Changed</h1>"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	var st PackStats
+	if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &st}); err != nil {
+		t.Fatalf("second pack: %v", err)
+	}
+	if st.ClustersCompressed == 0 {
+		t.Error("editing a page compressed nothing; expected the changed cluster to be recompressed")
+	}
+
+	r, err := zim.Open(out)
+	if err != nil {
+		t.Fatalf("reopen: %v", err)
+	}
+	defer func() { _ = r.Close() }()
+	about, err := r.Get(zim.NamespaceContent, "about/index.html")
+	if err != nil {
+		t.Fatalf("about page missing after re-pack: %v", err)
+	}
+	if !bytes.Contains(about.Data, []byte("Changed")) {
+		t.Errorf("re-packed page did not carry the edit: %q", about.Data)
+	}
+}
+
+// TestClusterCacheSurvivesCorruptSidecar checks that a damaged cache file is
+// treated as a cold start rather than failing the pack.
+func TestClusterCacheSurvivesCorruptSidecar(t *testing.T) {
+	host := writeMirror(t)
+	dir := t.TempDir()
+	out := filepath.Join(dir, "example.zim")
+	cache := out + ".kagecache"
+	if err := os.WriteFile(cache, []byte("not a real cache file"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	var st PackStats
+	if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &st}); err != nil {
+		t.Fatalf("pack over corrupt cache: %v", err)
+	}
+	if st.ClustersReused != 0 {
+		t.Errorf("corrupt cache should reuse nothing, reused %d", st.ClustersReused)
+	}
+	if st.ClustersCompressed == 0 {
+		t.Error("pack over corrupt cache compressed nothing")
+	}
+}
diff --git a/pack/zim.go b/pack/zim.go
index f68eabd..aff28e3 100644
--- a/pack/zim.go
+++ b/pack/zim.go
@@ -31,6 +31,24 @@ type ZIMOptions struct {
 	Language    string // M/Language (default "eng")
 	Date        string // M/Date, e.g. "2026-06-14"
 	Version     string // kage version, recorded as M/Scraper
+
+	// CachePath, when set, points at a content-addressed cluster cache sidecar.
+	// Compression of unchanged clusters is reused from it, and the cache is
+	// rewritten after a successful pack. Empty disables the cache. It has no
+	// effect with NoCompress, where nothing is compressed.
+	CachePath string
+	// Stats, when non-nil, is filled with how many clusters were reused from the
+	// cache versus compressed fresh. It lets the caller report incremental gains
+	// without changing the function's return signature.
+	Stats *PackStats
+}
+
+// PackStats reports how a pack reused cached compression. ClustersReused is the
+// number of clusters whose compressed bytes came straight from the cache;
+// ClustersCompressed is the number that were zstd-compressed this run.
+type PackStats struct {
+	ClustersReused     int
+	ClustersCompressed int
 }
 
 // BuildZIM walks mirrorDir, turns every file into a C/ content entry, infers the
@@ -38,7 +56,7 @@ type ZIMOptions struct {
 // redirect, and writes a .zim to opts.Out. It returns the output path and the
 // number of bytes written.
 func BuildZIM(mirrorDir string, opts ZIMOptions) (string, int64, error) {
-	w, err := buildWriter(mirrorDir, opts)
+	w, cache, err := buildWriter(mirrorDir, opts)
 	if err != nil {
 		return "", 0, err
 	}
@@ -60,14 +78,21 @@ func BuildZIM(mirrorDir string, opts ZIMOptions) (string, int64, error) {
 		_ = f.Close()
 		return out, n, err
 	}
-	return out, n, f.Close()
+	if err := f.Close(); err != nil {
+		return out, n, err
+	}
+	if err := persistCache(opts.CachePath, cache); err != nil {
+		return out, n, err
+	}
+	fillStats(opts.Stats, cache)
+	return out, n, nil
 }
 
 // BuildZIMBytes is the buffer-returning sibling of BuildZIM: it runs the same
 // walk and returns the archive in memory, which the binary path appends to a
 // base executable without writing the ZIM to disk first.
 func BuildZIMBytes(mirrorDir string, opts ZIMOptions) ([]byte, error) {
-	w, err := buildWriter(mirrorDir, opts)
+	w, cache, err := buildWriter(mirrorDir, opts)
 	if err != nil {
 		return nil, err
 	}
@@ -75,18 +100,41 @@ func BuildZIMBytes(mirrorDir string, opts ZIMOptions) ([]byte, error) {
 	if _, err := w.WriteTo(&buf); err != nil {
 		return nil, err
 	}
+	if err := persistCache(opts.CachePath, cache); err != nil {
+		return nil, err
+	}
+	fillStats(opts.Stats, cache)
 	return buf.Bytes(), nil
 }
 
+// persistCache writes the cluster cache back to its sidecar after a successful
+// pack. It is a no-op when caching is off.
+func persistCache(path string, c *clusterCache) error {
+	if c == nil || path == "" {
+		return nil
+	}
+	return c.save(path)
+}
+
+// fillStats copies the cache's reuse counters into the caller's PackStats when
+// both are present, so the CLI can report incremental gains.
+func fillStats(dst *PackStats, c *clusterCache) {
+	if dst == nil || c == nil {
+		return
+	}
+	dst.ClustersReused = c.reused
+	dst.ClustersCompressed = c.compressed
+}
+
 // buildWriter does the shared work of both BuildZIM and BuildZIMBytes: it loads
 // every file under mirrorDir into a zim.Writer with metadata and a main page.
-func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) {
+func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, *clusterCache, error) {
 	info, err := os.Stat(mirrorDir)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	if !info.IsDir() {
-		return nil, fmt.Errorf("pack: %q is not a directory", mirrorDir)
+		return nil, nil, fmt.Errorf("pack: %q is not a directory", mirrorDir)
 	}
 
 	w := zim.NewWriter()
@@ -94,6 +142,14 @@ func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) {
 		w.SetNoCompress(true)
 	}
 
+	// The cluster cache only helps when clusters are compressed; with NoCompress
+	// there is nothing to reuse, so it is skipped.
+	var cache *clusterCache
+	if opts.CachePath != "" && !opts.NoCompress {
+		cache = loadClusterCache(opts.CachePath)
+		w.SetCompress(cache.Compress)
+	}
+
 	skip := urlx.DefaultReserved + "/state.json"
 	var htmlPages []string
 	counts := map[string]int{}
@@ -122,7 +178,7 @@ func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) {
 		return nil
 	})
 	if walkErr != nil {
-		return nil, walkErr
+		return nil, nil, walkErr
 	}
 
 	main := pickMainPage(htmlPages)
@@ -152,7 +208,7 @@ func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) {
 	if png, ok := Favicon48(mirrorDir); ok {
 		w.AddMetadataBytes("Illustrator_48x48@1", "image/png", png)
 	}
-	return w, nil
+	return w, cache, nil
 }
 
 // pickMainPage chooses the archive's entry point: the root index if present,
diff --git a/zim/codec.go b/zim/codec.go
index 89be75e..fed567d 100644
--- a/zim/codec.go
+++ b/zim/codec.go
@@ -26,6 +26,12 @@ func zstdEncode(p []byte) []byte {
 	return zstdEnc.EncodeAll(p, nil)
 }
 
+// Compress zstd-compresses p with the exact codec the writer uses for its
+// clusters. It is exported so a caller can cache cluster compression across
+// packs and feed the result back through Writer.SetCompress; a cached cluster is
+// then byte-for-byte what a fresh compression would have produced.
+func Compress(p []byte) []byte { return zstdEncode(p) }
+
 func zstdDecode(p []byte) ([]byte, error) {
 	initZstd()
 	return zstdDec.DecodeAll(p, nil)
diff --git a/zim/writer.go b/zim/writer.go
index d45000b..6bac26f 100644
--- a/zim/writer.go
+++ b/zim/writer.go
@@ -22,6 +22,11 @@ type Writer struct {
 	byKey      map[string]*entry
 	mainKey    string
 	noCompress bool
+	// compress turns a cluster's uncompressed data section into its stored bytes.
+	// It defaults to the built-in zstd codec; a caller can replace it with a
+	// caching compressor so a re-pack reuses the compression of unchanged
+	// clusters instead of running zstd again.
+	compress func([]byte) []byte
 }
 
 type entry struct {
@@ -47,13 +52,24 @@ func key(ns byte, url string) string { return string(ns) + url }
 
 // NewWriter returns an empty Writer.
 func NewWriter() *Writer {
-	return &Writer{byKey: map[string]*entry{}}
+	return &Writer{byKey: map[string]*entry{}, compress: zstdEncode}
 }
 
 // SetNoCompress stores every cluster uncompressed. Useful when the input is
 // already compressed or when a reader without zstd must open the file.
 func (w *Writer) SetNoCompress(v bool) { w.noCompress = v }
 
+// SetCompress replaces the cluster compressor. The function must return
+// zstd-compressed bytes (the writer marks those clusters as zstd), so a caching
+// wrapper can short-circuit unchanged clusters while still producing a valid,
+// byte-identical archive. A nil function restores the built-in codec.
+func (w *Writer) SetCompress(f func([]byte) []byte) {
+	if f == nil {
+		f = zstdEncode
+	}
+	w.compress = f
+}
+
 // AddContent adds a content entry. A later add with the same namespace and url
 // replaces the earlier one. An empty title defaults to the url.
 func (w *Writer) AddContent(namespace byte, url, title, mime string, data []byte) {
@@ -185,7 +201,7 @@ func (w *Writer) buildPlan() (plan, error) {
 	clusters := w.packClusters(ents)
 	p.clusters = make([][]byte, len(clusters))
 	for i, c := range clusters {
-		p.clusters[i] = c.encode(w.noCompress)
+		p.clusters[i] = c.encode(w.noCompress, w.compress)
 	}
 
 	// 5. Directory entry bytes (URL order).
@@ -303,7 +319,10 @@ func indexOf(cs []*clusterBuf, c *clusterBuf) int {
 // encode renders a cluster: an info byte followed by the (optionally zstd)
 // data section, which is an offset table of (N+1) uint32 values then the N
 // concatenated blobs. Offsets are relative to the start of the data section.
-func (c *clusterBuf) encode(noCompress bool) []byte {
+func (c *clusterBuf) encode(noCompress bool, compress func([]byte) []byte) []byte {
+	if compress == nil {
+		compress = zstdEncode
+	}
 	n := len(c.blobs)
 	tableLen := 4 * (n + 1)
 	total := tableLen
@@ -327,7 +346,7 @@ func (c *clusterBuf) encode(noCompress bool) []byte {
 	}
 	payload := data
 	if comp == compZstd {
-		payload = zstdEncode(data)
+		payload = compress(data)
 	} else {
 		comp = compNone
 	}