diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a5a5a1..2eec67b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,14 @@ All notable changes to kage are recorded here. The format follows ### Added +- `kage pack --incremental` keeps a small cache sidecar next to the output and + reuses the compression of any cluster whose bytes have not changed since the + last pack. Compressing clusters with zstd is the dominant cost of packing a + large mirror, so re-packing after a small change (a `--refresh`, a handful of + edited pages) only compresses what actually changed instead of the whole + archive. A cached cluster is byte-for-byte what a fresh compression produces, + so the archive stays deterministic and valid. The pack reports how many + clusters it reused versus compressed. - Identical pages are now stored once. When a rendered page's bytes match a page already written, kage stores it as a hard link to the first copy instead of a second full file. This collapses the duplicate content a faceted site spawns diff --git a/cli/pack.go b/cli/pack.go index c40be60..a7ae674 100644 --- a/cli/pack.go +++ b/cli/pack.go @@ -28,8 +28,13 @@ type packFlags struct { description string language string date string + incremental bool } +// cacheSuffix names the cluster-cache sidecar kage writes next to a packed +// artifact when --incremental is set. +const cacheSuffix = ".kagecache" + func newPackCmd() *cobra.Command { f := &packFlags{} cmd := &cobra.Command{ @@ -40,7 +45,9 @@ func newPackCmd() *cobra.Command { "ZIM reader can browse. With --format binary it appends that archive to a copy\n" + "of kage, producing a single executable that serves the site offline when run.\n" + "Add --app to wrap that executable in a double-click desktop app (a .app bundle\n" + - "on macOS, an AppImage-style .AppDir on Linux) with the site's favicon as the icon.", + "on macOS, an AppImage-style .AppDir on Linux) with the site's favicon as the icon.\n" + + "Add --incremental to keep a cache sidecar so re-packing a mirror only compresses\n" + + "the clusters that changed, not the whole archive.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return runPack(args[0], f) @@ -53,6 +60,7 @@ func newPackCmd() *cobra.Command { fs.BoolVar(&f.app, "app", false, "wrap the viewer in a double-click desktop app (.app on macOS, .AppImage/.AppDir on Linux)") fs.StringVar(&f.icon, "icon", "", "icon file for --app (default the site's favicon)") fs.BoolVar(&f.noCompress, "no-compress", false, "store every cluster raw, no zstd") + fs.BoolVar(&f.incremental, "incremental", false, "reuse compression from a cache sidecar so re-packing a mirror only compresses what changed") fs.StringVar(&f.title, "title", "", "archive title (default the main page's )") fs.StringVar(&f.description, "description", "", "archive description") fs.StringVar(&f.language, "language", "eng", "archive language code") @@ -80,19 +88,26 @@ func runPack(mirrorArg string, f *packFlags) error { switch f.format { case "zim": - out, size, err := pack.BuildZIM(dir, zopts) + out := f.out + if out == "" { + out = filepath.Base(dir) + ".zim" + } + zopts.Out = out + var st pack.PackStats + if f.incremental { + zopts.CachePath = out + cacheSuffix + zopts.Stats = &st + } + outPath, size, err := pack.BuildZIM(dir, zopts) if err != nil { return err } - printPackResult(out, size) - fmt.Fprintf(os.Stderr, " open %s\n", styleAccent.Render("kage open "+out)) + printPackResult(outPath, size) + printCacheLine(f.incremental, st) + fmt.Fprintf(os.Stderr, " open %s\n", styleAccent.Render("kage open "+outPath)) return nil case "binary": - zbytes, err := pack.BuildZIMBytes(dir, zopts) - if err != nil { - return err - } target := resolveTargetOS(f.base) out := f.out if out == "" { @@ -103,11 +118,21 @@ func runPack(mirrorArg string, f *packFlags) error { if target == "windows" && !strings.HasSuffix(strings.ToLower(out), ".exe") { out += ".exe" } + var st pack.PackStats + if f.incremental { + zopts.CachePath = out + cacheSuffix + zopts.Stats = &st + } + zbytes, err := pack.BuildZIMBytes(dir, zopts) + if err != nil { + return err + } path, size, err := pack.BuildBinary(zbytes, pack.BinaryOptions{Out: out, Base: f.base}) if err != nil { return err } printPackResult(path, size) + printCacheLine(f.incremental, st) printRunHint(path, target) return nil @@ -142,10 +167,16 @@ func runPackApp(dir string, f *packFlags, zopts pack.ZIMOptions) error { if err != nil { return err } + var st pack.PackStats + if f.incremental { + zopts.CachePath = prog + cacheSuffix + zopts.Stats = &st + } zbytes, err := pack.BuildZIMBytes(dir, zopts) if err != nil { return err } + printCacheLine(f.incremental, st) switch target { case "darwin": @@ -345,6 +376,18 @@ func printPackResult(path string, size int64) { fmt.Fprintf(os.Stderr, " %s %s\n", styleAccent.Render("size"), humanBytes(size)) } +// printCacheLine reports how much compression the incremental cache reused. On +// the first pack everything is compressed fresh; on a re-pack after a small +// change most clusters are reused and only the rest are compressed. +func printCacheLine(incremental bool, st pack.PackStats) { + if !incremental { + return + } + total := st.ClustersReused + st.ClustersCompressed + fmt.Fprintf(os.Stderr, " %s %d reused, %d compressed of %d clusters\n", + styleDim.Render("cache"), st.ClustersReused, st.ClustersCompressed, total) +} + func printRunHint(path, target string) { rel := path if !strings.ContainsAny(path, "/\\") { diff --git a/pack/cache.go b/pack/cache.go new file mode 100644 index 0000000..e105b11 --- /dev/null +++ b/pack/cache.go @@ -0,0 +1,134 @@ +package pack + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/binary" + "io" + "os" + "sort" + + "github.com/tamnd/kage/zim" +) + +// cacheMagic tags the sidecar so a stale or foreign file is recognised and +// ignored rather than misread. The trailing digit is the format version. +var cacheMagic = [8]byte{'k', 'a', 'g', 'e', 'z', 'c', 'h', '1'} + +// clusterCache is a content-addressed store of compressed ZIM clusters, kept in +// a sidecar next to the archive. Compressing clusters with zstd is the dominant +// cost of packing a large mirror, so a re-pack reuses the compression of every +// cluster whose uncompressed bytes are unchanged and only compresses the rest. +// +// A cache hit returns exactly what a fresh compression would have produced, so +// the archive stays deterministic and valid; the cache only saves CPU. Clusters +// are keyed by the SHA-256 of their uncompressed data section, which the zim +// writer assembles before compression. +type clusterCache struct { + prev map[[32]byte][]byte // clusters loaded from the previous pack + used map[[32]byte][]byte // clusters touched this pack, written back on save + + reused int // clusters served from the cache this pack + compressed int // clusters compressed fresh this pack +} + +func newClusterCache() *clusterCache { + return &clusterCache{prev: map[[32]byte][]byte{}, used: map[[32]byte][]byte{}} +} + +// loadClusterCache reads the sidecar at path. A missing, unreadable, or +// truncated cache is not an error: packing starts cold and rebuilds it. +func loadClusterCache(path string) *clusterCache { + c := newClusterCache() + f, err := os.Open(path) + if err != nil { + return c + } + defer func() { _ = f.Close() }() + + r := bufio.NewReader(f) + var magic [8]byte + if _, err := io.ReadFull(r, magic[:]); err != nil || magic != cacheMagic { + return c + } + var count uint32 + if err := binary.Read(r, binary.LittleEndian, &count); err != nil { + return c + } + for i := uint32(0); i < count; i++ { + var h [32]byte + if _, err := io.ReadFull(r, h[:]); err != nil { + return newClusterCache() // truncated: drop the partial load + } + var n uint32 + if err := binary.Read(r, binary.LittleEndian, &n); err != nil { + return newClusterCache() + } + b := make([]byte, n) + if _, err := io.ReadFull(r, b); err != nil { + return newClusterCache() + } + c.prev[h] = b + } + return c +} + +// Compress returns the stored bytes for an uncompressed cluster: the cached +// compression when the cluster's bytes are unchanged, a fresh zstd compression +// on a miss. It is the function handed to zim.Writer.SetCompress. +func (c *clusterCache) Compress(data []byte) []byte { + h := sha256.Sum256(data) + if b, ok := c.used[h]; ok { + return b + } + if b, ok := c.prev[h]; ok { + c.used[h] = b + c.reused++ + return b + } + b := zim.Compress(data) + c.used[h] = b + c.compressed++ + return b +} + +// save writes the clusters touched this pack to path, replacing the previous +// sidecar. Only touched clusters are written, so clusters that left the mirror +// drop out and the cache cannot grow without bound. Entries are sorted by hash +// so the sidecar itself is reproducible. The write goes through a temp file and +// a rename so a crash mid-write cannot corrupt an existing cache. +func (c *clusterCache) save(path string) error { + hashes := make([][32]byte, 0, len(c.used)) + for h := range c.used { + hashes = append(hashes, h) + } + sort.Slice(hashes, func(i, j int) bool { + return bytes.Compare(hashes[i][:], hashes[j][:]) < 0 + }) + + tmp := path + ".tmp" + f, err := os.Create(tmp) + if err != nil { + return err + } + w := bufio.NewWriter(f) + _, _ = w.Write(cacheMagic[:]) + _ = binary.Write(w, binary.LittleEndian, uint32(len(hashes))) + for _, h := range hashes { + b := c.used[h] + _, _ = w.Write(h[:]) + _ = binary.Write(w, binary.LittleEndian, uint32(len(b))) + _, _ = w.Write(b) + } + if err := w.Flush(); err != nil { + _ = f.Close() + _ = os.Remove(tmp) + return err + } + if err := f.Close(); err != nil { + _ = os.Remove(tmp) + return err + } + return os.Rename(tmp, path) +} diff --git a/pack/cache_test.go b/pack/cache_test.go new file mode 100644 index 0000000..bc88fba --- /dev/null +++ b/pack/cache_test.go @@ -0,0 +1,122 @@ +package pack + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/tamnd/kage/zim" +) + +// TestIncrementalPackReusesCompression checks the core promise of the cache: a +// second pack of an unchanged mirror compresses nothing, reuses every cluster, +// and produces a byte-identical archive. +func TestIncrementalPackReusesCompression(t *testing.T) { + host := writeMirror(t) + dir := t.TempDir() + out := filepath.Join(dir, "example.zim") + cache := out + ".kagecache" + + var first PackStats + a, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &first}) + if err != nil { + t.Fatalf("first pack: %v", err) + } + if first.ClustersCompressed == 0 { + t.Fatal("first pack reported no clusters compressed") + } + if first.ClustersReused != 0 { + t.Errorf("first pack reused %d clusters, want 0", first.ClustersReused) + } + if _, err := os.Stat(cache); err != nil { + t.Fatalf("cache sidecar not written: %v", err) + } + bytesA, err := os.ReadFile(a) + if err != nil { + t.Fatal(err) + } + + var second PackStats + b, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &second}) + if err != nil { + t.Fatalf("second pack: %v", err) + } + if second.ClustersCompressed != 0 { + t.Errorf("second pack compressed %d clusters, want 0 (all reused)", second.ClustersCompressed) + } + if second.ClustersReused != first.ClustersCompressed { + t.Errorf("second pack reused %d clusters, want %d", second.ClustersReused, first.ClustersCompressed) + } + bytesB, err := os.ReadFile(b) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(bytesA, bytesB) { + t.Error("a cached re-pack produced a different archive than the cold pack") + } +} + +// TestIncrementalPackRecompressesChange checks that editing one page forces a +// fresh compression of the cluster that holds it while the rest are reused, and +// that the archive still opens and serves the new content. +func TestIncrementalPackRecompressesChange(t *testing.T) { + host := writeMirror(t) + dir := t.TempDir() + out := filepath.Join(dir, "example.zim") + cache := out + ".kagecache" + + if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache}); err != nil { + t.Fatalf("first pack: %v", err) + } + + // Change one page. Its text cluster must be recompressed. + edited := filepath.Join(host, "about", "index.html") + if err := os.WriteFile(edited, []byte("<!doctype html><title>About

Changed

"), 0o644); err != nil { + t.Fatal(err) + } + + var st PackStats + if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &st}); err != nil { + t.Fatalf("second pack: %v", err) + } + if st.ClustersCompressed == 0 { + t.Error("editing a page compressed nothing; expected the changed cluster to be recompressed") + } + + r, err := zim.Open(out) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer func() { _ = r.Close() }() + about, err := r.Get(zim.NamespaceContent, "about/index.html") + if err != nil { + t.Fatalf("about page missing after re-pack: %v", err) + } + if !bytes.Contains(about.Data, []byte("Changed")) { + t.Errorf("re-packed page did not carry the edit: %q", about.Data) + } +} + +// TestClusterCacheSurvivesCorruptSidecar checks that a damaged cache file is +// treated as a cold start rather than failing the pack. +func TestClusterCacheSurvivesCorruptSidecar(t *testing.T) { + host := writeMirror(t) + dir := t.TempDir() + out := filepath.Join(dir, "example.zim") + cache := out + ".kagecache" + if err := os.WriteFile(cache, []byte("not a real cache file"), 0o644); err != nil { + t.Fatal(err) + } + + var st PackStats + if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", CachePath: cache, Stats: &st}); err != nil { + t.Fatalf("pack over corrupt cache: %v", err) + } + if st.ClustersReused != 0 { + t.Errorf("corrupt cache should reuse nothing, reused %d", st.ClustersReused) + } + if st.ClustersCompressed == 0 { + t.Error("pack over corrupt cache compressed nothing") + } +} diff --git a/pack/zim.go b/pack/zim.go index f68eabd..aff28e3 100644 --- a/pack/zim.go +++ b/pack/zim.go @@ -31,6 +31,24 @@ type ZIMOptions struct { Language string // M/Language (default "eng") Date string // M/Date, e.g. "2026-06-14" Version string // kage version, recorded as M/Scraper + + // CachePath, when set, points at a content-addressed cluster cache sidecar. + // Compression of unchanged clusters is reused from it, and the cache is + // rewritten after a successful pack. Empty disables the cache. It has no + // effect with NoCompress, where nothing is compressed. + CachePath string + // Stats, when non-nil, is filled with how many clusters were reused from the + // cache versus compressed fresh. It lets the caller report incremental gains + // without changing the function's return signature. + Stats *PackStats +} + +// PackStats reports how a pack reused cached compression. ClustersReused is the +// number of clusters whose compressed bytes came straight from the cache; +// ClustersCompressed is the number that were zstd-compressed this run. +type PackStats struct { + ClustersReused int + ClustersCompressed int } // BuildZIM walks mirrorDir, turns every file into a C/ content entry, infers the @@ -38,7 +56,7 @@ type ZIMOptions struct { // redirect, and writes a .zim to opts.Out. It returns the output path and the // number of bytes written. func BuildZIM(mirrorDir string, opts ZIMOptions) (string, int64, error) { - w, err := buildWriter(mirrorDir, opts) + w, cache, err := buildWriter(mirrorDir, opts) if err != nil { return "", 0, err } @@ -60,14 +78,21 @@ func BuildZIM(mirrorDir string, opts ZIMOptions) (string, int64, error) { _ = f.Close() return out, n, err } - return out, n, f.Close() + if err := f.Close(); err != nil { + return out, n, err + } + if err := persistCache(opts.CachePath, cache); err != nil { + return out, n, err + } + fillStats(opts.Stats, cache) + return out, n, nil } // BuildZIMBytes is the buffer-returning sibling of BuildZIM: it runs the same // walk and returns the archive in memory, which the binary path appends to a // base executable without writing the ZIM to disk first. func BuildZIMBytes(mirrorDir string, opts ZIMOptions) ([]byte, error) { - w, err := buildWriter(mirrorDir, opts) + w, cache, err := buildWriter(mirrorDir, opts) if err != nil { return nil, err } @@ -75,18 +100,41 @@ func BuildZIMBytes(mirrorDir string, opts ZIMOptions) ([]byte, error) { if _, err := w.WriteTo(&buf); err != nil { return nil, err } + if err := persistCache(opts.CachePath, cache); err != nil { + return nil, err + } + fillStats(opts.Stats, cache) return buf.Bytes(), nil } +// persistCache writes the cluster cache back to its sidecar after a successful +// pack. It is a no-op when caching is off. +func persistCache(path string, c *clusterCache) error { + if c == nil || path == "" { + return nil + } + return c.save(path) +} + +// fillStats copies the cache's reuse counters into the caller's PackStats when +// both are present, so the CLI can report incremental gains. +func fillStats(dst *PackStats, c *clusterCache) { + if dst == nil || c == nil { + return + } + dst.ClustersReused = c.reused + dst.ClustersCompressed = c.compressed +} + // buildWriter does the shared work of both BuildZIM and BuildZIMBytes: it loads // every file under mirrorDir into a zim.Writer with metadata and a main page. -func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) { +func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, *clusterCache, error) { info, err := os.Stat(mirrorDir) if err != nil { - return nil, err + return nil, nil, err } if !info.IsDir() { - return nil, fmt.Errorf("pack: %q is not a directory", mirrorDir) + return nil, nil, fmt.Errorf("pack: %q is not a directory", mirrorDir) } w := zim.NewWriter() @@ -94,6 +142,14 @@ func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) { w.SetNoCompress(true) } + // The cluster cache only helps when clusters are compressed; with NoCompress + // there is nothing to reuse, so it is skipped. + var cache *clusterCache + if opts.CachePath != "" && !opts.NoCompress { + cache = loadClusterCache(opts.CachePath) + w.SetCompress(cache.Compress) + } + skip := urlx.DefaultReserved + "/state.json" var htmlPages []string counts := map[string]int{} @@ -122,7 +178,7 @@ func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) { return nil }) if walkErr != nil { - return nil, walkErr + return nil, nil, walkErr } main := pickMainPage(htmlPages) @@ -152,7 +208,7 @@ func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) { if png, ok := Favicon48(mirrorDir); ok { w.AddMetadataBytes("Illustrator_48x48@1", "image/png", png) } - return w, nil + return w, cache, nil } // pickMainPage chooses the archive's entry point: the root index if present, diff --git a/zim/codec.go b/zim/codec.go index 89be75e..fed567d 100644 --- a/zim/codec.go +++ b/zim/codec.go @@ -26,6 +26,12 @@ func zstdEncode(p []byte) []byte { return zstdEnc.EncodeAll(p, nil) } +// Compress zstd-compresses p with the exact codec the writer uses for its +// clusters. It is exported so a caller can cache cluster compression across +// packs and feed the result back through Writer.SetCompress; a cached cluster is +// then byte-for-byte what a fresh compression would have produced. +func Compress(p []byte) []byte { return zstdEncode(p) } + func zstdDecode(p []byte) ([]byte, error) { initZstd() return zstdDec.DecodeAll(p, nil) diff --git a/zim/writer.go b/zim/writer.go index d45000b..6bac26f 100644 --- a/zim/writer.go +++ b/zim/writer.go @@ -22,6 +22,11 @@ type Writer struct { byKey map[string]*entry mainKey string noCompress bool + // compress turns a cluster's uncompressed data section into its stored bytes. + // It defaults to the built-in zstd codec; a caller can replace it with a + // caching compressor so a re-pack reuses the compression of unchanged + // clusters instead of running zstd again. + compress func([]byte) []byte } type entry struct { @@ -47,13 +52,24 @@ func key(ns byte, url string) string { return string(ns) + url } // NewWriter returns an empty Writer. func NewWriter() *Writer { - return &Writer{byKey: map[string]*entry{}} + return &Writer{byKey: map[string]*entry{}, compress: zstdEncode} } // SetNoCompress stores every cluster uncompressed. Useful when the input is // already compressed or when a reader without zstd must open the file. func (w *Writer) SetNoCompress(v bool) { w.noCompress = v } +// SetCompress replaces the cluster compressor. The function must return +// zstd-compressed bytes (the writer marks those clusters as zstd), so a caching +// wrapper can short-circuit unchanged clusters while still producing a valid, +// byte-identical archive. A nil function restores the built-in codec. +func (w *Writer) SetCompress(f func([]byte) []byte) { + if f == nil { + f = zstdEncode + } + w.compress = f +} + // AddContent adds a content entry. A later add with the same namespace and url // replaces the earlier one. An empty title defaults to the url. func (w *Writer) AddContent(namespace byte, url, title, mime string, data []byte) { @@ -185,7 +201,7 @@ func (w *Writer) buildPlan() (plan, error) { clusters := w.packClusters(ents) p.clusters = make([][]byte, len(clusters)) for i, c := range clusters { - p.clusters[i] = c.encode(w.noCompress) + p.clusters[i] = c.encode(w.noCompress, w.compress) } // 5. Directory entry bytes (URL order). @@ -303,7 +319,10 @@ func indexOf(cs []*clusterBuf, c *clusterBuf) int { // encode renders a cluster: an info byte followed by the (optionally zstd) // data section, which is an offset table of (N+1) uint32 values then the N // concatenated blobs. Offsets are relative to the start of the data section. -func (c *clusterBuf) encode(noCompress bool) []byte { +func (c *clusterBuf) encode(noCompress bool, compress func([]byte) []byte) []byte { + if compress == nil { + compress = zstdEncode + } n := len(c.blobs) tableLen := 4 * (n + 1) total := tableLen @@ -327,7 +346,7 @@ func (c *clusterBuf) encode(noCompress bool) []byte { } payload := data if comp == compZstd { - payload = zstdEncode(data) + payload = compress(data) } else { comp = compNone }