From f3704021cdba96610ba7fc115b54c9e4272ebb4d Mon Sep 17 00:00:00 2001 From: Tam Nguyen Duc <1218621+tamnd@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:30:45 +0700 Subject: [PATCH] Add mandatory ZIM metadata for zimcheck (#14) * Add mandatory ZIM metadata for zimcheck ZIM archives were missing two pieces of metadata that the spec and zimcheck treat as mandatory: a Description and the Illustrator_48x48@1 favicon Kiwix shows as the book icon. A Name was missing too. Every archive now writes a Name and a Description, defaulting the description to a host-derived line when --description is not given. When the mirror has a usable icon, the favicon is rescaled to a 48x48 PNG and stored as Illustrator_48x48@1 with an image/png MIME, reusing the icon discovery and square-fit scaling the app packer already uses. AddMetadataBytes is added to the zim writer so a binary metadata value can carry its own MIME instead of being forced to text/plain. Verified by reading the output back through the libzim engine: all mandatory keys are present and the illustrator decodes as a 48x48 PNG. * Update docs for ZIM metadata and current flags Document the new mandatory metadata in the packing guide and the Kiwix compatibility note, and default --description in the CLI reference. While in the reference, bring it back in line with the code: add the --app and --icon pack flags (shipped in v0.2.0 but never documented), drop the --max-asset-mb clone flag that does not exist, and fix a stale --resume mention in the configuration layout. Add the v0.2.1 release notes and cut the changelog entry. --- CHANGELOG.md | 9 ++- docs/content/guides/packing-a-mirror.md | 6 +- docs/content/reference/cli.md | 5 +- docs/content/reference/configuration.md | 2 +- docs/content/reference/release-notes.md | 7 ++ pack/icon.go | 18 +++++- pack/pack_test.go | 85 +++++++++++++++++++++++++ pack/zim.go | 12 +++- zim/writer.go | 10 +++ 9 files changed, 144 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c59a1b5..fec618c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ All notable changes to kage are recorded here. The format follows ## [Unreleased] +## [0.2.1] - 2026-06-15 + +### Added + +- ZIM archives now carry the metadata Kiwix and `zimcheck` treat as mandatory. Every archive gets a `Name` and a `Description` (a host-derived line when `--description` is not given), and, when the mirror has a usable favicon, an `Illustrator_48x48@1` entry: the icon rescaled to a 48x48 PNG, which is the book icon Kiwix shows in its library. + ## [0.2.0] - 2026-06-15 ### Added @@ -109,7 +115,8 @@ can browse offline, with every script stripped out. a multi-arch container image on GHCR (Chromium bundled), checksums, SBOMs, and a cosign signature, all cut from one version tag by GoReleaser. -[Unreleased]: https://github.com/tamnd/kage/compare/v0.2.0...HEAD +[Unreleased]: https://github.com/tamnd/kage/compare/v0.2.1...HEAD +[0.2.1]: https://github.com/tamnd/kage/compare/v0.2.0...v0.2.1 [0.2.0]: https://github.com/tamnd/kage/compare/v0.1.2...v0.2.0 [0.1.2]: https://github.com/tamnd/kage/compare/v0.1.1...v0.1.2 [0.1.1]: https://github.com/tamnd/kage/compare/v0.1.0...v0.1.1 diff --git a/docs/content/guides/packing-a-mirror.md b/docs/content/guides/packing-a-mirror.md index cb0b939..5fae350 100644 --- a/docs/content/guides/packing-a-mirror.md +++ b/docs/content/guides/packing-a-mirror.md @@ -41,7 +41,7 @@ kage open paulgraham.com.zim # read it back with kage kiwix-serve paulgraham.com.zim # or serve it with Kiwix at http://localhost ``` -You can also double-click the file in the [Kiwix desktop app](https://kiwix.org/en/applications/), or load it on Kiwix for Android or iOS to read your mirror on your phone. One caveat: kage writes a structurally valid archive with the standard metadata, but it does not write the full-text search index that Kiwix's own packs ship with, so browsing works everywhere while in-reader search is limited. +You can also double-click the file in the [Kiwix desktop app](https://kiwix.org/en/applications/), or load it on Kiwix for Android or iOS to read your mirror on your phone. kage writes the metadata the format and `zimcheck` treat as mandatory, including the title, description, and the favicon Kiwix shows as the book icon in its library, so the archive shows up properly rather than as an untitled, iconless entry. One caveat: kage does not write the full-text search index that Kiwix's own packs ship with, so browsing works everywhere while in-reader search is limited. ## A self-contained binary @@ -149,4 +149,6 @@ kage pack paulgraham.com \ --date 2026-06-14 ``` -`--title` defaults to the main page's ``, then the host name. `--date` defaults to today; pass a fixed value for a fully reproducible file. `--no-compress` stores every cluster raw, which packs fastest and lets a reader without zstd open the result. `-o/--out` overrides the output path for either format. +`--title` defaults to the main page's `<title>`, then the host name. `--description` defaults to a line derived from the host, since a description is mandatory metadata. `--date` defaults to today; pass a fixed value for a fully reproducible file. `--no-compress` stores every cluster raw, which packs fastest and lets a reader without zstd open the result. `-o/--out` overrides the output path for either format. + +Beyond these, kage fills in the rest of the mandatory metadata on its own: a unique `Name`, the `Language`, and an `Illustrator_48x48@1` icon. The icon is the site's favicon dug out of the mirror (an `apple-touch-icon.png`, `favicon.png`, or a PNG-based `favicon.ico`) and rescaled to a 48x48 PNG, the same discovery `--app` uses for the desktop icon. A site that ships only a legacy BMP `.ico` or no icon at all is packed without one rather than with a broken image. diff --git a/docs/content/reference/cli.md b/docs/content/reference/cli.md index cac9cb4..14d5585 100644 --- a/docs/content/reference/cli.md +++ b/docs/content/reference/cli.md @@ -69,7 +69,6 @@ images, and fonts, and writes a browsable mirror to `<out>/<host>/`. | `--workers` | `4` | Concurrent page render workers | | `--asset-workers` | `8` | Concurrent asset download workers | | `--browser-pages` | `4` | Chrome page-pool size | -| `--max-asset-mb` | `25` | Skip assets larger than N MB | | `--timeout` | `30s` | Per-request timeout | | `-q, --quiet` | `false` | Suppress per-page progress lines | @@ -102,9 +101,11 @@ works right after `kage clone example.com`. | `--format` | `zim` | Output format: `zim` or `binary` | | `-o, --out` | per format | Output path; `<host>.zim` for zim, `<host>` (or `<host>.exe`) for binary | | `--base` | this kage | Base kage binary to append to (`--format binary`); point at another platform's binary to build a viewer for it | +| `--app` | `false` | Wrap the viewer in a double-click desktop app (`.app` on macOS, `.AppImage`/`.AppDir` on Linux) with the site's favicon as the icon | +| `--icon` | site favicon | Icon file for `--app`, overriding the favicon found in the mirror | | `--no-compress` | `false` | Store every cluster raw, no zstd | | `--title` | main page `<title>` | Archive title | -| `--description` | | Archive description | +| `--description` | host-derived line | Archive description (mandatory metadata, defaulted when unset) | | `--language` | `eng` | Archive language code | | `--date` | today | Archive date (`YYYY-MM-DD`); pass a fixed value for a reproducible file | diff --git a/docs/content/reference/configuration.md b/docs/content/reference/configuration.md index ce0d72e..e271a89 100644 --- a/docs/content/reference/configuration.md +++ b/docs/content/reference/configuration.md @@ -36,7 +36,7 @@ $HOME/data/kage/example.com/ │ │ ├── logo.png │ │ └── fonts/body.woff2 │ ├── cdn.example.com/ # assets from other hosts, by host -│ └── state.json # visited set, for --resume +│ └── state.json # visited set, for resume └── ... ``` diff --git a/docs/content/reference/release-notes.md b/docs/content/reference/release-notes.md index 16d936b..c5250c2 100644 --- a/docs/content/reference/release-notes.md +++ b/docs/content/reference/release-notes.md @@ -6,6 +6,13 @@ weight: 40 The authoritative, commit-level history lives in [`CHANGELOG.md`](https://github.com/tamnd/kage/blob/main/CHANGELOG.md) and on the [releases page](https://github.com/tamnd/kage/releases). This page summarises each version. +## v0.2.1 + +Packed ZIM archives now carry the metadata Kiwix expects, so a mirror shows up in a ZIM reader's library with a title, a description, and an icon instead of as a blank entry. + +- **Mandatory metadata is always written.** Every archive now gets a `Name` and a `Description` (a line derived from the host when `--description` is not given), the two fields `zimcheck` flags as missing otherwise. +- **The favicon becomes the book icon.** When the mirror has a usable icon (an `apple-touch-icon.png`, `favicon.png`, or a PNG-based `favicon.ico`), kage rescales it to a 48x48 PNG and stores it as `Illustrator_48x48@1`, which is the icon Kiwix shows for the archive in its library. A site with no usable icon is packed without one rather than with a broken image. + ## v0.2.0 Double-click apps, so a packed mirror opens like a real desktop app instead of a terminal program. diff --git a/pack/icon.go b/pack/icon.go index 03beb58..40640c9 100644 --- a/pack/icon.go +++ b/pack/icon.go @@ -15,7 +15,7 @@ import ( // explicitly below. _ "image/gif" _ "image/jpeg" - _ "image/png" + "image/png" ) // iconNames ranks the file names sites use for their icon, best first. A large @@ -44,6 +44,22 @@ func FindIcon(mirrorDir string) (image.Image, string, bool) { return nil, "", false } +// Favicon48 finds the mirror's icon and renders it to a 48x48 PNG, the form the +// ZIM Illustrator_48x48@1 metadata takes and the icon Kiwix shows for the book. +// It returns ok=false when the mirror has no usable icon, in which case the +// archive simply ships without one rather than failing the pack. +func Favicon48(mirrorDir string) ([]byte, bool) { + img, _, ok := FindIcon(mirrorDir) + if !ok { + return nil, false + } + var buf bytes.Buffer + if err := png.Encode(&buf, scaleSquare(img, 48)); err != nil { + return nil, false + } + return buf.Bytes(), true +} + // globIcon returns every file under dir whose base name equals name, nearest the // root first. Clones store assets under rewritten paths, so the icon may sit a // few directories deep rather than at the mirror root. diff --git a/pack/pack_test.go b/pack/pack_test.go index c03f8c8..448e6a9 100644 --- a/pack/pack_test.go +++ b/pack/pack_test.go @@ -3,6 +3,9 @@ package pack import ( "bytes" "encoding/binary" + "image" + "image/color" + "image/png" "io" "net/http" "net/http/httptest" @@ -134,6 +137,88 @@ func TestBuildZIMDeterministic(t *testing.T) { } } +// TestBuildZIMMetadata checks the metadata zimcheck treats as mandatory: a +// Title, Name, Language, Description, and the Illustrator_48x48@1 favicon. The +// mirror carries a real PNG favicon so the icon path is exercised end to end. +func TestBuildZIMMetadata(t *testing.T) { + host := writeMirror(t) + + // Drop a 64x64 PNG favicon into the mirror so Favicon48 has something to + // find, decode, and rescale to 48x48. + icon := image.NewRGBA(image.Rect(0, 0, 64, 64)) + for y := 0; y < 64; y++ { + for x := 0; x < 64; x++ { + icon.Set(x, y, color.RGBA{R: 0x33, G: 0x66, B: 0x99, A: 0xff}) + } + } + var pngBuf bytes.Buffer + if err := png.Encode(&pngBuf, icon); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(host, "favicon.png"), pngBuf.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + + out := filepath.Join(t.TempDir(), "meta.zim") + if _, _, err := BuildZIM(host, ZIMOptions{Out: out, Date: "2026-06-14", Version: "test"}); err != nil { + t.Fatalf("BuildZIM: %v", err) + } + r, err := zim.Open(out) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer func() { _ = r.Close() }() + + wantText := map[string]string{ + "Name": "example.com", + "Language": "eng", + "Description": "Offline mirror of example.com, cloned by kage.", + } + for k, want := range wantText { + e, err := r.Get(zim.NamespaceMetadata, k) + if err != nil { + t.Errorf("M/%s: %v", k, err) + continue + } + if string(e.Data) != want { + t.Errorf("M/%s = %q, want %q", k, e.Data, want) + } + } + + // Illustrator_48x48@1 is a 48x48 PNG with an image/png MIME. + ill, err := r.Get(zim.NamespaceMetadata, "Illustrator_48x48@1") + if err != nil { + t.Fatalf("M/Illustrator_48x48@1: %v", err) + } + if ill.MimeType != "image/png" { + t.Errorf("illustrator mime = %q, want image/png", ill.MimeType) + } + img, format, err := image.Decode(bytes.NewReader(ill.Data)) + if err != nil { + t.Fatalf("decode illustrator: %v", err) + } + if format != "png" { + t.Errorf("illustrator format = %q, want png", format) + } + if b := img.Bounds(); b.Dx() != 48 || b.Dy() != 48 { + t.Errorf("illustrator size = %dx%d, want 48x48", b.Dx(), b.Dy()) + } + + // A caller-supplied description overrides the host-derived default. + out2 := filepath.Join(t.TempDir(), "meta2.zim") + if _, _, err := BuildZIM(host, ZIMOptions{Out: out2, Description: "Custom blurb.", Date: "2026-06-14"}); err != nil { + t.Fatalf("BuildZIM: %v", err) + } + r2, err := zim.Open(out2) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer func() { _ = r2.Close() }() + if d, err := r2.Get(zim.NamespaceMetadata, "Description"); err != nil || string(d.Data) != "Custom blurb." { + t.Errorf("M/Description = %q, %v; want %q", d.Data, err, "Custom blurb.") + } +} + func TestPickMainPage(t *testing.T) { cases := []struct { in []string diff --git a/pack/zim.go b/pack/zim.go index a849238..f68eabd 100644 --- a/pack/zim.go +++ b/pack/zim.go @@ -134,10 +134,11 @@ func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) { host := filepath.Base(mirrorDir) title := firstNonEmpty(opts.Title, htmlTitleOf(mirrorDir, main), host) w.AddMetadata("Title", title) + w.AddMetadata("Name", host) w.AddMetadata("Language", firstNonEmpty(opts.Language, "eng")) - if opts.Description != "" { - w.AddMetadata("Description", opts.Description) - } + // Description is mandatory metadata in the ZIM spec, so it is always written: + // the caller's text when given, otherwise a line derived from the host. + w.AddMetadata("Description", firstNonEmpty(opts.Description, "Offline mirror of "+host+", cloned by kage.")) w.AddMetadata("Creator", "kage") w.AddMetadata("Publisher", "kage") if opts.Date != "" { @@ -146,6 +147,11 @@ func buildWriter(mirrorDir string, opts ZIMOptions) (*zim.Writer, error) { w.AddMetadata("Scraper", strings.TrimSpace("kage "+opts.Version)) w.AddMetadata("Source", host) w.AddMetadata("Counter", counterString(counts)) + // Illustrator_48x48@1 is the 48x48 PNG favicon Kiwix shows as the archive's + // icon. When the mirror has no usable icon the archive ships without one. + if png, ok := Favicon48(mirrorDir); ok { + w.AddMetadataBytes("Illustrator_48x48@1", "image/png", png) + } return w, nil } diff --git a/zim/writer.go b/zim/writer.go index 95b8f2e..d45000b 100644 --- a/zim/writer.go +++ b/zim/writer.go @@ -71,6 +71,16 @@ func (w *Writer) AddMetadata(name, value string) { w.put(&entry{namespace: NamespaceMetadata, url: name, title: name, mime: "text/plain", data: []byte(value)}) } +// AddMetadataBytes adds an 'M' namespace entry with an explicit MIME, for binary +// metadata such as Illustrator_48x48@1, the 48x48 PNG favicon Kiwix shows as the +// archive's icon. +func (w *Writer) AddMetadataBytes(name, mime string, data []byte) { + if mime == "" { + mime = "application/octet-stream" + } + w.put(&entry{namespace: NamespaceMetadata, url: name, title: name, mime: mime, data: data}) +} + // AddRedirect adds a redirect from (namespace,url) to (targetNamespace,targetURL). func (w *Writer) AddRedirect(namespace byte, url, title string, targetNamespace byte, targetURL string) { if title == "" {